Fixed furumi dashboard and web-petting limits

k8s/apps/web-petting/deployment.yaml

@@ -45,8 +45,8 @@ spec:
           value: "info"
         resources:
           requests:
-            memory: "64Mi"
+            memory: "256Mi"
-            cpu: "50m"
+            cpu: "350m"
           limits:
-            memory: "128Mi"
+            memory: "1Gi"
-            cpu: "150m"
+            cpu: "1000m"

k8s/core/prom-stack/dashboards/furumi-dashboard-cm.yaml

@@ -206,12 +206,13 @@ data:
         },
         {
           "id": 32,
-          "title": "Listen Quality",
+          "title": "Listen Completion Ratio",
+          "description": "Share of listens completed during the rolling 1h window. The wider window avoids 0/100% jumps when traffic is sparse.",
           "type": "timeseries",
           "gridPos": { "h": 7, "w": 8, "x": 8, "y": 29 },
-          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2 }, "overrides": [] },
+          "fieldConfig": { "defaults": { "unit": "percentunit", "decimals": 2, "min": 0, "max": 1 }, "overrides": [] },
           "targets": [
-            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(furumusic_listens_total{namespace=~\"$namespace\",completed=\"true\"}[$__rate_interval])) / clamp_min(sum(rate(furumusic_listens_total{namespace=~\"$namespace\"}[$__rate_interval])), 0.001)", "legendFormat": "completion ratio", "refId": "A" }
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listens_total{namespace=~\"$namespace\",completed=\"true\"}[1h])) / clamp_min(sum(increase(furumusic_listens_total{namespace=~\"$namespace\"}[1h])), 1)", "legendFormat": "completed / total", "refId": "A" }
           ],
           "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
         },
@@ -229,13 +230,19 @@ data:
         },
         {
           "id": 34,
-          "title": "Listened Hours and History",
+          "title": "Listened Hours and Play History Activity",
+          "description": "Per-bucket listening time and newly written play-history rows. This shows activity instead of the ever-growing history counter.",
           "type": "timeseries",
           "gridPos": { "h": 7, "w": 12, "x": 0, "y": 36 },
-          "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+          "fieldConfig": {
+            "defaults": { "unit": "h", "decimals": 2 },
+            "overrides": [
+              { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 0 }, { "id": "custom.axisPlacement", "value": "right" } ] }
+            ]
+          },
           "targets": [
-            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listened_seconds_total{namespace=~\"$namespace\"}[$__range])) / 3600", "legendFormat": "listened hours in range", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_listened_seconds_total{namespace=~\"$namespace\"}[$__rate_interval])) / 3600", "legendFormat": "listened hours per interval", "refId": "A" },
-            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_play_history_total{namespace=~\"$namespace\"})", "legendFormat": "history rows", "refId": "B" }
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(furumusic_play_history_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "history rows per interval", "refId": "B" }
           ],
           "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
         },
@@ -245,7 +252,7 @@ data:
           "title": "Library Inventory",
           "type": "timeseries",
           "gridPos": { "h": 7, "w": 8, "x": 0, "y": 44 },
-          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
           "targets": [
             { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_tracks_total{namespace=~\"$namespace\"})", "legendFormat": "tracks", "refId": "A" },
             { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(furumusic_library_releases_total{namespace=~\"$namespace\"})", "legendFormat": "releases", "refId": "B" },
@@ -256,13 +263,12 @@ data:
         },
         {
           "id": 42,
-          "title": "Media Files and Bytes by Type",
+          "title": "Media Bytes by Type",
           "type": "timeseries",
           "gridPos": { "h": 7, "w": 8, "x": 8, "y": 44 },
-          "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] },
+          "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
           "targets": [
-            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (type) (furumusic_media_files_total{namespace=~\"$namespace\"})", "legendFormat": "files {{type}}", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (type) (furumusic_media_file_bytes_total{namespace=~\"$namespace\"})", "legendFormat": "{{type}}", "refId": "A" }
-            { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (type) (furumusic_media_file_bytes_total{namespace=~\"$namespace\"})", "legendFormat": "bytes {{type}}", "refId": "B" }
           ],
           "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }
         },
@@ -283,7 +289,7 @@ data:
           "title": "Agent Queue and Reviews",
           "type": "timeseries",
           "gridPos": { "h": 7, "w": 8, "x": 0, "y": 52 },
-          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
+          "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] },
           "targets": [
             { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_queue_depth{namespace=~\"$namespace\"})", "legendFormat": "queue {{status}}", "refId": "A" },
             { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (status) (furumusic_agent_reviews_total{namespace=~\"$namespace\"})", "legendFormat": "reviews {{status}}", "refId": "B" }
@@ -365,10 +371,17 @@ data:
         },
         {
           "id": 58,
-          "title": "LLM Tokens and Batch Pressure",
+          "title": "LLM Token Rate, Batch Splits and Parse Failures",
+          "description": "Token series are tokens per second by model and token type. Batch splits and parse failures are operational events per second and use the right axis.",
           "type": "timeseries",
           "gridPos": { "h": 7, "w": 8, "x": 16, "y": 66 },
-          "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] },
+          "fieldConfig": {
+            "defaults": { "unit": "suffix: tok/s" },
+            "overrides": [
+              { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] },
+              { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.axisPlacement", "value": "right" } ] }
+            ]
+          },
           "targets": [
             { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (model, type) (rate(furumusic_agent_llm_tokens_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "tokens {{model}} {{type}}", "refId": "A" },
             { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (reason) (rate(furumusic_agent_llm_batch_splits_total{namespace=~\"$namespace\"}[$__rate_interval]))", "legendFormat": "splits {{reason}}", "refId": "B" },

k8s/core/prom-stack/dashboards/furumi-server.json

@@ -1,647 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "id": null,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 0,
-        "y": 0
-      },
-      "id": 1,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calc": "lastNotNull",
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto"
-      },
-      "pluginVersion": "10.0.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "expr": "furumi_active_streams",
-          "refId": "A"
-        }
-      ],
-      "title": "Active Streams",
-      "type": "stat"
-    },
-    {
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 6,
-        "y": 0
-      },
-      "id": 2,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calc": "rate",
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto"
-      },
-      "pluginVersion": "10.0.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "expr": "rate(furumi_bytes_read_total[$__rate_interval])",
-          "refId": "A"
-        }
-      ],
-      "title": "Bytes Read / Sec",
-      "type": "stat",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "Bps"
-        },
-        "overrides": []
-      }
-    },
-    {
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 12,
-        "y": 0
-      },
-      "id": 3,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calc": "lastNotNull",
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto"
-      },
-      "pluginVersion": "10.0.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "expr": "sum(increase(furumi_file_open_errors_total[$__rate_interval]))",
-          "refId": "A"
-        }
-      ],
-      "title": "File Open Errors (Rate)",
-      "type": "stat",
-       "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-             "mode": "absolute",
-             "steps": [
-               { "color": "green", "value": null },
-               { "color": "red", "value": 1 }
-             ]
-           }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "gridPos": {
-        "h": 4,
-        "w": 6,
-        "x": 18,
-        "y": 0
-      },
-      "id": 4,
-      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calc": "lastNotNull",
-          "fields": "",
-          "values": false
-        },
-        "textMode": "auto"
-      },
-      "pluginVersion": "10.0.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "expr": "sum(increase(furumi_auth_failures_total[$__rate_interval]))",
-          "refId": "A"
-        }
-      ],
-      "title": "Auth Failures (Rate)",
-      "type": "stat",
-       "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-             "mode": "absolute",
-             "steps": [
-               { "color": "green", "value": null },
-               { "color": "red", "value": 1 }
-             ]
-           }
-        },
-        "overrides": []
-      }
-    },
-    {
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 4
-      },
-      "id": 5,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.0.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "expr": "sum by (method, status) (rate(furumi_grpc_requests_total[$__rate_interval]))",
-          "legendFormat": "{{method}} - {{status}}",
-          "refId": "A"
-        }
-      ],
-      "title": "gRPC Request Rate by Method & Status",
-      "type": "timeseries",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 2,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "reqps"
-        },
-        "overrides": []
-      }
-    },
-    {
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 4
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.0.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "expr": "histogram_quantile(0.95, sum(rate(furumi_grpc_request_duration_seconds_bucket[$__rate_interval])) by (le, method))",
-          "legendFormat": "p95 {{method}}",
-          "refId": "A"
-        },
-         {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "expr": "histogram_quantile(0.99, sum(rate(furumi_grpc_request_duration_seconds_bucket[$__rate_interval])) by (le, method))",
-          "legendFormat": "p99 {{method}}",
-          "refId": "B"
-        }
-      ],
-      "title": "gRPC Request Duration (p95, p99)",
-      "type": "timeseries",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 2,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          },
-          "unit": "s"
-        },
-        "overrides": []
-      }
-    },
-    {
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 12
-      },
-      "id": 7,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.0.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "expr": "sum(process_resident_memory_bytes) / 1024 / 1024",
-          "legendFormat": "Resident Memory",
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "expr": "sum(process_virtual_memory_bytes) / 1024 / 1024",
-          "legendFormat": "Virtual Memory",
-          "refId": "B"
-        }
-      ],
-      "title": "Process Memory Usage",
-      "type": "timeseries",
-       "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 15,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 2,
-             "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-               { "color": "green", "value": null }
-            ]
-          },
-          "unit": "megbytes"
-        },
-        "overrides": []
-      }
-    },
-    {
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 12
-      },
-      "id": 8,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "10.0.0",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "expr": "process_open_fds",
-          "legendFormat": "Open FDs",
-          "refId": "A"
-        },
-        {
-           "datasource": {
-            "type": "prometheus",
-            "uid": "${datasource}"
-          },
-          "expr": "process_max_fds",
-          "legendFormat": "Max FDs",
-          "refId": "B"
-        }
-      ],
-      "title": "Process File Descriptors",
-      "type": "timeseries",
-       "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-             "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 2,
-            "pointSize": 5,
-            "scaleDistribution": {
-               "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-             "thresholdsStyle": {
-              "mode": "off"
-             }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-               { "color": "green", "value": null }
-            ]
-          },
-          "unit": "short"
-        },
-        "overrides": []
-      }
-    }
-  ],
-  "refresh": "10s",
-  "schemaVersion": 38,
-  "style": "dark",
-  "tags": [
-    "furumi-server",
-    "grpc"
-  ],
-  "templating": {
-    "list": [
-      {
-        "current": {
-          "selected": false,
-          "text": "Prometheus",
-          "value": "Prometheus"
-        },
-        "hide": 0,
-        "includeAll": false,
-        "multi": false,
-        "name": "datasource",
-        "options": [],
-        "query": "prometheus",
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "type": "datasource"
-      }
-    ]
-  },
-  "time": {
-    "from": "now-1h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "Furumi Server Metrics",
-  "uid": "furumi-metrics",
-  "version": 1
-}