{ "dashboard": { "title": "SASE v2 — Observability", "description": "API traces, frontend RUM (Faro), endpoint performance", "tags": ["sase", "otel", "faro"], "timezone": "Europe/Istanbul", "refresh": "30s", "time": { "from": "now-1h", "to": "now" }, "templating": { "list": [ { "name": "service", "type": "custom", "current": { "text": "sase-api", "value": "sase-api" }, "options": [ { "text": "sase-api", "value": "sase-api", "selected": true }, { "text": "sase-worker", "value": "sase-worker", "selected": false } ], "query": "sase-api,sase-worker" } ] }, "panels": [ { "type": "row", "title": "Overview", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false }, { "type": "stat", "title": "Request Rate (req/s)", "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"$service\"} | rate()" } ], "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 0.1 }, { "color": "green", "value": 0.5 } ] }, "unit": "reqps", "decimals": 2 } }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "area" } }, { "type": "stat", "title": "Error Rate", "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"$service\" && status=error} | rate()" } ], "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.01 }, { "color": "red", "value": 0.1 } ] }, "unit": "reqps", "decimals": 3 } }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "area" } }, { "type": "stat", "title": "P95 Latency", "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"$service\"} | quantile_over_time(duration, 0.95)" } ], "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 500000000 }, { "color": "orange", "value": 1000000000 }, { "color": "red", "value": 3000000000 } ] }, "unit": "ns", "decimals": 0 } }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "area" } }, { "type": "stat", "title": "Service Info", "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, "datasource": { "type": "prometheus", "uid": "grafanacloud-prom" }, "targets": [ { "refId": "A", "expr": "count(target_info)", "legendFormat": "Active Services" } ], "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "blue" }, "unit": "short" } }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" } }, { "type": "row", "title": "Request Rate & Latency", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "collapsed": false }, { "type": "timeseries", "title": "Request Rate Over Time", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"$service\"} | rate()", "legendFormat": "Total" }, { "refId": "B", "queryType": "traceql", "query": "{resource.service.name=\"$service\" && status=error} | rate()", "legendFormat": "Errors" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "pointSize": 5, "lineWidth": 2, "showPoints": "auto" } }, "overrides": [ { "matcher": { "id": "byName", "options": "Errors" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] } ] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "Latency Percentiles", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"$service\"} | quantile_over_time(duration, 0.50)", "legendFormat": "P50" }, { "refId": "B", "queryType": "traceql", "query": "{resource.service.name=\"$service\"} | quantile_over_time(duration, 0.95)", "legendFormat": "P95" }, { "refId": "C", "queryType": "traceql", "query": "{resource.service.name=\"$service\"} | quantile_over_time(duration, 0.99)", "legendFormat": "P99" } ], "fieldConfig": { "defaults": { "unit": "ns", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10, "lineWidth": 2, "showPoints": "auto" } }, "overrides": [ { "matcher": { "id": "byName", "options": "P50" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] }, { "matcher": { "id": "byName", "options": "P95" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } }] }, { "matcher": { "id": "byName", "options": "P99" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] } ] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "row", "title": "Endpoints", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "collapsed": false }, { "type": "timeseries", "title": "Rate by Endpoint (Top 10)", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"$service\" && kind=server} | rate() by (name)" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "lineWidth": 2, "stacking": { "mode": "normal" } } } }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "P95 Latency by Endpoint", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"$service\" && kind=server} | quantile_over_time(duration, 0.95) by (name)" } ], "fieldConfig": { "defaults": { "unit": "ns", "custom": { "drawStyle": "bars", "fillOpacity": 50, "lineWidth": 1, "stacking": { "mode": "none" } } } }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi" } } }, { "type": "row", "title": "Errors & Slow Requests", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "collapsed": false }, { "type": "timeseries", "title": "Error Rate by Endpoint", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"$service\" && status=error && kind=server} | rate() by (name)" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "bars", "fillOpacity": 50, "lineWidth": 1 }, "color": { "mode": "palette-classic-by-name" } } }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, "tooltip": { "mode": "multi" } } }, { "type": "table", "title": "Recent Errors", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceqlSearch", "query": "{resource.service.name=\"$service\" && status=error}", "limit": 20, "tableType": "traces" } ], "options": { "showHeader": true, "sortBy": [{ "displayName": "Start time", "desc": true }] } }, { "type": "row", "title": "VIN Decode & Database", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "collapsed": false }, { "type": "timeseries", "title": "VIN Decode Rate", "gridPos": { "h": 8, "w": 8, "x": 0, "y": 33 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\"POST /api/vehicles/decode\"} | rate()", "legendFormat": "VIN Decode" }, { "refId": "B", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\"GET /api/vehicles/preview\"} | rate()", "legendFormat": "VIN Preview" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "lineWidth": 2 } } }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "VIN Decode Latency (P95)", "gridPos": { "h": 8, "w": 8, "x": 8, "y": 33 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\"POST /api/vehicles/decode\"} | quantile_over_time(duration, 0.95)", "legendFormat": "VIN Decode P95" }, { "refId": "B", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\"GET /api/vehicles/preview\"} | quantile_over_time(duration, 0.95)", "legendFormat": "VIN Preview P95" } ], "fieldConfig": { "defaults": { "unit": "ns", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "lineWidth": 2 } } }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "Database Queries", "gridPos": { "h": 8, "w": 8, "x": 16, "y": 33 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"$service\" && span.db.system=\"postgresql\"} | rate()", "legendFormat": "DB Queries/s" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "lineWidth": 2 }, "color": { "mode": "fixed", "fixedColor": "purple" } } }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "row", "title": "Worker & Background Jobs", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, "collapsed": false }, { "type": "timeseries", "title": "Worker Job Rate", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\".*emex-scrape.*\"} | rate()", "legendFormat": "EMEX Scrape" }, { "refId": "B", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\".*subscription-expiry.*\"} | rate()", "legendFormat": "Subscription Expiry" }, { "refId": "C", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\".*query-cleanup.*\"} | rate()", "legendFormat": "Query Cleanup" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "lineWidth": 2 } } }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "table", "title": "Recent Traces", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceqlSearch", "query": "{resource.service.name=\"$service\"}", "limit": 20, "tableType": "traces" } ], "options": { "showHeader": true, "sortBy": [{ "displayName": "Start time", "desc": true }] } }, { "type": "row", "title": "Auth & Subscription", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, "collapsed": false }, { "type": "timeseries", "title": "Auth Requests Rate", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\".*auth.*\"} | rate()", "legendFormat": "Auth" }, { "refId": "B", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\".*subscriptions.*\"} | rate()", "legendFormat": "Subscriptions" }, { "refId": "C", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\".*payments.*\"} | rate()", "legendFormat": "Payments" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "lineWidth": 2 } } }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "Auth & Payment Latency (P95)", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, "datasource": { "type": "tempo", "uid": "grafanacloud-traces" }, "targets": [ { "refId": "A", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\".*auth.*\"} | quantile_over_time(duration, 0.95)", "legendFormat": "Auth P95" }, { "refId": "B", "queryType": "traceql", "query": "{resource.service.name=\"sase-api\" && name=~\".*payments.*\"} | quantile_over_time(duration, 0.95)", "legendFormat": "Payments P95" } ], "fieldConfig": { "defaults": { "unit": "ns", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "lineWidth": 2 } } }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "row", "title": "Frontend — Faro RUM", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 59 }, "collapsed": false }, { "type": "stat", "title": "Page Views", "gridPos": { "h": 4, "w": 4, "x": 0, "y": 60 }, "datasource": { "type": "loki", "uid": "grafanacloud-logs" }, "targets": [ { "refId": "A", "expr": "count_over_time({service_name=\"saseweb\", kind=\"event\"} | logfmt | event_name=\"faro.performance.navigation\" [$__range])" } ], "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "blue" }, "unit": "short" } }, "options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background", "graphMode": "area" } }, { "type": "stat", "title": "User Actions", "gridPos": { "h": 4, "w": 4, "x": 4, "y": 60 }, "datasource": { "type": "loki", "uid": "grafanacloud-logs" }, "targets": [ { "refId": "A", "expr": "count_over_time({service_name=\"saseweb\", kind=\"event\"} | logfmt | event_name=\"faro.user.action\" [$__range])" } ], "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "green" }, "unit": "short" } }, "options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background", "graphMode": "area" } }, { "type": "stat", "title": "Fetch Requests", "gridPos": { "h": 4, "w": 4, "x": 8, "y": 60 }, "datasource": { "type": "loki", "uid": "grafanacloud-logs" }, "targets": [ { "refId": "A", "expr": "count_over_time({service_name=\"saseweb\", kind=\"event\"} | logfmt | event_name=\"faro.tracing.fetch\" [$__range])" } ], "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "purple" }, "unit": "short" } }, "options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background", "graphMode": "area" } }, { "type": "stat", "title": "Sessions", "gridPos": { "h": 4, "w": 4, "x": 12, "y": 60 }, "datasource": { "type": "loki", "uid": "grafanacloud-logs" }, "targets": [ { "refId": "A", "expr": "count_over_time({service_name=\"saseweb\", kind=\"event\"} | logfmt | event_name=\"session_start\" [$__range])" } ], "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "orange" }, "unit": "short" } }, "options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background", "graphMode": "area" } }, { "type": "stat", "title": "Web Vitals", "gridPos": { "h": 4, "w": 4, "x": 16, "y": 60 }, "datasource": { "type": "loki", "uid": "grafanacloud-logs" }, "targets": [ { "refId": "A", "expr": "count_over_time({service_name=\"saseweb\", kind=\"measurement\"} | logfmt | type=\"web-vitals\" [$__range])" } ], "fieldConfig": { "defaults": { "color": { "mode": "fixed", "fixedColor": "yellow" }, "unit": "short" } }, "options": { "reduceOptions": { "calcs": ["sum"] }, "colorMode": "background", "graphMode": "area" } }, { "type": "text", "title": "", "gridPos": { "h": 4, "w": 4, "x": 20, "y": 60 }, "options": { "mode": "markdown", "content": "### Faro Links\n\n[Frontend Observability](https://semihy.grafana.net/a/grafana-kowalski-app/apps)\n\n[Explore Logs](https://semihy.grafana.net/explore?left=%7B%22datasource%22:%22grafanacloud-logs%22%7D)" } }, { "type": "timeseries", "title": "User Actions Over Time", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 64 }, "datasource": { "type": "loki", "uid": "grafanacloud-logs" }, "targets": [ { "refId": "A", "expr": "sum by (action_name) (count_over_time({service_name=\"saseweb\", kind=\"event\"} | logfmt | event_name=\"faro.user.action\" [1m]))", "legendFormat": "{{action_name}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 60, "lineWidth": 1, "stacking": { "mode": "normal" } } } }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "Frontend Events Over Time", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 64 }, "datasource": { "type": "loki", "uid": "grafanacloud-logs" }, "targets": [ { "refId": "A", "expr": "sum by (event_name) (count_over_time({service_name=\"saseweb\", kind=\"event\"} | logfmt [1m]))", "legendFormat": "{{event_name}}" } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 15, "lineWidth": 2 } } }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, "tooltip": { "mode": "multi" } } }, { "type": "table", "title": "User Actions Log", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 72 }, "datasource": { "type": "loki", "uid": "grafanacloud-logs" }, "targets": [ { "refId": "A", "expr": "{service_name=\"saseweb\", kind=\"event\"} | logfmt | event_name=\"faro.user.action\" | label_format timestamp=`{{.timestamp}}` | line_format \"{{.action_name}} | duration={{.event_data_userActionDuration}}ms | trigger={{.event_data_userActionTrigger}} | page={{.page_id}}\"", "maxLines": 50 } ], "options": { "showHeader": true, "sortBy": [{ "displayName": "Time", "desc": true }] } }, { "type": "table", "title": "Page Navigations", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 72 }, "datasource": { "type": "loki", "uid": "grafanacloud-logs" }, "targets": [ { "refId": "A", "expr": "{service_name=\"saseweb\", kind=\"event\"} | logfmt | event_name=\"faro.performance.navigation\" | line_format \"page={{.page_url}}\"", "maxLines": 50 } ], "options": { "showHeader": true, "sortBy": [{ "displayName": "Time", "desc": true }] } } ], "schemaVersion": 39 }, "folderUid": "", "overwrite": true }