{
  "annotations": { "list": [] },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
      "id": 100,
      "title": "Overview",
      "type": "row"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 },
      "id": 1,
      "targets": [
        {
          "expr": "sum(rate(altcraft_procevent_events_processed_total{result=\"ok\"}[5m]))",
          "legendFormat": "ok/s",
          "refId": "A"
        }
      ],
      "title": "Events Processed/s (total)",
      "description": "Overall procevent throughput. Depends on load and settings. A sharp drop indicates an RMQ problem or a crashed process.",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 },
      "id": 11,
      "targets": [
        {
          "expr": "sum(rate(altcraft_procevent_events_processed_total{result=~\"error|retry\"}[5m]))",
          "legendFormat": "error+retry/s",
          "refId": "A"
        }
      ],
      "title": "Errors/Retries/s (total)",
      "description": "Errors and retries. >0 — processing problems (MongoDB, validation, channels).",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 },
      "id": 2,
      "targets": [
        {
          "expr": "histogram_quantile(0.5, sum(rate(altcraft_procevent_event_processing_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "p50",
          "refId": "A"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(altcraft_procevent_event_processing_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "p95",
          "refId": "B"
        },
        {
          "expr": "histogram_quantile(0.99, sum(rate(altcraft_procevent_event_processing_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "p99",
          "refId": "C"
        }
      ],
      "title": "Event Processing Duration p50/p95/p99",
      "description": "Time to process a single event. p95 >50ms — a bottleneck (usually MongoDB). p99 >100ms — critical.",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
      "id": 200,
      "title": "Channel Blocking",
      "type": "row"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "s",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "yellow", "value": 0.1 },
              { "color": "red", "value": 1 }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 10 },
      "id": 3,
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(altcraft_procevent_mongo_update_send_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "mongo_update",
          "refId": "A"
        }
      ],
      "title": "Mongo Update Send p95",
      "description": "Time to send to the mgoUpdater channel. >0.1s — the channel is loaded. >1s — workers are blocked.",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "s",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "yellow", "value": 0.1 },
              { "color": "red", "value": 1 }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 10 },
      "id": 4,
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(altcraft_procevent_ssdb_hb_send_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "ssdb_hb",
          "refId": "A"
        }
      ],
      "title": "SSDB HB Send p95",
      "description": "Time to send a heartbeat to SSDB. >0.1s — the channel is loaded. NaN — the channel is not in use.",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 10 },
      "id": 5,
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(altcraft_procevent_hook_check_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "p95",
          "refId": "A"
        }
      ],
      "title": "Hook Check Duration p95",
      "description": "Time of the hook check in actionCache. >1s — the cache is not working or MongoDB is slow.",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
      "id": 300,
      "title": "ClickHouse",
      "type": "row"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 6, "x": 0, "y": 19 },
      "id": 6,
      "targets": [
        {
          "expr": "sum(rate(altcraft_procevent_ch_batch_size_sum[5m])) / sum(rate(altcraft_procevent_ch_batch_insert_duration_seconds_sum[5m]))",
          "legendFormat": "events/s",
          "refId": "A"
        }
      ],
      "title": "CH Insert Throughput (events/s)",
      "description": "Average event insertion rate into ClickHouse **per batcher**. If there are multiple batchers, multiply by their number. Not all events are written to CH.",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 6, "x": 6, "y": 19 },
      "id": 7,
      "targets": [
        {
          "expr": "histogram_quantile(0.5, sum(rate(altcraft_procevent_ch_batch_insert_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "p50",
          "refId": "A"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(altcraft_procevent_ch_batch_insert_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "p95",
          "refId": "B"
        },
        {
          "expr": "histogram_quantile(0.99, sum(rate(altcraft_procevent_ch_batch_insert_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "p99",
          "refId": "C"
        }
      ],
      "title": "CH Batch Insert Duration p50/p95/p99",
      "description": "Time to insert a batch into CH. p95 >10s — a CH problem or the batch is too large.",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 6, "x": 12, "y": 19 },
      "id": 8,
      "targets": [
        {
          "expr": "histogram_quantile(0.5, sum(rate(altcraft_procevent_ch_batch_size_bucket[5m])) by (le))",
          "legendFormat": "p50",
          "refId": "A"
        },
        {
          "expr": "histogram_quantile(0.95, sum(rate(altcraft_procevent_ch_batch_size_bucket[5m])) by (le))",
          "legendFormat": "p95",
          "refId": "B"
        }
      ],
      "title": "CH Batch Size p50/p95",
      "description": "Batch size. p50 << chBatchSize — flush on timeout (low traffic). p50 ≈ chBatchSize — flush on size.",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "s",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "yellow", "value": 0.01 },
              { "color": "red", "value": 0.1 }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 6, "x": 18, "y": 19 },
      "id": 9,
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(altcraft_procevent_ch_insert_send_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "p95",
          "refId": "A"
        }
      ],
      "title": "CH Insert Send Duration p95",
      "description": "Time to send to the batch writer insertChan. >0.01s — the channel is loaded. >0.1s — the batch writer is falling behind.",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
      "id": 400,
      "title": "Caches",
      "type": "row"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "thresholds" },
          "unit": "percentunit",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "red", "value": null },
              { "color": "yellow", "value": 0.5 },
              { "color": "green", "value": 0.8 }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 28 },
      "id": 9,
      "targets": [
        {
          "expr": "sum(rate(altcraft_procevent_lead_cache_lookup_total{result=\"hit\"}[5m])) / sum(rate(altcraft_procevent_lead_cache_lookup_total[5m]))",
          "legendFormat": "lead_cache",
          "refId": "A"
        }
      ],
      "title": "Lead Cache Hit Rate",
      "description": "leadCache hit rate. 0% — the cache is not being populated (a leadCache.Set bug). Normal is >80%.",
      "type": "stat"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "thresholds" },
          "unit": "percentunit",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "red", "value": null },
              { "color": "yellow", "value": 0.5 },
              { "color": "green", "value": 0.8 }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 28 },
      "id": 10,
      "targets": [
        {
          "expr": "sum(rate(altcraft_procevent_gcg_cache_lookup_total{result=\"hit\"}[5m])) / sum(rate(altcraft_procevent_gcg_cache_lookup_total[5m]))",
          "legendFormat": "gcg_cache",
          "refId": "A"
        }
      ],
      "title": "GCG Cache Hit Rate",
      "description": "gcgCache hit rate. Normal is >95%.",
      "type": "stat"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "s",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "yellow", "value": 0.05 },
              { "color": "red", "value": 0.5 }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 28 },
      "id": 11,
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(altcraft_procevent_find_lead_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "p95",
          "refId": "A"
        }
      ],
      "title": "Find Lead Duration p95 (MongoDB on miss)",
      "description": "MongoDB lookup time on a leadCache miss. >0.05s — MongoDB is loaded. >0.5s — critical.",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 },
      "id": 500,
      "title": "Hook Check",
      "type": "row"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "percentunit",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "red", "value": null },
              { "color": "yellow", "value": 0.5 },
              { "color": "green", "value": 0.8 }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 36 },
      "id": 12,
      "targets": [
        {
          "expr": "sum(rate(altcraft_procevent_hook_check_total{result=\"hit\"}[5m])) / sum(rate(altcraft_procevent_hook_check_total[5m]))",
          "legendFormat": "hit rate",
          "refId": "A"
        }
      ],
      "title": "Hook Check Hit Rate (total)",
      "description": "Share of events with a configured hook. 0% — no hook configured. 100% — all events send hook actions.",
      "type": "stat"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 36 },
      "id": 13,
      "targets": [
        {
          "expr": "sum(rate(altcraft_procevent_hook_check_total[5m]))",
          "legendFormat": "checks/s",
          "refId": "A"
        }
      ],
      "title": "Hook Check Throughput (total)",
      "description": "Hook checks per second. A subset of the overall throughput.",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "prometheus-2" },
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 36 },
      "id": 14,
      "targets": [
        {
          "expr": "histogram_quantile(0.95, sum(rate(altcraft_procevent_hook_check_duration_seconds_bucket[5m])) by (le))",
          "legendFormat": "p95",
          "refId": "A"
        }
      ],
      "title": "Hook Check Duration p95",
      "description": "Hook check time. >1s — an actionCache problem.",
      "type": "timeseries"
    }
  ],
  "refresh": "10s",
  "schemaVersion": 38,
  "style": "dark",
  "tags": ["procevent", "altcraft"],
  "templating": { "list": [] },
  "time": { "from": "now-1h", "to": "now" },
  "timepicker": {},
  "timezone": "",
  "title": "Procevent Monitoring",
  "uid": "procevent-monitoring",
  "version": 1
}