diff --git a/pinecone-mixin/Makefile b/pinecone-mixin/Makefile new file mode 100644 index 000000000..37cc871c1 --- /dev/null +++ b/pinecone-mixin/Makefile @@ -0,0 +1 @@ +include ../Makefile_mixin \ No newline at end of file diff --git a/pinecone-mixin/README.md b/pinecone-mixin/README.md new file mode 100644 index 000000000..9ef96ba6d --- /dev/null +++ b/pinecone-mixin/README.md @@ -0,0 +1,126 @@ +# Pinecone Mixin + +The Pinecone mixin is a set of configurable Grafana dashboards and alerts based on the metrics exported by the [Pinecone Prometheus exporter](https://docs.pinecone.io/guides/production/monitoring#monitor-with-prometheus). + +## Dashboards + +- **Pinecone Overview** - Provides details on index capacity, operation metrics (upsert, query, fetch, delete), operation durations, and resource usage (read/write units). + +## Alerts + +| Alert | Summary | +| ----- | ------- | +| PineconeHighQueryLatency | Query latency exceeds baseline thresholds, indicating performance degradation in query operations. | +| PineconeHighUpsertLatency | Upsert latency exceeds baseline thresholds, indicating performance degradation in upsert operations. | +| PineconeHighErrorRate | Error rate exceeds configured thresholds, indicating increased request failures. | +| PineconeHighStorageUsage | Index storage usage is high, risking degraded performance. | +| PineconeHighUnitConsumption | Read or write unit consumption is increasing rapidly or nearing allocated limits. | + +Alert thresholds can be configured in `config.libsonnet`. See the generated `prometheus_alerts.yaml` for default values. + +## Configuration + +This mixin is designed to work with Pinecone's built-in Prometheus exporter, which is available on Standard and Enterprise plans. + +### Alloy + +To monitor all serverless indexes in a project using Alloy, add the following to your Alloy configuration: + +```alloy +prometheus.scrape "pinecone" { + targets = discovery.http "pinecone" { + url = "https://api.pinecone.io/prometheus/projects/PROJECT_ID/metrics/discovery" + refresh_interval = "1m" + authorization { + type = "Bearer" + credentials = "API_KEY" + } + }.targets + forward_to = [prometheus.remote_write.metrics.receiver] +} + +prometheus.remote_write "metrics" { + endpoint { + url = "YOUR_PROMETHEUS_REMOTE_WRITE_ENDPOINT" + } +} +``` + +Replace `PROJECT_ID` and `API_KEY` with your Pinecone project ID and API key. For more details, see the [Pinecone monitoring documentation](https://docs.pinecone.io/guides/production/monitoring#monitor-with-prometheus). + +**Note:** If you have more than one Pinecone project, you need to add separate scrape configurations for each project with different project IDs and targets. It is recommended to add a `project_id` label via relabeling to distinguish metrics from different projects. + +#### Alloy (Multiple Projects) + +```alloy +discovery.http "pinecone_project_1" { + url = "https://api.pinecone.io/prometheus/projects/PROJECT_ID_1/metrics/discovery" + refresh_interval = "1m" + authorization { + type = "Bearer" + credentials = "API_KEY_1" + } +} + +prometheus.scrape "pinecone_project_1" { + targets = discovery.http.pinecone_project_1.targets + forward_to = [prometheus.remote_write.metrics.receiver] + + relabel { + source_labels = ["__meta_http_sd_url"] + regex = ".*/projects/([^/]+)/.*" + target_label = "project_id" + replacement = "${1}" + } +} + +discovery.http "pinecone_project_2" { + url = "https://api.pinecone.io/prometheus/projects/PROJECT_ID_2/metrics/discovery" + refresh_interval = "1m" + authorization { + type = "Bearer" + credentials = "API_KEY_2" + } +} + +prometheus.scrape "pinecone_project_2" { + targets = discovery.http.pinecone_project_2.targets + forward_to = [prometheus.remote_write.metrics.receiver] + + relabel { + source_labels = ["__meta_http_sd_url"] + regex = ".*/projects/([^/]+)/.*" + target_label = "project_id" + replacement = "${1}" + } +} + +prometheus.remote_write "metrics" { + endpoint { + url = "YOUR_PROMETHEUS_REMOTE_WRITE_ENDPOINT" + } +} +``` + +## Install Tools + +To use this mixin, a working Golang toolchain is required, alongside having `mixtool` and `jsonnetfmt` installed. +To do so, run the following: + +```bash +go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest +go install github.com/monitoring-mixins/mixtool/cmd/mixtool@latest +go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest +``` + +## Generate Dashboards and Alerts + +Edit `config.libsonnet` if required and then build JSON dashboard files for Grafana: + +```bash +make +``` + +The files in `dashboards_out` need to be imported into your Grafana server. The `prometheus_alerts.yaml` file needs to be imported into Prometheus. + +For more advanced uses of mixins, see [Prometheus Monitoring Mixins docs](https://github.com/monitoring-mixins/docs). diff --git a/pinecone-mixin/alerts/alerts.libsonnet b/pinecone-mixin/alerts/alerts.libsonnet new file mode 100644 index 000000000..8468555c4 --- /dev/null +++ b/pinecone-mixin/alerts/alerts.libsonnet @@ -0,0 +1,14 @@ +{ + new(this): { + prometheusAlerts+:: { + groups+: [ + { + name: 'pinecone', + rules: [ + // Add your alert rules here + ], + }, + ], + }, + }, +} diff --git a/pinecone-mixin/config.libsonnet b/pinecone-mixin/config.libsonnet new file mode 100644 index 000000000..e1cc4eae6 --- /dev/null +++ b/pinecone-mixin/config.libsonnet @@ -0,0 +1,26 @@ +{ + local this = self, + dashboardTags: ['pinecone'], + dashboardPeriod: 'now-1h', + dashboardTimezone: 'default', + dashboardRefresh: '1m', + dashboardNamePrefix: 'Pinecone', + uid: 'pinecone', + + // Filtering and labels + filteringSelector: 'job=~"$job"', + groupLabels: ['job'], // Job label for filtering + instanceLabels: ['instance', 'index_name'], // Instance and index_name labels + + // Metrics source + metricsSource: ['prometheus'], + + // Signals + signals: { + operations: (import './signals/operations.libsonnet')(this), + overview: (import './signals/overview.libsonnet')(this), + }, + + // Feature flags + enableLokiLogs: false, +} diff --git a/pinecone-mixin/dashboards.libsonnet b/pinecone-mixin/dashboards.libsonnet new file mode 100644 index 000000000..1131b618f --- /dev/null +++ b/pinecone-mixin/dashboards.libsonnet @@ -0,0 +1,25 @@ +local g = import '../g.libsonnet'; + +{ + new(this): + { + 'pinecone-overview.json': + g.dashboard.new(this.config.dashboardNamePrefix + ' overview') + + g.dashboard.withUid(this.config.uid + '-overview') + + g.dashboard.withTags(this.config.dashboardTags) + + g.dashboard.withTimezone(this.config.dashboardTimezone) + + g.dashboard.withRefresh(this.config.dashboardRefresh) + + g.dashboard.time.withFrom(this.config.dashboardPeriod) + + g.dashboard.withVariables( + this.signals.operations.getVariablesMultiChoice() + ) + + g.dashboard.withPanels( + g.util.grid.wrapPanels( + this.grafana.rows.overview + + this.grafana.rows.writeOperations + + this.grafana.rows.readOperations + + this.grafana.rows.resourceUsage + ) + ), + }, +} diff --git a/pinecone-mixin/g.libsonnet b/pinecone-mixin/g.libsonnet new file mode 100644 index 000000000..e6a2060ee --- /dev/null +++ b/pinecone-mixin/g.libsonnet @@ -0,0 +1 @@ +import 'github.com/grafana/grafonnet/gen/grafonnet-v11.4.0/main.libsonnet' diff --git a/pinecone-mixin/jsonnetfile.json b/pinecone-mixin/jsonnetfile.json new file mode 100644 index 000000000..4ad412ad0 --- /dev/null +++ b/pinecone-mixin/jsonnetfile.json @@ -0,0 +1,26 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet.git", + "subdir": "gen/grafonnet-v11.4.0" + } + }, + "version": "main" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "common-lib" + } + }, + "version": "master" + } + ], + "legacyImports": true +} + + diff --git a/pinecone-mixin/main.libsonnet b/pinecone-mixin/main.libsonnet new file mode 100644 index 000000000..a13982fdf --- /dev/null +++ b/pinecone-mixin/main.libsonnet @@ -0,0 +1,56 @@ +local alerts = import './alerts/alerts.libsonnet'; +local config = import './config.libsonnet'; +local dashboards = import './dashboards.libsonnet'; +local panels = import './panels.libsonnet'; +local rows = import './rows.libsonnet'; +local commonlib = import 'common-lib/common/main.libsonnet'; + +{ + new(): { + local this = self, + config: config, + + signals: + { + operations: commonlib.signals.unmarshallJsonMulti( + this.config.signals.operations, + type=this.config.metricsSource + ), + overview: commonlib.signals.unmarshallJsonMulti( + this.config.signals.overview, + type=this.config.metricsSource + ), + }, + + grafana: { + variables: commonlib.variables.new( + filteringSelector=this.config.filteringSelector, + groupLabels=this.config.groupLabels, + instanceLabels=this.config.instanceLabels, + varMetric='pinecone_db_record_total', + customAllValue='.+', + enableLokiLogs=this.config.enableLokiLogs, + ), + annotations: {}, + links: {}, + panels: panels.new(this), + dashboards: dashboards.new(this), + rows: rows.new(this), + }, + + prometheus: { + alerts: alerts.new(this), + recordingRules: {}, + }, + + asMonitoringMixin(): { + grafanaDashboards+:: this.grafana.dashboards, + prometheusAlerts+:: this.prometheus.alerts, + prometheusRules+:: this.prometheus.recordingRules, + }, + }, + + withConfigMixin(config): { + config+: config, + }, +} diff --git a/pinecone-mixin/mixin.libsonnet b/pinecone-mixin/mixin.libsonnet new file mode 100644 index 000000000..6f095a256 --- /dev/null +++ b/pinecone-mixin/mixin.libsonnet @@ -0,0 +1 @@ +(import './main.libsonnet').new().asMonitoringMixin() diff --git a/pinecone-mixin/panels.libsonnet b/pinecone-mixin/panels.libsonnet new file mode 100644 index 000000000..97d8a39c8 --- /dev/null +++ b/pinecone-mixin/panels.libsonnet @@ -0,0 +1,171 @@ +local g = import './g.libsonnet'; +local commonlib = import 'common-lib/common/main.libsonnet'; + +{ + new(this): { + local signals = this.signals.operations, + local overviewSignals = this.signals.overview, + + // Overview stat panels + indexesCountStat: + overviewSignals.indexesCount.asStat() + + commonlib.panels.generic.stat.info.stylize(), + + totalRecordsStat: + overviewSignals.totalRecords.asStat() + + commonlib.panels.generic.stat.info.stylize(), + + totalStorageStat: + overviewSignals.totalStorage.asStat() + + commonlib.panels.generic.stat.info.stylize(), + + // Overview aggregate time series panels + totalOperationsPerSec: + overviewSignals.totalOperationsPerSec.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.standardOptions.color.withMode('fixed') + + g.panel.timeSeries.standardOptions.color.withFixedColor('light-purple') + + g.panel.timeSeries.panelOptions.withDescription('Total operations per second across all indexes (read + write).'), + + totalReadWriteOperations: + commonlib.panels.generic.timeSeries.base.new('Read vs Write Operations', targets=[]) + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.queryOptions.withDatasource('prometheus', '${datasource}') + + g.panel.timeSeries.standardOptions.withUnit('reqps') + + overviewSignals.totalReadOperationsPerSec.asPanelMixin() + + overviewSignals.totalWriteOperationsPerSec.asPanelMixin() + + g.panel.timeSeries.fieldConfig.defaults.custom.withAxisLabel('write(-) | read(+)') + + g.panel.timeSeries.fieldConfig.defaults.custom.withAxisCenteredZero(true) + + g.panel.timeSeries.fieldConfig.defaults.custom.withStacking({ mode: 'none' }) + + g.panel.timeSeries.fieldConfig.defaults.custom.withDrawStyle('line') + + g.panel.timeSeries.standardOptions.withOverrides( + g.panel.timeSeries.fieldOverride.byRegexp.new('/Write|write/') + + g.panel.timeSeries.fieldOverride.byRegexp.withPropertiesFromOptions( + g.panel.timeSeries.fieldConfig.defaults.custom.withTransform('negative-Y') + ) + ) + + g.panel.timeSeries.panelOptions.withDescription('Total read and write operations per second across all indexes. Writes are shown on the negative y-axis, reads on the positive y-axis.'), + + // Overview table panel showing all indexes + indexesTable: + commonlib.panels.generic.table.base.new( + 'Indexes Overview', + targets=[ + signals.recordTotal.asTableTarget(), + signals.storageSizeBytes.asTableTarget(), + ], + description='Overview of all indexes showing total records and storage size per index.', + ) + + g.panel.table.queryOptions.withTransformations([ + // Filter to include only needed columns + g.panel.table.queryOptions.transformation.withId('filterFieldsByName') + + g.panel.table.queryOptions.transformation.withOptions({ + include: { + names: [ + 'index_name', + 'instance', + 'Value #Total records', + 'Value #Storage size', + ], + }, + }), + // Merge the two queries + g.panel.table.queryOptions.transformation.withId('merge') + + g.panel.table.queryOptions.transformation.withOptions({}), + // Organize and rename columns + g.panel.table.queryOptions.transformation.withId('organize') + + g.panel.table.queryOptions.transformation.withOptions({ + excludeByName: {}, + indexByName: {}, + renameByName: { + index_name: 'Index Name', + instance: 'Instance', + 'Value #Total records': 'Total Records', + 'Value #Storage size': 'Storage Size', + }, + includeByName: {}, + }), + ]) + + g.panel.table.standardOptions.withOverridesMixin([ + // Configure Total Records column + g.panel.table.fieldOverride.byName.new('Total Records') + + g.panel.table.fieldOverride.byName.withPropertiesFromOptions( + g.panel.table.standardOptions.withUnit('short') + ), + // Configure Storage Size column + g.panel.table.fieldOverride.byName.new('Storage Size') + + g.panel.table.fieldOverride.byName.withPropertiesFromOptions( + g.panel.table.standardOptions.withUnit('decbytes') + ), + ]), + + // Write operation panels + upsertTotal: + signals.upsertTotal.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.standardOptions.color.withMode('continuous-BlPu') + + g.panel.timeSeries.panelOptions.withDescription('Rate of upsert requests per index.'), + + upsertDuration: + signals.upsertDuration.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.standardOptions.withUnit('ms') + + g.panel.timeSeries.standardOptions.color.withMode('continuous-BlPu') + + g.panel.timeSeries.panelOptions.withDescription('Average upsert operation duration per index.'), + + // Read operation panels + queryTotal: + signals.queryTotal.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.standardOptions.color.withMode('palette-classic-by-name') + + g.panel.timeSeries.panelOptions.withDescription('Rate of query requests per index.'), + + queryDuration: + signals.queryDuration.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.standardOptions.withUnit('ms') + + g.panel.timeSeries.standardOptions.color.withMode('palette-classic-by-name') + + g.panel.timeSeries.panelOptions.withDescription('Average query operation duration per index.'), + + // Fetch + fetchTotal: + signals.fetchTotal.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.standardOptions.color.withMode('continuous-BlPu') + + g.panel.timeSeries.panelOptions.withDescription('Rate of fetch requests per index.'), + + fetchDuration: + signals.fetchDuration.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.standardOptions.withUnit('ms') + + g.panel.timeSeries.standardOptions.color.withMode('continuous-BlPu') + + g.panel.timeSeries.panelOptions.withDescription('Average fetch operation duration per index.'), + + // Delete + deleteTotal: + signals.deleteTotal.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.standardOptions.color.withMode('palette-classic-by-name') + + g.panel.timeSeries.panelOptions.withDescription('Rate of delete requests per index.'), + + deleteDuration: + signals.deleteDuration.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.standardOptions.withUnit('ms') + + g.panel.timeSeries.standardOptions.color.withMode('palette-classic-by-name') + + g.panel.timeSeries.panelOptions.withDescription('Average delete operation duration per index.'), + + // Resource usage panels + writeUnitsTotal: + signals.writeUnitsTotal.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.standardOptions.color.withMode('continuous-BlPu') + + g.panel.timeSeries.panelOptions.withDescription('Write units consumed per index over time.'), + + readUnitsTotal: + signals.readUnitsTotal.asTimeSeries() + + commonlib.panels.generic.timeSeries.base.stylize() + + g.panel.timeSeries.standardOptions.color.withMode('palette-classic-by-name') + + g.panel.timeSeries.panelOptions.withDescription('Read units consumed per index over time.'), + }, +} diff --git a/pinecone-mixin/rows.libsonnet b/pinecone-mixin/rows.libsonnet new file mode 100644 index 000000000..5ba97c141 --- /dev/null +++ b/pinecone-mixin/rows.libsonnet @@ -0,0 +1,77 @@ +local g = import './g.libsonnet'; + +{ + new(this): { + local panels = this.grafana.panels, + // Overview row with stat panels, aggregate metrics, and table + overview: [ + g.panel.row.new('Overview') + + g.panel.row.withCollapsed(false), + // Stat panels - key metrics at a glance + panels.indexesCountStat + + g.panel.stat.gridPos.withW(8) + + g.panel.stat.gridPos.withH(4), + this.grafana.panels.totalRecordsStat { gridPos+: { h: 4, w: 8 } }, + this.grafana.panels.totalStorageStat + + g.panel.stat.gridPos.withW(8) + + g.panel.stat.gridPos.withH(4), + // Aggregate operations time series + this.grafana.panels.totalOperationsPerSec + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + this.grafana.panels.totalReadWriteOperations + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(8), + // Indexes table - detailed view + this.grafana.panels.indexesTable + + g.panel.table.gridPos.withW(24), + ], + + // Write operations (upsert, delete) + writeOperations: [ + g.panel.row.new('Write Operations'), + this.grafana.panels.upsertTotal + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(7), + this.grafana.panels.upsertDuration + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(7), + // second row + this.grafana.panels.deleteTotal + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(7), + this.grafana.panels.deleteDuration + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(7), + ], + + // Read operations (query, fetch) + readOperations: [ + g.panel.row.new('Read Operations'), + this.grafana.panels.queryTotal + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(7), + this.grafana.panels.queryDuration + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(7), + // second row + this.grafana.panels.fetchTotal + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(7), + this.grafana.panels.fetchDuration + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(7), + ], + + // Resource usage + resourceUsage: [ + g.panel.row.new('Resource Usage'), + this.grafana.panels.writeUnitsTotal + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(7), + this.grafana.panels.readUnitsTotal + + g.panel.timeSeries.gridPos.withW(12) + + g.panel.timeSeries.gridPos.withH(7), + ], + }, +} diff --git a/pinecone-mixin/signals/operations.libsonnet b/pinecone-mixin/signals/operations.libsonnet new file mode 100644 index 000000000..d7f78a1b3 --- /dev/null +++ b/pinecone-mixin/signals/operations.libsonnet @@ -0,0 +1,193 @@ +local commonlib = import 'common-lib/common/main.libsonnet'; + +function(this) + { + filteringSelector: this.filteringSelector, + groupLabels: this.groupLabels, + instanceLabels: this.instanceLabels, + enableLokiLogs: this.enableLokiLogs, + aggLevel: 'none', // Show each index separately without aggregation + aggFunction: 'sum', + discoveryMetric: { + prometheus: 'pinecone_db_record_total', + }, + signals: { + // Overview/Capacity signals + recordTotal: { + name: 'Total records', + nameShort: 'Records', + type: 'gauge', + unit: 'short', + description: 'The total number of records in the index.', + sources: { + prometheus: { + expr: 'pinecone_db_record_total{%(queriesSelector)s}', + }, + }, + }, + + storageSizeBytes: { + name: 'Storage size', + nameShort: 'Storage', + type: 'gauge', + unit: 'decbytes', + description: 'The total size of the index in bytes.', + sources: { + prometheus: { + expr: 'pinecone_db_storage_size_bytes{%(queriesSelector)s}', + }, + }, + }, + + // Upsert operations + upsertTotal: { + name: 'Upsert requests', + nameShort: 'Upserts', + type: 'counter', + unit: 'reqps', + description: 'The number of upsert requests made to an index.', + sources: { + prometheus: { + expr: 'pinecone_db_op_upsert_total{%(queriesSelector)s}', + }, + }, + }, + + upsertDuration: { + name: 'Upsert duration', + nameShort: 'Upsert dur', + type: 'raw', + unit: 'ms', + description: ||| + Average latency of upsert operations in milliseconds. + Calculated as rate of total duration divided by rate of requests. + Indicates how long each upsert operation takes on average. + |||, + sources: { + prometheus: { + expr: 'rate(pinecone_db_op_upsert_duration_total{%(queriesSelector)s}[$__rate_interval]) / rate(pinecone_db_op_upsert_total{%(queriesSelector)s}[$__rate_interval])', + }, + }, + }, + + // Query operations + queryTotal: { + name: 'Query requests', + nameShort: 'Queries', + type: 'counter', + unit: 'reqps', + description: 'The number of query requests made to an index.', + sources: { + prometheus: { + expr: 'pinecone_db_op_query_total{%(queriesSelector)s}', + }, + }, + }, + + queryDuration: { + name: 'Query duration', + nameShort: 'Query dur', + type: 'raw', + unit: 'ms', + description: ||| + Average latency of query operations in milliseconds. + Calculated as rate of total duration divided by rate of requests. + Indicates how long each query operation takes on average. + |||, + sources: { + prometheus: { + expr: 'rate(pinecone_db_op_query_duration_total{%(queriesSelector)s}[$__rate_interval]) / rate(pinecone_db_op_query_total{%(queriesSelector)s}[$__rate_interval])', + }, + }, + }, + + // Fetch operations + fetchTotal: { + name: 'Fetch requests', + nameShort: 'Fetches', + type: 'counter', + unit: 'reqps', + description: 'The number of fetch requests made to an index.', + sources: { + prometheus: { + expr: 'pinecone_db_op_fetch_total{%(queriesSelector)s}', + }, + }, + }, + + fetchDuration: { + name: 'Fetch duration', + nameShort: 'Fetch dur', + type: 'raw', + unit: 'ms', + description: ||| + Average latency of fetch operations in milliseconds. + Calculated as rate of total duration divided by rate of requests. + Indicates how long each fetch operation takes on average. + |||, + sources: { + prometheus: { + expr: 'rate(pinecone_db_op_fetch_duration_total{%(queriesSelector)s}[$__rate_interval]) / rate(pinecone_db_op_fetch_total{%(queriesSelector)s}[$__rate_interval])', + }, + }, + }, + + // Delete operations + deleteTotal: { + name: 'Delete requests', + nameShort: 'Deletes', + type: 'counter', + unit: 'reqps', + description: 'The number of delete requests made to an index.', + sources: { + prometheus: { + expr: 'pinecone_db_op_delete_total{%(queriesSelector)s}', + }, + }, + }, + + deleteDuration: { + name: 'Delete duration', + nameShort: 'Delete dur', + type: 'raw', + unit: 'ms', + description: ||| + Average latency of delete operations in milliseconds. + Calculated as rate of total duration divided by rate of requests. + Indicates how long each delete operation takes on average. + |||, + sources: { + prometheus: { + expr: 'rate(pinecone_db_op_delete_duration_total{%(queriesSelector)s}[$__rate_interval]) / rate(pinecone_db_op_delete_total{%(queriesSelector)s}[$__rate_interval])', + }, + }, + }, + + // Resource usage + writeUnitsTotal: { + name: 'Write units', + nameShort: 'Write units', + type: 'counter', + unit: 'short', + description: 'The total number of write units consumed by an index.', + sources: { + prometheus: { + expr: 'pinecone_db_write_unit_total{%(queriesSelector)s}', + }, + }, + }, + + readUnitsTotal: { + name: 'Read units', + nameShort: 'Read units', + type: 'counter', + unit: 'short', + description: 'The total number of read units consumed by an index.', + sources: { + prometheus: { + expr: 'pinecone_db_read_unit_total{%(queriesSelector)s}', + }, + }, + }, + }, + } diff --git a/pinecone-mixin/signals/overview.libsonnet b/pinecone-mixin/signals/overview.libsonnet new file mode 100644 index 000000000..727cc9f1f --- /dev/null +++ b/pinecone-mixin/signals/overview.libsonnet @@ -0,0 +1,134 @@ +local commonlib = import 'common-lib/common/main.libsonnet'; + +function(this) + { + filteringSelector: this.filteringSelector, + groupLabels: this.groupLabels, + instanceLabels: this.instanceLabels, // ['index_name'] + enableLokiLogs: this.enableLokiLogs, + aggLevel: 'group', // Aggregate at group level (sum across selected indexes) + aggFunction: 'sum', + discoveryMetric: { + prometheus: 'pinecone_db_record_total', + }, + signals: { + // Total number of indexes + indexesCount: { + name: 'Total indexes', + nameShort: 'Indexes', + type: 'raw', + unit: 'short', + description: ||| + Total number of Pinecone indexes being monitored. + Each index represents a separate vector database instance. + |||, + sources: { + prometheus: { + expr: 'count(count by (index_name) (pinecone_db_record_total{%(queriesSelector)s}))', + legendCustomTemplate: 'Indexes', + }, + }, + }, + + // Aggregate total records across all indexes + totalRecords: { + name: 'Total records', + nameShort: 'Total records', + type: 'raw', + unit: 'short', + description: ||| + Total number of records across all indexes. + Sum of all records stored in all monitored Pinecone indexes. + |||, + sources: { + prometheus: { + expr: 'sum(pinecone_db_record_total{%(queriesSelector)s})', + legendCustomTemplate: 'Records', + }, + }, + }, + + // Aggregate total storage across all indexes + totalStorage: { + name: 'Total storage', + nameShort: 'Total storage', + type: 'raw', + unit: 'decbytes', + description: ||| + Total storage size across all indexes in bytes. + Sum of storage used by all monitored Pinecone indexes. + |||, + sources: { + prometheus: { + expr: 'sum(pinecone_db_storage_size_bytes{%(queriesSelector)s})', + legendCustomTemplate: 'Total storage', + }, + }, + }, + + // Aggregate total operations (read + write) + totalOperationsPerSec: { + name: 'Total operations', + nameShort: 'Total ops', + type: 'raw', + unit: 'reqps', + description: ||| + Total operations per second across all indexes. + Sum of all read (query, fetch) and write (upsert, delete) operations. + |||, + sources: { + prometheus: { + expr: ||| + sum(rate(pinecone_db_op_query_total{%(queriesSelector)s}[$__rate_interval])) + + sum(rate(pinecone_db_op_fetch_total{%(queriesSelector)s}[$__rate_interval])) + + sum(rate(pinecone_db_op_upsert_total{%(queriesSelector)s}[$__rate_interval])) + + sum(rate(pinecone_db_op_delete_total{%(queriesSelector)s}[$__rate_interval])) + |||, + legendCustomTemplate: 'Total operations', + }, + }, + }, + + // Aggregate read operations + totalReadOperationsPerSec: { + name: 'Total read operations', + nameShort: 'Read ops', + type: 'raw', + unit: 'reqps', + description: ||| + Total read operations per second across all indexes. + Sum of query and fetch operations. + |||, + sources: { + prometheus: { + expr: ||| + sum(rate(pinecone_db_op_query_total{%(queriesSelector)s}[$__rate_interval])) + + sum(rate(pinecone_db_op_fetch_total{%(queriesSelector)s}[$__rate_interval])) + |||, + legendCustomTemplate: 'Read operations', + }, + }, + }, + + // Aggregate write operations + totalWriteOperationsPerSec: { + name: 'Total write operations', + nameShort: 'Write ops', + type: 'raw', + unit: 'reqps', + description: ||| + Total write operations per second across all indexes. + Sum of upsert, and delete operations. + |||, + sources: { + prometheus: { + expr: ||| + sum(rate(pinecone_db_op_upsert_total{%(queriesSelector)s}[$__rate_interval])) + + sum(rate(pinecone_db_op_delete_total{%(queriesSelector)s}[$__rate_interval])) + |||, + legendCustomTemplate: 'Write operations', + }, + }, + }, + }, + }