Rework ErrorGroupState queries/table for indexes

Author: matt-aitken

apps/webapp/app/presenters/v3/ErrorsListPresenter.server.ts

@@ -168,20 +168,47 @@ export class ErrorsListPresenter extends BasePresenter {
       (tasks !== undefined && tasks.length > 0) ||
       (versions !== undefined && versions.length > 0) ||
       (search !== undefined && search !== "") ||
-      (statuses !== undefined && statuses.length > 0) ||
-      !time.isDefault;
+      (statuses !== undefined && statuses.length > 0);
 
     const possibleTasksAsync = getAllTaskIdentifiers(this.replica, environmentId);
 
-    const [possibleTasks, displayableEnvironment] = await Promise.all([
+    // Pre-filter by status: since status lives in Postgres (ErrorGroupState) and the error
+    // list comes from ClickHouse, we resolve inclusion/exclusion sets upfront so that
+    // ClickHouse pagination operates on the correctly filtered dataset.
+    const statusFilterAsync = this.resolveStatusFilter(environmentId, statuses);
+
+    const [possibleTasks, displayableEnvironment, statusFilter] = await Promise.all([
       possibleTasksAsync,
       findDisplayableEnvironment(environmentId, userId),
+      statusFilterAsync,
     ]);
 
     if (!displayableEnvironment) {
       throw new ServiceValidationError("No environment found");
     }
 
+    if (statusFilter.empty) {
+      return {
+        errorGroups: [],
+        pagination: {
+          next: undefined,
+          previous: undefined,
+        },
+        filters: {
+          tasks,
+          versions,
+          statuses,
+          search,
+          period: time,
+          from: effectiveFrom,
+          to: effectiveTo,
+          hasFilters,
+          possibleTasks,
+          wasClampedByRetention,
+        },
+      };
+    }
+
     // Query the per-minute error_occurrences_v1 table for time-scoped counts
     const queryBuilder = this.clickhouse.errors.occurrencesListQueryBuilder();
 
@@ -205,6 +232,19 @@ export class ErrorsListPresenter extends BasePresenter {
       queryBuilder.where("task_version IN {versions: Array(String)}", { versions });
     }
 
+    if (statusFilter.includeKeys) {
+      queryBuilder.where(
+        "concat(task_identifier, '::', error_fingerprint) IN ({statusIncludeKeys: Array(String)})",
+        { statusIncludeKeys: statusFilter.includeKeys }
+      );
+    }
+    if (statusFilter.excludeKeys) {
+      queryBuilder.where(
+        "concat(task_identifier, '::', error_fingerprint) NOT IN ({statusExcludeKeys: Array(String)})",
+        { statusExcludeKeys: statusFilter.excludeKeys }
+      );
+    }
+
     queryBuilder.groupBy("error_fingerprint, task_identifier");
 
     // Text search via HAVING (operates on aggregated values)
@@ -292,12 +332,6 @@ export class ErrorsListPresenter extends BasePresenter {
       };
     });
 
-    if (statuses && statuses.length > 0) {
-      transformedErrorGroups = transformedErrorGroups.filter((g) =>
-        statuses.includes(g.status as ErrorGroupStatus)
-      );
-    }
-
     return {
       errorGroups: transformedErrorGroups,
       pagination: {
@@ -393,6 +427,61 @@ export class ErrorsListPresenter extends BasePresenter {
     return { data };
   }
 
+  /**
+   * Determines which (task, fingerprint) pairs to include or exclude from the ClickHouse
+   * query based on the requested status filter. Since status lives in Postgres and errors
+   * live in ClickHouse, we resolve the filter set here so ClickHouse pagination is correct.
+   *
+   * - UNRESOLVED is the default (no ErrorGroupState row), so filtering FOR it means
+   *   excluding groups with non-matching explicit statuses.
+   * - RESOLVED/IGNORED are explicit, so filtering for them means including only matching groups.
+   */
+  private async resolveStatusFilter(
+    environmentId: string,
+    statuses?: ErrorGroupStatus[]
+  ): Promise<{
+    includeKeys?: string[];
+    excludeKeys?: string[];
+    empty: boolean;
+  }> {
+    if (!statuses || statuses.length === 0) {
+      return { empty: false };
+    }
+
+    const allStatuses: ErrorGroupStatus[] = ["UNRESOLVED", "RESOLVED", "IGNORED"];
+    const excludedStatuses = allStatuses.filter((s) => !statuses.includes(s));
+
+    if (excludedStatuses.length === 0) {
+      return { empty: false };
+    }
+
+    if (statuses.includes("UNRESOLVED")) {
+      const excluded = await this.replica.errorGroupState.findMany({
+        where: { environmentId, status: { in: excludedStatuses } },
+        select: { taskIdentifier: true, errorFingerprint: true },
+      });
+      if (excluded.length === 0) {
+        return { empty: false };
+      }
+      return {
+        excludeKeys: excluded.map((g) => `${g.taskIdentifier}::${g.errorFingerprint}`),
+        empty: false,
+      };
+    }
+
+    const included = await this.replica.errorGroupState.findMany({
+      where: { environmentId, status: { in: statuses } },
+      select: { taskIdentifier: true, errorFingerprint: true },
+    });
+    if (included.length === 0) {
+      return { empty: true };
+    }
+    return {
+      includeKeys: included.map((g) => `${g.taskIdentifier}::${g.errorFingerprint}`),
+      empty: false,
+    };
+  }
+
   /**
    * Batch-fetch ErrorGroupState rows from Postgres for the given ClickHouse error groups.
    * Returns a map keyed by `${taskIdentifier}:${errorFingerprint}`.

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx

@@ -408,10 +408,7 @@ function FiltersBar({
     searchParams.has("status") ||
     searchParams.has("tasks") ||
     searchParams.has("versions") ||
-    searchParams.has("search") ||
-    searchParams.has("period") ||
-    searchParams.has("from") ||
-    searchParams.has("to");
+    searchParams.has("search");
 
   return (
     <div className="flex items-start justify-between gap-x-2 border-b border-grid-bright p-2">

apps/webapp/app/v3/services/alerts/errorAlertEvaluator.server.ts

@@ -93,7 +93,7 @@ export class ErrorAlertEvaluator {
       return;
     }
 
-    const states = await this.getErrorGroupStates(projectId, activeErrors, envIds);
+    const states = await this.getErrorGroupStates(activeErrors);
     const stateMap = this.buildStateMap(states);
 
     const occurrenceCounts = await this.getOccurrenceCountsSince(projectId, envIds, scheduledAt);
@@ -322,18 +322,17 @@ export class ErrorAlertEvaluator {
   }
 
   private async getErrorGroupStates(
-    projectId: string,
-    activeErrors: ActiveErrorsSinceQueryResult[],
-    envIds: string[]
+    activeErrors: ActiveErrorsSinceQueryResult[]
   ): Promise<ErrorGroupState[]> {
-    const fingerprints = [...new Set(activeErrors.map((e) => e.error_fingerprint))];
-    if (fingerprints.length === 0) return [];
+    if (activeErrors.length === 0) return [];
 
     return this._replica.errorGroupState.findMany({
       where: {
-        projectId,
-        errorFingerprint: { in: fingerprints },
-        environmentId: { in: envIds },
+        OR: activeErrors.map((e) => ({
+          environmentId: e.environment_id,
+          taskIdentifier: e.task_identifier,
+          errorFingerprint: e.error_fingerprint,
+        })),
       },
     });
   }

internal-packages/database/prisma/migrations/20260306102053_error_group_state/migration.sql

@@ -2,7 +2,7 @@
 CREATE TYPE "public"."ErrorGroupStatus" AS ENUM ('UNRESOLVED', 'RESOLVED', 'IGNORED');
 
 -- AlterEnum
-ALTER TYPE "public"."ProjectAlertType" ADD VALUE 'ERROR_GROUP';
+ALTER TYPE "public"."ProjectAlertType" ADD VALUE IF NOT EXISTS 'ERROR_GROUP';
 
 -- CreateTable
 CREATE TABLE
@@ -17,6 +17,7 @@ CREATE TABLE
         "ignoredUntil" TIMESTAMP(3),
         "ignoredUntilOccurrenceRate" INTEGER,
         "ignoredUntilTotalOccurrences" INTEGER,
+        "ignoredAtOccurrenceCount" BIGINT,
         "ignoredAt" TIMESTAMP(3),
         "ignoredReason" TEXT,
         "ignoredByUserId" TEXT,
@@ -28,24 +29,25 @@ CREATE TABLE
         CONSTRAINT "ErrorGroupState_pkey" PRIMARY KEY ("id")
     );
 
--- CreateIndex
-CREATE INDEX "ErrorGroupState_status_idx" ON "public"."ErrorGroupState" ("status");
-
--- CreateIndex
-CREATE INDEX "ErrorGroupState_ignoredUntil_idx" ON "public"."ErrorGroupState" ("ignoredUntil");
-
 -- CreateIndex
 CREATE UNIQUE INDEX "ErrorGroupState_environmentId_taskIdentifier_errorFingerpri_key" ON "public"."ErrorGroupState" (
     "environmentId",
     "taskIdentifier",
     "errorFingerprint"
 );
 
+-- CreateIndex
+CREATE INDEX "ErrorGroupState_environmentId_status_idx" ON "public"."ErrorGroupState" ("environmentId", "status");
+
 -- AddForeignKey
 ALTER TABLE "public"."ErrorGroupState" ADD CONSTRAINT "ErrorGroupState_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "public"."Organization" ("id") ON DELETE CASCADE ON UPDATE CASCADE;
 
 -- AddForeignKey
 ALTER TABLE "public"."ErrorGroupState" ADD CONSTRAINT "ErrorGroupState_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "public"."Project" ("id") ON DELETE CASCADE ON UPDATE CASCADE;
 
 -- AddForeignKey
-ALTER TABLE "public"."ErrorGroupState" ADD CONSTRAINT "ErrorGroupState_environmentId_fkey" FOREIGN KEY ("environmentId") REFERENCES "public"."RuntimeEnvironment" ("id") ON DELETE CASCADE ON UPDATE CASCADE;
+ALTER TABLE "public"."ErrorGroupState" ADD CONSTRAINT "ErrorGroupState_environmentId_fkey" FOREIGN KEY ("environmentId") REFERENCES "public"."RuntimeEnvironment" ("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AlterTable
+ALTER TABLE "public"."ProjectAlertChannel"
+ADD COLUMN "errorAlertConfig" JSONB;

internal-packages/database/prisma/migrations/20260308181657_add_error_alert_config_to_project_alert_channel/migration.sql

@@ -2717,6 +2717,5 @@ model ErrorGroupState {
   updatedAt DateTime @updatedAt
 
   @@unique([environmentId, taskIdentifier, errorFingerprint])
-  @@index([status])
-  @@index([ignoredUntil])
+  @@index([environmentId, status])
 }