chore(service): allow to retrieve envoy gateway logs

libs/domains/service-logs/data-access/src/lib/domains-service-logs-data-access.spec.ts

@@ -0,0 +1,216 @@
+import { type LogFilters, buildLokiQuery } from './domains-service-logs-data-access'
+
+describe('buildLokiQuery', () => {
+  const baseFilters: LogFilters = {
+    serviceId: 'test-service-id',
+  }
+
+  describe('service logs (default)', () => {
+    it('should build query for service logs with only serviceId', () => {
+      const query = buildLokiQuery(baseFilters)
+      expect(query).toBe('{qovery_com_service_id="test-service-id"}')
+    })
+
+    it('should build query for service logs with explicit logType', () => {
+      const query = buildLokiQuery(baseFilters, 'service')
+      expect(query).toBe('{qovery_com_service_id="test-service-id"}')
+    })
+
+    it('should build query with level filter', () => {
+      const query = buildLokiQuery({ ...baseFilters, level: 'error' })
+      expect(query).toBe('{qovery_com_service_id="test-service-id",level="error"}')
+    })
+
+    it('should build query with instance filter', () => {
+      const query = buildLokiQuery({ ...baseFilters, instance: 'pod-123' })
+      expect(query).toBe('{qovery_com_service_id="test-service-id",pod="pod-123"}')
+    })
+
+    it('should build query with container filter', () => {
+      const query = buildLokiQuery({ ...baseFilters, container: 'main' })
+      expect(query).toBe('{qovery_com_service_id="test-service-id",container="main"}')
+    })
+
+    it('should build query with namespace filter', () => {
+      const query = buildLokiQuery({ ...baseFilters, namespace: 'production' })
+      expect(query).toBe('{qovery_com_service_id="test-service-id",namespace="production"}')
+    })
+
+    it('should build query with version filter', () => {
+      const query = buildLokiQuery({ ...baseFilters, version: 'v1.0.0' })
+      expect(query).toBe('{qovery_com_service_id="test-service-id",app="v1.0.0"}')
+    })
+
+    it('should build query with deploymentId filter', () => {
+      const query = buildLokiQuery({ ...baseFilters, deploymentId: 'deploy-123' })
+      expect(query).toBe('{qovery_com_service_id="test-service-id",qovery_com_deployment_id="deploy-123"}')
+    })
+
+    it('should build query with message search', () => {
+      const query = buildLokiQuery({ ...baseFilters, message: 'error occurred' })
+      expect(query).toBe('{qovery_com_service_id="test-service-id"} |= "error occurred"')
+    })
+
+    it('should build query with search text', () => {
+      const query = buildLokiQuery({ ...baseFilters, search: 'exception' })
+      expect(query).toBe('{qovery_com_service_id="test-service-id"} |= "exception"')
+    })
+
+    it('should build query with multiple filters', () => {
+      const query = buildLokiQuery({
+        ...baseFilters,
+        level: 'error',
+        instance: 'pod-123',
+        container: 'main',
+        message: 'failed',
+      })
+      expect(query).toBe(
+        '{qovery_com_service_id="test-service-id",level="error",pod="pod-123",container="main"} |= "failed"'
+      )
+    })
+
+    it('should combine message and search filters', () => {
+      const query = buildLokiQuery({
+        ...baseFilters,
+        message: 'error',
+        search: 'timeout',
+      })
+      expect(query).toBe('{qovery_com_service_id="test-service-id"} |= "errortimeout"')
+    })
+  })
+
+  describe('nginx logs', () => {
+    it('should build query for nginx logs', () => {
+      const query = buildLokiQuery(baseFilters, 'nginx')
+      expect(query).toBe(
+        '({app="ingress-nginx"} | level=~"error|warn" or (qovery_com_associated_service_id="test-service-id" and level!~"error|warn"))'
+      )
+    })
+
+    it('should build nginx query with level filter', () => {
+      const query = buildLokiQuery({ ...baseFilters, level: 'error' }, 'nginx')
+      expect(query).toBe(
+        '({app="ingress-nginx"} | level=~"error|warn" or (qovery_com_associated_service_id="test-service-id" and level!~"error|warn"),level="error")'
+      )
+    })
+
+    it('should build nginx query with instance filter', () => {
+      const query = buildLokiQuery({ ...baseFilters, instance: 'nginx-pod-123' }, 'nginx')
+      expect(query).toBe(
+        '({app="ingress-nginx"} | level=~"error|warn" or (qovery_com_associated_service_id="test-service-id" and level!~"error|warn"),pod="nginx-pod-123")'
+      )
+    })
+
+    it('should build nginx query with message search', () => {
+      const query = buildLokiQuery({ ...baseFilters, message: '404' }, 'nginx')
+      expect(query).toBe(
+        '({app="ingress-nginx"} | level=~"error|warn" or (qovery_com_associated_service_id="test-service-id" and level!~"error|warn")) |= "404"'
+      )
+    })
+
+    it('should build nginx query with multiple filters', () => {
+      const query = buildLokiQuery(
+        {
+          ...baseFilters,
+          level: 'error',
+          instance: 'nginx-pod-123',
+          message: '500',
+        },
+        'nginx'
+      )
+      expect(query).toBe(
+        '({app="ingress-nginx"} | level=~"error|warn" or (qovery_com_associated_service_id="test-service-id" and level!~"error|warn"),level="error",pod="nginx-pod-123") |= "500"'
+      )
+    })
+  })
+
+  describe('envoy logs', () => {
+    it('should build query for envoy logs', () => {
+      const query = buildLokiQuery(baseFilters, 'envoy')
+      expect(query).toBe(
+        '({app="envoy"} | level=~"error|warn" or (qovery_com_associated_service_id="test-service-id" and level!~"error|warn"))'
+      )
+    })
+
+    it('should build envoy query with level filter', () => {
+      const query = buildLokiQuery({ ...baseFilters, level: 'error' }, 'envoy')
+      expect(query).toBe(
+        '({app="envoy"} | level=~"error|warn" or (qovery_com_associated_service_id="test-service-id" and level!~"error|warn"),level="error")'
+      )
+    })
+
+    it('should build envoy query with instance filter', () => {
+      const query = buildLokiQuery({ ...baseFilters, instance: 'envoy-pod-123' }, 'envoy')
+      expect(query).toBe(
+        '({app="envoy"} | level=~"error|warn" or (qovery_com_associated_service_id="test-service-id" and level!~"error|warn"),pod="envoy-pod-123")'
+      )
+    })
+
+    it('should build envoy query with message search', () => {
+      const query = buildLokiQuery({ ...baseFilters, message: 'upstream error' }, 'envoy')
+      expect(query).toBe(
+        '({app="envoy"} | level=~"error|warn" or (qovery_com_associated_service_id="test-service-id" and level!~"error|warn")) |= "upstream error"'
+      )
+    })
+
+    it('should build envoy query with multiple filters', () => {
+      const query = buildLokiQuery(
+        {
+          ...baseFilters,
+          level: 'warn',
+          instance: 'envoy-pod-456',
+          container: 'gateway',
+          message: 'timeout',
+        },
+        'envoy'
+      )
+      expect(query).toBe(
+        '({app="envoy"} | level=~"error|warn" or (qovery_com_associated_service_id="test-service-id" and level!~"error|warn"),level="warn",pod="envoy-pod-456",container="gateway") |= "timeout"'
+      )
+    })
+
+    it('should build envoy query with deploymentId', () => {
+      const query = buildLokiQuery({ ...baseFilters, deploymentId: 'deploy-envoy-123' }, 'envoy')
+      expect(query).toBe(
+        '({app="envoy"} | level=~"error|warn" or (qovery_com_associated_service_id="test-service-id" and level!~"error|warn"),qovery_com_deployment_id="deploy-envoy-123")'
+      )
+    })
+  })
+
+  describe('log type comparison', () => {
+    it('should use parentheses for nginx and envoy, curly braces for service', () => {
+      const serviceQuery = buildLokiQuery(baseFilters, 'service')
+      const nginxQuery = buildLokiQuery(baseFilters, 'nginx')
+      const envoyQuery = buildLokiQuery(baseFilters, 'envoy')
+
+      expect(serviceQuery).toMatch(/^\{.*\}$/)
+      expect(nginxQuery).toMatch(/^\(.*\)$/)
+      expect(envoyQuery).toMatch(/^\(.*\)$/)
+    })
+
+    it('should use different app labels for nginx and envoy', () => {
+      const nginxQuery = buildLokiQuery(baseFilters, 'nginx')
+      const envoyQuery = buildLokiQuery(baseFilters, 'envoy')
+
+      expect(nginxQuery).toContain('app="ingress-nginx"')
+      expect(envoyQuery).toContain('app="envoy"')
+      expect(nginxQuery).not.toContain('app="envoy"')
+      expect(envoyQuery).not.toContain('app="ingress-nginx"')
+    })
+
+    it('should use qovery_com_associated_service_id for nginx and envoy', () => {
+      const serviceQuery = buildLokiQuery(baseFilters, 'service')
+      const nginxQuery = buildLokiQuery(baseFilters, 'nginx')
+      const envoyQuery = buildLokiQuery(baseFilters, 'envoy')
+
+      expect(serviceQuery).toContain('qovery_com_service_id="test-service-id"')
+      expect(serviceQuery).not.toContain('qovery_com_associated_service_id')
+
+      expect(nginxQuery).toContain('qovery_com_associated_service_id="test-service-id"')
+      expect(nginxQuery).not.toContain('qovery_com_service_id="test-service-id"')
+
+      expect(envoyQuery).toContain('qovery_com_associated_service_id="test-service-id"')
+      expect(envoyQuery).not.toContain('qovery_com_service_id="test-service-id"')
+    })
+  })
+})

libs/domains/service-logs/data-access/src/lib/domains-service-logs-data-access.ts

@@ -1,5 +1,6 @@
 import { createQueryKeys } from '@lukemorales/query-key-factory'
 import { ClustersApi } from 'qovery-typescript-axios'
+import { match } from 'ts-pattern'
 
 const clusterApi = new ClustersApi()
 
@@ -26,12 +27,18 @@ export interface LogFilters {
   deploymentId?: string
 }
 
-export function buildLokiQuery(filters: LogFilters, isNginx = false): string {
-  const labels: string[] = isNginx
-    ? [
-        `{app="ingress-nginx"} | level=~"error|warn" or (qovery_com_associated_service_id="${filters.serviceId}" and level!~"error|warn")`,
-      ]
-    : [`qovery_com_service_id="${filters.serviceId}"`]
+export type LogType = 'service' | 'nginx' | 'envoy'
+
+export function buildLokiQuery(filters: LogFilters, logType: LogType = 'service'): string {
+  const labels: string[] = match(logType)
+    .with('nginx', () => [
+      `{app="ingress-nginx"} | level=~"error|warn" or (qovery_com_associated_service_id="${filters.serviceId}" and level!~"error|warn")`,
+    ])
+    .with('envoy', () => [
+      `{app="envoy"} | level=~"error|warn" or (qovery_com_associated_service_id="${filters.serviceId}" and level!~"error|warn")`,
+    ])
+    .with('service', () => [`qovery_com_service_id="${filters.serviceId}"`])
+    .exhaustive()
 
   if (filters.level) {
     labels.push(`level="${filters.level}"`)
@@ -57,7 +64,9 @@ export function buildLokiQuery(filters: LogFilters, isNginx = false): string {
     labels.push(`qovery_com_deployment_id="${filters.deploymentId}"`)
   }
 
-  let query = isNginx ? `(${labels.join(',')})` : `{${labels.join(',')}}`
+  let query = match(logType)
+    .with('service', () => `{${labels.join(',')}}`)
+    .otherwise(() => `(${labels.join(',')})`)
 
   if (filters.message || filters.search) {
     query += ` |= "${filters.message ? filters.message : ''}${filters.search ? `${filters.search}` : ''}"`
@@ -187,7 +196,7 @@ export const serviceLogs = createQueryKeys('serviceLogs', {
     filters,
     limit,
     direction,
-    isNginx = false,
+    logType = 'service',
   }: {
     clusterId: string
     serviceId: string
@@ -197,16 +206,16 @@ export const serviceLogs = createQueryKeys('serviceLogs', {
     filters?: Omit<LogFilters, 'serviceId'>
     limit?: number
     direction?: 'forward' | 'backward'
-    isNginx?: boolean
+    logType?: LogType
   }) => ({
-    queryKey: [clusterId, timeRange, startDate, endDate, serviceId, filters, limit, direction, isNginx],
+    queryKey: [clusterId, timeRange, startDate, endDate, serviceId, filters, limit, direction, logType],
     async queryFn() {
       // Convert Date objects to nanosecond Unix epoch format for Loki API
       // https://grafana.com/docs/loki/latest/reference/loki-http-api/#timestamps
       const startTimestamp = startDate ? (startDate.getTime() * 1000000).toString() : undefined
       const endTimestamp = endDate ? (endDate.getTime() * 1000000).toString() : undefined
 
-      const query = buildLokiQuery({ serviceId, ...filters }, isNginx)
+      const query = buildLokiQuery({ serviceId, ...filters }, logType)
 
       const response = await clusterApi.getClusterLogs(
         clusterId,

libs/domains/service-logs/feature/src/lib/hooks/use-service-history-logs/use-service-history-logs.ts

@@ -97,52 +97,79 @@ export function useServiceHistoryLogs({ clusterId, serviceId, enabled = false }:
       filters,
       direction: 'backward',
       limit: LOGS_PER_BATCH,
-      isNginx: Boolean(queryParams.nginx) ?? false,
+      logType: 'nginx',
     }),
-    enabled: Boolean(clusterId) && Boolean(serviceId) && isHistoryMode && enabled,
+    enabled: Boolean(clusterId) && Boolean(serviceId) && isHistoryMode && Boolean(queryParams.nginx) && enabled,
+  })
+
+  const {
+    data: envoyLogs = [],
+    isFetched: isEnvoyFetched,
+    isFetching: isEnvoyLoading,
+    refetch: refetchEnvoyLogs,
+  } = useQuery({
+    ...serviceLogs.serviceLogs({
+      clusterId,
+      serviceId,
+      startDate,
+      endDate: currentEndDate ?? undefined,
+      filters,
+      direction: 'backward',
+      limit: LOGS_PER_BATCH,
+      logType: 'envoy',
+    }),
+    enabled: Boolean(clusterId) && Boolean(serviceId) && isHistoryMode && Boolean(queryParams.envoy) && enabled,
   })
 
-  const isFetched = isFetchedLogs || isNginxFetched
+  const isFetched = useMemo(
+    () => isFetchedLogs || isNginxFetched || isEnvoyFetched,
+    [isFetchedLogs, isNginxFetched, isEnvoyFetched]
+  )
 
   useEffect(() => {
-    if (isFetched && (logs.length > 0 || nginxLogs.length > 0)) {
+    if (isFetched && (logs.length > 0 || nginxLogs.length > 0 || envoyLogs.length > 0)) {
       setAccumulatedLogs((prev) => {
         const existingKeys = new Set(prev.map((log) => `${log.timestamp}|${log.message}`))
         const newLogs = logs.filter((log) => !existingKeys.has(`${log.timestamp}|${log.message}`))
         const newNginxLogs = nginxLogs.filter((log) => !existingKeys.has(`${log.timestamp}|${log.message}`))
-        return [...newLogs, ...newNginxLogs, ...prev]
+        const newEnvoyLogs = envoyLogs.filter((log) => !existingKeys.has(`${log.timestamp}|${log.message}`))
+        return [...newLogs, ...newNginxLogs, ...newEnvoyLogs, ...prev]
       })
       setIsPaginationLoading(false)
     }
-  }, [isFetched, logs, nginxLogs, resetCounter])
+  }, [isFetched, logs, nginxLogs, envoyLogs, resetCounter])
 
   useEffect(() => {
     if (isFetched && isPaginationLoading) {
-      if (logs.length === 0 && nginxLogs.length === 0) {
+      if (logs.length === 0 && nginxLogs.length === 0 && envoyLogs.length === 0) {
         setHasMoreLogs(false)
         setIsPaginationLoading(false)
         return
       }
 
-      if (logs.length > 0 || nginxLogs.length > 0) {
+      if (logs.length > 0 || nginxLogs.length > 0 || envoyLogs.length > 0) {
         const existingKeys = new Set(accumulatedLogs.map((log) => `${log.timestamp}|${log.message}`))
         const hasNewAppLogs = logs.some((log) => !existingKeys.has(`${log.timestamp}|${log.message}`))
         const hasNewNginxLogs = nginxLogs.some((log) => !existingKeys.has(`${log.timestamp}|${log.message}`))
+        const hasNewEnvoyLogs = envoyLogs.some((log) => !existingKeys.has(`${log.timestamp}|${log.message}`))
 
-        if (!hasNewAppLogs && !hasNewNginxLogs) {
+        if (!hasNewAppLogs && !hasNewNginxLogs && !hasNewEnvoyLogs) {
           setHasMoreLogs(false)
           setIsPaginationLoading(false)
         }
       }
     }
-  }, [isFetched, logs, nginxLogs, accumulatedLogs, isPaginationLoading])
+  }, [isFetched, logs, nginxLogs, envoyLogs, accumulatedLogs, isPaginationLoading])
 
   const refetch = useCallback(() => {
     refetchLogs()
     if (queryParams.nginx) {
       refetchNginxLogs()
     }
-  }, [refetchLogs, refetchNginxLogs, queryParams.nginx])
+    if (queryParams.envoy) {
+      refetchEnvoyLogs()
+    }
+  }, [refetchLogs, refetchNginxLogs, refetchEnvoyLogs, queryParams.nginx, queryParams.envoy])
 
   const loadPreviousLogs = useCallback(async () => {
     if (accumulatedLogs.length === 0 || !hasMoreLogs || isPaginationLoading) return
@@ -202,11 +229,16 @@ export function useServiceHistoryLogs({ clusterId, serviceId, enabled = false }:
     }
   }, [isFetched, logs.length, accumulatedLogs.length])
 
+  const isLoading = useMemo(
+    () => isLoadingLogs || isNginxLoading || isEnvoyLoading || isPaginationLoading,
+    [isLoadingLogs, isNginxLoading, isEnvoyLoading, isPaginationLoading]
+  )
+
   return {
     data: normalizedLogs,
     refetch,
     isFetched,
-    isLoading: isLoadingLogs || isNginxLoading || isPaginationLoading,
+    isLoading,
     loadPreviousLogs,
     hasMoreLogs,
   }