diff --git a/.github/workflows/cross-images.yml b/.github/workflows/cross-images.yml.bak similarity index 100% rename from .github/workflows/cross-images.yml rename to .github/workflows/cross-images.yml.bak diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml.bak similarity index 100% rename from .github/workflows/drivers-tests.yml rename to .github/workflows/drivers-tests.yml.bak diff --git a/.github/workflows/examples-publish.yml b/.github/workflows/examples-publish.yml.bak similarity index 100% rename from .github/workflows/examples-publish.yml rename to .github/workflows/examples-publish.yml.bak diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml.bak similarity index 100% rename from .github/workflows/issue-labeler.yml rename to .github/workflows/issue-labeler.yml.bak diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml.bak similarity index 100% rename from .github/workflows/master.yml rename to .github/workflows/master.yml.bak diff --git a/.github/workflows/post-release.yml b/.github/workflows/post-release.yml.bak similarity index 100% rename from .github/workflows/post-release.yml rename to .github/workflows/post-release.yml.bak diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml.bak similarity index 100% rename from .github/workflows/pr.yml rename to .github/workflows/pr.yml.bak diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml.bak similarity index 100% rename from .github/workflows/publish.yml rename to .github/workflows/publish.yml.bak diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml.bak similarity index 100% rename from .github/workflows/push.yml rename to .github/workflows/push.yml.bak diff --git a/.github/workflows/rust-cubesql.yml b/.github/workflows/rust-cubesql.yml.bak similarity index 100% rename from .github/workflows/rust-cubesql.yml rename to .github/workflows/rust-cubesql.yml.bak diff --git a/.github/workflows/rust-cubestore-master.yml b/.github/workflows/rust-cubestore-master.yml.bak similarity index 100% rename from .github/workflows/rust-cubestore-master.yml rename to .github/workflows/rust-cubestore-master.yml.bak diff --git a/.github/workflows/rust-cubestore.yml b/.github/workflows/rust-cubestore.yml.bak similarity index 100% rename from .github/workflows/rust-cubestore.yml rename to .github/workflows/rust-cubestore.yml.bak diff --git a/.github/workflows/wf-cube.yml b/.github/workflows/wf-cube.yml new file mode 100644 index 0000000000000..2c2e656a1532c --- /dev/null +++ b/.github/workflows/wf-cube.yml @@ -0,0 +1,33 @@ +on: + push: + branches: + - master + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + tag: 'transcube-${{ github.sha }}' + +jobs: + build-and-push-image: + name: Build Latest - TransAI Cube + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + attestations: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Copy yarn.lock file + run: cp yarn.lock packages/cubejs-docker + - name: Build & push + uses: xip-online-applications/gha-workflow-templates/ghcr-build-push@main + with: + repository: '${{ github.repository }}/cube' + tag: ${{ env.tag }} + context: ./ + dockerfile: ./packages/cubejs-docker/transai.Dockerfile diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml.bak similarity index 100% rename from .github/workflows/workflow-lint.yml rename to .github/workflows/workflow-lint.yml.bak diff --git a/package.json b/package.json index 0aa9879e618f4..bc2b9287e1621 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "lint:fix": "lerna run lint:fix", "tsc": "tsc --build", "tsc:watch": "tsc --build --watch", - "clean": "rimraf packages/*/{tsconfig.tsbuildinfo,lib,dist}" + "clean": "rimraf packages/*/{tsconfig.tsbuildinfo,lib,dist}", + "link:transai": "cd ./packages/cubejs-backend-shared && yarn link && yarn install && cd ../cubejs-backend-cloud && yarn link && yarn install && cd ../cubejs-backend-native && yarn link && yarn install && cd ../cubejs-server && yarn link && yarn install && yarn install && cd ../cubejs-server-core && yarn link && yarn install && cd ../cubejs-api-gateway && yarn link && yarn install && cd ../cubejs-schema-compiler && yarn link && yarn install && cd ../cubejs-query-orchestrator && yarn link && yarn install && cd ../cubejs-postgres-driver && yarn link && yarn install && cd ../cubejs-cubestore-driver && yarn link && yarn install && cd ../cubejs-event-emitter && yarn link && yarn install" }, "author": "Cube Dev, Inc.", "dependencies": { diff --git a/packages/cubejs-api-gateway/package.json b/packages/cubejs-api-gateway/package.json index 609c0c1009491..7ab29308af0e4 100644 --- a/packages/cubejs-api-gateway/package.json +++ b/packages/cubejs-api-gateway/package.json @@ -27,6 +27,7 @@ "dist/src/*" ], "dependencies": { + "@cubejs-backend/event-emitter": "file:../cubejs-event-emitter", "@cubejs-backend/native": "1.3.78", "@cubejs-backend/shared": "1.3.78", "@ungap/structured-clone": "^0.3.4", @@ -48,6 +49,7 @@ "nexus": "^1.1.0", "node-fetch": "^2.6.1", "ramda": "^0.27.0", + "rxjs": "^7.8.1", "uuid": "^8.3.2" }, "devDependencies": { diff --git a/packages/cubejs-api-gateway/src/LocalSubscriptionStore.ts b/packages/cubejs-api-gateway/src/LocalSubscriptionStore.ts index a94225d9368ec..7e8cdbc15d8bd 100644 --- a/packages/cubejs-api-gateway/src/LocalSubscriptionStore.ts +++ b/packages/cubejs-api-gateway/src/LocalSubscriptionStore.ts @@ -2,6 +2,24 @@ interface LocalSubscriptionStoreOptions { heartBeatInterval?: number; } +const haveCommonElement = (arr1: string[], arr2: string[]): boolean => { + return arr1.some(element => arr2.includes(element)); +} + +// TODO: Check whether this is the correct way to get cube names +const getCubeNames = (query) => { + if (!query) { + return []; + } + + const allColumns = [ + ...(query.measures || []).map(m => m.split('.')[0]), + ...(query.dimensions || []).map(d => d.split('.')[0]), + ]; + + return Array.from(new Set(allColumns)); +}; + export class LocalSubscriptionStore { protected connections = {}; @@ -20,6 +38,7 @@ export class LocalSubscriptionStore { const connection = this.getConnection(connectionId); connection.subscriptions[subscriptionId] = { ...subscription, + cubes: getCubeNames(subscription.message?.params?.query), timestamp: new Date() }; } @@ -45,6 +64,11 @@ export class LocalSubscriptionStore { }).reduce((a, b) => a.concat(b), []); } + public async getSubscriptionsByCubeName(cubes: Array) { + // TODO: Implement cube filtering by auth context + return (await this.getAllSubscriptions()).filter(subscription => haveCommonElement(cubes, subscription.cubes)); + } + public async cleanupSubscriptions(connectionId: string) { delete this.connections[connectionId]; } diff --git a/packages/cubejs-api-gateway/src/SubscriptionServer.ts b/packages/cubejs-api-gateway/src/SubscriptionServer.ts index 15cbe057c0b52..daa0275fb66aa 100644 --- a/packages/cubejs-api-gateway/src/SubscriptionServer.ts +++ b/packages/cubejs-api-gateway/src/SubscriptionServer.ts @@ -1,5 +1,7 @@ import { v4 as uuidv4 } from 'uuid'; - +import { EventEmitterInterface } from '@cubejs-backend/event-emitter'; +import { Subject } from 'rxjs'; +import { map, filter, bufferTime, tap } from 'rxjs/operators'; import { UserError } from './UserError'; import type { ApiGateway } from './gateway'; import type { LocalSubscriptionStore } from './LocalSubscriptionStore'; @@ -20,13 +22,35 @@ const calcMessageLength = (message: unknown) => Buffer.byteLength( export type WebSocketSendMessageFn = (connectionId: string, message: any) => Promise; +const ensureArray = (value: any) => (Array.isArray(value) ? value : [value]); + export class SubscriptionServer { + readonly #cubeRenewSubject = new Subject(); + + readonly #cubeRenewedPipe = this.#cubeRenewSubject.pipe( + map((val) => ensureArray(val)), + // Map only the renewedCube property + map((val) => val as Array<{ renewedCube: string | undefined }>), + map((val) => val.map(v => v.renewedCube)), + // Buffer the values for 300ms + bufferTime(100), + map((renewedCubes) => renewedCubes.flat()), + map((val) => val.filter(v => v !== undefined)), + // Filter out any empty arrays + filter((renewedCubes) => renewedCubes.length > 0), + // Convert the array of arrays to an array of unique arrays + map((renewedCubes) => Array.from(new Set(renewedCubes))), + ); + public constructor( protected readonly apiGateway: ApiGateway, protected readonly sendMessage: WebSocketSendMessageFn, protected readonly subscriptionStore: LocalSubscriptionStore, protected readonly contextAcceptor: ContextAcceptorFn, + protected readonly eventEmitter: EventEmitterInterface ) { + this.eventEmitter.on('cubeRenewed', (val: unknown) => this.#cubeRenewSubject.next(val)); + this.#cubeRenewedPipe.subscribe(this.renewCubes.bind(this)); } public resultFn(connectionId: string, messageId: string, requestId: string | undefined) { @@ -154,4 +178,21 @@ export class SubscriptionServer { public clear() { this.subscriptionStore.clear(); } + + public async renewCubes(cubes) { + if (cubes.length === 0) { + return; + } + + const subs = await this.subscriptionStore.getSubscriptionsByCubeName(cubes); + + if (subs.length === 0) { + return; + } + + console.log('Renewing subs based on changed cubes', cubes, subs.length); + subs.map(async subscription => { + this.processMessage(subscription.connectionId, subscription.message, true); + }); + } } diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 8e2cfce793f26..1df52e17773ce 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -1,5 +1,6 @@ /* eslint-disable no-restricted-syntax */ import * as stream from 'stream'; +import { EventEmitterInterface } from '@cubejs-backend/event-emitter'; import { assertNever } from 'assert-never'; import jwt, { Algorithm as JWTAlgorithm } from 'jsonwebtoken'; import R from 'ramda'; @@ -180,6 +181,7 @@ class ApiGateway { protected readonly compilerApi: (ctx: RequestContext) => Promise, protected readonly adapterApi: (ctx: RequestContext) => Promise, protected readonly logger: any, + protected readonly eventEmitter: EventEmitterInterface, protected readonly options: ApiGatewayOptions, ) { this.dataSourceStorage = options.dataSourceStorage; @@ -465,6 +467,18 @@ class ApiGateway { * jobs scope * *************************************************************** */ + app.get( + `${this.basePath}/v1/run-scheduled-refresh`, + userMiddlewares, + userAsyncHandler(async (req, res) => { + await this.runScheduledRefresh({ + queryingOptions: req.query.queryingOptions, + context: req.context, + res: this.resToResultFn(res) + }); + }) + ); + app.post( `${this.basePath}/v1/pre-aggregations/jobs`, userMiddlewares, @@ -569,13 +583,35 @@ class ApiGateway { } public initSubscriptionServer(sendMessage: WebSocketSendMessageFn) { - return new SubscriptionServer(this, sendMessage, this.subscriptionStore, this.wsContextAcceptor); + return new SubscriptionServer(this, sendMessage, this.subscriptionStore, this.wsContextAcceptor, this.eventEmitter); } protected duration(requestStarted) { return requestStarted && (new Date().getTime() - requestStarted.getTime()); } + public async runScheduledRefresh({ context, res, queryingOptions }: { + context: RequestContext, + res: ResponseResultFn, + queryingOptions: any + }) { + const requestStarted = new Date(); + try { + await this.assertApiScope('jobs', context.securityContext); + const refreshScheduler = this.refreshScheduler(); + res(await refreshScheduler.runScheduledRefresh(context, { + ...this.parseQueryParam(queryingOptions || {}), + throwErrors: true, + forceNoCache: true, + isJob: true, + })); + } catch (e: any) { + this.handleError({ + e, context, res, requestStarted + }); + } + } + private filterVisibleItemsInMeta(context: RequestContext, cubes: any[]) { const isDevMode = getEnv('devMode'); function visibilityFilter(item) { @@ -2090,6 +2126,7 @@ class ApiGateway { return; } + // TODO: RENE // TODO subscribe to refreshKeys instead of constantly firing load await this.load({ query, diff --git a/packages/cubejs-api-gateway/test/auth.test.ts b/packages/cubejs-api-gateway/test/auth.test.ts index 5ee4f47903483..2fbfe0eb00f4e 100644 --- a/packages/cubejs-api-gateway/test/auth.test.ts +++ b/packages/cubejs-api-gateway/test/auth.test.ts @@ -3,6 +3,7 @@ import express, { Application as ExpressApplication, RequestHandler } from 'expr // eslint-disable-next-line import/no-extraneous-dependencies import request from 'supertest'; import jwt from 'jsonwebtoken'; +import { DefaultEventEmitter } from '@cubejs-backend/event-emitter'; import { pausePromise } from '@cubejs-backend/shared'; import { resetLogger } from '@cubejs-backend/native'; @@ -60,7 +61,9 @@ function createApiGateway(handler: RequestHandler, logger: () => any, options: P } } - const apiGateway = new ApiGatewayFake('secret', null, () => adapterApi, logger, { + const eventEmitter = new DefaultEventEmitter(); + + const apiGateway = new ApiGatewayFake('secret', null, () => adapterApi, logger, eventEmitter, { standalone: true, dataSourceStorage, basePath: '/cubejs-api', diff --git a/packages/cubejs-api-gateway/test/index.test.ts b/packages/cubejs-api-gateway/test/index.test.ts index 2a8d5f03bab0f..8037cefa1e35b 100644 --- a/packages/cubejs-api-gateway/test/index.test.ts +++ b/packages/cubejs-api-gateway/test/index.test.ts @@ -5,6 +5,7 @@ import express from 'express'; // eslint-disable-next-line import/no-extraneous-dependencies import request from 'supertest'; import jwt from 'jsonwebtoken'; +import { DefaultEventEmitter } from '@cubejs-backend/event-emitter'; import * as console from 'console'; import { ApiGateway, ApiGatewayOptions, Query, QueryRequest, Request } from '../src'; @@ -52,7 +53,8 @@ async function createApiGateway( ) { process.env.NODE_ENV = 'production'; - const apiGateway = new ApiGateway(API_SECRET, compilerApi, async () => adapterApi, logger, { + const eventEmitter = new DefaultEventEmitter(); + const apiGateway = new ApiGateway(API_SECRET, compilerApi, async () => adapterApi, logger, eventEmitter, { standalone: true, dataSourceStorage, basePath: '/cubejs-api', diff --git a/packages/cubejs-api-gateway/test/permissions.test.ts b/packages/cubejs-api-gateway/test/permissions.test.ts index b1a850f15437c..1c36ffd80e362 100644 --- a/packages/cubejs-api-gateway/test/permissions.test.ts +++ b/packages/cubejs-api-gateway/test/permissions.test.ts @@ -6,6 +6,7 @@ import { DataSourceStorageMock, AdapterApiMock } from './mocks'; +import { DefaultEventEmitter } from '@cubejs-backend/event-emitter'; const API_SECRET = 'secret'; const AUTH_TOKEN = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.e30.t-IDcSemACt8x4iTMCda8Yhe3iZaWbvV5XKSTbuAn0M'; @@ -18,7 +19,9 @@ function createApiGateway( const app = express(); const adapterApi: any = new AdapterApiMock(); const dataSourceStorage: any = new DataSourceStorageMock(); - const apiGateway = new ApiGateway(API_SECRET, compilerApi, () => adapterApi, logger, { + const eventEmitter = new DefaultEventEmitter(); + + const apiGateway = new ApiGateway(API_SECRET, compilerApi, () => adapterApi, logger, eventEmitter, { standalone: true, dataSourceStorage, basePath: '/cubejs-api', diff --git a/packages/cubejs-backend-shared/src/env.ts b/packages/cubejs-backend-shared/src/env.ts index 8491e3d7b5b81..3507d544f69ff 100644 --- a/packages/cubejs-backend-shared/src/env.ts +++ b/packages/cubejs-backend-shared/src/env.ts @@ -142,7 +142,7 @@ const variables: Record any> = { .default('false') .asBoolStrict(), webSockets: () => get('CUBEJS_WEB_SOCKETS') - .default('false') + .default('true') .asBoolStrict(), serverHeadersTimeout: () => get('CUBEJS_SERVER_HEADERS_TIMEOUT') .asInt(), @@ -2020,6 +2020,28 @@ const variables: Record any> = { cubeStoreNoHeartBeatTimeout: () => get('CUBEJS_CUBESTORE_NO_HEART_BEAT_TIMEOUT') .default('30') .asInt(), + + // EventEmitter + eventEmitterSetting: () => get('CUBEJS_EVENT_EMITTER_TYPE') + .default('memory') + .asString(), + eventEmitterRedisUrl: () => get('CUBEJS_EVENT_EMITTER_REDIS_URL') + .default('') + .asString(), + + // Redis + redisPoolMin: () => get('CUBEJS_REDIS_POOL_MIN') + .default('2') + .asInt(), + redisPoolMax: () => get('CUBEJS_REDIS_POOL_MAX') + .default('1000') + .asInt(), + redisUseIORedis: () => get('CUBEJS_REDIS_USE_IOREDIS') + .default('false') + .asBoolStrict(), + redisAcquireTimeout: () => get('CUBEJS_REDIS_ACQUIRE_TIMEOUT') + .default('5000') + .asInt(), cubeStoreRollingWindowJoin: () => get('CUBEJS_CUBESTORE_ROLLING_WINDOW_JOIN') .default('false') .asBoolStrict(), @@ -2144,10 +2166,6 @@ const variables: Record any> = { // Support for Redis as queue & cache driver was removed in 0.36 // This code is used to detect Redis and throw an error // TODO(ovr): Remove in after 1.0 + LTS - redisUseIORedis: () => get('CUBEJS_REDIS_USE_IOREDIS') - .default('false') - .asBoolStrict(), - // TODO(ovr): Remove in after 1.0 + LTS redisUrl: () => { const redisUrl = get('CUBEJS_REDIS_URL') .asString(); diff --git a/packages/cubejs-docker/package.json.transai b/packages/cubejs-docker/package.json.transai new file mode 100644 index 0000000000000..5ff3a6758f291 --- /dev/null +++ b/packages/cubejs-docker/package.json.transai @@ -0,0 +1,28 @@ +{ + "name": "@cubejs-backend/docker", + "version": "0.31.20", + "description": "Cube.js In Docker (virtual package)", + "author": "Cube Dev, Inc.", + "license": "Apache-2.0", + "private": true, + "engines": { + "node": "^14.0.0 || ^16.0.0 || >=17.0.0" + }, + "scripts": { + "link:dev": "yarn link @cubejs-backend/shared @cubejs-backend/server @cubejs-backend/server-core @cubejs-backend/api-gateway @cubejs-backend/schema-compiler @cubejs-backend/query-orchestrator @cubejs-backend/postgres-driver @cubejs-backend/event-emitter", + "link:transai": "cd ./packages/cubejs-backend-shared && yarn link && cd ../cubejs-backend-cloud && yarn link && cd ../cubejs-backend-native && yarn link && cd ../cubejs-server && yarn link && cd ../cubejs-server-core && yarn link && cd ../cubejs-api-gateway && yarn link && cd ../cubejs-schema-compiler && yarn link && cd ../cubejs-query-orchestrator && yarn link && cd ../cubejs-postgres-driver && yarn link && cd ../cubejs-cubestore-driver && yarn link && cd ../cubejs-client-core && yarn link && cd ../cubejs-client-ws-transport && yarn link && cd ../cubejs-event-emitter && yarn link" + }, + "dependencies": { + "@cubejs-backend/postgres-driver": "file:/cube-build/packages/cubejs-postgres-driver", + "@cubejs-backend/server": "file:/cube-build/packages/cubejs-server", + "cubejs-cli": "file:/cube-build/packages/cubejs-cli", + "typescript": "~4.1.5", + "@typescript-eslint/eslint-plugin": "^4.17.0", + "core-js": "^3.34.0", + "lerna": "^4.0.0", + "redis": "^4.6.14" + }, + "resolutions": { + "colors": "1.4.0" + } +} diff --git a/packages/cubejs-docker/transai.Dockerfile b/packages/cubejs-docker/transai.Dockerfile new file mode 100644 index 0000000000000..649cc12d3d8e4 --- /dev/null +++ b/packages/cubejs-docker/transai.Dockerfile @@ -0,0 +1,133 @@ +FROM node:20.17.0-bookworm-slim AS build + +ARG IMAGE_VERSION=dev + +ENV CUBEJS_DOCKER_IMAGE_VERSION=$IMAGE_VERSION +ENV CUBEJS_DOCKER_IMAGE_TAG=dev +ENV CI=0 + +RUN DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get install -y --no-install-recommends libssl3 curl \ + cmake python3.11 libpython3.11-dev gcc g++ make cmake openjdk-17-jdk-headless \ + && rm -rf /var/lib/apt/lists/* + +ENV CUBESTORE_SKIP_POST_INSTALL=true +ENV NODE_ENV development + +RUN yarn policies set-version v1.22.19 +# Yarn v1 uses aggressive timeouts with summing time spending on fs, https://github.com/yarnpkg/yarn/issues/4890 +RUN yarn config set network-timeout 120000 -g + +WORKDIR /cubejs + +COPY package.json . +COPY lerna.json . +COPY yarn.lock . +COPY tsconfig.base.json . +COPY rollup.config.js . +COPY packages/cubejs-linter packages/cubejs-linter + +# Backend +COPY packages/cubejs-backend-shared/ packages/cubejs-backend-shared/ +COPY packages/cubejs-base-driver/ packages/cubejs-base-driver/ +COPY packages/cubejs-backend-native/ packages/cubejs-backend-native/ +COPY packages/cubejs-testing-shared/ packages/cubejs-testing-shared/ +COPY packages/cubejs-backend-cloud/ packages/cubejs-backend-cloud/ +COPY packages/cubejs-api-gateway/ packages/cubejs-api-gateway/ +#COPY packages/cubejs-athena-driver/ packages/cubejs-athena-driver/ +#COPY packages/cubejs-bigquery-driver/ packages/cubejs-bigquery-driver/ +COPY packages/cubejs-cli/ packages/cubejs-cli/ +#COPY packages/cubejs-clickhouse-driver/ packages/cubejs-clickhouse-driver/ +#COPY packages/cubejs-crate-driver/ packages/cubejs-crate-driver/ +#COPY packages/cubejs-dremio-driver/ packages/cubejs-dremio-driver/ +#COPY packages/cubejs-druid-driver/ packages/cubejs-druid-driver/ +#COPY packages/cubejs-duckdb-driver/ packages/cubejs-duckdb-driver/ +#COPY packages/cubejs-elasticsearch-driver/ packages/cubejs-elasticsearch-driver/ +#COPY packages/cubejs-firebolt-driver/ packages/cubejs-firebolt-driver/ +#COPY packages/cubejs-hive-driver/ packages/cubejs-hive-driver/ +#COPY packages/cubejs-mongobi-driver/ packages/cubejs-mongobi-driver/ +#COPY packages/cubejs-mssql-driver/ packages/cubejs-mssql-driver/ +#COPY packages/cubejs-mysql-driver/ packages/cubejs-mysql-driver/ +COPY packages/cubejs-cubestore-driver/ packages/cubejs-cubestore-driver/ +#COPY packages/cubejs-oracle-driver/ packages/cubejs-oracle-driver/ +#COPY packages/cubejs-redshift-driver/ packages/cubejs-redshift-driver/ +COPY packages/cubejs-postgres-driver/ packages/cubejs-postgres-driver/ +#COPY packages/cubejs-questdb-driver/ packages/cubejs-questdb-driver/ +#COPY packages/cubejs-materialize-driver/ packages/cubejs-materialize-driver/ +#COPY packages/cubejs-prestodb-driver/ packages/cubejs-prestodb-driver/ +#COPY packages/cubejs-trino-driver/ packages/cubejs-trino-driver/ +COPY packages/cubejs-query-orchestrator/ packages/cubejs-query-orchestrator/ +COPY packages/cubejs-schema-compiler/ packages/cubejs-schema-compiler/ +COPY packages/cubejs-server/ packages/cubejs-server/ +COPY packages/cubejs-server-core/ packages/cubejs-server-core/ +#COPY packages/cubejs-snowflake-driver/ packages/cubejs-snowflake-driver/ +#COPY packages/cubejs-sqlite-driver/ packages/cubejs-sqlite-driver/ +#COPY packages/cubejs-ksql-driver/ packages/cubejs-ksql-driver/ +#COPY packages/cubejs-dbt-schema-extension/ packages/cubejs-dbt-schema-extension/ +#COPY packages/cubejs-jdbc-driver/ packages/cubejs-jdbc-driver/ +#COPY packages/cubejs-databricks-jdbc-driver/ packages/cubejs-databricks-jdbc-driver/ +COPY packages/cubejs-event-emitter/ packages/cubejs-event-emitter/ +# Skip +# COPY packages/cubejs-testing/ packages/cubejs-testing/ +# COPY packages/cubejs-docker/ packages/cubejs-docker/ +# Frontend +#COPY packages/cubejs-templates/ packages/cubejs-templates/ +#COPY packages/cubejs-client-core/ packages/cubejs-client-core/ +#COPY packages/cubejs-client-react/ packages/cubejs-client-react/ +#COPY packages/cubejs-client-vue/ packages/cubejs-client-vue/ +#COPY packages/cubejs-client-vue3/ packages/cubejs-client-vue3/ +#COPY packages/cubejs-client-ngx/ packages/cubejs-client-ngx/ +#COPY packages/cubejs-client-ws-transport/ packages/cubejs-client-ws-transport/ +#COPY packages/cubejs-playground/ packages/cubejs-playground/ + +RUN yarn install +RUN yarn lerna run build + +RUN find . -name 'node_modules' -type d -prune -exec rm -rf '{}' + + +FROM node:20.17.0-bookworm-slim AS final + +ARG IMAGE_VERSION=dev + +ENV CUBEJS_DOCKER_IMAGE_VERSION=$IMAGE_VERSION +ENV CUBEJS_DOCKER_IMAGE_TAG=latest + +RUN DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get install -y --no-install-recommends libssl3 curl \ + cmake python3.11 libpython3.11-dev gcc g++ make cmake openjdk-17-jdk-headless \ + && rm -rf /var/lib/apt/lists/* + +ENV TERM rxvt-unicode +ENV NODE_ENV production + +WORKDIR /cube +COPY packages/cubejs-docker/bin bin +# Unlike latest.Dockerfile, this one doesn't install the latest cubejs from +# npm, but rather copies all the artifacts from the dev image and links them to +# the /cube directory +COPY --from=build /cubejs /cube-build +RUN cd /cube-build && yarn run link:transai +COPY packages/cubejs-docker/package.json.transai package.json + +RUN yarn policies set-version v1.22.19 +# Yarn v1 uses aggressive timeouts with summing time spending on fs, https://github.com/yarnpkg/yarn/issues/4890 +RUN yarn config set network-timeout 120000 -g + +# We are copying root yarn.lock file to the context folder during the Publish GH +# action. So, a process will use the root lock file here. +RUN yarn install --prod && yarn cache clean && yarn link:dev + +# By default Node dont search in parent directory from /cube/conf, @todo Reaserch a little bit more +ENV NODE_PATH /cube/conf/node_modules:/cube/node_modules +ENV PYTHONUNBUFFERED=1 +RUN ln -s /cube/node_modules/.bin/cubejs /usr/local/bin/cubejs +RUN ln -s /cube/node_modules/.bin/cubestore-dev /usr/local/bin/cubestore-dev + +WORKDIR /cube/conf + +EXPOSE 4000 + +CMD ["cubejs", "server"] + diff --git a/packages/cubejs-event-emitter/.gitignore b/packages/cubejs-event-emitter/.gitignore new file mode 100644 index 0000000000000..d160856ee977d --- /dev/null +++ b/packages/cubejs-event-emitter/.gitignore @@ -0,0 +1,3 @@ +storage +dist +test/.* diff --git a/packages/cubejs-event-emitter/CHANGELOG.md b/packages/cubejs-event-emitter/CHANGELOG.md new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/cubejs-event-emitter/LICENSE b/packages/cubejs-event-emitter/LICENSE new file mode 100644 index 0000000000000..da67a9a3301be --- /dev/null +++ b/packages/cubejs-event-emitter/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018-2020 Cube Dev, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/packages/cubejs-event-emitter/README.md b/packages/cubejs-event-emitter/README.md new file mode 100644 index 0000000000000..9b385caa3bfab --- /dev/null +++ b/packages/cubejs-event-emitter/README.md @@ -0,0 +1,14 @@ +

Cube.js

+ +[Website](https://cube.dev) • [Docs](https://cube.dev/docs) • [Blog](https://cube.dev/blog) • [Slack](https://slack.cube.dev) • [Twitter](https://twitter.com/the_cube_dev) + +[![npm version](https://badge.fury.io/js/%40cubejs-backend%2Fserver.svg)](https://badge.fury.io/js/%40cubejs-backend%2Fserver) +[![GitHub Actions](https://github.com/cube-js/cube.js/workflows/Build/badge.svg)](https://github.com/cube-js/cube.js/actions?query=workflow%3ABuild+branch%3Amaster) + +# Cube Cloud package + +Shared code for Cube Cloud integration + +### License + +Cube.js Druid Database Driver is [Apache 2.0 licensed](./LICENSE). diff --git a/packages/cubejs-event-emitter/package.json b/packages/cubejs-event-emitter/package.json new file mode 100644 index 0000000000000..dff23d2dc2405 --- /dev/null +++ b/packages/cubejs-event-emitter/package.json @@ -0,0 +1,63 @@ +{ + "name": "@cubejs-backend/event-emitter", + "version": "0.35.67", + "description": "Cube Cloud package", + "main": "dist/src/index.js", + "typings": "dist/src/index.d.ts", + "scripts": { + "build": "rm -rf dist && npm run tsc", + "tsc": "tsc", + "watch": "tsc -w", + "test": "npm run unit", + "unit": "jest dist/test", + "lint": "eslint --debug src/* --ext .ts", + "lint:fix": "eslint --fix src/* --ext .ts" + }, + "files": [ + "README.md", + "dist/src/*" + ], + "engines": { + "node": "^14.0.0 || ^16.0.0 || >=17.0.0" + }, + "author": "Cube Dev, Inc.", + "license": "Apache-2.0", + "devDependencies": { + "@cubejs-backend/linter": "^0.35.0", + "@types/fs-extra": "^9.0.8", + "@types/jest": "^27", + "@types/request-promise": "^4.1.46", + "jest": "^27", + "typescript": "~5.2.2" + }, + "dependencies": { + "@cubejs-backend/dotenv": "^9.0.2", + "@cubejs-backend/shared": "^0.35.67", + "chokidar": "^3.5.1", + "env-var": "^6.3.0", + "fs-extra": "^9.1.0", + "jsonwebtoken": "^8.5.1", + "request": "^2.88.2", + "request-promise": "^4.2.5", + "redis": "^4.6.14", + "rxjs": "^7.8.1" + }, + "publishConfig": { + "access": "public" + }, + "eslintConfig": { + "extends": "../cubejs-linter" + }, + "jest": { + "testEnvironment": "node", + "collectCoverage": false, + "coverageDirectory": "coverage/", + "collectCoverageFrom": [ + "dist/src/**/*.js", + "dist/src/**/*.ts" + ], + "coveragePathIgnorePatterns": [ + ".*\\.d\\.ts" + ] + } +} diff --git a/packages/cubejs-event-emitter/src/DefaultEventEmitter.ts b/packages/cubejs-event-emitter/src/DefaultEventEmitter.ts new file mode 100644 index 0000000000000..b0fff479fb977 --- /dev/null +++ b/packages/cubejs-event-emitter/src/DefaultEventEmitter.ts @@ -0,0 +1,8 @@ +import { EventEmitter } from 'events'; +import { EventEmitterInterface, EventEmitterOptions } from './EventEmitter.interface'; + +export interface DefaultEventEmitterOptions extends EventEmitterOptions { + type: 'memory'; +} + +export class DefaultEventEmitter extends EventEmitter implements EventEmitterInterface {} diff --git a/packages/cubejs-event-emitter/src/EventEmitter.interface.ts b/packages/cubejs-event-emitter/src/EventEmitter.interface.ts new file mode 100644 index 0000000000000..e5d4914ceed6f --- /dev/null +++ b/packages/cubejs-event-emitter/src/EventEmitter.interface.ts @@ -0,0 +1,10 @@ +export type EventEmitterType = 'redis' | 'memory'; + +export type EventEmitterOptions = { + type: EventEmitterType +}; + +export interface EventEmitterInterface { + on (event: string, listener: (args: any) => void): void + emit (event: string, ...args: any): boolean +} diff --git a/packages/cubejs-event-emitter/src/RedisEventEmitter.ts b/packages/cubejs-event-emitter/src/RedisEventEmitter.ts new file mode 100644 index 0000000000000..6ffca2720539a --- /dev/null +++ b/packages/cubejs-event-emitter/src/RedisEventEmitter.ts @@ -0,0 +1,70 @@ +import {createClient, RedisClientType} from 'redis'; +import { Subject } from 'rxjs'; +import { take } from 'rxjs/operators'; +import { EventEmitterInterface, EventEmitterOptions } from './EventEmitter.interface'; + +export interface RedisEventEmitterOptions extends EventEmitterOptions { + type: 'redis'; + url: string; +} + +export class RedisEventEmitter implements EventEmitterInterface { + #sub: RedisClientType | null = null; + + #pub: RedisClientType | null = null; + + #initialized = false; + + readonly #url: string; + + readonly #initSubject = new Subject(); + + public constructor(url: string) { + this.#url = url; + this.init().then(() => { + console.log('🔗 TransAI Redis Client is initialized'); + }).catch((error) => { + console.error('🔗 TransAI Redis Client initialization failed', error); + }); + } + + public async init() { + const options = { url: this.#url }; + this.#sub = createClient(options); + this.#pub = createClient(options); + + await this.#sub.connect(); + await this.#pub.connect(); + + this.#initialized = true; + this.#initSubject.next(null); + this.#initSubject.complete(); + } + + public on(event: string, listener: (...args: any[]) => void) { + if (!this.#initialized || !this.#sub) { + this.#initSubject + .pipe( + take(1) + ) + .subscribe(() => { + this.on(event, listener); + }); + return; + } + + console.log('Subscribing to', event); + this.#sub.subscribe(event, (message) => { + listener(JSON.parse(message)); + }); + } + + public emit(event: string, ...args: any[]): boolean { + if (!this.#initialized || !this.#pub) { + return false; + } + + this.#pub.publish(event, JSON.stringify(args)); + return true; + } +} diff --git a/packages/cubejs-event-emitter/src/index.ts b/packages/cubejs-event-emitter/src/index.ts new file mode 100644 index 0000000000000..ab7f041e23807 --- /dev/null +++ b/packages/cubejs-event-emitter/src/index.ts @@ -0,0 +1,3 @@ +export * from './DefaultEventEmitter'; +export * from './EventEmitter.interface'; +export * from './RedisEventEmitter'; diff --git a/packages/cubejs-event-emitter/src/utils.ts b/packages/cubejs-event-emitter/src/utils.ts new file mode 100644 index 0000000000000..e08850a355ef6 --- /dev/null +++ b/packages/cubejs-event-emitter/src/utils.ts @@ -0,0 +1,177 @@ +/* eslint-disable no-restricted-syntax */ +import * as querystring from 'querystring'; + +function parseHostPort(addr: string): { host: string, port: number } { + if (addr.includes(':')) { + const parts = addr.split(':'); + + if (parts.length === 2) { + return { + host: parts[0], + port: parseInt(parts[1], 10), + }; + } + + throw new Error( + `Unsupported host:port part inside REDIS_URL: ${addr}` + ); + } + + return { + host: addr, + port: 6379, + }; +} + +function parseAddrPart(addr: string): { host: string, port: number, username?: string, password?: string } { + if (addr.includes('@')) { + const parts = addr.split('@'); + if (parts.length !== 2) { + throw new Error( + `Unsupported host part inside REDIS_URL: ${addr}` + ); + } + + const credentials = parts[0].split(':'); + if (credentials.length !== 2) { + throw new Error( + `Unsupported credentials part inside REDIS_URL: ${addr}` + ); + } + + return { + username: credentials[0], + password: credentials[1], + ...parseHostPort(parts[1]), + }; + } + + return parseHostPort(addr); +} + +export interface RedisParsedResult { + ssl: boolean, + password?: string, + username?: string, + host?: string, + port?: number, + /** + * Local domain socket path. If set the port, host and family will be ignored. + */ + path?: string, + sentinels?: { host: string, port: number }[], + db?: number, + name?: string, +} + +function parseHostPartBasic(addUrl: string, result: RedisParsedResult) { + const { host, port, password, username } = parseAddrPart(addUrl); + + result.password = password; + result.username = username; + result.host = host; + result.port = port; + + return result; +} + +function parseHostPartSentinel(addUrl: string, result: RedisParsedResult) { + const servers = addUrl.split(','); + + result.sentinels = servers.map((addr) => parseHostPort(addr)); + + return result; +} + +function parseUrl( + url: string, + result: RedisParsedResult, + parseAddPartFn: (addr: string, result: RedisParsedResult) => RedisParsedResult, +): RedisParsedResult { + if (url.includes('/')) { + const parts = url.split('/'); + if (parts.length === 2) { + result.db = parseInt(parts[1], 10); + } else if (parts.length === 3) { + result.name = parts[1]; + result.db = parseInt(parts[2], 10); + } else { + throw new Error( + `Unsupported REDIS_URL: "${url}"` + ); + } + + return parseAddPartFn(parts[0], result); + } + + return parseAddPartFn(url, result); +} + +function parseUnixUrl(url: string, result: RedisParsedResult) { + if (url.includes('?')) { + const parts = url.split('?'); + if (parts.length === 2) { + const query = querystring.parse(parts[1]); + + for (const key of Object.keys(query)) { + switch (key.toLowerCase()) { + case 'db': + result.db = parseInt(query[key], 10); + break; + default: + break; + } + } + + return { + ...result, + path: parts[0], + }; + } + + throw new Error( + `Unsupported REDIS_URL: "${url}"` + ); + } + + result.path = url; + + return result; +} + +export function parseRedisUrl(url: Readonly): RedisParsedResult { + const result: RedisParsedResult = { + username: undefined, + password: undefined, + host: undefined, + port: undefined, + ssl: false, + sentinels: undefined, + db: undefined, + name: undefined, + }; + + if (!url) { + return result; + } + + if (url.startsWith('redis://')) { + return parseUrl(url.slice('redis://'.length), result, parseHostPartBasic); + } + + if (url.startsWith('rediss://')) { + result.ssl = true; + + return parseUrl(url.slice('rediss://'.length), result, parseHostPartBasic); + } + + if (url.startsWith('redis+sentinel://')) { + return parseUrl(url.slice('redis+sentinel://'.length), result, parseHostPartSentinel); + } + + if (url.startsWith('unix://')) { + return parseUnixUrl(url.slice('unix://'.length), result); + } + + return parseUrl(url, result, parseHostPartBasic); +} diff --git a/packages/cubejs-event-emitter/tsconfig.json b/packages/cubejs-event-emitter/tsconfig.json new file mode 100644 index 0000000000000..c8d824bc302e7 --- /dev/null +++ b/packages/cubejs-event-emitter/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../tsconfig.base.json", + "include": [ + "src", + "test" + ], + "compilerOptions": { + "outDir": "dist", + "rootDir": ".", + "baseUrl": ".", + "allowJs": false, + "noImplicitAny": false, + "strictNullChecks": false, + "strictFunctionTypes": false, + "strictBindCallApply": false, + "strictPropertyInitialization": false, + }, +} diff --git a/packages/cubejs-playground/src/QueryBuilderV2/utils/cube-sql-converter.ts b/packages/cubejs-playground/src/QueryBuilderV2/utils/cube-sql-converter.ts index 4b3711bd48fb1..a388641745e6c 100644 --- a/packages/cubejs-playground/src/QueryBuilderV2/utils/cube-sql-converter.ts +++ b/packages/cubejs-playground/src/QueryBuilderV2/utils/cube-sql-converter.ts @@ -379,7 +379,7 @@ export class CubeSQLConverter { } if (filter.operator === 'startsWith') { - return repeatFilter(`starts_with(${filter.member}, '{{value}}')`, filter.values, false); + return repeatFilter(`${filter.member} LIKE '{{value}}%'`, filter.values, false); } if (filter.operator === 'notStartsWith') { diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts index a492e974aa510..2a28b951048ab 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts @@ -2,6 +2,7 @@ import crypto from 'crypto'; import csvWriter from 'csv-write-stream'; import { LRUCache } from 'lru-cache'; import { pipeline } from 'stream'; +import { EventEmitterInterface } from '@cubejs-backend/event-emitter'; import { AsyncDebounce, getEnv, MaybeCancelablePromise, streamToArray } from '@cubejs-backend/shared'; import { CubeStoreCacheDriver, CubeStoreDriver } from '@cubejs-backend/cubestore-driver'; import { @@ -37,27 +38,36 @@ export type QueryWithParams = [ export type LoadRefreshKeyOptions = { requestId?: string; skipRefreshKeyWaitForRenew?: boolean; - dataSource: string + dataSource: string; + renewedCube?: string; + requestContext?: any; + forceNoCache?: boolean; }; export type Query = { requestId?: string; + requestContext?: any; + cube?: string; dataSource: string; preAggregations?: PreAggregationDescription[]; groupedPartitionPreAggregations?: PreAggregationDescription[][]; preAggregationsLoadCacheByDataSource?: any; renewQuery?: boolean; + forceNoCache?: boolean; + securityContext?: any; compilerCacheFn?: (subKey: string[], cacheFn: () => T) => T; }; export type QueryBody = { dataSource?: string; + cube?: string; persistent?: boolean; query?: string; values?: string[]; continueWait?: boolean; renewQuery?: boolean; requestId?: string; + requestContext?: any; external?: boolean; isJob?: boolean; forceNoCache?: boolean; @@ -133,6 +143,7 @@ export class QueryCache { protected readonly cachePrefix: string, protected readonly driverFactory: DriverFactoryByDataSource, protected readonly logger: any, + protected readonly eventEmitter: EventEmitterInterface, public readonly options: QueryCacheOptions ) { switch (options.cacheAndQueueDriver || 'memory') { @@ -790,7 +801,7 @@ export class QueryCache { )); } - public async loadRefreshKeysFromQuery(query: Query) { + public async loadRefreshKeysFromQuery(query: QueryBody) { return Promise.all( this.loadRefreshKeys( this.cacheKeyQueriesFrom(query), @@ -798,6 +809,9 @@ export class QueryCache { { requestId: query.requestId, dataSource: query.dataSource, + renewedCube: query.cube, + requestContext: query.requestContext, + forceNoCache: query.forceNoCache, } ) ); @@ -828,6 +842,10 @@ export class QueryCache { dataSource: options.dataSource, useInMemory: true, external: queryOptions?.external, + renewedCube: options.renewedCube, + requestContext: options.requestContext, + isScheduledRefresh: true, + forceNoCache: options.forceNoCache, }, ); } @@ -858,6 +876,9 @@ export class QueryCache { persistent?: boolean, primaryQuery?: boolean, renewCycle?: boolean, + renewedCube?: string, + requestContext?: any, + isScheduledRefresh?: boolean, } ) { const spanId = crypto.randomBytes(16).toString('hex'); @@ -918,11 +939,6 @@ export class QueryCache { }) ); - if (options.forceNoCache) { - this.logger('Force no cache for', { cacheKey, requestId: options.requestId, spanId, primaryQuery, renewCycle }); - return fetchNew(); - } - let res; const inMemoryCacheDisablePeriod = 5 * 60 * 1000; @@ -931,7 +947,6 @@ export class QueryCache { const inMemoryValue = this.memoryCache.get(redisKey); if (inMemoryValue) { const renewedAgo = (new Date()).getTime() - inMemoryValue.time; - if ( renewalKey && ( !renewalThreshold || @@ -965,6 +980,13 @@ export class QueryCache { res = await this.cacheDriver.get(redisKey); } + if (options.forceNoCache) { + // this.logger('Force no cache for', { cacheKey, requestId: options.requestId, spanId, primaryQuery, renewCycle }); + const newRes = await fetchNew(); + this.emitEventWhenUpdatedUpdated(res?.result ?? null, newRes, options); + return newRes; + } + if (res) { const parsedResult = res; const renewedAgo = (new Date()).getTime() - parsedResult.time; @@ -988,16 +1010,23 @@ export class QueryCache { parsedResult.renewalKey !== renewalKey ) ) { + // RENE CHECK if (options.waitForRenew) { this.logger('Waiting for renew', { cacheKey, renewalThreshold, requestId: options.requestId, spanId, primaryQuery, renewCycle }); - return fetchNew(); + const newRes = await fetchNew(); + this.emitEventWhenUpdatedUpdated(parsedResult.result, newRes, options); + return newRes; } else { this.logger('Renewing existing key', { cacheKey, renewalThreshold, requestId: options.requestId, spanId, primaryQuery, renewCycle }); - fetchNew().catch(e => { - if (!(e instanceof ContinueWaitError)) { - this.logger('Error renewing', { cacheKey, error: e.stack || e, requestId: options.requestId, spanId, primaryQuery, renewCycle }); - } - }); + fetchNew() + .then(newRes => { + this.emitEventWhenUpdatedUpdated(parsedResult.result, newRes, options); + }) + .catch(e => { + if (!(e instanceof ContinueWaitError)) { + this.logger('Error renewing', { cacheKey, error: e.stack || e, requestId: options.requestId, spanId, primaryQuery, renewCycle }); + } + }); } } this.logger('Using cache for', { cacheKey, requestId: options.requestId, spanId, primaryQuery, renewCycle }); @@ -1007,7 +1036,9 @@ export class QueryCache { return parsedResult.result; } else { this.logger('Missing cache for', { cacheKey, requestId: options.requestId, spanId, primaryQuery, renewCycle }); - return fetchNew(); + const newRes = await fetchNew(); + this.emitEventWhenUpdatedUpdated(null, newRes, options); + return newRes; } } @@ -1039,4 +1070,29 @@ export class QueryCache { public async testConnection() { return this.cacheDriver.testConnection(); } + + private emitEventWhenUpdatedUpdated( + prevRes: unknown, + res: unknown, + options: { + renewedCube?: string, + requestContext?: any, + isScheduledRefresh?: boolean, + } + ) { + if (!options.renewedCube) { + return; + } + + const prevResHash = getCacheHash(JSON.stringify(prevRes)); + const resHash = getCacheHash(JSON.stringify(res)); + + if (prevResHash !== resHash) { + console.log(`Emitting Cube Renewed ${options.renewedCube} with options: ${options.requestContext?.authInfo?.tenantId}`); + this.eventEmitter.emit('cubeRenewed', { + renewedCube: options.renewedCube, + requestContext: options.requestContext, + }); + } + } } diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts b/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts index fe53c59a55888..050629f92c784 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts @@ -1,5 +1,6 @@ import * as stream from 'stream'; import R from 'ramda'; +import { EventEmitterInterface } from '@cubejs-backend/event-emitter'; import { getEnv } from '@cubejs-backend/shared'; import { CubeStoreDriver } from '@cubejs-backend/cubestore-driver'; import { @@ -9,7 +10,7 @@ import { QueryKey } from '@cubejs-backend/base-driver'; -import { QueryCache, QueryBody, TempTable, PreAggTableToTempTable, QueryWithParams, CacheKey } from './QueryCache'; +import { QueryCache, QueryBody, TempTable, Query, PreAggTableToTempTable, QueryWithParams, CacheKey } from './QueryCache'; import { PreAggregations, PreAggregationDescription, getLastUpdatedAtTimestamp } from './PreAggregations'; import { DriverFactory, DriverFactoryByDataSource } from './DriverFactory'; import { QueryStream } from './QueryStream'; @@ -48,10 +49,6 @@ function detectQueueAndCacheDriver(options: QueryOrchestratorOptions): CacheAndQ return cacheAndQueueDriver; } - if (getEnv('redisUrl') || getEnv('redisUseIORedis')) { - return 'redis'; - } - if (getEnv('nodeEnv') === 'production') { return 'cubestore'; } @@ -72,6 +69,7 @@ export class QueryOrchestrator { protected readonly redisPrefix: string, protected readonly driverFactory: DriverFactoryByDataSource, protected readonly logger: any, + protected readonly eventEmitter: EventEmitterInterface, options: QueryOrchestratorOptions = {} ) { this.rollupOnlyMode = options.rollupOnlyMode; @@ -104,6 +102,7 @@ export class QueryOrchestrator { this.redisPrefix, driverFactory, this.logger, + this.eventEmitter, { externalDriverFactory, cacheAndQueueDriver, @@ -288,7 +287,7 @@ export class QueryOrchestrator { }; } - public async loadRefreshKeys(query) { + public async loadRefreshKeys(query: QueryBody) { return this.queryCache.loadRefreshKeysFromQuery(query); } diff --git a/packages/cubejs-query-orchestrator/test/unit/QueryCache.abstract.ts b/packages/cubejs-query-orchestrator/test/unit/QueryCache.abstract.ts index 58403e93f6504..20a9d6bd06511 100644 --- a/packages/cubejs-query-orchestrator/test/unit/QueryCache.abstract.ts +++ b/packages/cubejs-query-orchestrator/test/unit/QueryCache.abstract.ts @@ -1,4 +1,5 @@ import crypto from 'crypto'; +import { DefaultEventEmitter } from '@cubejs-backend/event-emitter'; import { createCancelablePromise, pausePromise } from '@cubejs-backend/shared'; import { QueryCache, QueryCacheOptions } from '../../src'; @@ -10,12 +11,14 @@ export type QueryCacheTestOptions = QueryCacheOptions & { export const QueryCacheTest = (name: string, options: QueryCacheTestOptions) => { describe(`QueryQueue${name}`, () => { + const eventEmitter = new DefaultEventEmitter(); const cache = new QueryCache( crypto.randomBytes(16).toString('hex'), jest.fn(() => { throw new Error('It`s not implemented mock...'); }), jest.fn(), + eventEmitter, options, ); diff --git a/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts b/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts index f8b91ffccdb67..78838158ad701 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PostgresQuery.ts @@ -24,7 +24,7 @@ export class PostgresQuery extends BaseQuery { } public convertTz(field: string): string { - return `(${field}::timestamptz AT TIME ZONE '${this.timezone}')`; + return `((${field})::timestamptz AT TIME ZONE '${this.timezone}')`; } public timeGroupedColumn(granularity: string, dimension: string): string { diff --git a/packages/cubejs-server-core/package.json b/packages/cubejs-server-core/package.json index 0a0d342353f33..1c6febc305995 100644 --- a/packages/cubejs-server-core/package.json +++ b/packages/cubejs-server-core/package.json @@ -26,7 +26,8 @@ "lint": "eslint src/* test/* --ext .ts,.js", "lint:fix": "eslint --fix src/* test/* --ext .ts,.js", "test": "npm run unit", - "unit": "jest --runInBand --forceExit --coverage dist/test" + "unit": "jest --runInBand --forceExit --coverage dist/test", + "use:transai": "yarn link @cubejs-backend/shared && yarn link @cubejs-backend/cloud && yarn link @cubejs-backend/native && yarn link @cubejs-backend/server && yarn link @cubejs-backend/api-gateway && yarn link @cubejs-backend/schema-compiler && yarn link @cubejs-backend/query-orchestrator && yarn link @cubejs-backend/postgres-driver && yarn link @cubejs-backend/cubestore-driver && yarn link @cubejs-backend/event-emitter && yarn link" }, "dependencies": { "@cubejs-backend/api-gateway": "1.3.78", diff --git a/packages/cubejs-server-core/src/core/OrchestratorApi.ts b/packages/cubejs-server-core/src/core/OrchestratorApi.ts index 77caf0d556c3f..b8d0c625b5da1 100644 --- a/packages/cubejs-server-core/src/core/OrchestratorApi.ts +++ b/packages/cubejs-server-core/src/core/OrchestratorApi.ts @@ -9,7 +9,7 @@ import { QueryOrchestrator, QueryOrchestratorOptions, } from '@cubejs-backend/query-orchestrator'; - +import { EventEmitterInterface } from '@cubejs-backend/event-emitter'; import { DatabaseType, RequestContext } from './types'; export interface OrchestratorApiOptions extends QueryOrchestratorOptions { @@ -28,6 +28,7 @@ export class OrchestratorApi { public constructor( protected readonly driverFactory: DriverFactoryByDataSource, protected readonly logger, + protected readonly eventEmitter: EventEmitterInterface, protected readonly options: OrchestratorApiOptions ) { this.continueWaitTimeout = this.options.continueWaitTimeout || 5; @@ -36,6 +37,7 @@ export class OrchestratorApi { options.redisPrefix || 'STANDALONE', driverFactory, logger, + eventEmitter, options ); } diff --git a/packages/cubejs-server-core/src/core/RefreshScheduler.ts b/packages/cubejs-server-core/src/core/RefreshScheduler.ts index d9d947ec5b1b5..91ccbc82bbf06 100644 --- a/packages/cubejs-server-core/src/core/RefreshScheduler.ts +++ b/packages/cubejs-server-core/src/core/RefreshScheduler.ts @@ -27,7 +27,9 @@ type ScheduledRefreshQueryingOptions = Required { + const cubeNames = queryForEvaluation.cubeEvaluator.cubeNames(); + // console.log(`Running refresh Queries for ${context?.authInfo?.tenantId}`); + await Promise.all(cubeNames.map(async cube => { const cubeFromPath = queryForEvaluation.cubeEvaluator.cubeFromPath(cube); const measuresCount = Object.keys(cubeFromPath.measures || {}).length; const dimensionsCount = Object.keys(cubeFromPath.dimensions || {}).length; @@ -371,8 +374,12 @@ export class RefreshScheduler { continueWait: true, renewQuery: true, requestId: context.requestId, + requestContext: context, scheduledRefresh: true, + cube, loadRefreshKeysOnly: true, + forceNoCache: queryingOptions.forceNoCache ?? false, + isJob: queryingOptions.isJob ?? false, }); })); })); diff --git a/packages/cubejs-server-core/src/core/logger.js b/packages/cubejs-server-core/src/core/logger.js index 8f3a9852db5e5..211a0578059bc 100644 --- a/packages/cubejs-server-core/src/core/logger.js +++ b/packages/cubejs-server-core/src/core/logger.js @@ -81,7 +81,7 @@ export const devLogger = (level) => (type, { error, warning, ...message }) => { }; export const prodLogger = (level) => (msg, params) => { - const { error, warning } = params; + const { error, warning } = params ?? {}; const logMessage = () => console.log(JSON.stringify({ message: msg, ...params })); // eslint-disable-next-line default-case diff --git a/packages/cubejs-server-core/src/core/optionsValidate.ts b/packages/cubejs-server-core/src/core/optionsValidate.ts index 7a3c5b9e97027..3332a166a8c9d 100644 --- a/packages/cubejs-server-core/src/core/optionsValidate.ts +++ b/packages/cubejs-server-core/src/core/optionsValidate.ts @@ -72,7 +72,8 @@ const schemaOptions = Joi.object().keys({ externalDialectFactory: Joi.func(), externalDriverFactory: Joi.func(), // - cacheAndQueueDriver: Joi.string().valid('cubestore', 'memory'), + eventEmitterOptions: Joi.object(), + cacheAndQueueDriver: Joi.string().valid('cubestore', 'redis', 'memory'), contextToAppId: Joi.func(), contextToRoles: Joi.func(), contextToGroups: Joi.func(), diff --git a/packages/cubejs-server-core/src/core/server.ts b/packages/cubejs-server-core/src/core/server.ts index 1db5966d5af31..4dc1f3d44b045 100644 --- a/packages/cubejs-server-core/src/core/server.ts +++ b/packages/cubejs-server-core/src/core/server.ts @@ -26,6 +26,11 @@ import { import type { Application as ExpressApplication } from 'express'; import { BaseDriver, DriverFactoryByDataSource } from '@cubejs-backend/query-orchestrator'; +import { + DefaultEventEmitter, + EventEmitterInterface, + RedisEventEmitter, RedisEventEmitterOptions +} from '@cubejs-backend/event-emitter'; import { RefreshScheduler, ScheduledRefreshOptions } from './RefreshScheduler'; import { OrchestratorApi, OrchestratorApiOptions } from './OrchestratorApi'; import { CompilerApi } from './CompilerApi'; @@ -121,6 +126,8 @@ export class CubejsServerCore { */ public static getDriverMaxPool = getDriverMaxPool; + private readonly eventEmitter: EventEmitterInterface; + public repository: FileRepository; protected devServer: DevServer | undefined; @@ -193,6 +200,15 @@ export class CubejsServerCore { this.repository = new FileRepository(this.options.schemaPath); this.repositoryFactory = this.options.repositoryFactory || (() => this.repository); + if (opts.eventEmitterOptions?.type || getEnv('eventEmitterSetting') === 'redis') { + console.log('🔗 TransAI Redis Event Emitter'); + const options = opts.eventEmitterOptions as RedisEventEmitterOptions | undefined; + const url = options?.url || getEnv('eventEmitterRedisUrl'); + this.eventEmitter = new RedisEventEmitter(url); + } else { + console.log('Memory Event Emitter'); + this.eventEmitter = new DefaultEventEmitter(); + } this.contextToDbType = this.options.dbType; this.contextToExternalDbType = wrapToFnIfNeeded(this.options.externalDbType); this.preAggregationsSchema = wrapToFnIfNeeded(this.options.preAggregationsSchema); @@ -499,7 +515,7 @@ export class CubejsServerCore { logger: LoggerFn, options: ApiGatewayOptions ): ApiGateway { - return new ApiGateway(apiSecret, getCompilerApi, getOrchestratorApi, logger, options); + return new ApiGateway(apiSecret, getCompilerApi, getOrchestratorApi, logger, this.eventEmitter, options); } protected async contextRejectionMiddleware(req, res, next) { @@ -730,6 +746,7 @@ export class CubejsServerCore { return new OrchestratorApi( getDriver, this.logger, + this.eventEmitter, options ); } diff --git a/packages/cubejs-server-core/src/core/types.ts b/packages/cubejs-server-core/src/core/types.ts index 818f39e37445f..af1cfa7bfcbb2 100644 --- a/packages/cubejs-server-core/src/core/types.ts +++ b/packages/cubejs-server-core/src/core/types.ts @@ -11,6 +11,10 @@ import { } from '@cubejs-backend/api-gateway'; import { BaseDriver, CacheAndQueryDriverType } from '@cubejs-backend/query-orchestrator'; import { BaseQuery } from '@cubejs-backend/schema-compiler'; +import { + DefaultEventEmitterOptions, + RedisEventEmitterOptions +} from "@cubejs-backend/event-emitter"; export interface QueueOptions { concurrency?: number; @@ -165,8 +169,8 @@ export type DialectFactoryFn = (context: DialectContext) => BaseQuery; // external export type ExternalDbTypeFn = (context: RequestContext) => DatabaseType; -export type ExternalDriverFactoryFn = (context: RequestContext) => Promise | BaseDriver; export type ExternalDialectFactoryFn = (context: RequestContext) => BaseQuery; +export type ExternalDriverFactoryFn = (context: RequestContext) => Promise | BaseDriver; export type LoggerFnParams = { // It's possible to fill timestamp at the place of logging, otherwise, it will be filled in automatically @@ -189,6 +193,7 @@ export interface CreateOptions { devServer?: boolean; apiSecret?: string; logger?: LoggerFn; + eventEmitterOptions?: RedisEventEmitterOptions | DefaultEventEmitterOptions; driverFactory?: DriverFactoryFn; dialectFactory?: DialectFactoryFn; externalDriverFactory?: ExternalDriverFactoryFn; diff --git a/packages/cubejs-server/src/server/container.ts b/packages/cubejs-server/src/server/container.ts index bf2b693e74406..60476b61132c0 100644 --- a/packages/cubejs-server/src/server/container.ts +++ b/packages/cubejs-server/src/server/container.ts @@ -240,7 +240,7 @@ export class ServerContainer { try { const { version, port } = await server.listen(); - console.log(`🚀 Cube API server (${version}) is listening on ${port}`); + console.log(`🚀 Cube API server (${version}, TransAI build) is listening on ${port}`); } catch (e: any) { console.error('Fatal error during server start: '); console.error(e.stack || e); diff --git a/rust/cubeorchestrator/src/query_result_transform.rs b/rust/cubeorchestrator/src/query_result_transform.rs index a29b12ff56562..8f2756d492277 100644 --- a/rust/cubeorchestrator/src/query_result_transform.rs +++ b/rust/cubeorchestrator/src/query_result_transform.rs @@ -15,6 +15,7 @@ use std::{ fmt::Display, sync::{Arc, LazyLock}, }; +use regex::Regex; pub const COMPARE_DATE_RANGE_FIELD: &str = "compareDateRange"; pub const COMPARE_DATE_RANGE_SEPARATOR: &str = " - "; @@ -47,8 +48,12 @@ pub fn transform_value(value: DBResponseValue, type_: &str) -> DBResponsePrimiti ) } DBResponseValue::Primitive(DBResponsePrimitive::String(ref s)) if type_ == "time" => { - let formatted = DateTime::parse_from_rfc3339(s) - .map(|dt| dt.format("%Y-%m-%dT%H:%M:%S%.3f").to_string()) + let re = Regex::new(r"([+-]\d{2})$").unwrap(); + let s_norm = re.replace(s, "$1:00"); + + + let formatted = DateTime::parse_from_rfc3339(s_norm) + .map(|dt| dt.with_timezone(&Utc).format("%Y-%m-%dT%H:%M:%S%.3f").to_string()) .or_else(|_| { NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.3f").map(|dt| { Utc.from_utc_datetime(&dt) diff --git a/tsconfig.base.json b/tsconfig.base.json index e2007451bb64b..467f797215cff 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -20,7 +20,8 @@ "packages/cubejs-query-orchestrator/src", "packages/cubejs-schema-compiler/src", "packages/cubejs-server/src", - "packages/cubejs-server-core/src" + "packages/cubejs-server-core/src", + "packages/cubejs-event-emitter/src" ] }, "composite": true, diff --git a/tsconfig.json b/tsconfig.json index aa6ec37cefcb6..3fb67acf816fb 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,6 +10,9 @@ { "path": "packages/cubejs-backend-shared" }, + { + "path": "packages/cubejs-event-emitter" + }, { "path": "packages/cubejs-templates" }, diff --git a/yarn.lock b/yarn.lock index 919c694a59df5..6c94b0dc26819 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2899,6 +2899,20 @@ resolved "https://registry.yarnpkg.com/@cubejs-backend/dotenv/-/dotenv-9.0.2.tgz#c3679091b702f0fd38de120c5a63943fcdc0dcbf" integrity sha512-yC1juhXEjM7K97KfXubDm7WGipd4Lpxe+AT8XeTRE9meRULrKlw0wtE2E8AQkGOfTBn+P1SCkePQ/BzIbOh1VA== +"@cubejs-backend/event-emitter@file:packages/cubejs-event-emitter": + version "0.35.67" + dependencies: + "@cubejs-backend/dotenv" "^9.0.2" + "@cubejs-backend/shared" "^0.35.67" + chokidar "^3.5.1" + env-var "^6.3.0" + fs-extra "^9.1.0" + jsonwebtoken "^8.5.1" + redis "^4.6.14" + request "^2.88.2" + request-promise "^4.2.5" + rxjs "^7.8.1" + "@cubejs-backend/jdbc@^0.8.1": version "0.8.1" resolved "https://registry.yarnpkg.com/@cubejs-backend/jdbc/-/jdbc-0.8.1.tgz#32e21fcbf35f91028864396503620c83c2779261" @@ -2910,6 +2924,18 @@ uuid "^11.1.0" winston "^3.17.0" +"@cubejs-backend/linter@^0.35.0": + version "0.35.0" + resolved "https://registry.yarnpkg.com/@cubejs-backend/linter/-/linter-0.35.0.tgz#3913d49087a08d31edf1fb04e432d1581824e435" + integrity sha512-t7Kivqe2tfyHMs8EOJ53Ai6qWJWD+j8vI38c8ZYNUSO25syyVl3RtVagRmGTPzab9LfAVvO1lGv0yAJuYd5i7Q== + dependencies: + "@typescript-eslint/eslint-plugin" "^6.12.0" + "@typescript-eslint/parser" "^6.12.0" + eslint "^8.54.0" + eslint-config-airbnb-base "^14.2.1" + eslint-plugin-import "^2.22.1" + eslint-plugin-node "^9.2.0" + "@cubejs-backend/node-java-maven@^0.1.3": version "0.1.3" resolved "https://registry.yarnpkg.com/@cubejs-backend/node-java-maven/-/node-java-maven-0.1.3.tgz#d7d85204804ccce4e988019472867d1b6da3a8ae" @@ -2940,6 +2966,26 @@ throttle-debounce "^3.0.1" uuid "^8.3.2" +"@cubejs-backend/shared@^0.35.67": + version "0.35.67" + resolved "https://registry.yarnpkg.com/@cubejs-backend/shared/-/shared-0.35.67.tgz#4092762f880a57dc0daa8c328569eafcb1bebcd5" + integrity sha512-aNc0vrOA87hI1LrmD/SxiGoEx12xyIZEh3VdchYYxhSANUjNB8rmmz3EWVvA5YRoLQ5Sl1AxJPZJTGKjLNNn7Q== + dependencies: + "@oclif/color" "^0.1.2" + bytes "^3.1.0" + cli-progress "^3.9.0" + cross-spawn "^7.0.3" + decompress "^4.2.1" + env-var "^6.3.0" + fs-extra "^9.1.0" + http-proxy-agent "^4.0.1" + moment-range "^4.0.1" + moment-timezone "^0.5.33" + node-fetch "^2.6.1" + shelljs "^0.8.5" + throttle-debounce "^3.0.1" + uuid "^8.3.2" + "@cubejs-infra/post-installer@^0.0.7": version "0.0.7" resolved "https://registry.yarnpkg.com/@cubejs-infra/post-installer/-/post-installer-0.0.7.tgz#a28d2d03e5b7b69a64020d75194a7078cf911d2d" @@ -6851,6 +6897,40 @@ "@react-types/overlays" "^3.8.10" "@react-types/shared" "^3.25.0" +"@redis/bloom@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" + integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== + +"@redis/client@1.6.1": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.6.1.tgz#c4636b7cb34e96008a988409b7e787364ae761a2" + integrity sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw== + dependencies: + cluster-key-slot "1.1.2" + generic-pool "3.9.0" + yallist "4.0.0" + +"@redis/graph@1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.1.tgz#8c10df2df7f7d02741866751764031a957a170ea" + integrity sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw== + +"@redis/json@1.0.7": + version "1.0.7" + resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.7.tgz#016257fcd933c4cbcb9c49cde8a0961375c6893b" + integrity sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ== + +"@redis/search@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.2.0.tgz#50976fd3f31168f585666f7922dde111c74567b8" + integrity sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw== + +"@redis/time-series@1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.1.0.tgz#cba454c05ec201bd5547aaf55286d44682ac8eb5" + integrity sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g== + "@rollup/plugin-alias@^3.1.2": version "3.1.8" resolved "https://registry.yarnpkg.com/@rollup/plugin-alias/-/plugin-alias-3.1.8.tgz#645fd84659e08d3d1b059408fcdf69c1dd435a6b" @@ -8150,6 +8230,11 @@ resolved "https://registry.yarnpkg.com/@types/big.js/-/big.js-6.2.2.tgz#69422ec9ef59df1330ccfde2106d9e1159a083c3" integrity sha512-e2cOW9YlVzFY2iScnGBBkplKsrn2CsObHQ2Hiw4V1sSyiGbgWL8IyqE3zFi1Pt5o1pdAtYkDAIsF3KKUPjdzaA== +"@types/bluebird@*": + version "3.5.42" + resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.42.tgz#7ec05f1ce9986d920313c1377a5662b1b563d366" + integrity sha512-Jhy+MWRlro6UjVi578V/4ZGNfeCOcNCp0YaFNIUGFKlImowqwb1O/22wDVk3FDGMLqxdpOV3qQHD5fPEH4hK6A== + "@types/body-parser@*", "@types/body-parser@^1.19.0": version "1.19.2" resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.2.tgz#aea2059e28b7658639081347ac4fab3de166e6f0" @@ -8749,6 +8834,24 @@ "@types/node" "*" safe-buffer "~5.1.1" +"@types/request-promise@^4.1.46": + version "4.1.51" + resolved "https://registry.yarnpkg.com/@types/request-promise/-/request-promise-4.1.51.tgz#a6bb43289569de84055073757d37a8fd6146bfe3" + integrity sha512-qVcP9Fuzh9oaAh8oPxiSoWMFGnWKkJDknnij66vi09Yiy62bsSDqtd+fG5kIM9wLLgZsRP3Y6acqj9O/v2ZtRw== + dependencies: + "@types/bluebird" "*" + "@types/request" "*" + +"@types/request@*": + version "2.48.13" + resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.13.tgz#abdf4256524e801ea8fdda54320f083edb5a6b80" + integrity sha512-FGJ6udDNUCjd19pp0Q3iTiDkwhYup7J8hpMW9c4k53NrccQFFWKRho6hvtPPEhnXWKvukfwAlB6DbDz4yhH5Gg== + dependencies: + "@types/caseless" "*" + "@types/node" "*" + "@types/tough-cookie" "*" + form-data "^2.5.5" + "@types/request@^2.48.8": version "2.48.12" resolved "https://registry.yarnpkg.com/@types/request/-/request-2.48.12.tgz#0f590f615a10f87da18e9790ac94c29ec4c5ef30" @@ -9975,7 +10078,7 @@ ajv@8.17.1, ajv@^8.0.0, ajv@^8.0.1, ajv@^8.12.0, ajv@^8.9.0: json-schema-traverse "^1.0.0" require-from-string "^2.0.2" -ajv@^6.10.0, ajv@^6.12.4, ajv@^6.12.5, ajv@^6.12.6: +ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.4, ajv@^6.12.5, ajv@^6.12.6: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -10871,7 +10974,7 @@ blob@0.0.5: resolved "https://registry.yarnpkg.com/blob/-/blob-0.0.5.tgz#d680eeef25f8cd91ad533f5b01eed48e64caf683" integrity sha512-gaqbzQPqOoamawKg0LGVd7SzLgXS+JH61oWprSLH+P+abTczqJbhTR8CmJ2u9/bUYNmHTGJx/UEmn6doAvvuig== -bluebird@^3.1.1, bluebird@^3.7.2: +bluebird@^3.1.1, bluebird@^3.5.0, bluebird@^3.7.2: version "3.7.2" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== @@ -11630,6 +11733,11 @@ clsx@^2.0.0: resolved "https://registry.yarnpkg.com/clsx/-/clsx-2.1.0.tgz#e851283bcb5c80ee7608db18487433f7b23f77cb" integrity sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg== +cluster-key-slot@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + cmd-shim@6.0.3, cmd-shim@^6.0.0: version "6.0.3" resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-6.0.3.tgz#c491e9656594ba17ac83c4bd931590a9d6e26033" @@ -13983,6 +14091,14 @@ eslint-plugin-cypress@^2.12.1: dependencies: globals "^11.12.0" +eslint-plugin-es@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/eslint-plugin-es/-/eslint-plugin-es-1.4.1.tgz#12acae0f4953e76ba444bfd1b2271081ac620998" + integrity sha512-5fa/gR2yR3NxQf+UXkeLeP8FBBl6tSgdrAz1+cF84v1FMM4twGwQoqTnn+QxFLcPOrF4pdKEJKDB/q9GoyJrCA== + dependencies: + eslint-utils "^1.4.2" + regexpp "^2.0.1" + eslint-plugin-es@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/eslint-plugin-es/-/eslint-plugin-es-2.0.0.tgz#0f5f5da5f18aa21989feebe8a73eadefb3432976" @@ -14040,6 +14156,18 @@ eslint-plugin-node@^10.0.0: resolve "^1.10.1" semver "^6.1.0" +eslint-plugin-node@^9.2.0: + version "9.2.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-node/-/eslint-plugin-node-9.2.0.tgz#b1911f111002d366c5954a6d96d3cd5bf2a3036a" + integrity sha512-2abNmzAH/JpxI4gEOwd6K8wZIodK3BmHbTxz4s79OIYwwIt2gkpEXlAouJXu4H1c9ySTnRso0tsuthSOZbUMlA== + dependencies: + eslint-plugin-es "^1.4.1" + eslint-utils "^1.4.2" + ignore "^5.1.1" + minimatch "^3.0.4" + resolve "^1.10.1" + semver "^6.1.0" + eslint-plugin-react@^7.15.1, eslint-plugin-react@^7.20.0: version "7.27.1" resolved "https://registry.yarnpkg.com/eslint-plugin-react/-/eslint-plugin-react-7.27.1.tgz#469202442506616f77a854d91babaae1ec174b45" @@ -14950,6 +15078,18 @@ form-data@^2.3.1, form-data@^2.5.0: combined-stream "^1.0.6" mime-types "^2.1.12" +form-data@^2.5.5: + version "2.5.5" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.5.tgz#a5f6364ad7e4e67e95b4a07e2d8c6f711c74f624" + integrity sha512-jqdObeR2rxZZbPSGL+3VckHMYtu+f9//KXBsVny6JSX/pa38Fy+bGjuG8eW/H6USNQWhLi8Num++cU2yOCNz4A== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + es-set-tostringtag "^2.1.0" + hasown "^2.0.2" + mime-types "^2.1.35" + safe-buffer "^5.2.1" + form-data@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.1.tgz#ebd53791b78356a99af9a300d4282c4d5eb9755f" @@ -14970,6 +15110,15 @@ form-data@^4.0.0, form-data@^4.0.4, form-data@~4.0.0: hasown "^2.0.2" mime-types "^2.1.12" +form-data@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" + integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.6" + mime-types "^2.1.12" + formdata-polyfill@^4.0.10: version "4.0.10" resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423" @@ -15175,7 +15324,7 @@ generate-object-property@^1.0.0: dependencies: is-property "^1.0.0" -generic-pool@^3.8.2, generic-pool@^3.9.0: +generic-pool@3.9.0, generic-pool@^3.8.2, generic-pool@^3.9.0: version "3.9.0" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== @@ -15653,6 +15802,19 @@ handlebars@^4.7.7: optionalDependencies: uglify-js "^3.1.4" +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + integrity sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q== + +har-validator@~5.1.3: + version "5.1.5" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.5.tgz#1f0803b9f8cb20c0fa13822df1ecddb36bde1efd" + integrity sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w== + dependencies: + ajv "^6.12.3" + har-schema "^2.0.0" + hard-rejection@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/hard-rejection/-/hard-rejection-2.1.0.tgz#1c6eda5c1685c63942766d79bb40ae773cecd883" @@ -16047,6 +16209,15 @@ http-proxy@^1.18.1: follow-redirects "^1.0.0" requires-port "^1.0.0" +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + integrity sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ== + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + http-signature@~1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.4.0.tgz#dee5a9ba2bf49416abc544abd6d967f6a94c8c3f" @@ -18144,6 +18315,22 @@ jsonparse@^1.2.0, jsonparse@^1.3.1: resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA= +jsonwebtoken@^8.5.1: + version "8.5.1" + resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d" + integrity sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w== + dependencies: + jws "^3.2.2" + lodash.includes "^4.3.0" + lodash.isboolean "^3.0.3" + lodash.isinteger "^4.0.4" + lodash.isnumber "^3.0.3" + lodash.isplainobject "^4.0.6" + lodash.isstring "^4.0.1" + lodash.once "^4.0.0" + ms "^2.1.1" + semver "^5.6.0" + jsonwebtoken@^9.0.0, jsonwebtoken@^9.0.2: version "9.0.2" resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz#65ff91f4abef1784697d40952bb1998c504caaf3" @@ -18160,6 +18347,16 @@ jsonwebtoken@^9.0.0, jsonwebtoken@^9.0.2: ms "^2.1.1" semver "^7.5.4" +jsprim@^1.2.2: + version "1.4.2" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.2.tgz#712c65533a15c878ba59e9ed5f0e26d5b77c5feb" + integrity sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw== + dependencies: + assert-plus "1.0.0" + extsprintf "1.3.0" + json-schema "0.4.0" + verror "1.10.0" + jsprim@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-2.0.2.tgz#77ca23dbcd4135cd364800d22ff82c2185803d4d" @@ -19196,7 +19393,7 @@ mime-db@1.52.0, "mime-db@>= 1.43.0 < 2": resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.1.12, mime-types@^2.1.27, mime-types@^2.1.29, mime-types@^2.1.31, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34: +mime-types@^2.1.12, mime-types@^2.1.27, mime-types@^2.1.29, mime-types@^2.1.31, mime-types@^2.1.35, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24, mime-types@~2.1.34: version "2.1.35" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== @@ -20208,6 +20405,11 @@ nwsapi@^2.2.0, nwsapi@^2.2.7: "@nx/nx-win32-arm64-msvc" "20.8.2" "@nx/nx-win32-x64-msvc" "20.8.2" +oauth-sign@~0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" + integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== + oauth4webapi@^3.0.1: version "3.7.0" resolved "https://registry.yarnpkg.com/oauth4webapi/-/oauth4webapi-3.7.0.tgz#66e4af4674f6fc61ad7e55fe54a777508b17c82f" @@ -21792,6 +21994,13 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM= +psl@^1.1.28: + version "1.15.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.15.0.tgz#bdace31896f1d97cec6a79e8224898ce93d974c6" + integrity sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w== + dependencies: + punycode "^2.3.1" + psl@^1.1.33: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" @@ -21865,6 +22074,11 @@ qs@6.13.1, qs@^6.5.1: dependencies: side-channel "^1.0.6" +qs@~6.5.2: + version "6.5.3" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad" + integrity sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA== + querystring@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" @@ -22754,6 +22968,18 @@ redeyed@~2.1.0: dependencies: esprima "~4.0.0" +redis@^4.6.14: + version "4.7.1" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.7.1.tgz#08588a30936be0e7ad9c0f3e1ac6a85ccaf73e94" + integrity sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ== + dependencies: + "@redis/bloom" "1.2.0" + "@redis/client" "1.6.1" + "@redis/graph" "1.1.1" + "@redis/json" "1.0.7" + "@redis/search" "1.2.0" + "@redis/time-series" "1.1.0" + redux@^4.0.0, redux@^4.0.4: version "4.1.2" resolved "https://registry.yarnpkg.com/redux/-/redux-4.1.2.tgz#140f35426d99bb4729af760afcf79eaaac407104" @@ -22815,6 +23041,11 @@ regexp.prototype.flags@^1.3.1, regexp.prototype.flags@^1.5.2: es-errors "^1.3.0" set-function-name "^2.0.1" +regexpp@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-2.0.1.tgz#8d19d31cf632482b589049f8281f93dbcba4d07f" + integrity sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw== + regexpp@^3.0.0, regexpp@^3.1.0: version "3.2.0" resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.2.0.tgz#0425a2768d8f23bad70ca4b90461fa2f1213e1b2" @@ -22872,6 +23103,49 @@ request-progress@^3.0.0: dependencies: throttleit "^1.0.0" +request-promise-core@1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.4.tgz#3eedd4223208d419867b78ce815167d10593a22f" + integrity sha512-TTbAfBBRdWD7aNNOoVOBH4pN/KigV6LyapYNNlAPA8JwbovRti1E88m3sYAwsLi5ryhPKsE9APwnjFTgdUjTpw== + dependencies: + lodash "^4.17.19" + +request-promise@^4.2.5: + version "4.2.6" + resolved "https://registry.yarnpkg.com/request-promise/-/request-promise-4.2.6.tgz#7e7e5b9578630e6f598e3813c0f8eb342a27f0a2" + integrity sha512-HCHI3DJJUakkOr8fNoCc73E5nU5bqITjOYFMDrKHYOXWXrgD/SBaC7LjwuPymUprRyuF06UK7hd/lMHkmUXglQ== + dependencies: + bluebird "^3.5.0" + request-promise-core "1.1.4" + stealthy-require "^1.1.1" + tough-cookie "^2.3.3" + +request@^2.88.2: + version "2.88.2" + resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" + integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.8.0" + caseless "~0.12.0" + combined-stream "~1.0.6" + extend "~3.0.2" + forever-agent "~0.6.1" + form-data "~2.3.2" + har-validator "~5.1.3" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.19" + oauth-sign "~0.9.0" + performance-now "^2.1.0" + qs "~6.5.2" + safe-buffer "^5.1.2" + tough-cookie "~2.5.0" + tunnel-agent "^0.6.0" + uuid "^3.3.2" + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" @@ -23185,7 +23459,7 @@ safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.2.0: +safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@^5.2.1, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -24054,7 +24328,7 @@ ssh2@^1.15.0, ssh2@^1.4.0: cpu-features "~0.0.10" nan "^2.20.0" -sshpk@^1.18.0: +sshpk@^1.18.0, sshpk@^1.7.0: version "1.18.0" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.18.0.tgz#1663e55cddf4d688b86a46b77f0d5fe363aba028" integrity sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ== @@ -24137,6 +24411,11 @@ std-env@^3.3.3: resolved "https://registry.yarnpkg.com/std-env/-/std-env-3.4.3.tgz#326f11db518db751c83fd58574f449b7c3060910" integrity sha512-f9aPhy8fYBuMN+sNfakZV18U39PbalgjXG3lLB9WkaYTxijru61wb57V9wxxNthXM5Sd88ETBWi29qLAsHO52Q== +stealthy-require@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b" + integrity sha512-ZnWpYnYugiOVEY5GkcuJK1io5V8QmNYChG62gSit9pQVGErXtrKuPC55ITaVSukmMta5qpMU7vqLt2Lnni4f/g== + stoppable@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/stoppable/-/stoppable-1.1.0.tgz#32da568e83ea488b08e4d7ea2c3bcc9d75015d5b" @@ -24192,16 +24471,7 @@ string-length@^5.0.1: char-regex "^2.0.0" strip-ansi "^7.0.1" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -24292,7 +24562,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -24320,13 +24590,6 @@ strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1, strip-ansi@^7.1.0: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -25019,6 +25282,14 @@ totalist@^3.0.0: resolved "https://registry.yarnpkg.com/totalist/-/totalist-3.0.1.tgz#ba3a3d600c915b1a97872348f79c127475f6acf8" integrity sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ== +tough-cookie@^2.3.3, tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + tough-cookie@^4.0.0, tough-cookie@^4.1.3: version "4.1.4" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.4.tgz#945f1461b45b5a8c76821c33ea49c3ac192c1b36" @@ -25638,6 +25909,11 @@ uuid@^11.1.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-11.1.0.tgz#9549028be1753bb934fc96e2bca09bb4105ae912" integrity sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A== +uuid@^3.3.2: + version "3.4.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" + integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== + uuid@^8.0.0, uuid@^8.3.0, uuid@^8.3.2: version "8.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" @@ -26415,7 +26691,7 @@ workerpool@^9.2.0: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-9.2.0.tgz#f74427cbb61234708332ed8ab9cbf56dcb1c4371" integrity sha512-PKZqBOCo6CYkVOwAxWxQaSF2Fvb5Iv2fCeTP7buyWI2GiynWr46NcXSgK/idoV6e60dgCBfgYc+Un3HMvmqP8w== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -26441,15 +26717,6 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" @@ -26640,6 +26907,11 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== +yallist@4.0.0, yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yallist@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" @@ -26650,11 +26922,6 @@ yallist@^3.0.2: resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== -yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - yallist@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/yallist/-/yallist-5.0.0.tgz#00e2de443639ed0d78fd87de0d27469fbcffb533"