From 59c5e363a168825db929101095d1a6c6822f7497 Mon Sep 17 00:00:00 2001 From: Jayababu Date: Sun, 7 Sep 2025 23:37:37 +0530 Subject: [PATCH] feat: add GCP connectors (bucket list, table creation) --- packages/core/package.json | 2 + .../core/src/Components/GCPBigQuery.class.ts | 56 +++++++++++++++++++ .../core/src/Components/GCPBucket.class.ts | 52 +++++++++++++++++ packages/core/src/Components/index.ts | 7 +++ 4 files changed, 117 insertions(+) create mode 100644 packages/core/src/Components/GCPBigQuery.class.ts create mode 100644 packages/core/src/Components/GCPBucket.class.ts diff --git a/packages/core/package.json b/packages/core/package.json index d09776ab..f4caa290 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -61,6 +61,8 @@ "@aws-sdk/client-s3": "^3.826.0", "@aws-sdk/client-secrets-manager": "^3.826.0", "@faker-js/faker": "^9.8.0", + "@google-cloud/bigquery": "^7.0.0", + "@google-cloud/storage": "^7.10.0", "@google-cloud/vertexai": "^1.7.0", "@google/genai": "^1.10.0", "@google/generative-ai": "^0.14.1", diff --git a/packages/core/src/Components/GCPBigQuery.class.ts b/packages/core/src/Components/GCPBigQuery.class.ts new file mode 100644 index 00000000..23a2b52e --- /dev/null +++ b/packages/core/src/Components/GCPBigQuery.class.ts @@ -0,0 +1,56 @@ +import { BigQuery } from "@google-cloud/bigquery"; +import { Component } from "./Component.class"; +import { IAgent as Agent } from "@sre/types/Agent.types"; +import Joi from "joi"; +import { TemplateStringHelper } from "@sre/helpers/TemplateString.helper"; + +export class GCPBigQuery extends Component { + protected configSchema = Joi.object({ + projectId: Joi.string().max(200).required().label("GCP Project ID"), + keyFilename: Joi.string().max(500).allow("").label("Path to service account key JSON"), + datasetId: Joi.string().max(200).required(), + tableId: Joi.string().max(200).required(), + schema: Joi.string().required().label("Table schema as JSON"), + name: Joi.string().max(100).required(), + displayName: Joi.string().max(100).required(), + desc: Joi.string().max(5000).allow(""), + logoUrl: Joi.string().max(500).allow(""), + }); + + constructor() { + super(); + } + + init() {} + + async process(input, config, agent: Agent) { + await super.process(input, config, agent); + + const logger = this.createComponentLogger(agent, config); + logger.debug("=== GCP BigQuery Connector ==="); + + try { + const teamId = agent?.teamId; + const projectId = (await TemplateStringHelper.create(config?.data?.projectId).parseTeamKeysAsync(teamId).asyncResult) as string; + const keyFilename = config?.data?.keyFilename || undefined; + const datasetId = config?.data?.datasetId; + const tableId = config?.data?.tableId; + const schemaJson = JSON.parse(config?.data?.schema); + + if (!projectId || !datasetId || !tableId || !schemaJson) { + return { _error: "Please provide projectId, datasetId, tableId, and schema", _debug: logger.output }; + } + + const bigquery = new BigQuery({ projectId, keyFilename }); + + const [table] = await bigquery.dataset(datasetId).createTable(tableId, { schema: schemaJson }); + + logger.debug(`Created table: ${datasetId}.${tableId}`); + + return { Output: `Table ${table.id} created successfully`, _debug: logger.output }; + } catch (error: any) { + logger.debug("Error: ", error?.message); + return { _error: error?.message || JSON.stringify(error), _debug: logger.output }; + } + } +} diff --git a/packages/core/src/Components/GCPBucket.class.ts b/packages/core/src/Components/GCPBucket.class.ts new file mode 100644 index 00000000..0da6a5fb --- /dev/null +++ b/packages/core/src/Components/GCPBucket.class.ts @@ -0,0 +1,52 @@ +import { Storage } from "@google-cloud/storage"; +import { Component } from "./Component.class"; +import { IAgent as Agent } from "@sre/types/Agent.types"; +import Joi from "joi"; +import { TemplateStringHelper } from "@sre/helpers/TemplateString.helper"; + +export class GCPBucket extends Component { + protected configSchema = Joi.object({ + projectId: Joi.string().max(200).required().label("GCP Project ID"), + keyFilename: Joi.string().max(500).allow("").label("Path to service account key JSON"), + name: Joi.string().max(100).required(), + displayName: Joi.string().max(100).required(), + desc: Joi.string().max(5000).allow(""), + logoUrl: Joi.string().max(500).allow(""), + }); + + constructor() { + super(); + } + + init() {} + + async process(input, config, agent: Agent) { + await super.process(input, config, agent); + + const logger = this.createComponentLogger(agent, config); + logger.debug("=== GCP Bucket Connector ==="); + + try { + // Resolve project + keyfile from config (supports Smyth team key substitution) + const teamId = agent?.teamId; + const projectId = (await TemplateStringHelper.create(config?.data?.projectId).parseTeamKeysAsync(teamId).asyncResult) as string; + const keyFilename = config?.data?.keyFilename || undefined; + + if (!projectId) { + return { _error: "Please provide a valid GCP Project ID", _debug: logger.output }; + } + + const storage = new Storage({ projectId, keyFilename }); + + const [buckets] = await storage.getBuckets(); + const bucketNames = buckets.map(b => b.name); + + logger.debug("Buckets: ", bucketNames); + + return { Output: bucketNames, _debug: logger.output }; + } catch (error: any) { + logger.debug("Error: ", error?.message); + return { _error: error?.message || JSON.stringify(error), _debug: logger.output }; + } + } +} diff --git a/packages/core/src/Components/index.ts b/packages/core/src/Components/index.ts index 07da7c49..29af5433 100644 --- a/packages/core/src/Components/index.ts +++ b/packages/core/src/Components/index.ts @@ -43,6 +43,10 @@ import { MemoryReadKeyVal } from './MemoryReadKeyVal.class'; import { MemoryDeleteKeyVal } from './MemoryDeleteKeyVal.class'; import { MemoryWriteObject } from './MemoryWriteObject.class'; +import { GCPBucket } from './GCPBucket.class'; +import { GCPBigQuery } from './GCPBigQuery.class'; + + const components = { Component: new Component(), Note: new Component(), //this is a fake component @@ -92,6 +96,9 @@ const components = { MemoryReadKeyVal: new MemoryReadKeyVal(), MemoryDeleteKeyVal: new MemoryDeleteKeyVal(), MemoryWriteObject: new MemoryWriteObject(), + + GCPBucket: new GCPBucket(), + GCPBigQuery: new GCPBigQuery(), }; export const ComponentInstances = components;