From ff01ea058f2c5eacd2ec89c1501e3ad663a3b0a4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 00:38:43 +0000 Subject: [PATCH 01/20] feat: Add metadata models package with dynamic schema download - Add airbyte_cdk.metadata_models package with auto-generated Pydantic models - Models are generated from JSON schemas in airbytehq/airbyte repository - Schemas are downloaded on-demand during build process (no submodules) - Uses pydantic.v1 compatibility layer for consistency with declarative models - Includes comprehensive README with usage examples - Adds py.typed marker for type hint compliance The metadata models enable validation of connector metadata.yaml files using the same schemas maintained in the main Airbyte repository. Co-Authored-By: AJ Steers --- airbyte_cdk/metadata_models/README.md | 79 +++ airbyte_cdk/metadata_models/__init__.py | 1 + .../ActorDefinitionResourceRequirements.py | 48 ++ .../generated/AirbyteInternal.py | 39 ++ .../metadata_models/generated/AllowedHosts.py | 18 + .../generated/ConnectorBreakingChanges.py | 70 +++ .../generated/ConnectorBuildOptions.py | 15 + .../generated/ConnectorIPCOptions.py | 36 ++ .../ConnectorMetadataDefinitionV0.py | 495 ++++++++++++++++++ .../generated/ConnectorMetrics.py | 36 ++ .../generated/ConnectorPackageInfo.py | 12 + .../ConnectorRegistryDestinationDefinition.py | 439 ++++++++++++++++ .../generated/ConnectorRegistryReleases.py | 438 ++++++++++++++++ .../ConnectorRegistrySourceDefinition.py | 439 ++++++++++++++++ .../generated/ConnectorRegistryV0.py | 445 ++++++++++++++++ .../generated/ConnectorReleases.py | 103 ++++ .../generated/ConnectorTestSuiteOptions.py | 63 +++ .../generated/GeneratedFields.py | 74 +++ .../metadata_models/generated/GitInfo.py | 31 ++ .../metadata_models/generated/JobType.py | 16 + ...ormalizationDestinationDefinitionConfig.py | 24 + .../generated/RegistryOverrides.py | 105 ++++ .../metadata_models/generated/ReleaseStage.py | 13 + .../generated/RemoteRegistries.py | 23 + .../generated/ResourceRequirements.py | 18 + .../generated/RolloutConfiguration.py | 29 + .../metadata_models/generated/Secret.py | 33 ++ .../metadata_models/generated/SecretStore.py | 21 + .../generated/SourceFileInfo.py | 16 + .../generated/SuggestedStreams.py | 18 + .../metadata_models/generated/SupportLevel.py | 12 + .../generated/TestConnections.py | 14 + .../metadata_models/generated/__init__.py | 31 ++ airbyte_cdk/metadata_models/py.typed | 1 + bin/generate_component_manifest_files.py | 186 +++++-- 35 files changed, 3400 insertions(+), 41 deletions(-) create mode 100644 airbyte_cdk/metadata_models/README.md create mode 100644 airbyte_cdk/metadata_models/__init__.py create mode 100644 airbyte_cdk/metadata_models/generated/ActorDefinitionResourceRequirements.py create mode 100644 airbyte_cdk/metadata_models/generated/AirbyteInternal.py create mode 100644 airbyte_cdk/metadata_models/generated/AllowedHosts.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorBreakingChanges.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorBuildOptions.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorIPCOptions.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorMetadataDefinitionV0.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorMetrics.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorPackageInfo.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorRegistryDestinationDefinition.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorRegistryReleases.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorRegistrySourceDefinition.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorRegistryV0.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorReleases.py create mode 100644 airbyte_cdk/metadata_models/generated/ConnectorTestSuiteOptions.py create mode 100644 airbyte_cdk/metadata_models/generated/GeneratedFields.py create mode 100644 airbyte_cdk/metadata_models/generated/GitInfo.py create mode 100644 airbyte_cdk/metadata_models/generated/JobType.py create mode 100644 airbyte_cdk/metadata_models/generated/NormalizationDestinationDefinitionConfig.py create mode 100644 airbyte_cdk/metadata_models/generated/RegistryOverrides.py create mode 100644 airbyte_cdk/metadata_models/generated/ReleaseStage.py create mode 100644 airbyte_cdk/metadata_models/generated/RemoteRegistries.py create mode 100644 airbyte_cdk/metadata_models/generated/ResourceRequirements.py create mode 100644 airbyte_cdk/metadata_models/generated/RolloutConfiguration.py create mode 100644 airbyte_cdk/metadata_models/generated/Secret.py create mode 100644 airbyte_cdk/metadata_models/generated/SecretStore.py create mode 100644 airbyte_cdk/metadata_models/generated/SourceFileInfo.py create mode 100644 airbyte_cdk/metadata_models/generated/SuggestedStreams.py create mode 100644 airbyte_cdk/metadata_models/generated/SupportLevel.py create mode 100644 airbyte_cdk/metadata_models/generated/TestConnections.py create mode 100644 airbyte_cdk/metadata_models/generated/__init__.py create mode 100644 airbyte_cdk/metadata_models/py.typed diff --git a/airbyte_cdk/metadata_models/README.md b/airbyte_cdk/metadata_models/README.md new file mode 100644 index 000000000..1883fc7f5 --- /dev/null +++ b/airbyte_cdk/metadata_models/README.md @@ -0,0 +1,79 @@ +# Airbyte Metadata Models + +This package contains Pydantic models for validating Airbyte connector `metadata.yaml` files. + +## Overview + +The models are automatically generated from JSON Schema YAML files maintained in the [airbytehq/airbyte](https://github.com/airbytehq/airbyte) repository at: +``` +airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ +``` + +During the CDK build process (`poetry run poe build`), these schemas are downloaded from GitHub and used to generate Pydantic models via `datamodel-code-generator`. + +## Usage + +### Validating a metadata.yaml file + +```python +from pathlib import Path +import yaml +from airbyte_cdk.metadata_models import ConnectorMetadataDefinitionV0 + +# Load metadata.yaml +metadata_path = Path("path/to/metadata.yaml") +metadata_dict = yaml.safe_load(metadata_path.read_text()) + +# Validate using Pydantic +try: + metadata = ConnectorMetadataDefinitionV0(**metadata_dict) + print("✓ Metadata is valid!") +except Exception as e: + print(f"✗ Validation failed: {e}") +``` + +### Accessing metadata fields + +```python +from airbyte_cdk.metadata_models import ConnectorMetadataDefinitionV0 + +metadata = ConnectorMetadataDefinitionV0(**metadata_dict) + +# Access fields with full type safety +print(f"Connector: {metadata.data.name}") +print(f"Docker repository: {metadata.data.dockerRepository}") +print(f"Docker image tag: {metadata.data.dockerImageTag}") +print(f"Support level: {metadata.data.supportLevel}") +``` + +### Available models + +The main model is `ConnectorMetadataDefinitionV0`, which includes nested models for: + +- `ConnectorType` - Source or destination +- `ConnectorSubtype` - API, database, file, etc. +- `SupportLevel` - Community, certified, etc. +- `ReleaseStage` - Alpha, beta, generally_available +- `ConnectorBreakingChanges` - Breaking change definitions +- `ConnectorReleases` - Release information +- `AllowedHosts` - Network access configuration +- And many more... + +## Regenerating Models + +Models are regenerated automatically when you run: + +```bash +poetry run poe build +``` + +This command: +1. Downloads the latest schema YAML files from the airbyte repository +2. Generates Pydantic models using `datamodel-code-generator` +3. Outputs models to `airbyte_cdk/metadata_models/generated/` + +## Schema Source + +The authoritative schemas are maintained in the [airbyte monorepo](https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src). + +Any changes to metadata validation should be made there, and will be automatically picked up by the CDK build process. diff --git a/airbyte_cdk/metadata_models/__init__.py b/airbyte_cdk/metadata_models/__init__.py new file mode 100644 index 000000000..e37b3fef6 --- /dev/null +++ b/airbyte_cdk/metadata_models/__init__.py @@ -0,0 +1 @@ +from .generated import * diff --git a/airbyte_cdk/metadata_models/generated/ActorDefinitionResourceRequirements.py b/airbyte_cdk/metadata_models/generated/ActorDefinitionResourceRequirements.py new file mode 100644 index 000000000..eb4784a3d --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ActorDefinitionResourceRequirements.py @@ -0,0 +1,48 @@ +# generated by datamodel-codegen: +# filename: ActorDefinitionResourceRequirements.yaml + +from __future__ import annotations + +from enum import Enum +from typing import List, Optional + +from pydantic.v1 import BaseModel, Extra, Field + + +class ResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + cpu_request: Optional[str] = None + cpu_limit: Optional[str] = None + memory_request: Optional[str] = None + memory_limit: Optional[str] = None + + +class JobType(Enum): + get_spec = "get_spec" + check_connection = "check_connection" + discover_schema = "discover_schema" + sync = "sync" + reset_connection = "reset_connection" + connection_updater = "connection_updater" + replicate = "replicate" + + +class JobTypeResourceLimit(BaseModel): + class Config: + extra = Extra.forbid + + jobType: JobType + resourceRequirements: ResourceRequirements + + +class ActorDefinitionResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + default: Optional[ResourceRequirements] = Field( + None, + description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + ) + jobSpecific: Optional[List[JobTypeResourceLimit]] = None diff --git a/airbyte_cdk/metadata_models/generated/AirbyteInternal.py b/airbyte_cdk/metadata_models/generated/AirbyteInternal.py new file mode 100644 index 000000000..5882c86e3 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/AirbyteInternal.py @@ -0,0 +1,39 @@ +# generated by datamodel-codegen: +# filename: AirbyteInternal.yaml + +from __future__ import annotations + +from enum import Enum +from typing import Optional + +from pydantic.v1 import BaseModel, Extra, Field + + +class Sl(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + + +class Ql(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + integer_400 = 400 + integer_500 = 500 + integer_600 = 600 + + +class AirbyteInternal(BaseModel): + class Config: + extra = Extra.allow + + sl: Optional[Sl] = None + ql: Optional[Ql] = None + isEnterprise: Optional[bool] = False + requireVersionIncrementsInPullRequests: Optional[bool] = Field( + True, + description="When false, version increment checks will be skipped for this connector", + ) diff --git a/airbyte_cdk/metadata_models/generated/AllowedHosts.py b/airbyte_cdk/metadata_models/generated/AllowedHosts.py new file mode 100644 index 000000000..eb7318522 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/AllowedHosts.py @@ -0,0 +1,18 @@ +# generated by datamodel-codegen: +# filename: AllowedHosts.yaml + +from __future__ import annotations + +from typing import List, Optional + +from pydantic.v1 import BaseModel, Extra, Field + + +class AllowedHosts(BaseModel): + class Config: + extra = Extra.allow + + hosts: Optional[List[str]] = Field( + None, + description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + ) diff --git a/airbyte_cdk/metadata_models/generated/ConnectorBreakingChanges.py b/airbyte_cdk/metadata_models/generated/ConnectorBreakingChanges.py new file mode 100644 index 000000000..4d73847f3 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorBreakingChanges.py @@ -0,0 +1,70 @@ +# generated by datamodel-codegen: +# filename: ConnectorBreakingChanges.yaml + +from __future__ import annotations + +from datetime import date +from enum import Enum +from typing import Any, Dict, List, Optional + +from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, constr + + +class DeadlineAction(Enum): + auto_upgrade = "auto_upgrade" + disable = "disable" + + +class StreamBreakingChangeScope(BaseModel): + class Config: + extra = Extra.forbid + + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( + ..., + description="List of streams that are impacted by the breaking change.", + min_items=1, + ) + + +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( + ..., + description="A scope that can be used to limit the impact of a breaking change.", + ) + + +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + deadlineAction: Optional[DeadlineAction] = Field( + None, description="Action to do when the deadline is reached." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + title="ConnectorBreakingChanges", + ) diff --git a/airbyte_cdk/metadata_models/generated/ConnectorBuildOptions.py b/airbyte_cdk/metadata_models/generated/ConnectorBuildOptions.py new file mode 100644 index 000000000..32e00539c --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorBuildOptions.py @@ -0,0 +1,15 @@ +# generated by datamodel-codegen: +# filename: ConnectorBuildOptions.yaml + +from __future__ import annotations + +from typing import Optional + +from pydantic.v1 import BaseModel, Extra + + +class ConnectorBuildOptions(BaseModel): + class Config: + extra = Extra.forbid + + baseImage: Optional[str] = None diff --git a/airbyte_cdk/metadata_models/generated/ConnectorIPCOptions.py b/airbyte_cdk/metadata_models/generated/ConnectorIPCOptions.py new file mode 100644 index 000000000..e5ae22e8a --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorIPCOptions.py @@ -0,0 +1,36 @@ +# generated by datamodel-codegen: +# filename: ConnectorIPCOptions.yaml + +from __future__ import annotations + +from enum import Enum +from typing import List + +from pydantic.v1 import BaseModel, Extra + + +class SupportedSerializationEnum(Enum): + JSONL = "JSONL" + PROTOBUF = "PROTOBUF" + FLATBUFFERS = "FLATBUFFERS" + + +class SupportedTransportEnum(Enum): + STDIO = "STDIO" + SOCKET = "SOCKET" + + +class DataChannel(BaseModel): + class Config: + extra = Extra.forbid + + version: str + supportedSerialization: List[SupportedSerializationEnum] + supportedTransport: List[SupportedTransportEnum] + + +class ConnectorIPCOptions(BaseModel): + class Config: + extra = Extra.forbid + + dataChannel: DataChannel diff --git a/airbyte_cdk/metadata_models/generated/ConnectorMetadataDefinitionV0.py b/airbyte_cdk/metadata_models/generated/ConnectorMetadataDefinitionV0.py new file mode 100644 index 000000000..6c414b955 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorMetadataDefinitionV0.py @@ -0,0 +1,495 @@ +# generated by datamodel-codegen: +# filename: ConnectorMetadataDefinitionV0.yaml + +from __future__ import annotations + +from datetime import date, datetime +from enum import Enum +from typing import Any, Dict, List, Literal, Optional, Union +from uuid import UUID + +from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr + + +class ConnectorType(Enum): + destination = "destination" + source = "source" + + +class ConnectorSubtype(Enum): + api = "api" + database = "database" + datalake = "datalake" + file = "file" + custom = "custom" + message_queue = "message_queue" + unknown = "unknown" + vectorstore = "vectorstore" + + +class ConnectorBuildOptions(BaseModel): + class Config: + extra = Extra.forbid + + baseImage: Optional[str] = None + + +class Suite(Enum): + unitTests = "unitTests" + integrationTests = "integrationTests" + acceptanceTests = "acceptanceTests" + liveTests = "liveTests" + + +class SecretStore(BaseModel): + class Config: + extra = Extra.forbid + + alias: Optional[str] = Field( + None, + description="The alias of the secret store which can map to its actual secret address", + ) + type: Optional[Literal["GSM"]] = Field( + None, description="The type of the secret store" + ) + + +class TestConnections(BaseModel): + class Config: + extra = Extra.forbid + + name: str = Field(..., description="The connection name") + id: str = Field(..., description="The connection ID") + + +class ReleaseStage(Enum): + alpha = "alpha" + beta = "beta" + generally_available = "generally_available" + custom = "custom" + + +class SupportLevel(Enum): + community = "community" + certified = "certified" + archived = "archived" + + +class AllowedHosts(BaseModel): + class Config: + extra = Extra.allow + + hosts: Optional[List[str]] = Field( + None, + description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + ) + + +class NormalizationDestinationDefinitionConfig(BaseModel): + class Config: + extra = Extra.allow + + normalizationRepository: str = Field( + ..., + description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", + ) + normalizationTag: str = Field( + ..., + description="a field indicating the tag of the docker repository to be used for normalization.", + ) + normalizationIntegrationType: str = Field( + ..., + description="a field indicating the type of integration dialect to use for normalization.", + ) + + +class SuggestedStreams(BaseModel): + class Config: + extra = Extra.allow + + streams: Optional[List[str]] = Field( + None, + description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", + ) + + +class ResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + cpu_request: Optional[str] = None + cpu_limit: Optional[str] = None + memory_request: Optional[str] = None + memory_limit: Optional[str] = None + + +class JobType(Enum): + get_spec = "get_spec" + check_connection = "check_connection" + discover_schema = "discover_schema" + sync = "sync" + reset_connection = "reset_connection" + connection_updater = "connection_updater" + replicate = "replicate" + + +class RolloutConfiguration(BaseModel): + class Config: + extra = Extra.forbid + + enableProgressiveRollout: Optional[bool] = Field( + False, description="Whether to enable progressive rollout for the connector." + ) + initialPercentage: Optional[conint(ge=0, le=100)] = Field( + 0, + description="The percentage of users that should receive the new version initially.", + ) + maxPercentage: Optional[conint(ge=0, le=100)] = Field( + 50, + description="The percentage of users who should receive the release candidate during the test phase before full rollout.", + ) + advanceDelayMinutes: Optional[conint(ge=10)] = Field( + 10, + description="The number of minutes to wait before advancing the rollout percentage.", + ) + + +class DeadlineAction(Enum): + auto_upgrade = "auto_upgrade" + disable = "disable" + + +class StreamBreakingChangeScope(BaseModel): + class Config: + extra = Extra.forbid + + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( + ..., + description="List of streams that are impacted by the breaking change.", + min_items=1, + ) + + +class Sl(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + + +class Ql(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + integer_400 = 400 + integer_500 = 500 + integer_600 = 600 + + +class AirbyteInternal(BaseModel): + class Config: + extra = Extra.allow + + sl: Optional[Sl] = None + ql: Optional[Ql] = None + isEnterprise: Optional[bool] = False + requireVersionIncrementsInPullRequests: Optional[bool] = Field( + True, + description="When false, version increment checks will be skipped for this connector", + ) + + +class PyPi(BaseModel): + class Config: + extra = Extra.forbid + + enabled: bool + packageName: str = Field(..., description="The name of the package on PyPi.") + + +class GitInfo(BaseModel): + class Config: + extra = Extra.forbid + + commit_sha: Optional[str] = Field( + None, + description="The git commit sha of the last commit that modified this file.", + ) + commit_timestamp: Optional[datetime] = Field( + None, + description="The git commit timestamp of the last commit that modified this file.", + ) + commit_author: Optional[str] = Field( + None, + description="The git commit author of the last commit that modified this file.", + ) + commit_author_email: Optional[str] = Field( + None, + description="The git commit author email of the last commit that modified this file.", + ) + + +class SourceFileInfo(BaseModel): + metadata_etag: Optional[str] = None + metadata_file_path: Optional[str] = None + metadata_bucket_name: Optional[str] = None + metadata_last_modified: Optional[str] = None + registry_entry_generated_at: Optional[str] = None + + +class ConnectorMetrics(BaseModel): + all: Optional[Any] = None + cloud: Optional[Any] = None + oss: Optional[Any] = None + + +class Usage(Enum): + low = "low" + medium = "medium" + high = "high" + + +class SyncSuccessRate(Enum): + low = "low" + medium = "medium" + high = "high" + + +class ConnectorMetric(BaseModel): + class Config: + extra = Extra.allow + + usage: Optional[Union[str, Usage]] = None + sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None + connector_version: Optional[str] = None + + +class SupportedSerializationEnum(Enum): + JSONL = "JSONL" + PROTOBUF = "PROTOBUF" + FLATBUFFERS = "FLATBUFFERS" + + +class SupportedTransportEnum(Enum): + STDIO = "STDIO" + SOCKET = "SOCKET" + + +class DataChannel(BaseModel): + class Config: + extra = Extra.forbid + + version: str + supportedSerialization: List[SupportedSerializationEnum] + supportedTransport: List[SupportedTransportEnum] + + +class ConnectorIPCOptions(BaseModel): + class Config: + extra = Extra.forbid + + dataChannel: DataChannel + + +class Secret(BaseModel): + class Config: + extra = Extra.forbid + + name: str = Field(..., description="The secret name in the secret store") + fileName: Optional[str] = Field( + None, + description="The name of the file to which the secret value would be persisted", + ) + secretStore: SecretStore + + +class JobTypeResourceLimit(BaseModel): + class Config: + extra = Extra.forbid + + jobType: JobType + resourceRequirements: ResourceRequirements + + +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( + ..., + description="A scope that can be used to limit the impact of a breaking change.", + ) + + +class RemoteRegistries(BaseModel): + class Config: + extra = Extra.forbid + + pypi: Optional[PyPi] = None + + +class GeneratedFields(BaseModel): + git: Optional[GitInfo] = None + source_file_info: Optional[SourceFileInfo] = None + metrics: Optional[ConnectorMetrics] = None + sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") + + +class ConnectorTestSuiteOptions(BaseModel): + class Config: + extra = Extra.forbid + + suite: Suite = Field(..., description="Name of the configured test suite") + testSecrets: Optional[List[Secret]] = Field( + None, description="List of secrets required to run the test suite" + ) + testConnections: Optional[List[TestConnections]] = Field( + None, + description="List of sandbox cloud connections that tests can be run against", + ) + + +class ActorDefinitionResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + default: Optional[ResourceRequirements] = Field( + None, + description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + ) + jobSpecific: Optional[List[JobTypeResourceLimit]] = None + + +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + deadlineAction: Optional[DeadlineAction] = Field( + None, description="Action to do when the deadline is reached." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class RegistryOverrides(BaseModel): + class Config: + extra = Extra.forbid + + enabled: bool + name: Optional[str] = None + dockerRepository: Optional[str] = None + dockerImageTag: Optional[str] = None + supportsDbt: Optional[bool] = None + supportsNormalization: Optional[bool] = None + license: Optional[str] = None + documentationUrl: Optional[AnyUrl] = None + connectorSubtype: Optional[str] = None + allowedHosts: Optional[AllowedHosts] = None + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + suggestedStreams: Optional[SuggestedStreams] = None + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + title="ConnectorBreakingChanges", + ) + + +class RegistryOverridesModel(BaseModel): + class Config: + extra = Extra.forbid + + oss: Optional[RegistryOverrides] = None + cloud: Optional[RegistryOverrides] = None + + +class ConnectorReleases(BaseModel): + class Config: + extra = Extra.forbid + + rolloutConfiguration: Optional[RolloutConfiguration] = None + breakingChanges: Optional[ConnectorBreakingChanges] = None + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + ) + + +class Data(BaseModel): + class Config: + extra = Extra.forbid + + name: str + icon: Optional[str] = None + definitionId: UUID + connectorBuildOptions: Optional[ConnectorBuildOptions] = None + connectorTestSuitesOptions: Optional[List[ConnectorTestSuiteOptions]] = None + connectorType: ConnectorType + dockerRepository: str + dockerImageTag: str + supportsDbt: Optional[bool] = None + supportsNormalization: Optional[bool] = None + license: str + documentationUrl: AnyUrl + githubIssueLabel: str + maxSecondsBetweenMessages: Optional[int] = Field( + None, + description="Maximum delay between 2 airbyte protocol messages, in second. The source will timeout if this delay is reached", + ) + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + erdUrl: Optional[str] = Field( + None, description="The URL where you can visualize the ERD" + ) + connectorSubtype: ConnectorSubtype + releaseStage: ReleaseStage + supportLevel: Optional[SupportLevel] = None + tags: Optional[List[str]] = Field( + [], + description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", + ) + registryOverrides: Optional[RegistryOverridesModel] = None + allowedHosts: Optional[AllowedHosts] = None + releases: Optional[ConnectorReleases] = None + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + suggestedStreams: Optional[SuggestedStreams] = None + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + ab_internal: Optional[AirbyteInternal] = None + remoteRegistries: Optional[RemoteRegistries] = None + supportsRefreshes: Optional[bool] = False + generated: Optional[GeneratedFields] = None + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + connectorIPCOptions: Optional[ConnectorIPCOptions] = None + + +class ConnectorMetadataDefinitionV0(BaseModel): + class Config: + extra = Extra.forbid + + metadataSpecVersion: str + data: Data diff --git a/airbyte_cdk/metadata_models/generated/ConnectorMetrics.py b/airbyte_cdk/metadata_models/generated/ConnectorMetrics.py new file mode 100644 index 000000000..2b3d44a80 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorMetrics.py @@ -0,0 +1,36 @@ +# generated by datamodel-codegen: +# filename: ConnectorMetrics.yaml + +from __future__ import annotations + +from enum import Enum +from typing import Any, Optional, Union + +from pydantic.v1 import BaseModel, Extra + + +class ConnectorMetrics(BaseModel): + all: Optional[Any] = None + cloud: Optional[Any] = None + oss: Optional[Any] = None + + +class Usage(Enum): + low = "low" + medium = "medium" + high = "high" + + +class SyncSuccessRate(Enum): + low = "low" + medium = "medium" + high = "high" + + +class ConnectorMetric(BaseModel): + class Config: + extra = Extra.allow + + usage: Optional[Union[str, Usage]] = None + sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None + connector_version: Optional[str] = None diff --git a/airbyte_cdk/metadata_models/generated/ConnectorPackageInfo.py b/airbyte_cdk/metadata_models/generated/ConnectorPackageInfo.py new file mode 100644 index 000000000..2a41dbc10 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorPackageInfo.py @@ -0,0 +1,12 @@ +# generated by datamodel-codegen: +# filename: ConnectorPackageInfo.yaml + +from __future__ import annotations + +from typing import Optional + +from pydantic.v1 import BaseModel + + +class ConnectorPackageInfo(BaseModel): + cdk_version: Optional[str] = None diff --git a/airbyte_cdk/metadata_models/generated/ConnectorRegistryDestinationDefinition.py b/airbyte_cdk/metadata_models/generated/ConnectorRegistryDestinationDefinition.py new file mode 100644 index 000000000..d6acd3a7a --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorRegistryDestinationDefinition.py @@ -0,0 +1,439 @@ +# generated by datamodel-codegen: +# filename: ConnectorRegistryDestinationDefinition.yaml + +from __future__ import annotations + +from datetime import date, datetime +from enum import Enum +from typing import Any, Dict, List, Optional, Union +from uuid import UUID + +from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr + + +class ReleaseStage(Enum): + alpha = "alpha" + beta = "beta" + generally_available = "generally_available" + custom = "custom" + + +class SupportLevel(Enum): + community = "community" + certified = "certified" + archived = "archived" + + +class ResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + cpu_request: Optional[str] = None + cpu_limit: Optional[str] = None + memory_request: Optional[str] = None + memory_limit: Optional[str] = None + + +class JobType(Enum): + get_spec = "get_spec" + check_connection = "check_connection" + discover_schema = "discover_schema" + sync = "sync" + reset_connection = "reset_connection" + connection_updater = "connection_updater" + replicate = "replicate" + + +class NormalizationDestinationDefinitionConfig(BaseModel): + class Config: + extra = Extra.allow + + normalizationRepository: str = Field( + ..., + description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", + ) + normalizationTag: str = Field( + ..., + description="a field indicating the tag of the docker repository to be used for normalization.", + ) + normalizationIntegrationType: str = Field( + ..., + description="a field indicating the type of integration dialect to use for normalization.", + ) + + +class AllowedHosts(BaseModel): + class Config: + extra = Extra.allow + + hosts: Optional[List[str]] = Field( + None, + description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + ) + + +class RolloutConfiguration(BaseModel): + class Config: + extra = Extra.forbid + + enableProgressiveRollout: Optional[bool] = Field( + False, description="Whether to enable progressive rollout for the connector." + ) + initialPercentage: Optional[conint(ge=0, le=100)] = Field( + 0, + description="The percentage of users that should receive the new version initially.", + ) + maxPercentage: Optional[conint(ge=0, le=100)] = Field( + 50, + description="The percentage of users who should receive the release candidate during the test phase before full rollout.", + ) + advanceDelayMinutes: Optional[conint(ge=10)] = Field( + 10, + description="The number of minutes to wait before advancing the rollout percentage.", + ) + + +class DeadlineAction(Enum): + auto_upgrade = "auto_upgrade" + disable = "disable" + + +class StreamBreakingChangeScope(BaseModel): + class Config: + extra = Extra.forbid + + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( + ..., + description="List of streams that are impacted by the breaking change.", + min_items=1, + ) + + +class SourceType(Enum): + api = "api" + file = "file" + database = "database" + custom = "custom" + + +class SuggestedStreams(BaseModel): + class Config: + extra = Extra.allow + + streams: Optional[List[str]] = Field( + None, + description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", + ) + + +class Sl(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + + +class Ql(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + integer_400 = 400 + integer_500 = 500 + integer_600 = 600 + + +class AirbyteInternal(BaseModel): + class Config: + extra = Extra.allow + + sl: Optional[Sl] = None + ql: Optional[Ql] = None + isEnterprise: Optional[bool] = False + requireVersionIncrementsInPullRequests: Optional[bool] = Field( + True, + description="When false, version increment checks will be skipped for this connector", + ) + + +class GitInfo(BaseModel): + class Config: + extra = Extra.forbid + + commit_sha: Optional[str] = Field( + None, + description="The git commit sha of the last commit that modified this file.", + ) + commit_timestamp: Optional[datetime] = Field( + None, + description="The git commit timestamp of the last commit that modified this file.", + ) + commit_author: Optional[str] = Field( + None, + description="The git commit author of the last commit that modified this file.", + ) + commit_author_email: Optional[str] = Field( + None, + description="The git commit author email of the last commit that modified this file.", + ) + + +class SourceFileInfo(BaseModel): + metadata_etag: Optional[str] = None + metadata_file_path: Optional[str] = None + metadata_bucket_name: Optional[str] = None + metadata_last_modified: Optional[str] = None + registry_entry_generated_at: Optional[str] = None + + +class ConnectorMetrics(BaseModel): + all: Optional[Any] = None + cloud: Optional[Any] = None + oss: Optional[Any] = None + + +class Usage(Enum): + low = "low" + medium = "medium" + high = "high" + + +class SyncSuccessRate(Enum): + low = "low" + medium = "medium" + high = "high" + + +class ConnectorMetric(BaseModel): + class Config: + extra = Extra.allow + + usage: Optional[Union[str, Usage]] = None + sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None + connector_version: Optional[str] = None + + +class ConnectorPackageInfo(BaseModel): + cdk_version: Optional[str] = None + + +class JobTypeResourceLimit(BaseModel): + class Config: + extra = Extra.forbid + + jobType: JobType + resourceRequirements: ResourceRequirements + + +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( + ..., + description="A scope that can be used to limit the impact of a breaking change.", + ) + + +class GeneratedFields(BaseModel): + git: Optional[GitInfo] = None + source_file_info: Optional[SourceFileInfo] = None + metrics: Optional[ConnectorMetrics] = None + sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") + + +class ActorDefinitionResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + default: Optional[ResourceRequirements] = Field( + None, + description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + ) + jobSpecific: Optional[List[JobTypeResourceLimit]] = None + + +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + deadlineAction: Optional[DeadlineAction] = Field( + None, description="Action to do when the deadline is reached." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + title="ConnectorBreakingChanges", + ) + + +class ConnectorRegistryDestinationDefinition(BaseModel): + class Config: + extra = Extra.allow + + destinationDefinitionId: UUID + name: str + dockerRepository: str + dockerImageTag: str + documentationUrl: str + icon: Optional[str] = None + iconUrl: Optional[str] = None + spec: Dict[str, Any] + tombstone: Optional[bool] = Field( + False, + description="if false, the configuration is active. if true, then this configuration is permanently off.", + ) + public: Optional[bool] = Field( + False, + description="true if this connector definition is available to all workspaces", + ) + custom: Optional[bool] = Field( + False, description="whether this is a custom connector definition" + ) + releaseStage: Optional[ReleaseStage] = None + supportLevel: Optional[SupportLevel] = None + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + tags: Optional[List[str]] = Field( + None, + description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", + ) + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + supportsDbt: Optional[bool] = Field( + None, + description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", + ) + allowedHosts: Optional[AllowedHosts] = None + releases: Optional[ConnectorRegistryReleases] = None + ab_internal: Optional[AirbyteInternal] = None + supportsRefreshes: Optional[bool] = False + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + generated: Optional[GeneratedFields] = None + packageInfo: Optional[ConnectorPackageInfo] = None + language: Optional[str] = Field( + None, description="The language the connector is written in" + ) + + +class ConnectorRegistryReleases(BaseModel): + class Config: + extra = Extra.forbid + + releaseCandidates: Optional[ConnectorReleaseCandidates] = None + rolloutConfiguration: Optional[RolloutConfiguration] = None + breakingChanges: Optional[ConnectorBreakingChanges] = None + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + ) + + +class ConnectorReleaseCandidates(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[ + constr(regex=r"^\d+\.\d+\.\d+(-[0-9A-Za-z-.]+)?$"), VersionReleaseCandidate + ] = Field( + ..., + description="Each entry denotes a release candidate version of a connector.", + ) + + +class VersionReleaseCandidate(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Union[ + ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition + ] = Field( + ..., + description="Contains information about a release candidate version of a connector.", + ) + + +class ConnectorRegistrySourceDefinition(BaseModel): + class Config: + extra = Extra.allow + + sourceDefinitionId: UUID + name: str + dockerRepository: str + dockerImageTag: str + documentationUrl: str + icon: Optional[str] = None + iconUrl: Optional[str] = None + sourceType: Optional[SourceType] = None + spec: Dict[str, Any] + tombstone: Optional[bool] = Field( + False, + description="if false, the configuration is active. if true, then this configuration is permanently off.", + ) + public: Optional[bool] = Field( + False, + description="true if this connector definition is available to all workspaces", + ) + custom: Optional[bool] = Field( + False, description="whether this is a custom connector definition" + ) + releaseStage: Optional[ReleaseStage] = None + supportLevel: Optional[SupportLevel] = None + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + allowedHosts: Optional[AllowedHosts] = None + suggestedStreams: Optional[SuggestedStreams] = None + maxSecondsBetweenMessages: Optional[int] = Field( + None, + description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", + ) + erdUrl: Optional[str] = Field( + None, description="The URL where you can visualize the ERD" + ) + releases: Optional[ConnectorRegistryReleases] = None + ab_internal: Optional[AirbyteInternal] = None + generated: Optional[GeneratedFields] = None + packageInfo: Optional[ConnectorPackageInfo] = None + language: Optional[str] = Field( + None, description="The language the connector is written in" + ) + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + + +ConnectorRegistryDestinationDefinition.update_forward_refs() +ConnectorRegistryReleases.update_forward_refs() +ConnectorReleaseCandidates.update_forward_refs() +VersionReleaseCandidate.update_forward_refs() diff --git a/airbyte_cdk/metadata_models/generated/ConnectorRegistryReleases.py b/airbyte_cdk/metadata_models/generated/ConnectorRegistryReleases.py new file mode 100644 index 000000000..097fc061f --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorRegistryReleases.py @@ -0,0 +1,438 @@ +# generated by datamodel-codegen: +# filename: ConnectorRegistryReleases.yaml + +from __future__ import annotations + +from datetime import date, datetime +from enum import Enum +from typing import Any, Dict, List, Optional, Union +from uuid import UUID + +from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr + + +class RolloutConfiguration(BaseModel): + class Config: + extra = Extra.forbid + + enableProgressiveRollout: Optional[bool] = Field( + False, description="Whether to enable progressive rollout for the connector." + ) + initialPercentage: Optional[conint(ge=0, le=100)] = Field( + 0, + description="The percentage of users that should receive the new version initially.", + ) + maxPercentage: Optional[conint(ge=0, le=100)] = Field( + 50, + description="The percentage of users who should receive the release candidate during the test phase before full rollout.", + ) + advanceDelayMinutes: Optional[conint(ge=10)] = Field( + 10, + description="The number of minutes to wait before advancing the rollout percentage.", + ) + + +class DeadlineAction(Enum): + auto_upgrade = "auto_upgrade" + disable = "disable" + + +class StreamBreakingChangeScope(BaseModel): + class Config: + extra = Extra.forbid + + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( + ..., + description="List of streams that are impacted by the breaking change.", + min_items=1, + ) + + +class SourceType(Enum): + api = "api" + file = "file" + database = "database" + custom = "custom" + + +class ReleaseStage(Enum): + alpha = "alpha" + beta = "beta" + generally_available = "generally_available" + custom = "custom" + + +class SupportLevel(Enum): + community = "community" + certified = "certified" + archived = "archived" + + +class ResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + cpu_request: Optional[str] = None + cpu_limit: Optional[str] = None + memory_request: Optional[str] = None + memory_limit: Optional[str] = None + + +class JobType(Enum): + get_spec = "get_spec" + check_connection = "check_connection" + discover_schema = "discover_schema" + sync = "sync" + reset_connection = "reset_connection" + connection_updater = "connection_updater" + replicate = "replicate" + + +class AllowedHosts(BaseModel): + class Config: + extra = Extra.allow + + hosts: Optional[List[str]] = Field( + None, + description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + ) + + +class SuggestedStreams(BaseModel): + class Config: + extra = Extra.allow + + streams: Optional[List[str]] = Field( + None, + description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", + ) + + +class Sl(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + + +class Ql(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + integer_400 = 400 + integer_500 = 500 + integer_600 = 600 + + +class AirbyteInternal(BaseModel): + class Config: + extra = Extra.allow + + sl: Optional[Sl] = None + ql: Optional[Ql] = None + isEnterprise: Optional[bool] = False + requireVersionIncrementsInPullRequests: Optional[bool] = Field( + True, + description="When false, version increment checks will be skipped for this connector", + ) + + +class GitInfo(BaseModel): + class Config: + extra = Extra.forbid + + commit_sha: Optional[str] = Field( + None, + description="The git commit sha of the last commit that modified this file.", + ) + commit_timestamp: Optional[datetime] = Field( + None, + description="The git commit timestamp of the last commit that modified this file.", + ) + commit_author: Optional[str] = Field( + None, + description="The git commit author of the last commit that modified this file.", + ) + commit_author_email: Optional[str] = Field( + None, + description="The git commit author email of the last commit that modified this file.", + ) + + +class SourceFileInfo(BaseModel): + metadata_etag: Optional[str] = None + metadata_file_path: Optional[str] = None + metadata_bucket_name: Optional[str] = None + metadata_last_modified: Optional[str] = None + registry_entry_generated_at: Optional[str] = None + + +class ConnectorMetrics(BaseModel): + all: Optional[Any] = None + cloud: Optional[Any] = None + oss: Optional[Any] = None + + +class Usage(Enum): + low = "low" + medium = "medium" + high = "high" + + +class SyncSuccessRate(Enum): + low = "low" + medium = "medium" + high = "high" + + +class ConnectorMetric(BaseModel): + class Config: + extra = Extra.allow + + usage: Optional[Union[str, Usage]] = None + sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None + connector_version: Optional[str] = None + + +class ConnectorPackageInfo(BaseModel): + cdk_version: Optional[str] = None + + +class NormalizationDestinationDefinitionConfig(BaseModel): + class Config: + extra = Extra.allow + + normalizationRepository: str = Field( + ..., + description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", + ) + normalizationTag: str = Field( + ..., + description="a field indicating the tag of the docker repository to be used for normalization.", + ) + normalizationIntegrationType: str = Field( + ..., + description="a field indicating the type of integration dialect to use for normalization.", + ) + + +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( + ..., + description="A scope that can be used to limit the impact of a breaking change.", + ) + + +class JobTypeResourceLimit(BaseModel): + class Config: + extra = Extra.forbid + + jobType: JobType + resourceRequirements: ResourceRequirements + + +class GeneratedFields(BaseModel): + git: Optional[GitInfo] = None + source_file_info: Optional[SourceFileInfo] = None + metrics: Optional[ConnectorMetrics] = None + sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") + + +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + deadlineAction: Optional[DeadlineAction] = Field( + None, description="Action to do when the deadline is reached." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class ActorDefinitionResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + default: Optional[ResourceRequirements] = Field( + None, + description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + ) + jobSpecific: Optional[List[JobTypeResourceLimit]] = None + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + title="ConnectorBreakingChanges", + ) + + +class ConnectorRegistryReleases(BaseModel): + class Config: + extra = Extra.forbid + + releaseCandidates: Optional[ConnectorReleaseCandidates] = None + rolloutConfiguration: Optional[RolloutConfiguration] = None + breakingChanges: Optional[ConnectorBreakingChanges] = None + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + ) + + +class ConnectorReleaseCandidates(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[ + constr(regex=r"^\d+\.\d+\.\d+(-[0-9A-Za-z-.]+)?$"), VersionReleaseCandidate + ] = Field( + ..., + description="Each entry denotes a release candidate version of a connector.", + ) + + +class VersionReleaseCandidate(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Union[ + ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition + ] = Field( + ..., + description="Contains information about a release candidate version of a connector.", + ) + + +class ConnectorRegistrySourceDefinition(BaseModel): + class Config: + extra = Extra.allow + + sourceDefinitionId: UUID + name: str + dockerRepository: str + dockerImageTag: str + documentationUrl: str + icon: Optional[str] = None + iconUrl: Optional[str] = None + sourceType: Optional[SourceType] = None + spec: Dict[str, Any] + tombstone: Optional[bool] = Field( + False, + description="if false, the configuration is active. if true, then this configuration is permanently off.", + ) + public: Optional[bool] = Field( + False, + description="true if this connector definition is available to all workspaces", + ) + custom: Optional[bool] = Field( + False, description="whether this is a custom connector definition" + ) + releaseStage: Optional[ReleaseStage] = None + supportLevel: Optional[SupportLevel] = None + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + allowedHosts: Optional[AllowedHosts] = None + suggestedStreams: Optional[SuggestedStreams] = None + maxSecondsBetweenMessages: Optional[int] = Field( + None, + description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", + ) + erdUrl: Optional[str] = Field( + None, description="The URL where you can visualize the ERD" + ) + releases: Optional[ConnectorRegistryReleases] = None + ab_internal: Optional[AirbyteInternal] = None + generated: Optional[GeneratedFields] = None + packageInfo: Optional[ConnectorPackageInfo] = None + language: Optional[str] = Field( + None, description="The language the connector is written in" + ) + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + + +class ConnectorRegistryDestinationDefinition(BaseModel): + class Config: + extra = Extra.allow + + destinationDefinitionId: UUID + name: str + dockerRepository: str + dockerImageTag: str + documentationUrl: str + icon: Optional[str] = None + iconUrl: Optional[str] = None + spec: Dict[str, Any] + tombstone: Optional[bool] = Field( + False, + description="if false, the configuration is active. if true, then this configuration is permanently off.", + ) + public: Optional[bool] = Field( + False, + description="true if this connector definition is available to all workspaces", + ) + custom: Optional[bool] = Field( + False, description="whether this is a custom connector definition" + ) + releaseStage: Optional[ReleaseStage] = None + supportLevel: Optional[SupportLevel] = None + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + tags: Optional[List[str]] = Field( + None, + description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", + ) + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + supportsDbt: Optional[bool] = Field( + None, + description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", + ) + allowedHosts: Optional[AllowedHosts] = None + releases: Optional[ConnectorRegistryReleases] = None + ab_internal: Optional[AirbyteInternal] = None + supportsRefreshes: Optional[bool] = False + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + generated: Optional[GeneratedFields] = None + packageInfo: Optional[ConnectorPackageInfo] = None + language: Optional[str] = Field( + None, description="The language the connector is written in" + ) + + +ConnectorRegistryReleases.update_forward_refs() +ConnectorReleaseCandidates.update_forward_refs() +VersionReleaseCandidate.update_forward_refs() diff --git a/airbyte_cdk/metadata_models/generated/ConnectorRegistrySourceDefinition.py b/airbyte_cdk/metadata_models/generated/ConnectorRegistrySourceDefinition.py new file mode 100644 index 000000000..002a4aac7 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorRegistrySourceDefinition.py @@ -0,0 +1,439 @@ +# generated by datamodel-codegen: +# filename: ConnectorRegistrySourceDefinition.yaml + +from __future__ import annotations + +from datetime import date, datetime +from enum import Enum +from typing import Any, Dict, List, Optional, Union +from uuid import UUID + +from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr + + +class SourceType(Enum): + api = "api" + file = "file" + database = "database" + custom = "custom" + + +class ReleaseStage(Enum): + alpha = "alpha" + beta = "beta" + generally_available = "generally_available" + custom = "custom" + + +class SupportLevel(Enum): + community = "community" + certified = "certified" + archived = "archived" + + +class ResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + cpu_request: Optional[str] = None + cpu_limit: Optional[str] = None + memory_request: Optional[str] = None + memory_limit: Optional[str] = None + + +class JobType(Enum): + get_spec = "get_spec" + check_connection = "check_connection" + discover_schema = "discover_schema" + sync = "sync" + reset_connection = "reset_connection" + connection_updater = "connection_updater" + replicate = "replicate" + + +class AllowedHosts(BaseModel): + class Config: + extra = Extra.allow + + hosts: Optional[List[str]] = Field( + None, + description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + ) + + +class SuggestedStreams(BaseModel): + class Config: + extra = Extra.allow + + streams: Optional[List[str]] = Field( + None, + description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", + ) + + +class RolloutConfiguration(BaseModel): + class Config: + extra = Extra.forbid + + enableProgressiveRollout: Optional[bool] = Field( + False, description="Whether to enable progressive rollout for the connector." + ) + initialPercentage: Optional[conint(ge=0, le=100)] = Field( + 0, + description="The percentage of users that should receive the new version initially.", + ) + maxPercentage: Optional[conint(ge=0, le=100)] = Field( + 50, + description="The percentage of users who should receive the release candidate during the test phase before full rollout.", + ) + advanceDelayMinutes: Optional[conint(ge=10)] = Field( + 10, + description="The number of minutes to wait before advancing the rollout percentage.", + ) + + +class DeadlineAction(Enum): + auto_upgrade = "auto_upgrade" + disable = "disable" + + +class StreamBreakingChangeScope(BaseModel): + class Config: + extra = Extra.forbid + + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( + ..., + description="List of streams that are impacted by the breaking change.", + min_items=1, + ) + + +class NormalizationDestinationDefinitionConfig(BaseModel): + class Config: + extra = Extra.allow + + normalizationRepository: str = Field( + ..., + description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", + ) + normalizationTag: str = Field( + ..., + description="a field indicating the tag of the docker repository to be used for normalization.", + ) + normalizationIntegrationType: str = Field( + ..., + description="a field indicating the type of integration dialect to use for normalization.", + ) + + +class Sl(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + + +class Ql(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + integer_400 = 400 + integer_500 = 500 + integer_600 = 600 + + +class AirbyteInternal(BaseModel): + class Config: + extra = Extra.allow + + sl: Optional[Sl] = None + ql: Optional[Ql] = None + isEnterprise: Optional[bool] = False + requireVersionIncrementsInPullRequests: Optional[bool] = Field( + True, + description="When false, version increment checks will be skipped for this connector", + ) + + +class GitInfo(BaseModel): + class Config: + extra = Extra.forbid + + commit_sha: Optional[str] = Field( + None, + description="The git commit sha of the last commit that modified this file.", + ) + commit_timestamp: Optional[datetime] = Field( + None, + description="The git commit timestamp of the last commit that modified this file.", + ) + commit_author: Optional[str] = Field( + None, + description="The git commit author of the last commit that modified this file.", + ) + commit_author_email: Optional[str] = Field( + None, + description="The git commit author email of the last commit that modified this file.", + ) + + +class SourceFileInfo(BaseModel): + metadata_etag: Optional[str] = None + metadata_file_path: Optional[str] = None + metadata_bucket_name: Optional[str] = None + metadata_last_modified: Optional[str] = None + registry_entry_generated_at: Optional[str] = None + + +class ConnectorMetrics(BaseModel): + all: Optional[Any] = None + cloud: Optional[Any] = None + oss: Optional[Any] = None + + +class Usage(Enum): + low = "low" + medium = "medium" + high = "high" + + +class SyncSuccessRate(Enum): + low = "low" + medium = "medium" + high = "high" + + +class ConnectorMetric(BaseModel): + class Config: + extra = Extra.allow + + usage: Optional[Union[str, Usage]] = None + sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None + connector_version: Optional[str] = None + + +class ConnectorPackageInfo(BaseModel): + cdk_version: Optional[str] = None + + +class JobTypeResourceLimit(BaseModel): + class Config: + extra = Extra.forbid + + jobType: JobType + resourceRequirements: ResourceRequirements + + +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( + ..., + description="A scope that can be used to limit the impact of a breaking change.", + ) + + +class GeneratedFields(BaseModel): + git: Optional[GitInfo] = None + source_file_info: Optional[SourceFileInfo] = None + metrics: Optional[ConnectorMetrics] = None + sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") + + +class ActorDefinitionResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + default: Optional[ResourceRequirements] = Field( + None, + description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + ) + jobSpecific: Optional[List[JobTypeResourceLimit]] = None + + +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + deadlineAction: Optional[DeadlineAction] = Field( + None, description="Action to do when the deadline is reached." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + title="ConnectorBreakingChanges", + ) + + +class ConnectorRegistrySourceDefinition(BaseModel): + class Config: + extra = Extra.allow + + sourceDefinitionId: UUID + name: str + dockerRepository: str + dockerImageTag: str + documentationUrl: str + icon: Optional[str] = None + iconUrl: Optional[str] = None + sourceType: Optional[SourceType] = None + spec: Dict[str, Any] + tombstone: Optional[bool] = Field( + False, + description="if false, the configuration is active. if true, then this configuration is permanently off.", + ) + public: Optional[bool] = Field( + False, + description="true if this connector definition is available to all workspaces", + ) + custom: Optional[bool] = Field( + False, description="whether this is a custom connector definition" + ) + releaseStage: Optional[ReleaseStage] = None + supportLevel: Optional[SupportLevel] = None + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + allowedHosts: Optional[AllowedHosts] = None + suggestedStreams: Optional[SuggestedStreams] = None + maxSecondsBetweenMessages: Optional[int] = Field( + None, + description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", + ) + erdUrl: Optional[str] = Field( + None, description="The URL where you can visualize the ERD" + ) + releases: Optional[ConnectorRegistryReleases] = None + ab_internal: Optional[AirbyteInternal] = None + generated: Optional[GeneratedFields] = None + packageInfo: Optional[ConnectorPackageInfo] = None + language: Optional[str] = Field( + None, description="The language the connector is written in" + ) + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + + +class ConnectorRegistryReleases(BaseModel): + class Config: + extra = Extra.forbid + + releaseCandidates: Optional[ConnectorReleaseCandidates] = None + rolloutConfiguration: Optional[RolloutConfiguration] = None + breakingChanges: Optional[ConnectorBreakingChanges] = None + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + ) + + +class ConnectorReleaseCandidates(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[ + constr(regex=r"^\d+\.\d+\.\d+(-[0-9A-Za-z-.]+)?$"), VersionReleaseCandidate + ] = Field( + ..., + description="Each entry denotes a release candidate version of a connector.", + ) + + +class VersionReleaseCandidate(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Union[ + ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition + ] = Field( + ..., + description="Contains information about a release candidate version of a connector.", + ) + + +class ConnectorRegistryDestinationDefinition(BaseModel): + class Config: + extra = Extra.allow + + destinationDefinitionId: UUID + name: str + dockerRepository: str + dockerImageTag: str + documentationUrl: str + icon: Optional[str] = None + iconUrl: Optional[str] = None + spec: Dict[str, Any] + tombstone: Optional[bool] = Field( + False, + description="if false, the configuration is active. if true, then this configuration is permanently off.", + ) + public: Optional[bool] = Field( + False, + description="true if this connector definition is available to all workspaces", + ) + custom: Optional[bool] = Field( + False, description="whether this is a custom connector definition" + ) + releaseStage: Optional[ReleaseStage] = None + supportLevel: Optional[SupportLevel] = None + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + tags: Optional[List[str]] = Field( + None, + description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", + ) + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + supportsDbt: Optional[bool] = Field( + None, + description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", + ) + allowedHosts: Optional[AllowedHosts] = None + releases: Optional[ConnectorRegistryReleases] = None + ab_internal: Optional[AirbyteInternal] = None + supportsRefreshes: Optional[bool] = False + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + generated: Optional[GeneratedFields] = None + packageInfo: Optional[ConnectorPackageInfo] = None + language: Optional[str] = Field( + None, description="The language the connector is written in" + ) + + +ConnectorRegistrySourceDefinition.update_forward_refs() +ConnectorRegistryReleases.update_forward_refs() +ConnectorReleaseCandidates.update_forward_refs() +VersionReleaseCandidate.update_forward_refs() diff --git a/airbyte_cdk/metadata_models/generated/ConnectorRegistryV0.py b/airbyte_cdk/metadata_models/generated/ConnectorRegistryV0.py new file mode 100644 index 000000000..9fa1ac0c9 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorRegistryV0.py @@ -0,0 +1,445 @@ +# generated by datamodel-codegen: +# filename: ConnectorRegistryV0.yaml + +from __future__ import annotations + +from datetime import date, datetime +from enum import Enum +from typing import Any, Dict, List, Optional, Union +from uuid import UUID + +from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr + + +class ReleaseStage(Enum): + alpha = "alpha" + beta = "beta" + generally_available = "generally_available" + custom = "custom" + + +class SupportLevel(Enum): + community = "community" + certified = "certified" + archived = "archived" + + +class ResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + cpu_request: Optional[str] = None + cpu_limit: Optional[str] = None + memory_request: Optional[str] = None + memory_limit: Optional[str] = None + + +class JobType(Enum): + get_spec = "get_spec" + check_connection = "check_connection" + discover_schema = "discover_schema" + sync = "sync" + reset_connection = "reset_connection" + connection_updater = "connection_updater" + replicate = "replicate" + + +class NormalizationDestinationDefinitionConfig(BaseModel): + class Config: + extra = Extra.allow + + normalizationRepository: str = Field( + ..., + description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", + ) + normalizationTag: str = Field( + ..., + description="a field indicating the tag of the docker repository to be used for normalization.", + ) + normalizationIntegrationType: str = Field( + ..., + description="a field indicating the type of integration dialect to use for normalization.", + ) + + +class AllowedHosts(BaseModel): + class Config: + extra = Extra.allow + + hosts: Optional[List[str]] = Field( + None, + description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + ) + + +class RolloutConfiguration(BaseModel): + class Config: + extra = Extra.forbid + + enableProgressiveRollout: Optional[bool] = Field( + False, description="Whether to enable progressive rollout for the connector." + ) + initialPercentage: Optional[conint(ge=0, le=100)] = Field( + 0, + description="The percentage of users that should receive the new version initially.", + ) + maxPercentage: Optional[conint(ge=0, le=100)] = Field( + 50, + description="The percentage of users who should receive the release candidate during the test phase before full rollout.", + ) + advanceDelayMinutes: Optional[conint(ge=10)] = Field( + 10, + description="The number of minutes to wait before advancing the rollout percentage.", + ) + + +class DeadlineAction(Enum): + auto_upgrade = "auto_upgrade" + disable = "disable" + + +class StreamBreakingChangeScope(BaseModel): + class Config: + extra = Extra.forbid + + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( + ..., + description="List of streams that are impacted by the breaking change.", + min_items=1, + ) + + +class SourceType(Enum): + api = "api" + file = "file" + database = "database" + custom = "custom" + + +class SuggestedStreams(BaseModel): + class Config: + extra = Extra.allow + + streams: Optional[List[str]] = Field( + None, + description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", + ) + + +class Sl(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + + +class Ql(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + integer_400 = 400 + integer_500 = 500 + integer_600 = 600 + + +class AirbyteInternal(BaseModel): + class Config: + extra = Extra.allow + + sl: Optional[Sl] = None + ql: Optional[Ql] = None + isEnterprise: Optional[bool] = False + requireVersionIncrementsInPullRequests: Optional[bool] = Field( + True, + description="When false, version increment checks will be skipped for this connector", + ) + + +class GitInfo(BaseModel): + class Config: + extra = Extra.forbid + + commit_sha: Optional[str] = Field( + None, + description="The git commit sha of the last commit that modified this file.", + ) + commit_timestamp: Optional[datetime] = Field( + None, + description="The git commit timestamp of the last commit that modified this file.", + ) + commit_author: Optional[str] = Field( + None, + description="The git commit author of the last commit that modified this file.", + ) + commit_author_email: Optional[str] = Field( + None, + description="The git commit author email of the last commit that modified this file.", + ) + + +class SourceFileInfo(BaseModel): + metadata_etag: Optional[str] = None + metadata_file_path: Optional[str] = None + metadata_bucket_name: Optional[str] = None + metadata_last_modified: Optional[str] = None + registry_entry_generated_at: Optional[str] = None + + +class ConnectorMetrics(BaseModel): + all: Optional[Any] = None + cloud: Optional[Any] = None + oss: Optional[Any] = None + + +class Usage(Enum): + low = "low" + medium = "medium" + high = "high" + + +class SyncSuccessRate(Enum): + low = "low" + medium = "medium" + high = "high" + + +class ConnectorMetric(BaseModel): + class Config: + extra = Extra.allow + + usage: Optional[Union[str, Usage]] = None + sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None + connector_version: Optional[str] = None + + +class ConnectorPackageInfo(BaseModel): + cdk_version: Optional[str] = None + + +class JobTypeResourceLimit(BaseModel): + class Config: + extra = Extra.forbid + + jobType: JobType + resourceRequirements: ResourceRequirements + + +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( + ..., + description="A scope that can be used to limit the impact of a breaking change.", + ) + + +class GeneratedFields(BaseModel): + git: Optional[GitInfo] = None + source_file_info: Optional[SourceFileInfo] = None + metrics: Optional[ConnectorMetrics] = None + sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") + + +class ActorDefinitionResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + default: Optional[ResourceRequirements] = Field( + None, + description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + ) + jobSpecific: Optional[List[JobTypeResourceLimit]] = None + + +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + deadlineAction: Optional[DeadlineAction] = Field( + None, description="Action to do when the deadline is reached." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + title="ConnectorBreakingChanges", + ) + + +class ConnectorRegistryV0(BaseModel): + destinations: List[ConnectorRegistryDestinationDefinition] + sources: List[ConnectorRegistrySourceDefinition] + + +class ConnectorRegistryDestinationDefinition(BaseModel): + class Config: + extra = Extra.allow + + destinationDefinitionId: UUID + name: str + dockerRepository: str + dockerImageTag: str + documentationUrl: str + icon: Optional[str] = None + iconUrl: Optional[str] = None + spec: Dict[str, Any] + tombstone: Optional[bool] = Field( + False, + description="if false, the configuration is active. if true, then this configuration is permanently off.", + ) + public: Optional[bool] = Field( + False, + description="true if this connector definition is available to all workspaces", + ) + custom: Optional[bool] = Field( + False, description="whether this is a custom connector definition" + ) + releaseStage: Optional[ReleaseStage] = None + supportLevel: Optional[SupportLevel] = None + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + tags: Optional[List[str]] = Field( + None, + description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", + ) + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + supportsDbt: Optional[bool] = Field( + None, + description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", + ) + allowedHosts: Optional[AllowedHosts] = None + releases: Optional[ConnectorRegistryReleases] = None + ab_internal: Optional[AirbyteInternal] = None + supportsRefreshes: Optional[bool] = False + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + generated: Optional[GeneratedFields] = None + packageInfo: Optional[ConnectorPackageInfo] = None + language: Optional[str] = Field( + None, description="The language the connector is written in" + ) + + +class ConnectorRegistryReleases(BaseModel): + class Config: + extra = Extra.forbid + + releaseCandidates: Optional[ConnectorReleaseCandidates] = None + rolloutConfiguration: Optional[RolloutConfiguration] = None + breakingChanges: Optional[ConnectorBreakingChanges] = None + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + ) + + +class ConnectorReleaseCandidates(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[ + constr(regex=r"^\d+\.\d+\.\d+(-[0-9A-Za-z-.]+)?$"), VersionReleaseCandidate + ] = Field( + ..., + description="Each entry denotes a release candidate version of a connector.", + ) + + +class VersionReleaseCandidate(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Union[ + ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition + ] = Field( + ..., + description="Contains information about a release candidate version of a connector.", + ) + + +class ConnectorRegistrySourceDefinition(BaseModel): + class Config: + extra = Extra.allow + + sourceDefinitionId: UUID + name: str + dockerRepository: str + dockerImageTag: str + documentationUrl: str + icon: Optional[str] = None + iconUrl: Optional[str] = None + sourceType: Optional[SourceType] = None + spec: Dict[str, Any] + tombstone: Optional[bool] = Field( + False, + description="if false, the configuration is active. if true, then this configuration is permanently off.", + ) + public: Optional[bool] = Field( + False, + description="true if this connector definition is available to all workspaces", + ) + custom: Optional[bool] = Field( + False, description="whether this is a custom connector definition" + ) + releaseStage: Optional[ReleaseStage] = None + supportLevel: Optional[SupportLevel] = None + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + allowedHosts: Optional[AllowedHosts] = None + suggestedStreams: Optional[SuggestedStreams] = None + maxSecondsBetweenMessages: Optional[int] = Field( + None, + description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", + ) + erdUrl: Optional[str] = Field( + None, description="The URL where you can visualize the ERD" + ) + releases: Optional[ConnectorRegistryReleases] = None + ab_internal: Optional[AirbyteInternal] = None + generated: Optional[GeneratedFields] = None + packageInfo: Optional[ConnectorPackageInfo] = None + language: Optional[str] = Field( + None, description="The language the connector is written in" + ) + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + + +ConnectorRegistryV0.update_forward_refs() +ConnectorRegistryDestinationDefinition.update_forward_refs() +ConnectorRegistryReleases.update_forward_refs() +ConnectorReleaseCandidates.update_forward_refs() +VersionReleaseCandidate.update_forward_refs() diff --git a/airbyte_cdk/metadata_models/generated/ConnectorReleases.py b/airbyte_cdk/metadata_models/generated/ConnectorReleases.py new file mode 100644 index 000000000..c0498dc30 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorReleases.py @@ -0,0 +1,103 @@ +# generated by datamodel-codegen: +# filename: ConnectorReleases.yaml + +from __future__ import annotations + +from datetime import date +from enum import Enum +from typing import Any, Dict, List, Optional + +from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr + + +class RolloutConfiguration(BaseModel): + class Config: + extra = Extra.forbid + + enableProgressiveRollout: Optional[bool] = Field( + False, description="Whether to enable progressive rollout for the connector." + ) + initialPercentage: Optional[conint(ge=0, le=100)] = Field( + 0, + description="The percentage of users that should receive the new version initially.", + ) + maxPercentage: Optional[conint(ge=0, le=100)] = Field( + 50, + description="The percentage of users who should receive the release candidate during the test phase before full rollout.", + ) + advanceDelayMinutes: Optional[conint(ge=10)] = Field( + 10, + description="The number of minutes to wait before advancing the rollout percentage.", + ) + + +class DeadlineAction(Enum): + auto_upgrade = "auto_upgrade" + disable = "disable" + + +class StreamBreakingChangeScope(BaseModel): + class Config: + extra = Extra.forbid + + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( + ..., + description="List of streams that are impacted by the breaking change.", + min_items=1, + ) + + +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( + ..., + description="A scope that can be used to limit the impact of a breaking change.", + ) + + +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + deadlineAction: Optional[DeadlineAction] = Field( + None, description="Action to do when the deadline is reached." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + title="ConnectorBreakingChanges", + ) + + +class ConnectorReleases(BaseModel): + class Config: + extra = Extra.forbid + + rolloutConfiguration: Optional[RolloutConfiguration] = None + breakingChanges: Optional[ConnectorBreakingChanges] = None + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + ) diff --git a/airbyte_cdk/metadata_models/generated/ConnectorTestSuiteOptions.py b/airbyte_cdk/metadata_models/generated/ConnectorTestSuiteOptions.py new file mode 100644 index 000000000..6f511f53c --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ConnectorTestSuiteOptions.py @@ -0,0 +1,63 @@ +# generated by datamodel-codegen: +# filename: ConnectorTestSuiteOptions.yaml + +from __future__ import annotations + +from enum import Enum +from typing import List, Literal, Optional + +from pydantic.v1 import BaseModel, Extra, Field + + +class Suite(Enum): + unitTests = "unitTests" + integrationTests = "integrationTests" + acceptanceTests = "acceptanceTests" + liveTests = "liveTests" + + +class SecretStore(BaseModel): + class Config: + extra = Extra.forbid + + alias: Optional[str] = Field( + None, + description="The alias of the secret store which can map to its actual secret address", + ) + type: Optional[Literal["GSM"]] = Field( + None, description="The type of the secret store" + ) + + +class TestConnections(BaseModel): + class Config: + extra = Extra.forbid + + name: str = Field(..., description="The connection name") + id: str = Field(..., description="The connection ID") + + +class Secret(BaseModel): + class Config: + extra = Extra.forbid + + name: str = Field(..., description="The secret name in the secret store") + fileName: Optional[str] = Field( + None, + description="The name of the file to which the secret value would be persisted", + ) + secretStore: SecretStore + + +class ConnectorTestSuiteOptions(BaseModel): + class Config: + extra = Extra.forbid + + suite: Suite = Field(..., description="Name of the configured test suite") + testSecrets: Optional[List[Secret]] = Field( + None, description="List of secrets required to run the test suite" + ) + testConnections: Optional[List[TestConnections]] = Field( + None, + description="List of sandbox cloud connections that tests can be run against", + ) diff --git a/airbyte_cdk/metadata_models/generated/GeneratedFields.py b/airbyte_cdk/metadata_models/generated/GeneratedFields.py new file mode 100644 index 000000000..f290484f4 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/GeneratedFields.py @@ -0,0 +1,74 @@ +# generated by datamodel-codegen: +# filename: GeneratedFields.yaml + +from __future__ import annotations + +from datetime import datetime +from enum import Enum +from typing import Any, Optional, Union + +from pydantic.v1 import BaseModel, Extra, Field + + +class GitInfo(BaseModel): + class Config: + extra = Extra.forbid + + commit_sha: Optional[str] = Field( + None, + description="The git commit sha of the last commit that modified this file.", + ) + commit_timestamp: Optional[datetime] = Field( + None, + description="The git commit timestamp of the last commit that modified this file.", + ) + commit_author: Optional[str] = Field( + None, + description="The git commit author of the last commit that modified this file.", + ) + commit_author_email: Optional[str] = Field( + None, + description="The git commit author email of the last commit that modified this file.", + ) + + +class SourceFileInfo(BaseModel): + metadata_etag: Optional[str] = None + metadata_file_path: Optional[str] = None + metadata_bucket_name: Optional[str] = None + metadata_last_modified: Optional[str] = None + registry_entry_generated_at: Optional[str] = None + + +class ConnectorMetrics(BaseModel): + all: Optional[Any] = None + cloud: Optional[Any] = None + oss: Optional[Any] = None + + +class Usage(Enum): + low = "low" + medium = "medium" + high = "high" + + +class SyncSuccessRate(Enum): + low = "low" + medium = "medium" + high = "high" + + +class ConnectorMetric(BaseModel): + class Config: + extra = Extra.allow + + usage: Optional[Union[str, Usage]] = None + sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None + connector_version: Optional[str] = None + + +class GeneratedFields(BaseModel): + git: Optional[GitInfo] = None + source_file_info: Optional[SourceFileInfo] = None + metrics: Optional[ConnectorMetrics] = None + sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") diff --git a/airbyte_cdk/metadata_models/generated/GitInfo.py b/airbyte_cdk/metadata_models/generated/GitInfo.py new file mode 100644 index 000000000..d59317ec2 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/GitInfo.py @@ -0,0 +1,31 @@ +# generated by datamodel-codegen: +# filename: GitInfo.yaml + +from __future__ import annotations + +from datetime import datetime +from typing import Optional + +from pydantic.v1 import BaseModel, Extra, Field + + +class GitInfo(BaseModel): + class Config: + extra = Extra.forbid + + commit_sha: Optional[str] = Field( + None, + description="The git commit sha of the last commit that modified this file.", + ) + commit_timestamp: Optional[datetime] = Field( + None, + description="The git commit timestamp of the last commit that modified this file.", + ) + commit_author: Optional[str] = Field( + None, + description="The git commit author of the last commit that modified this file.", + ) + commit_author_email: Optional[str] = Field( + None, + description="The git commit author email of the last commit that modified this file.", + ) diff --git a/airbyte_cdk/metadata_models/generated/JobType.py b/airbyte_cdk/metadata_models/generated/JobType.py new file mode 100644 index 000000000..943ff83bc --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/JobType.py @@ -0,0 +1,16 @@ +# generated by datamodel-codegen: +# filename: JobType.yaml + +from __future__ import annotations + +from enum import Enum + + +class JobType(Enum): + get_spec = "get_spec" + check_connection = "check_connection" + discover_schema = "discover_schema" + sync = "sync" + reset_connection = "reset_connection" + connection_updater = "connection_updater" + replicate = "replicate" diff --git a/airbyte_cdk/metadata_models/generated/NormalizationDestinationDefinitionConfig.py b/airbyte_cdk/metadata_models/generated/NormalizationDestinationDefinitionConfig.py new file mode 100644 index 000000000..c35edf057 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/NormalizationDestinationDefinitionConfig.py @@ -0,0 +1,24 @@ +# generated by datamodel-codegen: +# filename: NormalizationDestinationDefinitionConfig.yaml + +from __future__ import annotations + +from pydantic.v1 import BaseModel, Extra, Field + + +class NormalizationDestinationDefinitionConfig(BaseModel): + class Config: + extra = Extra.allow + + normalizationRepository: str = Field( + ..., + description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", + ) + normalizationTag: str = Field( + ..., + description="a field indicating the tag of the docker repository to be used for normalization.", + ) + normalizationIntegrationType: str = Field( + ..., + description="a field indicating the type of integration dialect to use for normalization.", + ) diff --git a/airbyte_cdk/metadata_models/generated/RegistryOverrides.py b/airbyte_cdk/metadata_models/generated/RegistryOverrides.py new file mode 100644 index 000000000..c4060c877 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/RegistryOverrides.py @@ -0,0 +1,105 @@ +# generated by datamodel-codegen: +# filename: RegistryOverrides.yaml + +from __future__ import annotations + +from enum import Enum +from typing import List, Optional + +from pydantic.v1 import AnyUrl, BaseModel, Extra, Field + + +class AllowedHosts(BaseModel): + class Config: + extra = Extra.allow + + hosts: Optional[List[str]] = Field( + None, + description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + ) + + +class NormalizationDestinationDefinitionConfig(BaseModel): + class Config: + extra = Extra.allow + + normalizationRepository: str = Field( + ..., + description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", + ) + normalizationTag: str = Field( + ..., + description="a field indicating the tag of the docker repository to be used for normalization.", + ) + normalizationIntegrationType: str = Field( + ..., + description="a field indicating the type of integration dialect to use for normalization.", + ) + + +class SuggestedStreams(BaseModel): + class Config: + extra = Extra.allow + + streams: Optional[List[str]] = Field( + None, + description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", + ) + + +class ResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + cpu_request: Optional[str] = None + cpu_limit: Optional[str] = None + memory_request: Optional[str] = None + memory_limit: Optional[str] = None + + +class JobType(Enum): + get_spec = "get_spec" + check_connection = "check_connection" + discover_schema = "discover_schema" + sync = "sync" + reset_connection = "reset_connection" + connection_updater = "connection_updater" + replicate = "replicate" + + +class JobTypeResourceLimit(BaseModel): + class Config: + extra = Extra.forbid + + jobType: JobType + resourceRequirements: ResourceRequirements + + +class ActorDefinitionResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + default: Optional[ResourceRequirements] = Field( + None, + description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + ) + jobSpecific: Optional[List[JobTypeResourceLimit]] = None + + +class RegistryOverrides(BaseModel): + class Config: + extra = Extra.forbid + + enabled: bool + name: Optional[str] = None + dockerRepository: Optional[str] = None + dockerImageTag: Optional[str] = None + supportsDbt: Optional[bool] = None + supportsNormalization: Optional[bool] = None + license: Optional[str] = None + documentationUrl: Optional[AnyUrl] = None + connectorSubtype: Optional[str] = None + allowedHosts: Optional[AllowedHosts] = None + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + suggestedStreams: Optional[SuggestedStreams] = None + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None diff --git a/airbyte_cdk/metadata_models/generated/ReleaseStage.py b/airbyte_cdk/metadata_models/generated/ReleaseStage.py new file mode 100644 index 000000000..8ef69075c --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ReleaseStage.py @@ -0,0 +1,13 @@ +# generated by datamodel-codegen: +# filename: ReleaseStage.yaml + +from __future__ import annotations + +from enum import Enum + + +class ReleaseStage(Enum): + alpha = "alpha" + beta = "beta" + generally_available = "generally_available" + custom = "custom" diff --git a/airbyte_cdk/metadata_models/generated/RemoteRegistries.py b/airbyte_cdk/metadata_models/generated/RemoteRegistries.py new file mode 100644 index 000000000..7a587f9d3 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/RemoteRegistries.py @@ -0,0 +1,23 @@ +# generated by datamodel-codegen: +# filename: RemoteRegistries.yaml + +from __future__ import annotations + +from typing import Optional + +from pydantic.v1 import BaseModel, Extra, Field + + +class PyPi(BaseModel): + class Config: + extra = Extra.forbid + + enabled: bool + packageName: str = Field(..., description="The name of the package on PyPi.") + + +class RemoteRegistries(BaseModel): + class Config: + extra = Extra.forbid + + pypi: Optional[PyPi] = None diff --git a/airbyte_cdk/metadata_models/generated/ResourceRequirements.py b/airbyte_cdk/metadata_models/generated/ResourceRequirements.py new file mode 100644 index 000000000..8f079d99f --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/ResourceRequirements.py @@ -0,0 +1,18 @@ +# generated by datamodel-codegen: +# filename: ResourceRequirements.yaml + +from __future__ import annotations + +from typing import Optional + +from pydantic.v1 import BaseModel, Extra + + +class ResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + cpu_request: Optional[str] = None + cpu_limit: Optional[str] = None + memory_request: Optional[str] = None + memory_limit: Optional[str] = None diff --git a/airbyte_cdk/metadata_models/generated/RolloutConfiguration.py b/airbyte_cdk/metadata_models/generated/RolloutConfiguration.py new file mode 100644 index 000000000..aae003a64 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/RolloutConfiguration.py @@ -0,0 +1,29 @@ +# generated by datamodel-codegen: +# filename: RolloutConfiguration.yaml + +from __future__ import annotations + +from typing import Optional + +from pydantic.v1 import BaseModel, Extra, Field, conint + + +class RolloutConfiguration(BaseModel): + class Config: + extra = Extra.forbid + + enableProgressiveRollout: Optional[bool] = Field( + False, description="Whether to enable progressive rollout for the connector." + ) + initialPercentage: Optional[conint(ge=0, le=100)] = Field( + 0, + description="The percentage of users that should receive the new version initially.", + ) + maxPercentage: Optional[conint(ge=0, le=100)] = Field( + 50, + description="The percentage of users who should receive the release candidate during the test phase before full rollout.", + ) + advanceDelayMinutes: Optional[conint(ge=10)] = Field( + 10, + description="The number of minutes to wait before advancing the rollout percentage.", + ) diff --git a/airbyte_cdk/metadata_models/generated/Secret.py b/airbyte_cdk/metadata_models/generated/Secret.py new file mode 100644 index 000000000..298f803fc --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/Secret.py @@ -0,0 +1,33 @@ +# generated by datamodel-codegen: +# filename: Secret.yaml + +from __future__ import annotations + +from typing import Literal, Optional + +from pydantic.v1 import BaseModel, Extra, Field + + +class SecretStore(BaseModel): + class Config: + extra = Extra.forbid + + alias: Optional[str] = Field( + None, + description="The alias of the secret store which can map to its actual secret address", + ) + type: Optional[Literal["GSM"]] = Field( + None, description="The type of the secret store" + ) + + +class Secret(BaseModel): + class Config: + extra = Extra.forbid + + name: str = Field(..., description="The secret name in the secret store") + fileName: Optional[str] = Field( + None, + description="The name of the file to which the secret value would be persisted", + ) + secretStore: SecretStore diff --git a/airbyte_cdk/metadata_models/generated/SecretStore.py b/airbyte_cdk/metadata_models/generated/SecretStore.py new file mode 100644 index 000000000..5e004a9df --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/SecretStore.py @@ -0,0 +1,21 @@ +# generated by datamodel-codegen: +# filename: SecretStore.yaml + +from __future__ import annotations + +from typing import Literal, Optional + +from pydantic.v1 import BaseModel, Extra, Field + + +class SecretStore(BaseModel): + class Config: + extra = Extra.forbid + + alias: Optional[str] = Field( + None, + description="The alias of the secret store which can map to its actual secret address", + ) + type: Optional[Literal["GSM"]] = Field( + None, description="The type of the secret store" + ) diff --git a/airbyte_cdk/metadata_models/generated/SourceFileInfo.py b/airbyte_cdk/metadata_models/generated/SourceFileInfo.py new file mode 100644 index 000000000..03b3410d3 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/SourceFileInfo.py @@ -0,0 +1,16 @@ +# generated by datamodel-codegen: +# filename: SourceFileInfo.yaml + +from __future__ import annotations + +from typing import Optional + +from pydantic.v1 import BaseModel + + +class SourceFileInfo(BaseModel): + metadata_etag: Optional[str] = None + metadata_file_path: Optional[str] = None + metadata_bucket_name: Optional[str] = None + metadata_last_modified: Optional[str] = None + registry_entry_generated_at: Optional[str] = None diff --git a/airbyte_cdk/metadata_models/generated/SuggestedStreams.py b/airbyte_cdk/metadata_models/generated/SuggestedStreams.py new file mode 100644 index 000000000..5969de281 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/SuggestedStreams.py @@ -0,0 +1,18 @@ +# generated by datamodel-codegen: +# filename: SuggestedStreams.yaml + +from __future__ import annotations + +from typing import List, Optional + +from pydantic.v1 import BaseModel, Extra, Field + + +class SuggestedStreams(BaseModel): + class Config: + extra = Extra.allow + + streams: Optional[List[str]] = Field( + None, + description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", + ) diff --git a/airbyte_cdk/metadata_models/generated/SupportLevel.py b/airbyte_cdk/metadata_models/generated/SupportLevel.py new file mode 100644 index 000000000..5b369890d --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/SupportLevel.py @@ -0,0 +1,12 @@ +# generated by datamodel-codegen: +# filename: SupportLevel.yaml + +from __future__ import annotations + +from enum import Enum + + +class SupportLevel(Enum): + community = "community" + certified = "certified" + archived = "archived" diff --git a/airbyte_cdk/metadata_models/generated/TestConnections.py b/airbyte_cdk/metadata_models/generated/TestConnections.py new file mode 100644 index 000000000..9450a3c19 --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/TestConnections.py @@ -0,0 +1,14 @@ +# generated by datamodel-codegen: +# filename: TestConnections.yaml + +from __future__ import annotations + +from pydantic.v1 import BaseModel, Extra, Field + + +class TestConnections(BaseModel): + class Config: + extra = Extra.forbid + + name: str = Field(..., description="The connection name") + id: str = Field(..., description="The connection ID") diff --git a/airbyte_cdk/metadata_models/generated/__init__.py b/airbyte_cdk/metadata_models/generated/__init__.py new file mode 100644 index 000000000..d026bba2a --- /dev/null +++ b/airbyte_cdk/metadata_models/generated/__init__.py @@ -0,0 +1,31 @@ +# generated by bin/generate_component_manifest_files.py +from .ActorDefinitionResourceRequirements import * +from .AirbyteInternal import * +from .AllowedHosts import * +from .ConnectorBreakingChanges import * +from .ConnectorBuildOptions import * +from .ConnectorIPCOptions import * +from .ConnectorMetadataDefinitionV0 import * +from .ConnectorMetrics import * +from .ConnectorPackageInfo import * +from .ConnectorRegistryDestinationDefinition import * +from .ConnectorRegistryReleases import * +from .ConnectorRegistrySourceDefinition import * +from .ConnectorRegistryV0 import * +from .ConnectorReleases import * +from .ConnectorTestSuiteOptions import * +from .GeneratedFields import * +from .GitInfo import * +from .JobType import * +from .NormalizationDestinationDefinitionConfig import * +from .RegistryOverrides import * +from .ReleaseStage import * +from .RemoteRegistries import * +from .ResourceRequirements import * +from .RolloutConfiguration import * +from .Secret import * +from .SecretStore import * +from .SourceFileInfo import * +from .SuggestedStreams import * +from .SupportLevel import * +from .TestConnections import * diff --git a/airbyte_cdk/metadata_models/py.typed b/airbyte_cdk/metadata_models/py.typed new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/airbyte_cdk/metadata_models/py.typed @@ -0,0 +1 @@ + diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index 51b3d8efb..2b6431eab 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -2,16 +2,21 @@ import re import sys +import tempfile from glob import glob from pathlib import Path import anyio import dagger +import httpx PYTHON_IMAGE = "python:3.10" LOCAL_YAML_DIR_PATH = "airbyte_cdk/sources/declarative" LOCAL_OUTPUT_DIR_PATH = "airbyte_cdk/sources/declarative/models" +METADATA_SCHEMAS_GITHUB_URL = "https://api.github.com/repos/airbytehq/airbyte/contents/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" +METADATA_SCHEMAS_RAW_URL_BASE = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" +LOCAL_METADATA_OUTPUT_DIR_PATH = "airbyte_cdk/metadata_models/generated" PIP_DEPENDENCIES = [ "datamodel_code_generator==0.26.3", @@ -22,13 +27,55 @@ def get_all_yaml_files_without_ext() -> list[str]: return [Path(f).stem for f in glob(f"{LOCAL_YAML_DIR_PATH}/*.yaml")] -def generate_init_module_content() -> str: +def get_all_yaml_files_from_dir(directory: str) -> list[str]: + return [Path(f).stem for f in glob(f"{directory}/*.yaml")] + + +def generate_init_module_content(yaml_files: list[str]) -> str: header = "# generated by bin/generate_component_manifest_files.py\n" - for module_name in get_all_yaml_files_without_ext(): + for module_name in yaml_files: header += f"from .{module_name} import *\n" return header +async def download_metadata_schemas(temp_dir: Path) -> list[str]: + """Download metadata schema YAML files from GitHub to a temporary directory.""" + headers = { + "User-Agent": "airbyte-python-cdk-build", + "Accept": "application/vnd.github.v3+json", + } + + async with httpx.AsyncClient(headers=headers, timeout=30.0) as client: + try: + response = await client.get(METADATA_SCHEMAS_GITHUB_URL) + response.raise_for_status() + files_info = response.json() + except httpx.HTTPStatusError as e: + if e.response.status_code == 403: + print( + "Warning: GitHub API rate limit exceeded. Using cached schemas if available.", + file=sys.stderr, + ) + raise + raise + + yaml_files = [] + for file_info in files_info: + if file_info["name"].endswith(".yaml"): + file_name = file_info["name"] + file_url = f"{METADATA_SCHEMAS_RAW_URL_BASE}/{file_name}" + + print(f"Downloading {file_name}...", file=sys.stderr) + file_response = await client.get(file_url) + file_response.raise_for_status() + + file_path = temp_dir / file_name + file_path.write_text(file_response.text) + yaml_files.append(Path(file_name).stem) + + return yaml_files + + def replace_base_model_for_classes_with_deprecated_fields(post_processed_content: str) -> str: """ Replace the base model for classes with deprecated fields. @@ -110,49 +157,106 @@ async def post_process_codegen(codegen_container: dagger.Container): return codegen_container -async def main(): - init_module_content = generate_init_module_content() - - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - codegen_container = ( - dagger_client.container() - .from_(PYTHON_IMAGE) - .with_exec(["mkdir", "/generated"], use_entrypoint=True) - .with_exec(["pip", "install", " ".join(PIP_DEPENDENCIES)], use_entrypoint=True) - .with_mounted_directory( - "/yaml", dagger_client.host().directory(LOCAL_YAML_DIR_PATH, include=["*.yaml"]) +async def post_process_metadata_models(codegen_container: dagger.Container): + """Post-process metadata models to use pydantic.v1 compatibility layer.""" + codegen_container = codegen_container.with_exec( + ["mkdir", "/generated_post_processed"], use_entrypoint=True + ) + for generated_file in await codegen_container.directory("/generated").entries(): + if generated_file.endswith(".py"): + original_content = await codegen_container.file( + f"/generated/{generated_file}" + ).contents() + + post_processed_content = original_content.replace( + "from pydantic", "from pydantic.v1" ) - .with_new_file("/generated/__init__.py", contents=init_module_content) - ) - for yaml_file in get_all_yaml_files_without_ext(): - codegen_container = codegen_container.with_exec( - [ - "datamodel-codegen", - "--input", - f"/yaml/{yaml_file}.yaml", - "--output", - f"/generated/{yaml_file}.py", - "--disable-timestamp", - "--enum-field-as-literal", - "one", - "--set-default-enum-member", - "--use-double-quotes", - "--remove-special-field-name-prefix", - # allow usage of the extra key such as `deprecated`, etc. - "--field-extra-keys", - # account the `deprecated` flag provided for the field. - "deprecated", - # account the `deprecation_message` provided for the field. - "deprecation_message", - ], - use_entrypoint=True, + + codegen_container = codegen_container.with_new_file( + f"/generated_post_processed/{generated_file}", contents=post_processed_content ) + return codegen_container + + +async def generate_models_from_schemas( + dagger_client: dagger.Client, + yaml_dir_path: str, + output_dir_path: str, + yaml_files: list[str], + post_process: bool = False, + metadata_models: bool = False, +) -> None: + """Generate Pydantic models from YAML schemas using datamodel-codegen.""" + init_module_content = generate_init_module_content(yaml_files) + + codegen_container = ( + dagger_client.container() + .from_(PYTHON_IMAGE) + .with_exec(["mkdir", "/generated"], use_entrypoint=True) + .with_exec(["pip", "install", " ".join(PIP_DEPENDENCIES)], use_entrypoint=True) + .with_mounted_directory( + "/yaml", dagger_client.host().directory(yaml_dir_path, include=["*.yaml"]) + ) + .with_new_file("/generated/__init__.py", contents=init_module_content) + ) + + for yaml_file in yaml_files: + codegen_container = codegen_container.with_exec( + [ + "datamodel-codegen", + "--input", + f"/yaml/{yaml_file}.yaml", + "--output", + f"/generated/{yaml_file}.py", + "--disable-timestamp", + "--enum-field-as-literal", + "one", + "--set-default-enum-member", + "--use-double-quotes", + "--remove-special-field-name-prefix", + "--field-extra-keys", + "deprecated", + "deprecation_message", + ], + use_entrypoint=True, + ) + + if post_process: + codegen_container = await post_process_codegen(codegen_container) + await codegen_container.directory("/generated_post_processed").export(output_dir_path) + elif metadata_models: + codegen_container = await post_process_metadata_models(codegen_container) + await codegen_container.directory("/generated_post_processed").export(output_dir_path) + else: + await codegen_container.directory("/generated").export(output_dir_path) + - await ( - (await post_process_codegen(codegen_container)) - .directory("/generated_post_processed") - .export(LOCAL_OUTPUT_DIR_PATH) +async def main(): + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: + print("Generating declarative component models...", file=sys.stderr) + declarative_yaml_files = get_all_yaml_files_without_ext() + await generate_models_from_schemas( + dagger_client=dagger_client, + yaml_dir_path=LOCAL_YAML_DIR_PATH, + output_dir_path=LOCAL_OUTPUT_DIR_PATH, + yaml_files=declarative_yaml_files, + post_process=True, ) + + print("\nGenerating metadata models...", file=sys.stderr) + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + metadata_yaml_files = await download_metadata_schemas(temp_path) + + await generate_models_from_schemas( + dagger_client=dagger_client, + yaml_dir_path=str(temp_path), + output_dir_path=LOCAL_METADATA_OUTPUT_DIR_PATH, + yaml_files=metadata_yaml_files, + metadata_models=True, + ) + + print("\nModel generation complete!", file=sys.stderr) anyio.run(main) From 63930c69494c551dd1a91bba5d645783d98dfe3f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 01:03:25 +0000 Subject: [PATCH 02/20] refactor: Move metadata models to airbyte_cdk.test.models.connector_metadata - Moved from airbyte_cdk.metadata_models.generated to airbyte_cdk.test.models.connector_metadata - Updated build script to output to new location - Updated README with new import paths Co-Authored-By: AJ Steers --- airbyte_cdk/metadata_models/__init__.py | 1 - .../ActorDefinitionResourceRequirements.py | 0 .../models/connector_metadata}/AirbyteInternal.py | 0 .../models/connector_metadata}/AllowedHosts.py | 0 .../connector_metadata}/ConnectorBreakingChanges.py | 0 .../models/connector_metadata}/ConnectorBuildOptions.py | 0 .../models/connector_metadata}/ConnectorIPCOptions.py | 0 .../connector_metadata}/ConnectorMetadataDefinitionV0.py | 0 .../models/connector_metadata}/ConnectorMetrics.py | 0 .../models/connector_metadata}/ConnectorPackageInfo.py | 0 .../ConnectorRegistryDestinationDefinition.py | 0 .../connector_metadata}/ConnectorRegistryReleases.py | 0 .../ConnectorRegistrySourceDefinition.py | 0 .../models/connector_metadata}/ConnectorRegistryV0.py | 0 .../models/connector_metadata}/ConnectorReleases.py | 0 .../connector_metadata}/ConnectorTestSuiteOptions.py | 0 .../models/connector_metadata}/GeneratedFields.py | 0 .../models/connector_metadata}/GitInfo.py | 0 .../models/connector_metadata}/JobType.py | 0 .../NormalizationDestinationDefinitionConfig.py | 0 .../models/connector_metadata}/README.md | 8 ++++---- .../models/connector_metadata}/RegistryOverrides.py | 0 .../models/connector_metadata}/ReleaseStage.py | 0 .../models/connector_metadata}/RemoteRegistries.py | 0 .../models/connector_metadata}/ResourceRequirements.py | 0 .../models/connector_metadata}/RolloutConfiguration.py | 0 .../models/connector_metadata}/Secret.py | 0 .../models/connector_metadata}/SecretStore.py | 0 .../models/connector_metadata}/SourceFileInfo.py | 0 .../models/connector_metadata}/SuggestedStreams.py | 0 .../models/connector_metadata}/SupportLevel.py | 0 .../models/connector_metadata}/TestConnections.py | 0 .../models/connector_metadata}/__init__.py | 0 .../models/connector_metadata}/py.typed | 0 bin/generate_component_manifest_files.py | 2 +- 35 files changed, 5 insertions(+), 6 deletions(-) delete mode 100644 airbyte_cdk/metadata_models/__init__.py rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ActorDefinitionResourceRequirements.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/AirbyteInternal.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/AllowedHosts.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorBreakingChanges.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorBuildOptions.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorIPCOptions.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorMetadataDefinitionV0.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorMetrics.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorPackageInfo.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorRegistryDestinationDefinition.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorRegistryReleases.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorRegistrySourceDefinition.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorRegistryV0.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorReleases.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ConnectorTestSuiteOptions.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/GeneratedFields.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/GitInfo.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/JobType.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/NormalizationDestinationDefinitionConfig.py (100%) rename airbyte_cdk/{metadata_models => test/models/connector_metadata}/README.md (89%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/RegistryOverrides.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ReleaseStage.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/RemoteRegistries.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/ResourceRequirements.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/RolloutConfiguration.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/Secret.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/SecretStore.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/SourceFileInfo.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/SuggestedStreams.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/SupportLevel.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/TestConnections.py (100%) rename airbyte_cdk/{metadata_models/generated => test/models/connector_metadata}/__init__.py (100%) rename airbyte_cdk/{metadata_models => test/models/connector_metadata}/py.typed (100%) diff --git a/airbyte_cdk/metadata_models/__init__.py b/airbyte_cdk/metadata_models/__init__.py deleted file mode 100644 index e37b3fef6..000000000 --- a/airbyte_cdk/metadata_models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .generated import * diff --git a/airbyte_cdk/metadata_models/generated/ActorDefinitionResourceRequirements.py b/airbyte_cdk/test/models/connector_metadata/ActorDefinitionResourceRequirements.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ActorDefinitionResourceRequirements.py rename to airbyte_cdk/test/models/connector_metadata/ActorDefinitionResourceRequirements.py diff --git a/airbyte_cdk/metadata_models/generated/AirbyteInternal.py b/airbyte_cdk/test/models/connector_metadata/AirbyteInternal.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/AirbyteInternal.py rename to airbyte_cdk/test/models/connector_metadata/AirbyteInternal.py diff --git a/airbyte_cdk/metadata_models/generated/AllowedHosts.py b/airbyte_cdk/test/models/connector_metadata/AllowedHosts.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/AllowedHosts.py rename to airbyte_cdk/test/models/connector_metadata/AllowedHosts.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorBreakingChanges.py b/airbyte_cdk/test/models/connector_metadata/ConnectorBreakingChanges.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorBreakingChanges.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorBreakingChanges.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorBuildOptions.py b/airbyte_cdk/test/models/connector_metadata/ConnectorBuildOptions.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorBuildOptions.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorBuildOptions.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorIPCOptions.py b/airbyte_cdk/test/models/connector_metadata/ConnectorIPCOptions.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorIPCOptions.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorIPCOptions.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorMetadataDefinitionV0.py b/airbyte_cdk/test/models/connector_metadata/ConnectorMetadataDefinitionV0.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorMetadataDefinitionV0.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorMetadataDefinitionV0.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorMetrics.py b/airbyte_cdk/test/models/connector_metadata/ConnectorMetrics.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorMetrics.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorMetrics.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorPackageInfo.py b/airbyte_cdk/test/models/connector_metadata/ConnectorPackageInfo.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorPackageInfo.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorPackageInfo.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorRegistryDestinationDefinition.py b/airbyte_cdk/test/models/connector_metadata/ConnectorRegistryDestinationDefinition.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorRegistryDestinationDefinition.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorRegistryDestinationDefinition.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorRegistryReleases.py b/airbyte_cdk/test/models/connector_metadata/ConnectorRegistryReleases.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorRegistryReleases.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorRegistryReleases.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorRegistrySourceDefinition.py b/airbyte_cdk/test/models/connector_metadata/ConnectorRegistrySourceDefinition.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorRegistrySourceDefinition.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorRegistrySourceDefinition.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorRegistryV0.py b/airbyte_cdk/test/models/connector_metadata/ConnectorRegistryV0.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorRegistryV0.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorRegistryV0.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorReleases.py b/airbyte_cdk/test/models/connector_metadata/ConnectorReleases.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorReleases.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorReleases.py diff --git a/airbyte_cdk/metadata_models/generated/ConnectorTestSuiteOptions.py b/airbyte_cdk/test/models/connector_metadata/ConnectorTestSuiteOptions.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ConnectorTestSuiteOptions.py rename to airbyte_cdk/test/models/connector_metadata/ConnectorTestSuiteOptions.py diff --git a/airbyte_cdk/metadata_models/generated/GeneratedFields.py b/airbyte_cdk/test/models/connector_metadata/GeneratedFields.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/GeneratedFields.py rename to airbyte_cdk/test/models/connector_metadata/GeneratedFields.py diff --git a/airbyte_cdk/metadata_models/generated/GitInfo.py b/airbyte_cdk/test/models/connector_metadata/GitInfo.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/GitInfo.py rename to airbyte_cdk/test/models/connector_metadata/GitInfo.py diff --git a/airbyte_cdk/metadata_models/generated/JobType.py b/airbyte_cdk/test/models/connector_metadata/JobType.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/JobType.py rename to airbyte_cdk/test/models/connector_metadata/JobType.py diff --git a/airbyte_cdk/metadata_models/generated/NormalizationDestinationDefinitionConfig.py b/airbyte_cdk/test/models/connector_metadata/NormalizationDestinationDefinitionConfig.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/NormalizationDestinationDefinitionConfig.py rename to airbyte_cdk/test/models/connector_metadata/NormalizationDestinationDefinitionConfig.py diff --git a/airbyte_cdk/metadata_models/README.md b/airbyte_cdk/test/models/connector_metadata/README.md similarity index 89% rename from airbyte_cdk/metadata_models/README.md rename to airbyte_cdk/test/models/connector_metadata/README.md index 1883fc7f5..37ce58912 100644 --- a/airbyte_cdk/metadata_models/README.md +++ b/airbyte_cdk/test/models/connector_metadata/README.md @@ -1,4 +1,4 @@ -# Airbyte Metadata Models +# Airbyte Connector Metadata Models This package contains Pydantic models for validating Airbyte connector `metadata.yaml` files. @@ -18,7 +18,7 @@ During the CDK build process (`poetry run poe build`), these schemas are downloa ```python from pathlib import Path import yaml -from airbyte_cdk.metadata_models import ConnectorMetadataDefinitionV0 +from airbyte_cdk.test.models.connector_metadata import ConnectorMetadataDefinitionV0 # Load metadata.yaml metadata_path = Path("path/to/metadata.yaml") @@ -35,7 +35,7 @@ except Exception as e: ### Accessing metadata fields ```python -from airbyte_cdk.metadata_models import ConnectorMetadataDefinitionV0 +from airbyte_cdk.test.models.connector_metadata import ConnectorMetadataDefinitionV0 metadata = ConnectorMetadataDefinitionV0(**metadata_dict) @@ -70,7 +70,7 @@ poetry run poe build This command: 1. Downloads the latest schema YAML files from the airbyte repository 2. Generates Pydantic models using `datamodel-code-generator` -3. Outputs models to `airbyte_cdk/metadata_models/generated/` +3. Outputs models to `airbyte_cdk/test/models/connector_metadata/` ## Schema Source diff --git a/airbyte_cdk/metadata_models/generated/RegistryOverrides.py b/airbyte_cdk/test/models/connector_metadata/RegistryOverrides.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/RegistryOverrides.py rename to airbyte_cdk/test/models/connector_metadata/RegistryOverrides.py diff --git a/airbyte_cdk/metadata_models/generated/ReleaseStage.py b/airbyte_cdk/test/models/connector_metadata/ReleaseStage.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ReleaseStage.py rename to airbyte_cdk/test/models/connector_metadata/ReleaseStage.py diff --git a/airbyte_cdk/metadata_models/generated/RemoteRegistries.py b/airbyte_cdk/test/models/connector_metadata/RemoteRegistries.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/RemoteRegistries.py rename to airbyte_cdk/test/models/connector_metadata/RemoteRegistries.py diff --git a/airbyte_cdk/metadata_models/generated/ResourceRequirements.py b/airbyte_cdk/test/models/connector_metadata/ResourceRequirements.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/ResourceRequirements.py rename to airbyte_cdk/test/models/connector_metadata/ResourceRequirements.py diff --git a/airbyte_cdk/metadata_models/generated/RolloutConfiguration.py b/airbyte_cdk/test/models/connector_metadata/RolloutConfiguration.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/RolloutConfiguration.py rename to airbyte_cdk/test/models/connector_metadata/RolloutConfiguration.py diff --git a/airbyte_cdk/metadata_models/generated/Secret.py b/airbyte_cdk/test/models/connector_metadata/Secret.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/Secret.py rename to airbyte_cdk/test/models/connector_metadata/Secret.py diff --git a/airbyte_cdk/metadata_models/generated/SecretStore.py b/airbyte_cdk/test/models/connector_metadata/SecretStore.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/SecretStore.py rename to airbyte_cdk/test/models/connector_metadata/SecretStore.py diff --git a/airbyte_cdk/metadata_models/generated/SourceFileInfo.py b/airbyte_cdk/test/models/connector_metadata/SourceFileInfo.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/SourceFileInfo.py rename to airbyte_cdk/test/models/connector_metadata/SourceFileInfo.py diff --git a/airbyte_cdk/metadata_models/generated/SuggestedStreams.py b/airbyte_cdk/test/models/connector_metadata/SuggestedStreams.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/SuggestedStreams.py rename to airbyte_cdk/test/models/connector_metadata/SuggestedStreams.py diff --git a/airbyte_cdk/metadata_models/generated/SupportLevel.py b/airbyte_cdk/test/models/connector_metadata/SupportLevel.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/SupportLevel.py rename to airbyte_cdk/test/models/connector_metadata/SupportLevel.py diff --git a/airbyte_cdk/metadata_models/generated/TestConnections.py b/airbyte_cdk/test/models/connector_metadata/TestConnections.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/TestConnections.py rename to airbyte_cdk/test/models/connector_metadata/TestConnections.py diff --git a/airbyte_cdk/metadata_models/generated/__init__.py b/airbyte_cdk/test/models/connector_metadata/__init__.py similarity index 100% rename from airbyte_cdk/metadata_models/generated/__init__.py rename to airbyte_cdk/test/models/connector_metadata/__init__.py diff --git a/airbyte_cdk/metadata_models/py.typed b/airbyte_cdk/test/models/connector_metadata/py.typed similarity index 100% rename from airbyte_cdk/metadata_models/py.typed rename to airbyte_cdk/test/models/connector_metadata/py.typed diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index 2b6431eab..35d0b4168 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -16,7 +16,7 @@ METADATA_SCHEMAS_GITHUB_URL = "https://api.github.com/repos/airbytehq/airbyte/contents/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" METADATA_SCHEMAS_RAW_URL_BASE = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" -LOCAL_METADATA_OUTPUT_DIR_PATH = "airbyte_cdk/metadata_models/generated" +LOCAL_METADATA_OUTPUT_DIR_PATH = "airbyte_cdk/test/models/connector_metadata" PIP_DEPENDENCIES = [ "datamodel_code_generator==0.26.3", From 62902f68a23e98204d2b1626e78612a869f06593 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 01:21:18 +0000 Subject: [PATCH 03/20] refactor: Move models to generated subdirectory and add convenience imports - Moved all generated models to connector_metadata/generated/ - Added __init__.py in connector_metadata to export key models - Updated airbyte_cdk/test/models/__init__.py to import: - ConnectorMetadataDefinitionV0 (top-level manifest model) - ConnectorTestSuiteOptions (test suites definition model) - Updated README with new import paths and examples - Updated build script to output to new location This reduces the API surface area while keeping all models accessible through the generated submodule. Co-Authored-By: AJ Steers --- airbyte_cdk/test/models/__init__.py | 6 +++ .../test/models/connector_metadata/README.md | 19 ++++++++-- .../models/connector_metadata/__init__.py | 38 ++++--------------- .../ActorDefinitionResourceRequirements.py | 0 .../{ => generated}/AirbyteInternal.py | 0 .../{ => generated}/AllowedHosts.py | 0 .../ConnectorBreakingChanges.py | 0 .../{ => generated}/ConnectorBuildOptions.py | 0 .../{ => generated}/ConnectorIPCOptions.py | 0 .../ConnectorMetadataDefinitionV0.py | 0 .../{ => generated}/ConnectorMetrics.py | 0 .../{ => generated}/ConnectorPackageInfo.py | 0 .../ConnectorRegistryDestinationDefinition.py | 0 .../ConnectorRegistryReleases.py | 0 .../ConnectorRegistrySourceDefinition.py | 0 .../{ => generated}/ConnectorRegistryV0.py | 0 .../{ => generated}/ConnectorReleases.py | 0 .../ConnectorTestSuiteOptions.py | 0 .../{ => generated}/GeneratedFields.py | 0 .../{ => generated}/GitInfo.py | 0 .../{ => generated}/JobType.py | 0 ...ormalizationDestinationDefinitionConfig.py | 0 .../{ => generated}/RegistryOverrides.py | 0 .../{ => generated}/ReleaseStage.py | 0 .../{ => generated}/RemoteRegistries.py | 0 .../{ => generated}/ResourceRequirements.py | 0 .../{ => generated}/RolloutConfiguration.py | 0 .../{ => generated}/Secret.py | 0 .../{ => generated}/SecretStore.py | 0 .../{ => generated}/SourceFileInfo.py | 0 .../{ => generated}/SuggestedStreams.py | 0 .../{ => generated}/SupportLevel.py | 0 .../{ => generated}/TestConnections.py | 0 .../connector_metadata/generated/__init__.py | 31 +++++++++++++++ bin/generate_component_manifest_files.py | 2 +- 35 files changed, 61 insertions(+), 35 deletions(-) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ActorDefinitionResourceRequirements.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/AirbyteInternal.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/AllowedHosts.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorBreakingChanges.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorBuildOptions.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorIPCOptions.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorMetadataDefinitionV0.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorMetrics.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorPackageInfo.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorRegistryDestinationDefinition.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorRegistryReleases.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorRegistrySourceDefinition.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorRegistryV0.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorReleases.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ConnectorTestSuiteOptions.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/GeneratedFields.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/GitInfo.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/JobType.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/NormalizationDestinationDefinitionConfig.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/RegistryOverrides.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ReleaseStage.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/RemoteRegistries.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/ResourceRequirements.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/RolloutConfiguration.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/Secret.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/SecretStore.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/SourceFileInfo.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/SuggestedStreams.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/SupportLevel.py (100%) rename airbyte_cdk/test/models/connector_metadata/{ => generated}/TestConnections.py (100%) create mode 100644 airbyte_cdk/test/models/connector_metadata/generated/__init__.py diff --git a/airbyte_cdk/test/models/__init__.py b/airbyte_cdk/test/models/__init__.py index 70e6a3600..14aba786e 100644 --- a/airbyte_cdk/test/models/__init__.py +++ b/airbyte_cdk/test/models/__init__.py @@ -1,10 +1,16 @@ # Copyright (c) 2025 Airbyte, Inc., all rights reserved. """Models used for standard tests.""" +from airbyte_cdk.test.models.connector_metadata import ( + ConnectorMetadataDefinitionV0, + ConnectorTestSuiteOptions, +) from airbyte_cdk.test.models.outcome import ExpectedOutcome from airbyte_cdk.test.models.scenario import ConnectorTestScenario __all__ = [ + "ConnectorMetadataDefinitionV0", "ConnectorTestScenario", + "ConnectorTestSuiteOptions", "ExpectedOutcome", ] diff --git a/airbyte_cdk/test/models/connector_metadata/README.md b/airbyte_cdk/test/models/connector_metadata/README.md index 37ce58912..ac8b33ae4 100644 --- a/airbyte_cdk/test/models/connector_metadata/README.md +++ b/airbyte_cdk/test/models/connector_metadata/README.md @@ -18,7 +18,7 @@ During the CDK build process (`poetry run poe build`), these schemas are downloa ```python from pathlib import Path import yaml -from airbyte_cdk.test.models.connector_metadata import ConnectorMetadataDefinitionV0 +from airbyte_cdk.test.models import ConnectorMetadataDefinitionV0 # Load metadata.yaml metadata_path = Path("path/to/metadata.yaml") @@ -35,7 +35,7 @@ except Exception as e: ### Accessing metadata fields ```python -from airbyte_cdk.test.models.connector_metadata import ConnectorMetadataDefinitionV0 +from airbyte_cdk.test.models import ConnectorMetadataDefinitionV0 metadata = ConnectorMetadataDefinitionV0(**metadata_dict) @@ -46,6 +46,19 @@ print(f"Docker image tag: {metadata.data.dockerImageTag}") print(f"Support level: {metadata.data.supportLevel}") ``` +### Accessing other models + +All generated models are available in the `generated` submodule: + +```python +from airbyte_cdk.test.models.connector_metadata.generated import ( + ConnectorBreakingChanges, + ConnectorReleases, + ReleaseStage, + SupportLevel, +) +``` + ### Available models The main model is `ConnectorMetadataDefinitionV0`, which includes nested models for: @@ -70,7 +83,7 @@ poetry run poe build This command: 1. Downloads the latest schema YAML files from the airbyte repository 2. Generates Pydantic models using `datamodel-code-generator` -3. Outputs models to `airbyte_cdk/test/models/connector_metadata/` +3. Outputs models to `airbyte_cdk/test/models/connector_metadata/generated/` ## Schema Source diff --git a/airbyte_cdk/test/models/connector_metadata/__init__.py b/airbyte_cdk/test/models/connector_metadata/__init__.py index d026bba2a..fef3ed025 100644 --- a/airbyte_cdk/test/models/connector_metadata/__init__.py +++ b/airbyte_cdk/test/models/connector_metadata/__init__.py @@ -1,31 +1,7 @@ -# generated by bin/generate_component_manifest_files.py -from .ActorDefinitionResourceRequirements import * -from .AirbyteInternal import * -from .AllowedHosts import * -from .ConnectorBreakingChanges import * -from .ConnectorBuildOptions import * -from .ConnectorIPCOptions import * -from .ConnectorMetadataDefinitionV0 import * -from .ConnectorMetrics import * -from .ConnectorPackageInfo import * -from .ConnectorRegistryDestinationDefinition import * -from .ConnectorRegistryReleases import * -from .ConnectorRegistrySourceDefinition import * -from .ConnectorRegistryV0 import * -from .ConnectorReleases import * -from .ConnectorTestSuiteOptions import * -from .GeneratedFields import * -from .GitInfo import * -from .JobType import * -from .NormalizationDestinationDefinitionConfig import * -from .RegistryOverrides import * -from .ReleaseStage import * -from .RemoteRegistries import * -from .ResourceRequirements import * -from .RolloutConfiguration import * -from .Secret import * -from .SecretStore import * -from .SourceFileInfo import * -from .SuggestedStreams import * -from .SupportLevel import * -from .TestConnections import * +from .generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 +from .generated.ConnectorTestSuiteOptions import ConnectorTestSuiteOptions + +__all__ = [ + "ConnectorMetadataDefinitionV0", + "ConnectorTestSuiteOptions", +] diff --git a/airbyte_cdk/test/models/connector_metadata/ActorDefinitionResourceRequirements.py b/airbyte_cdk/test/models/connector_metadata/generated/ActorDefinitionResourceRequirements.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ActorDefinitionResourceRequirements.py rename to airbyte_cdk/test/models/connector_metadata/generated/ActorDefinitionResourceRequirements.py diff --git a/airbyte_cdk/test/models/connector_metadata/AirbyteInternal.py b/airbyte_cdk/test/models/connector_metadata/generated/AirbyteInternal.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/AirbyteInternal.py rename to airbyte_cdk/test/models/connector_metadata/generated/AirbyteInternal.py diff --git a/airbyte_cdk/test/models/connector_metadata/AllowedHosts.py b/airbyte_cdk/test/models/connector_metadata/generated/AllowedHosts.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/AllowedHosts.py rename to airbyte_cdk/test/models/connector_metadata/generated/AllowedHosts.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorBreakingChanges.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorBreakingChanges.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorBreakingChanges.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorBreakingChanges.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorBuildOptions.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorBuildOptions.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorBuildOptions.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorBuildOptions.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorIPCOptions.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorIPCOptions.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorIPCOptions.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorIPCOptions.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorMetadataDefinitionV0.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetadataDefinitionV0.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorMetadataDefinitionV0.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetadataDefinitionV0.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorMetrics.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetrics.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorMetrics.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetrics.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorPackageInfo.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorPackageInfo.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorPackageInfo.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorPackageInfo.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorRegistryDestinationDefinition.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryDestinationDefinition.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorRegistryDestinationDefinition.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryDestinationDefinition.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorRegistryReleases.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryReleases.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorRegistryReleases.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryReleases.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorRegistrySourceDefinition.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistrySourceDefinition.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorRegistrySourceDefinition.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistrySourceDefinition.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorRegistryV0.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryV0.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorRegistryV0.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryV0.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorReleases.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorReleases.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorReleases.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorReleases.py diff --git a/airbyte_cdk/test/models/connector_metadata/ConnectorTestSuiteOptions.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorTestSuiteOptions.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ConnectorTestSuiteOptions.py rename to airbyte_cdk/test/models/connector_metadata/generated/ConnectorTestSuiteOptions.py diff --git a/airbyte_cdk/test/models/connector_metadata/GeneratedFields.py b/airbyte_cdk/test/models/connector_metadata/generated/GeneratedFields.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/GeneratedFields.py rename to airbyte_cdk/test/models/connector_metadata/generated/GeneratedFields.py diff --git a/airbyte_cdk/test/models/connector_metadata/GitInfo.py b/airbyte_cdk/test/models/connector_metadata/generated/GitInfo.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/GitInfo.py rename to airbyte_cdk/test/models/connector_metadata/generated/GitInfo.py diff --git a/airbyte_cdk/test/models/connector_metadata/JobType.py b/airbyte_cdk/test/models/connector_metadata/generated/JobType.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/JobType.py rename to airbyte_cdk/test/models/connector_metadata/generated/JobType.py diff --git a/airbyte_cdk/test/models/connector_metadata/NormalizationDestinationDefinitionConfig.py b/airbyte_cdk/test/models/connector_metadata/generated/NormalizationDestinationDefinitionConfig.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/NormalizationDestinationDefinitionConfig.py rename to airbyte_cdk/test/models/connector_metadata/generated/NormalizationDestinationDefinitionConfig.py diff --git a/airbyte_cdk/test/models/connector_metadata/RegistryOverrides.py b/airbyte_cdk/test/models/connector_metadata/generated/RegistryOverrides.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/RegistryOverrides.py rename to airbyte_cdk/test/models/connector_metadata/generated/RegistryOverrides.py diff --git a/airbyte_cdk/test/models/connector_metadata/ReleaseStage.py b/airbyte_cdk/test/models/connector_metadata/generated/ReleaseStage.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ReleaseStage.py rename to airbyte_cdk/test/models/connector_metadata/generated/ReleaseStage.py diff --git a/airbyte_cdk/test/models/connector_metadata/RemoteRegistries.py b/airbyte_cdk/test/models/connector_metadata/generated/RemoteRegistries.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/RemoteRegistries.py rename to airbyte_cdk/test/models/connector_metadata/generated/RemoteRegistries.py diff --git a/airbyte_cdk/test/models/connector_metadata/ResourceRequirements.py b/airbyte_cdk/test/models/connector_metadata/generated/ResourceRequirements.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/ResourceRequirements.py rename to airbyte_cdk/test/models/connector_metadata/generated/ResourceRequirements.py diff --git a/airbyte_cdk/test/models/connector_metadata/RolloutConfiguration.py b/airbyte_cdk/test/models/connector_metadata/generated/RolloutConfiguration.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/RolloutConfiguration.py rename to airbyte_cdk/test/models/connector_metadata/generated/RolloutConfiguration.py diff --git a/airbyte_cdk/test/models/connector_metadata/Secret.py b/airbyte_cdk/test/models/connector_metadata/generated/Secret.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/Secret.py rename to airbyte_cdk/test/models/connector_metadata/generated/Secret.py diff --git a/airbyte_cdk/test/models/connector_metadata/SecretStore.py b/airbyte_cdk/test/models/connector_metadata/generated/SecretStore.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/SecretStore.py rename to airbyte_cdk/test/models/connector_metadata/generated/SecretStore.py diff --git a/airbyte_cdk/test/models/connector_metadata/SourceFileInfo.py b/airbyte_cdk/test/models/connector_metadata/generated/SourceFileInfo.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/SourceFileInfo.py rename to airbyte_cdk/test/models/connector_metadata/generated/SourceFileInfo.py diff --git a/airbyte_cdk/test/models/connector_metadata/SuggestedStreams.py b/airbyte_cdk/test/models/connector_metadata/generated/SuggestedStreams.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/SuggestedStreams.py rename to airbyte_cdk/test/models/connector_metadata/generated/SuggestedStreams.py diff --git a/airbyte_cdk/test/models/connector_metadata/SupportLevel.py b/airbyte_cdk/test/models/connector_metadata/generated/SupportLevel.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/SupportLevel.py rename to airbyte_cdk/test/models/connector_metadata/generated/SupportLevel.py diff --git a/airbyte_cdk/test/models/connector_metadata/TestConnections.py b/airbyte_cdk/test/models/connector_metadata/generated/TestConnections.py similarity index 100% rename from airbyte_cdk/test/models/connector_metadata/TestConnections.py rename to airbyte_cdk/test/models/connector_metadata/generated/TestConnections.py diff --git a/airbyte_cdk/test/models/connector_metadata/generated/__init__.py b/airbyte_cdk/test/models/connector_metadata/generated/__init__.py new file mode 100644 index 000000000..d026bba2a --- /dev/null +++ b/airbyte_cdk/test/models/connector_metadata/generated/__init__.py @@ -0,0 +1,31 @@ +# generated by bin/generate_component_manifest_files.py +from .ActorDefinitionResourceRequirements import * +from .AirbyteInternal import * +from .AllowedHosts import * +from .ConnectorBreakingChanges import * +from .ConnectorBuildOptions import * +from .ConnectorIPCOptions import * +from .ConnectorMetadataDefinitionV0 import * +from .ConnectorMetrics import * +from .ConnectorPackageInfo import * +from .ConnectorRegistryDestinationDefinition import * +from .ConnectorRegistryReleases import * +from .ConnectorRegistrySourceDefinition import * +from .ConnectorRegistryV0 import * +from .ConnectorReleases import * +from .ConnectorTestSuiteOptions import * +from .GeneratedFields import * +from .GitInfo import * +from .JobType import * +from .NormalizationDestinationDefinitionConfig import * +from .RegistryOverrides import * +from .ReleaseStage import * +from .RemoteRegistries import * +from .ResourceRequirements import * +from .RolloutConfiguration import * +from .Secret import * +from .SecretStore import * +from .SourceFileInfo import * +from .SuggestedStreams import * +from .SupportLevel import * +from .TestConnections import * diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index 35d0b4168..e791792bb 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -16,7 +16,7 @@ METADATA_SCHEMAS_GITHUB_URL = "https://api.github.com/repos/airbytehq/airbyte/contents/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" METADATA_SCHEMAS_RAW_URL_BASE = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" -LOCAL_METADATA_OUTPUT_DIR_PATH = "airbyte_cdk/test/models/connector_metadata" +LOCAL_METADATA_OUTPUT_DIR_PATH = "airbyte_cdk/test/models/connector_metadata/generated" PIP_DEPENDENCIES = [ "datamodel_code_generator==0.26.3", From 3de3af0b78f51f4b8e4b8573aabdeeebce4e40d6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 01:37:16 +0000 Subject: [PATCH 04/20] refactor: Generate metadata models as single file with JSON schema output - Modified build script to generate all models into a single models.py file - Added consolidated JSON schema generation (metadata_schema.json) - Updated imports to reference generated.models module - Removed 30+ individual model files in favor of single-file approach - Updated README to document new structure and outputs - Added GitHub token support to avoid rate limiting Co-Authored-By: AJ Steers --- .../models/declarative_component_schema.py | 144 +++-- .../test/models/connector_metadata/README.md | 13 +- .../models/connector_metadata/__init__.py | 3 +- .../ActorDefinitionResourceRequirements.py | 48 -- .../generated/AirbyteInternal.py | 39 -- .../generated/AllowedHosts.py | 18 - .../generated/ConnectorBreakingChanges.py | 70 --- .../generated/ConnectorBuildOptions.py | 15 - .../generated/ConnectorIPCOptions.py | 36 -- .../ConnectorMetadataDefinitionV0.py | 495 ------------------ .../generated/ConnectorMetrics.py | 36 -- .../generated/ConnectorPackageInfo.py | 12 - .../ConnectorRegistryDestinationDefinition.py | 439 ---------------- .../generated/ConnectorRegistryReleases.py | 438 ---------------- .../ConnectorRegistrySourceDefinition.py | 439 ---------------- .../generated/ConnectorRegistryV0.py | 445 ---------------- .../generated/ConnectorReleases.py | 103 ---- .../generated/ConnectorTestSuiteOptions.py | 63 --- .../generated/GeneratedFields.py | 74 --- .../connector_metadata/generated/GitInfo.py | 31 -- .../connector_metadata/generated/JobType.py | 16 - ...ormalizationDestinationDefinitionConfig.py | 24 - .../generated/RegistryOverrides.py | 105 ---- .../generated/ReleaseStage.py | 13 - .../generated/RemoteRegistries.py | 23 - .../generated/ResourceRequirements.py | 18 - .../generated/RolloutConfiguration.py | 29 - .../connector_metadata/generated/Secret.py | 33 -- .../generated/SecretStore.py | 21 - .../generated/SourceFileInfo.py | 16 - .../generated/SuggestedStreams.py | 18 - .../generated/SupportLevel.py | 12 - .../generated/TestConnections.py | 14 - .../connector_metadata/generated/__init__.py | 31 -- bin/generate_component_manifest_files.py | 108 +++- 35 files changed, 197 insertions(+), 3245 deletions(-) delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ActorDefinitionResourceRequirements.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/AirbyteInternal.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/AllowedHosts.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorBreakingChanges.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorBuildOptions.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorIPCOptions.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetadataDefinitionV0.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetrics.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorPackageInfo.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryDestinationDefinition.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryReleases.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistrySourceDefinition.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryV0.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorReleases.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ConnectorTestSuiteOptions.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/GeneratedFields.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/GitInfo.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/JobType.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/NormalizationDestinationDefinitionConfig.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/RegistryOverrides.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ReleaseStage.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/RemoteRegistries.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/ResourceRequirements.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/RolloutConfiguration.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/Secret.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/SecretStore.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/SourceFileInfo.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/SuggestedStreams.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/SupportLevel.py delete mode 100644 airbyte_cdk/test/models/connector_metadata/generated/TestConnections.py diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 915f942d0..4c6f771c0 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -926,24 +926,28 @@ class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", - examples=[ - {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, - { - "app_id": { - "type": "string", - "path_in_connector_config": ["info", "app_id"], - } - }, - ], - title="OAuth user input", + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( + Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + examples=[ + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, + { + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], + } + }, + ], + title="OAuth user input", + ) ) - oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( - None, - description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', - title="DeclarativeOAuth Connector Specification", + oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = ( + Field( + None, + description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', + title="DeclarativeOAuth Connector Specification", + ) ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, @@ -961,7 +965,9 @@ class Config: complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", - examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], + examples=[ + {"client_id": {"type": "string"}, "client_secret": {"type": "string"}} + ], title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( @@ -1465,7 +1471,9 @@ class CustomConfigTransformation(BaseModel): class_name: str = Field( ..., description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_..`.", - examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], + examples=[ + "source_declarative_manifest.components.MyCustomConfigTransformation" + ], ) parameters: Optional[Dict[str, Any]] = Field( None, @@ -1883,7 +1891,9 @@ class OAuthAuthenticator(BaseModel): scopes: Optional[List[str]] = Field( None, description="List of scopes that should be granted to the access token.", - examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], + examples=[ + ["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"] + ], title="Scopes", ) token_expiry_date: Optional[str] = Field( @@ -2059,7 +2069,9 @@ class RecordSelector(BaseModel): description="Responsible for filtering records to be emitted by the Source.", title="Record Filter", ) - schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( + schema_normalization: Optional[ + Union[SchemaNormalization, CustomSchemaNormalization] + ] = Field( None, description="Responsible for normalization according to the schema.", title="Schema Normalization", @@ -2101,10 +2113,12 @@ class DpathValidator(BaseModel): ], title="Field Path", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( - ..., - description="The condition that the specified config value will be evaluated against", - title="Validation Strategy", + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( + Field( + ..., + description="The condition that the specified config value will be evaluated against", + title="Validation Strategy", + ) ) @@ -2121,10 +2135,12 @@ class PredicateValidator(BaseModel): ], title="Value", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( - ..., - description="The validation strategy to apply to the value.", - title="Validation Strategy", + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( + Field( + ..., + description="The validation strategy to apply to the value.", + title="Validation Strategy", + ) ) @@ -2149,12 +2165,12 @@ class ConfigAddFields(BaseModel): class CompositeErrorHandler(BaseModel): type: Literal["CompositeErrorHandler"] - error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( - Field( - ..., - description="List of error handlers to iterate on to determine how to handle a failed response.", - title="Error Handlers", - ) + error_handlers: List[ + Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler] + ] = Field( + ..., + description="List of error handlers to iterate on to determine how to handle a failed response.", + title="Error Handlers", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -2316,9 +2332,9 @@ class Config: type: Literal["DeclarativeSource"] check: Union[CheckStream, CheckDynamicStream] - streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( - None - ) + streams: Optional[ + List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] + ] = None dynamic_streams: List[DynamicDeclarativeStream] version: str = Field( ..., @@ -2443,16 +2459,20 @@ class Config: extra = Extra.allow type: Literal["DeclarativeStream"] - name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") + name: Optional[str] = Field( + "", description="The stream name.", example=["Users"], title="Name" + ) retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages.", title="Retriever", ) - incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( - None, - description="Component used to fetch data incrementally based on a time field in the data.", - title="Incremental Sync", + incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = ( + Field( + None, + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", + ) ) primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") schema_loader: Optional[ @@ -2626,18 +2646,20 @@ class HttpRequester(BaseModelWithDeprecations): description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", title="Query Properties", ) - request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( - None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", - examples=[ - {"unit": "day"}, - { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, - ], - title="Query Parameters", + request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = ( + Field( + None, + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + examples=[ + {"unit": "day"}, + { + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, + ], + title="Query Parameters", + ) ) request_headers: Optional[Union[Dict[str, str], str]] = Field( None, @@ -2804,7 +2826,9 @@ class QueryProperties(BaseModel): class StateDelegatingStream(BaseModel): type: Literal["StateDelegatingStream"] - name: str = Field(..., description="The stream name.", example=["Users"], title="Name") + name: str = Field( + ..., description="The stream name.", example=["Users"], title="Name" + ) full_refresh_stream: DeclarativeStream = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", @@ -2891,13 +2915,17 @@ class AsyncRetriever(BaseModel): status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( ..., description="Responsible for fetching the actual status of the async job." ) - download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( + download_target_extractor: Optional[ + Union[DpathExtractor, CustomRecordExtractor] + ] = Field( None, description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", ) download_extractor: Optional[ Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] - ] = Field(None, description="Responsible for fetching the records from provided urls.") + ] = Field( + None, description="Responsible for fetching the records from provided urls." + ) creation_requester: Union[HttpRequester, CustomRequester] = Field( ..., description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", diff --git a/airbyte_cdk/test/models/connector_metadata/README.md b/airbyte_cdk/test/models/connector_metadata/README.md index ac8b33ae4..e254e6da1 100644 --- a/airbyte_cdk/test/models/connector_metadata/README.md +++ b/airbyte_cdk/test/models/connector_metadata/README.md @@ -9,7 +9,7 @@ The models are automatically generated from JSON Schema YAML files maintained in airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ ``` -During the CDK build process (`poetry run poe build`), these schemas are downloaded from GitHub and used to generate Pydantic models via `datamodel-code-generator`. +During the CDK build process (`poetry run poe build`), these schemas are downloaded from GitHub and used to generate Pydantic models via `datamodel-code-generator`. All models are generated into a single Python file for simplicity and easier imports. ## Usage @@ -48,10 +48,10 @@ print(f"Support level: {metadata.data.supportLevel}") ### Accessing other models -All generated models are available in the `generated` submodule: +All generated models are available in the `generated.models` module: ```python -from airbyte_cdk.test.models.connector_metadata.generated import ( +from airbyte_cdk.test.models.connector_metadata.generated.models import ( ConnectorBreakingChanges, ConnectorReleases, ReleaseStage, @@ -82,8 +82,11 @@ poetry run poe build This command: 1. Downloads the latest schema YAML files from the airbyte repository -2. Generates Pydantic models using `datamodel-code-generator` -3. Outputs models to `airbyte_cdk/test/models/connector_metadata/generated/` +2. Generates all Pydantic models into a single file using `datamodel-code-generator` +3. Generates a consolidated JSON schema file for external validation tools +4. Outputs to `airbyte_cdk/test/models/connector_metadata/generated/`: + - `models.py` - All Pydantic models in a single file + - `metadata_schema.json` - Consolidated JSON schema ## Schema Source diff --git a/airbyte_cdk/test/models/connector_metadata/__init__.py b/airbyte_cdk/test/models/connector_metadata/__init__.py index fef3ed025..2828d2b8f 100644 --- a/airbyte_cdk/test/models/connector_metadata/__init__.py +++ b/airbyte_cdk/test/models/connector_metadata/__init__.py @@ -1,5 +1,4 @@ -from .generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 -from .generated.ConnectorTestSuiteOptions import ConnectorTestSuiteOptions +from .generated.models import ConnectorMetadataDefinitionV0, ConnectorTestSuiteOptions __all__ = [ "ConnectorMetadataDefinitionV0", diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ActorDefinitionResourceRequirements.py b/airbyte_cdk/test/models/connector_metadata/generated/ActorDefinitionResourceRequirements.py deleted file mode 100644 index eb4784a3d..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ActorDefinitionResourceRequirements.py +++ /dev/null @@ -1,48 +0,0 @@ -# generated by datamodel-codegen: -# filename: ActorDefinitionResourceRequirements.yaml - -from __future__ import annotations - -from enum import Enum -from typing import List, Optional - -from pydantic.v1 import BaseModel, Extra, Field - - -class ResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - cpu_request: Optional[str] = None - cpu_limit: Optional[str] = None - memory_request: Optional[str] = None - memory_limit: Optional[str] = None - - -class JobType(Enum): - get_spec = "get_spec" - check_connection = "check_connection" - discover_schema = "discover_schema" - sync = "sync" - reset_connection = "reset_connection" - connection_updater = "connection_updater" - replicate = "replicate" - - -class JobTypeResourceLimit(BaseModel): - class Config: - extra = Extra.forbid - - jobType: JobType - resourceRequirements: ResourceRequirements - - -class ActorDefinitionResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - default: Optional[ResourceRequirements] = Field( - None, - description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", - ) - jobSpecific: Optional[List[JobTypeResourceLimit]] = None diff --git a/airbyte_cdk/test/models/connector_metadata/generated/AirbyteInternal.py b/airbyte_cdk/test/models/connector_metadata/generated/AirbyteInternal.py deleted file mode 100644 index 5882c86e3..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/AirbyteInternal.py +++ /dev/null @@ -1,39 +0,0 @@ -# generated by datamodel-codegen: -# filename: AirbyteInternal.yaml - -from __future__ import annotations - -from enum import Enum -from typing import Optional - -from pydantic.v1 import BaseModel, Extra, Field - - -class Sl(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - - -class Ql(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - integer_400 = 400 - integer_500 = 500 - integer_600 = 600 - - -class AirbyteInternal(BaseModel): - class Config: - extra = Extra.allow - - sl: Optional[Sl] = None - ql: Optional[Ql] = None - isEnterprise: Optional[bool] = False - requireVersionIncrementsInPullRequests: Optional[bool] = Field( - True, - description="When false, version increment checks will be skipped for this connector", - ) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/AllowedHosts.py b/airbyte_cdk/test/models/connector_metadata/generated/AllowedHosts.py deleted file mode 100644 index eb7318522..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/AllowedHosts.py +++ /dev/null @@ -1,18 +0,0 @@ -# generated by datamodel-codegen: -# filename: AllowedHosts.yaml - -from __future__ import annotations - -from typing import List, Optional - -from pydantic.v1 import BaseModel, Extra, Field - - -class AllowedHosts(BaseModel): - class Config: - extra = Extra.allow - - hosts: Optional[List[str]] = Field( - None, - description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", - ) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorBreakingChanges.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorBreakingChanges.py deleted file mode 100644 index 4d73847f3..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorBreakingChanges.py +++ /dev/null @@ -1,70 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorBreakingChanges.yaml - -from __future__ import annotations - -from datetime import date -from enum import Enum -from typing import Any, Dict, List, Optional - -from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, constr - - -class DeadlineAction(Enum): - auto_upgrade = "auto_upgrade" - disable = "disable" - - -class StreamBreakingChangeScope(BaseModel): - class Config: - extra = Extra.forbid - - scopeType: Any = Field("stream", const=True) - impactedScopes: List[str] = Field( - ..., - description="List of streams that are impacted by the breaking change.", - min_items=1, - ) - - -class BreakingChangeScope(BaseModel): - __root__: StreamBreakingChangeScope = Field( - ..., - description="A scope that can be used to limit the impact of a breaking change.", - ) - - -class VersionBreakingChange(BaseModel): - class Config: - extra = Extra.forbid - - upgradeDeadline: date = Field( - ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." - ) - deadlineAction: Optional[DeadlineAction] = Field( - None, description="Action to do when the deadline is reached." - ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", - ) - scopedImpact: Optional[List[BreakingChangeScope]] = Field( - None, - description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", - min_items=1, - ) - - -class ConnectorBreakingChanges(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( - ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", - title="ConnectorBreakingChanges", - ) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorBuildOptions.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorBuildOptions.py deleted file mode 100644 index 32e00539c..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorBuildOptions.py +++ /dev/null @@ -1,15 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorBuildOptions.yaml - -from __future__ import annotations - -from typing import Optional - -from pydantic.v1 import BaseModel, Extra - - -class ConnectorBuildOptions(BaseModel): - class Config: - extra = Extra.forbid - - baseImage: Optional[str] = None diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorIPCOptions.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorIPCOptions.py deleted file mode 100644 index e5ae22e8a..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorIPCOptions.py +++ /dev/null @@ -1,36 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorIPCOptions.yaml - -from __future__ import annotations - -from enum import Enum -from typing import List - -from pydantic.v1 import BaseModel, Extra - - -class SupportedSerializationEnum(Enum): - JSONL = "JSONL" - PROTOBUF = "PROTOBUF" - FLATBUFFERS = "FLATBUFFERS" - - -class SupportedTransportEnum(Enum): - STDIO = "STDIO" - SOCKET = "SOCKET" - - -class DataChannel(BaseModel): - class Config: - extra = Extra.forbid - - version: str - supportedSerialization: List[SupportedSerializationEnum] - supportedTransport: List[SupportedTransportEnum] - - -class ConnectorIPCOptions(BaseModel): - class Config: - extra = Extra.forbid - - dataChannel: DataChannel diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetadataDefinitionV0.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetadataDefinitionV0.py deleted file mode 100644 index 6c414b955..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetadataDefinitionV0.py +++ /dev/null @@ -1,495 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorMetadataDefinitionV0.yaml - -from __future__ import annotations - -from datetime import date, datetime -from enum import Enum -from typing import Any, Dict, List, Literal, Optional, Union -from uuid import UUID - -from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr - - -class ConnectorType(Enum): - destination = "destination" - source = "source" - - -class ConnectorSubtype(Enum): - api = "api" - database = "database" - datalake = "datalake" - file = "file" - custom = "custom" - message_queue = "message_queue" - unknown = "unknown" - vectorstore = "vectorstore" - - -class ConnectorBuildOptions(BaseModel): - class Config: - extra = Extra.forbid - - baseImage: Optional[str] = None - - -class Suite(Enum): - unitTests = "unitTests" - integrationTests = "integrationTests" - acceptanceTests = "acceptanceTests" - liveTests = "liveTests" - - -class SecretStore(BaseModel): - class Config: - extra = Extra.forbid - - alias: Optional[str] = Field( - None, - description="The alias of the secret store which can map to its actual secret address", - ) - type: Optional[Literal["GSM"]] = Field( - None, description="The type of the secret store" - ) - - -class TestConnections(BaseModel): - class Config: - extra = Extra.forbid - - name: str = Field(..., description="The connection name") - id: str = Field(..., description="The connection ID") - - -class ReleaseStage(Enum): - alpha = "alpha" - beta = "beta" - generally_available = "generally_available" - custom = "custom" - - -class SupportLevel(Enum): - community = "community" - certified = "certified" - archived = "archived" - - -class AllowedHosts(BaseModel): - class Config: - extra = Extra.allow - - hosts: Optional[List[str]] = Field( - None, - description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", - ) - - -class NormalizationDestinationDefinitionConfig(BaseModel): - class Config: - extra = Extra.allow - - normalizationRepository: str = Field( - ..., - description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", - ) - normalizationTag: str = Field( - ..., - description="a field indicating the tag of the docker repository to be used for normalization.", - ) - normalizationIntegrationType: str = Field( - ..., - description="a field indicating the type of integration dialect to use for normalization.", - ) - - -class SuggestedStreams(BaseModel): - class Config: - extra = Extra.allow - - streams: Optional[List[str]] = Field( - None, - description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", - ) - - -class ResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - cpu_request: Optional[str] = None - cpu_limit: Optional[str] = None - memory_request: Optional[str] = None - memory_limit: Optional[str] = None - - -class JobType(Enum): - get_spec = "get_spec" - check_connection = "check_connection" - discover_schema = "discover_schema" - sync = "sync" - reset_connection = "reset_connection" - connection_updater = "connection_updater" - replicate = "replicate" - - -class RolloutConfiguration(BaseModel): - class Config: - extra = Extra.forbid - - enableProgressiveRollout: Optional[bool] = Field( - False, description="Whether to enable progressive rollout for the connector." - ) - initialPercentage: Optional[conint(ge=0, le=100)] = Field( - 0, - description="The percentage of users that should receive the new version initially.", - ) - maxPercentage: Optional[conint(ge=0, le=100)] = Field( - 50, - description="The percentage of users who should receive the release candidate during the test phase before full rollout.", - ) - advanceDelayMinutes: Optional[conint(ge=10)] = Field( - 10, - description="The number of minutes to wait before advancing the rollout percentage.", - ) - - -class DeadlineAction(Enum): - auto_upgrade = "auto_upgrade" - disable = "disable" - - -class StreamBreakingChangeScope(BaseModel): - class Config: - extra = Extra.forbid - - scopeType: Any = Field("stream", const=True) - impactedScopes: List[str] = Field( - ..., - description="List of streams that are impacted by the breaking change.", - min_items=1, - ) - - -class Sl(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - - -class Ql(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - integer_400 = 400 - integer_500 = 500 - integer_600 = 600 - - -class AirbyteInternal(BaseModel): - class Config: - extra = Extra.allow - - sl: Optional[Sl] = None - ql: Optional[Ql] = None - isEnterprise: Optional[bool] = False - requireVersionIncrementsInPullRequests: Optional[bool] = Field( - True, - description="When false, version increment checks will be skipped for this connector", - ) - - -class PyPi(BaseModel): - class Config: - extra = Extra.forbid - - enabled: bool - packageName: str = Field(..., description="The name of the package on PyPi.") - - -class GitInfo(BaseModel): - class Config: - extra = Extra.forbid - - commit_sha: Optional[str] = Field( - None, - description="The git commit sha of the last commit that modified this file.", - ) - commit_timestamp: Optional[datetime] = Field( - None, - description="The git commit timestamp of the last commit that modified this file.", - ) - commit_author: Optional[str] = Field( - None, - description="The git commit author of the last commit that modified this file.", - ) - commit_author_email: Optional[str] = Field( - None, - description="The git commit author email of the last commit that modified this file.", - ) - - -class SourceFileInfo(BaseModel): - metadata_etag: Optional[str] = None - metadata_file_path: Optional[str] = None - metadata_bucket_name: Optional[str] = None - metadata_last_modified: Optional[str] = None - registry_entry_generated_at: Optional[str] = None - - -class ConnectorMetrics(BaseModel): - all: Optional[Any] = None - cloud: Optional[Any] = None - oss: Optional[Any] = None - - -class Usage(Enum): - low = "low" - medium = "medium" - high = "high" - - -class SyncSuccessRate(Enum): - low = "low" - medium = "medium" - high = "high" - - -class ConnectorMetric(BaseModel): - class Config: - extra = Extra.allow - - usage: Optional[Union[str, Usage]] = None - sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None - connector_version: Optional[str] = None - - -class SupportedSerializationEnum(Enum): - JSONL = "JSONL" - PROTOBUF = "PROTOBUF" - FLATBUFFERS = "FLATBUFFERS" - - -class SupportedTransportEnum(Enum): - STDIO = "STDIO" - SOCKET = "SOCKET" - - -class DataChannel(BaseModel): - class Config: - extra = Extra.forbid - - version: str - supportedSerialization: List[SupportedSerializationEnum] - supportedTransport: List[SupportedTransportEnum] - - -class ConnectorIPCOptions(BaseModel): - class Config: - extra = Extra.forbid - - dataChannel: DataChannel - - -class Secret(BaseModel): - class Config: - extra = Extra.forbid - - name: str = Field(..., description="The secret name in the secret store") - fileName: Optional[str] = Field( - None, - description="The name of the file to which the secret value would be persisted", - ) - secretStore: SecretStore - - -class JobTypeResourceLimit(BaseModel): - class Config: - extra = Extra.forbid - - jobType: JobType - resourceRequirements: ResourceRequirements - - -class BreakingChangeScope(BaseModel): - __root__: StreamBreakingChangeScope = Field( - ..., - description="A scope that can be used to limit the impact of a breaking change.", - ) - - -class RemoteRegistries(BaseModel): - class Config: - extra = Extra.forbid - - pypi: Optional[PyPi] = None - - -class GeneratedFields(BaseModel): - git: Optional[GitInfo] = None - source_file_info: Optional[SourceFileInfo] = None - metrics: Optional[ConnectorMetrics] = None - sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") - - -class ConnectorTestSuiteOptions(BaseModel): - class Config: - extra = Extra.forbid - - suite: Suite = Field(..., description="Name of the configured test suite") - testSecrets: Optional[List[Secret]] = Field( - None, description="List of secrets required to run the test suite" - ) - testConnections: Optional[List[TestConnections]] = Field( - None, - description="List of sandbox cloud connections that tests can be run against", - ) - - -class ActorDefinitionResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - default: Optional[ResourceRequirements] = Field( - None, - description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", - ) - jobSpecific: Optional[List[JobTypeResourceLimit]] = None - - -class VersionBreakingChange(BaseModel): - class Config: - extra = Extra.forbid - - upgradeDeadline: date = Field( - ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." - ) - deadlineAction: Optional[DeadlineAction] = Field( - None, description="Action to do when the deadline is reached." - ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", - ) - scopedImpact: Optional[List[BreakingChangeScope]] = Field( - None, - description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", - min_items=1, - ) - - -class RegistryOverrides(BaseModel): - class Config: - extra = Extra.forbid - - enabled: bool - name: Optional[str] = None - dockerRepository: Optional[str] = None - dockerImageTag: Optional[str] = None - supportsDbt: Optional[bool] = None - supportsNormalization: Optional[bool] = None - license: Optional[str] = None - documentationUrl: Optional[AnyUrl] = None - connectorSubtype: Optional[str] = None - allowedHosts: Optional[AllowedHosts] = None - normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None - suggestedStreams: Optional[SuggestedStreams] = None - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None - - -class ConnectorBreakingChanges(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( - ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", - title="ConnectorBreakingChanges", - ) - - -class RegistryOverridesModel(BaseModel): - class Config: - extra = Extra.forbid - - oss: Optional[RegistryOverrides] = None - cloud: Optional[RegistryOverrides] = None - - -class ConnectorReleases(BaseModel): - class Config: - extra = Extra.forbid - - rolloutConfiguration: Optional[RolloutConfiguration] = None - breakingChanges: Optional[ConnectorBreakingChanges] = None - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", - ) - - -class Data(BaseModel): - class Config: - extra = Extra.forbid - - name: str - icon: Optional[str] = None - definitionId: UUID - connectorBuildOptions: Optional[ConnectorBuildOptions] = None - connectorTestSuitesOptions: Optional[List[ConnectorTestSuiteOptions]] = None - connectorType: ConnectorType - dockerRepository: str - dockerImageTag: str - supportsDbt: Optional[bool] = None - supportsNormalization: Optional[bool] = None - license: str - documentationUrl: AnyUrl - githubIssueLabel: str - maxSecondsBetweenMessages: Optional[int] = Field( - None, - description="Maximum delay between 2 airbyte protocol messages, in second. The source will timeout if this delay is reached", - ) - releaseDate: Optional[date] = Field( - None, - description="The date when this connector was first released, in yyyy-mm-dd format.", - ) - protocolVersion: Optional[str] = Field( - None, description="the Airbyte Protocol version supported by the connector" - ) - erdUrl: Optional[str] = Field( - None, description="The URL where you can visualize the ERD" - ) - connectorSubtype: ConnectorSubtype - releaseStage: ReleaseStage - supportLevel: Optional[SupportLevel] = None - tags: Optional[List[str]] = Field( - [], - description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", - ) - registryOverrides: Optional[RegistryOverridesModel] = None - allowedHosts: Optional[AllowedHosts] = None - releases: Optional[ConnectorReleases] = None - normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None - suggestedStreams: Optional[SuggestedStreams] = None - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None - ab_internal: Optional[AirbyteInternal] = None - remoteRegistries: Optional[RemoteRegistries] = None - supportsRefreshes: Optional[bool] = False - generated: Optional[GeneratedFields] = None - supportsFileTransfer: Optional[bool] = False - supportsDataActivation: Optional[bool] = False - connectorIPCOptions: Optional[ConnectorIPCOptions] = None - - -class ConnectorMetadataDefinitionV0(BaseModel): - class Config: - extra = Extra.forbid - - metadataSpecVersion: str - data: Data diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetrics.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetrics.py deleted file mode 100644 index 2b3d44a80..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorMetrics.py +++ /dev/null @@ -1,36 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorMetrics.yaml - -from __future__ import annotations - -from enum import Enum -from typing import Any, Optional, Union - -from pydantic.v1 import BaseModel, Extra - - -class ConnectorMetrics(BaseModel): - all: Optional[Any] = None - cloud: Optional[Any] = None - oss: Optional[Any] = None - - -class Usage(Enum): - low = "low" - medium = "medium" - high = "high" - - -class SyncSuccessRate(Enum): - low = "low" - medium = "medium" - high = "high" - - -class ConnectorMetric(BaseModel): - class Config: - extra = Extra.allow - - usage: Optional[Union[str, Usage]] = None - sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None - connector_version: Optional[str] = None diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorPackageInfo.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorPackageInfo.py deleted file mode 100644 index 2a41dbc10..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorPackageInfo.py +++ /dev/null @@ -1,12 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorPackageInfo.yaml - -from __future__ import annotations - -from typing import Optional - -from pydantic.v1 import BaseModel - - -class ConnectorPackageInfo(BaseModel): - cdk_version: Optional[str] = None diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryDestinationDefinition.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryDestinationDefinition.py deleted file mode 100644 index d6acd3a7a..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryDestinationDefinition.py +++ /dev/null @@ -1,439 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorRegistryDestinationDefinition.yaml - -from __future__ import annotations - -from datetime import date, datetime -from enum import Enum -from typing import Any, Dict, List, Optional, Union -from uuid import UUID - -from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr - - -class ReleaseStage(Enum): - alpha = "alpha" - beta = "beta" - generally_available = "generally_available" - custom = "custom" - - -class SupportLevel(Enum): - community = "community" - certified = "certified" - archived = "archived" - - -class ResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - cpu_request: Optional[str] = None - cpu_limit: Optional[str] = None - memory_request: Optional[str] = None - memory_limit: Optional[str] = None - - -class JobType(Enum): - get_spec = "get_spec" - check_connection = "check_connection" - discover_schema = "discover_schema" - sync = "sync" - reset_connection = "reset_connection" - connection_updater = "connection_updater" - replicate = "replicate" - - -class NormalizationDestinationDefinitionConfig(BaseModel): - class Config: - extra = Extra.allow - - normalizationRepository: str = Field( - ..., - description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", - ) - normalizationTag: str = Field( - ..., - description="a field indicating the tag of the docker repository to be used for normalization.", - ) - normalizationIntegrationType: str = Field( - ..., - description="a field indicating the type of integration dialect to use for normalization.", - ) - - -class AllowedHosts(BaseModel): - class Config: - extra = Extra.allow - - hosts: Optional[List[str]] = Field( - None, - description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", - ) - - -class RolloutConfiguration(BaseModel): - class Config: - extra = Extra.forbid - - enableProgressiveRollout: Optional[bool] = Field( - False, description="Whether to enable progressive rollout for the connector." - ) - initialPercentage: Optional[conint(ge=0, le=100)] = Field( - 0, - description="The percentage of users that should receive the new version initially.", - ) - maxPercentage: Optional[conint(ge=0, le=100)] = Field( - 50, - description="The percentage of users who should receive the release candidate during the test phase before full rollout.", - ) - advanceDelayMinutes: Optional[conint(ge=10)] = Field( - 10, - description="The number of minutes to wait before advancing the rollout percentage.", - ) - - -class DeadlineAction(Enum): - auto_upgrade = "auto_upgrade" - disable = "disable" - - -class StreamBreakingChangeScope(BaseModel): - class Config: - extra = Extra.forbid - - scopeType: Any = Field("stream", const=True) - impactedScopes: List[str] = Field( - ..., - description="List of streams that are impacted by the breaking change.", - min_items=1, - ) - - -class SourceType(Enum): - api = "api" - file = "file" - database = "database" - custom = "custom" - - -class SuggestedStreams(BaseModel): - class Config: - extra = Extra.allow - - streams: Optional[List[str]] = Field( - None, - description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", - ) - - -class Sl(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - - -class Ql(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - integer_400 = 400 - integer_500 = 500 - integer_600 = 600 - - -class AirbyteInternal(BaseModel): - class Config: - extra = Extra.allow - - sl: Optional[Sl] = None - ql: Optional[Ql] = None - isEnterprise: Optional[bool] = False - requireVersionIncrementsInPullRequests: Optional[bool] = Field( - True, - description="When false, version increment checks will be skipped for this connector", - ) - - -class GitInfo(BaseModel): - class Config: - extra = Extra.forbid - - commit_sha: Optional[str] = Field( - None, - description="The git commit sha of the last commit that modified this file.", - ) - commit_timestamp: Optional[datetime] = Field( - None, - description="The git commit timestamp of the last commit that modified this file.", - ) - commit_author: Optional[str] = Field( - None, - description="The git commit author of the last commit that modified this file.", - ) - commit_author_email: Optional[str] = Field( - None, - description="The git commit author email of the last commit that modified this file.", - ) - - -class SourceFileInfo(BaseModel): - metadata_etag: Optional[str] = None - metadata_file_path: Optional[str] = None - metadata_bucket_name: Optional[str] = None - metadata_last_modified: Optional[str] = None - registry_entry_generated_at: Optional[str] = None - - -class ConnectorMetrics(BaseModel): - all: Optional[Any] = None - cloud: Optional[Any] = None - oss: Optional[Any] = None - - -class Usage(Enum): - low = "low" - medium = "medium" - high = "high" - - -class SyncSuccessRate(Enum): - low = "low" - medium = "medium" - high = "high" - - -class ConnectorMetric(BaseModel): - class Config: - extra = Extra.allow - - usage: Optional[Union[str, Usage]] = None - sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None - connector_version: Optional[str] = None - - -class ConnectorPackageInfo(BaseModel): - cdk_version: Optional[str] = None - - -class JobTypeResourceLimit(BaseModel): - class Config: - extra = Extra.forbid - - jobType: JobType - resourceRequirements: ResourceRequirements - - -class BreakingChangeScope(BaseModel): - __root__: StreamBreakingChangeScope = Field( - ..., - description="A scope that can be used to limit the impact of a breaking change.", - ) - - -class GeneratedFields(BaseModel): - git: Optional[GitInfo] = None - source_file_info: Optional[SourceFileInfo] = None - metrics: Optional[ConnectorMetrics] = None - sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") - - -class ActorDefinitionResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - default: Optional[ResourceRequirements] = Field( - None, - description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", - ) - jobSpecific: Optional[List[JobTypeResourceLimit]] = None - - -class VersionBreakingChange(BaseModel): - class Config: - extra = Extra.forbid - - upgradeDeadline: date = Field( - ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." - ) - deadlineAction: Optional[DeadlineAction] = Field( - None, description="Action to do when the deadline is reached." - ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", - ) - scopedImpact: Optional[List[BreakingChangeScope]] = Field( - None, - description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", - min_items=1, - ) - - -class ConnectorBreakingChanges(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( - ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", - title="ConnectorBreakingChanges", - ) - - -class ConnectorRegistryDestinationDefinition(BaseModel): - class Config: - extra = Extra.allow - - destinationDefinitionId: UUID - name: str - dockerRepository: str - dockerImageTag: str - documentationUrl: str - icon: Optional[str] = None - iconUrl: Optional[str] = None - spec: Dict[str, Any] - tombstone: Optional[bool] = Field( - False, - description="if false, the configuration is active. if true, then this configuration is permanently off.", - ) - public: Optional[bool] = Field( - False, - description="true if this connector definition is available to all workspaces", - ) - custom: Optional[bool] = Field( - False, description="whether this is a custom connector definition" - ) - releaseStage: Optional[ReleaseStage] = None - supportLevel: Optional[SupportLevel] = None - releaseDate: Optional[date] = Field( - None, - description="The date when this connector was first released, in yyyy-mm-dd format.", - ) - tags: Optional[List[str]] = Field( - None, - description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", - ) - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None - protocolVersion: Optional[str] = Field( - None, description="the Airbyte Protocol version supported by the connector" - ) - normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None - supportsDbt: Optional[bool] = Field( - None, - description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", - ) - allowedHosts: Optional[AllowedHosts] = None - releases: Optional[ConnectorRegistryReleases] = None - ab_internal: Optional[AirbyteInternal] = None - supportsRefreshes: Optional[bool] = False - supportsFileTransfer: Optional[bool] = False - supportsDataActivation: Optional[bool] = False - generated: Optional[GeneratedFields] = None - packageInfo: Optional[ConnectorPackageInfo] = None - language: Optional[str] = Field( - None, description="The language the connector is written in" - ) - - -class ConnectorRegistryReleases(BaseModel): - class Config: - extra = Extra.forbid - - releaseCandidates: Optional[ConnectorReleaseCandidates] = None - rolloutConfiguration: Optional[RolloutConfiguration] = None - breakingChanges: Optional[ConnectorBreakingChanges] = None - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", - ) - - -class ConnectorReleaseCandidates(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[ - constr(regex=r"^\d+\.\d+\.\d+(-[0-9A-Za-z-.]+)?$"), VersionReleaseCandidate - ] = Field( - ..., - description="Each entry denotes a release candidate version of a connector.", - ) - - -class VersionReleaseCandidate(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Union[ - ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition - ] = Field( - ..., - description="Contains information about a release candidate version of a connector.", - ) - - -class ConnectorRegistrySourceDefinition(BaseModel): - class Config: - extra = Extra.allow - - sourceDefinitionId: UUID - name: str - dockerRepository: str - dockerImageTag: str - documentationUrl: str - icon: Optional[str] = None - iconUrl: Optional[str] = None - sourceType: Optional[SourceType] = None - spec: Dict[str, Any] - tombstone: Optional[bool] = Field( - False, - description="if false, the configuration is active. if true, then this configuration is permanently off.", - ) - public: Optional[bool] = Field( - False, - description="true if this connector definition is available to all workspaces", - ) - custom: Optional[bool] = Field( - False, description="whether this is a custom connector definition" - ) - releaseStage: Optional[ReleaseStage] = None - supportLevel: Optional[SupportLevel] = None - releaseDate: Optional[date] = Field( - None, - description="The date when this connector was first released, in yyyy-mm-dd format.", - ) - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None - protocolVersion: Optional[str] = Field( - None, description="the Airbyte Protocol version supported by the connector" - ) - allowedHosts: Optional[AllowedHosts] = None - suggestedStreams: Optional[SuggestedStreams] = None - maxSecondsBetweenMessages: Optional[int] = Field( - None, - description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", - ) - erdUrl: Optional[str] = Field( - None, description="The URL where you can visualize the ERD" - ) - releases: Optional[ConnectorRegistryReleases] = None - ab_internal: Optional[AirbyteInternal] = None - generated: Optional[GeneratedFields] = None - packageInfo: Optional[ConnectorPackageInfo] = None - language: Optional[str] = Field( - None, description="The language the connector is written in" - ) - supportsFileTransfer: Optional[bool] = False - supportsDataActivation: Optional[bool] = False - - -ConnectorRegistryDestinationDefinition.update_forward_refs() -ConnectorRegistryReleases.update_forward_refs() -ConnectorReleaseCandidates.update_forward_refs() -VersionReleaseCandidate.update_forward_refs() diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryReleases.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryReleases.py deleted file mode 100644 index 097fc061f..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryReleases.py +++ /dev/null @@ -1,438 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorRegistryReleases.yaml - -from __future__ import annotations - -from datetime import date, datetime -from enum import Enum -from typing import Any, Dict, List, Optional, Union -from uuid import UUID - -from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr - - -class RolloutConfiguration(BaseModel): - class Config: - extra = Extra.forbid - - enableProgressiveRollout: Optional[bool] = Field( - False, description="Whether to enable progressive rollout for the connector." - ) - initialPercentage: Optional[conint(ge=0, le=100)] = Field( - 0, - description="The percentage of users that should receive the new version initially.", - ) - maxPercentage: Optional[conint(ge=0, le=100)] = Field( - 50, - description="The percentage of users who should receive the release candidate during the test phase before full rollout.", - ) - advanceDelayMinutes: Optional[conint(ge=10)] = Field( - 10, - description="The number of minutes to wait before advancing the rollout percentage.", - ) - - -class DeadlineAction(Enum): - auto_upgrade = "auto_upgrade" - disable = "disable" - - -class StreamBreakingChangeScope(BaseModel): - class Config: - extra = Extra.forbid - - scopeType: Any = Field("stream", const=True) - impactedScopes: List[str] = Field( - ..., - description="List of streams that are impacted by the breaking change.", - min_items=1, - ) - - -class SourceType(Enum): - api = "api" - file = "file" - database = "database" - custom = "custom" - - -class ReleaseStage(Enum): - alpha = "alpha" - beta = "beta" - generally_available = "generally_available" - custom = "custom" - - -class SupportLevel(Enum): - community = "community" - certified = "certified" - archived = "archived" - - -class ResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - cpu_request: Optional[str] = None - cpu_limit: Optional[str] = None - memory_request: Optional[str] = None - memory_limit: Optional[str] = None - - -class JobType(Enum): - get_spec = "get_spec" - check_connection = "check_connection" - discover_schema = "discover_schema" - sync = "sync" - reset_connection = "reset_connection" - connection_updater = "connection_updater" - replicate = "replicate" - - -class AllowedHosts(BaseModel): - class Config: - extra = Extra.allow - - hosts: Optional[List[str]] = Field( - None, - description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", - ) - - -class SuggestedStreams(BaseModel): - class Config: - extra = Extra.allow - - streams: Optional[List[str]] = Field( - None, - description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", - ) - - -class Sl(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - - -class Ql(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - integer_400 = 400 - integer_500 = 500 - integer_600 = 600 - - -class AirbyteInternal(BaseModel): - class Config: - extra = Extra.allow - - sl: Optional[Sl] = None - ql: Optional[Ql] = None - isEnterprise: Optional[bool] = False - requireVersionIncrementsInPullRequests: Optional[bool] = Field( - True, - description="When false, version increment checks will be skipped for this connector", - ) - - -class GitInfo(BaseModel): - class Config: - extra = Extra.forbid - - commit_sha: Optional[str] = Field( - None, - description="The git commit sha of the last commit that modified this file.", - ) - commit_timestamp: Optional[datetime] = Field( - None, - description="The git commit timestamp of the last commit that modified this file.", - ) - commit_author: Optional[str] = Field( - None, - description="The git commit author of the last commit that modified this file.", - ) - commit_author_email: Optional[str] = Field( - None, - description="The git commit author email of the last commit that modified this file.", - ) - - -class SourceFileInfo(BaseModel): - metadata_etag: Optional[str] = None - metadata_file_path: Optional[str] = None - metadata_bucket_name: Optional[str] = None - metadata_last_modified: Optional[str] = None - registry_entry_generated_at: Optional[str] = None - - -class ConnectorMetrics(BaseModel): - all: Optional[Any] = None - cloud: Optional[Any] = None - oss: Optional[Any] = None - - -class Usage(Enum): - low = "low" - medium = "medium" - high = "high" - - -class SyncSuccessRate(Enum): - low = "low" - medium = "medium" - high = "high" - - -class ConnectorMetric(BaseModel): - class Config: - extra = Extra.allow - - usage: Optional[Union[str, Usage]] = None - sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None - connector_version: Optional[str] = None - - -class ConnectorPackageInfo(BaseModel): - cdk_version: Optional[str] = None - - -class NormalizationDestinationDefinitionConfig(BaseModel): - class Config: - extra = Extra.allow - - normalizationRepository: str = Field( - ..., - description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", - ) - normalizationTag: str = Field( - ..., - description="a field indicating the tag of the docker repository to be used for normalization.", - ) - normalizationIntegrationType: str = Field( - ..., - description="a field indicating the type of integration dialect to use for normalization.", - ) - - -class BreakingChangeScope(BaseModel): - __root__: StreamBreakingChangeScope = Field( - ..., - description="A scope that can be used to limit the impact of a breaking change.", - ) - - -class JobTypeResourceLimit(BaseModel): - class Config: - extra = Extra.forbid - - jobType: JobType - resourceRequirements: ResourceRequirements - - -class GeneratedFields(BaseModel): - git: Optional[GitInfo] = None - source_file_info: Optional[SourceFileInfo] = None - metrics: Optional[ConnectorMetrics] = None - sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") - - -class VersionBreakingChange(BaseModel): - class Config: - extra = Extra.forbid - - upgradeDeadline: date = Field( - ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." - ) - deadlineAction: Optional[DeadlineAction] = Field( - None, description="Action to do when the deadline is reached." - ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", - ) - scopedImpact: Optional[List[BreakingChangeScope]] = Field( - None, - description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", - min_items=1, - ) - - -class ActorDefinitionResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - default: Optional[ResourceRequirements] = Field( - None, - description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", - ) - jobSpecific: Optional[List[JobTypeResourceLimit]] = None - - -class ConnectorBreakingChanges(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( - ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", - title="ConnectorBreakingChanges", - ) - - -class ConnectorRegistryReleases(BaseModel): - class Config: - extra = Extra.forbid - - releaseCandidates: Optional[ConnectorReleaseCandidates] = None - rolloutConfiguration: Optional[RolloutConfiguration] = None - breakingChanges: Optional[ConnectorBreakingChanges] = None - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", - ) - - -class ConnectorReleaseCandidates(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[ - constr(regex=r"^\d+\.\d+\.\d+(-[0-9A-Za-z-.]+)?$"), VersionReleaseCandidate - ] = Field( - ..., - description="Each entry denotes a release candidate version of a connector.", - ) - - -class VersionReleaseCandidate(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Union[ - ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition - ] = Field( - ..., - description="Contains information about a release candidate version of a connector.", - ) - - -class ConnectorRegistrySourceDefinition(BaseModel): - class Config: - extra = Extra.allow - - sourceDefinitionId: UUID - name: str - dockerRepository: str - dockerImageTag: str - documentationUrl: str - icon: Optional[str] = None - iconUrl: Optional[str] = None - sourceType: Optional[SourceType] = None - spec: Dict[str, Any] - tombstone: Optional[bool] = Field( - False, - description="if false, the configuration is active. if true, then this configuration is permanently off.", - ) - public: Optional[bool] = Field( - False, - description="true if this connector definition is available to all workspaces", - ) - custom: Optional[bool] = Field( - False, description="whether this is a custom connector definition" - ) - releaseStage: Optional[ReleaseStage] = None - supportLevel: Optional[SupportLevel] = None - releaseDate: Optional[date] = Field( - None, - description="The date when this connector was first released, in yyyy-mm-dd format.", - ) - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None - protocolVersion: Optional[str] = Field( - None, description="the Airbyte Protocol version supported by the connector" - ) - allowedHosts: Optional[AllowedHosts] = None - suggestedStreams: Optional[SuggestedStreams] = None - maxSecondsBetweenMessages: Optional[int] = Field( - None, - description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", - ) - erdUrl: Optional[str] = Field( - None, description="The URL where you can visualize the ERD" - ) - releases: Optional[ConnectorRegistryReleases] = None - ab_internal: Optional[AirbyteInternal] = None - generated: Optional[GeneratedFields] = None - packageInfo: Optional[ConnectorPackageInfo] = None - language: Optional[str] = Field( - None, description="The language the connector is written in" - ) - supportsFileTransfer: Optional[bool] = False - supportsDataActivation: Optional[bool] = False - - -class ConnectorRegistryDestinationDefinition(BaseModel): - class Config: - extra = Extra.allow - - destinationDefinitionId: UUID - name: str - dockerRepository: str - dockerImageTag: str - documentationUrl: str - icon: Optional[str] = None - iconUrl: Optional[str] = None - spec: Dict[str, Any] - tombstone: Optional[bool] = Field( - False, - description="if false, the configuration is active. if true, then this configuration is permanently off.", - ) - public: Optional[bool] = Field( - False, - description="true if this connector definition is available to all workspaces", - ) - custom: Optional[bool] = Field( - False, description="whether this is a custom connector definition" - ) - releaseStage: Optional[ReleaseStage] = None - supportLevel: Optional[SupportLevel] = None - releaseDate: Optional[date] = Field( - None, - description="The date when this connector was first released, in yyyy-mm-dd format.", - ) - tags: Optional[List[str]] = Field( - None, - description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", - ) - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None - protocolVersion: Optional[str] = Field( - None, description="the Airbyte Protocol version supported by the connector" - ) - normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None - supportsDbt: Optional[bool] = Field( - None, - description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", - ) - allowedHosts: Optional[AllowedHosts] = None - releases: Optional[ConnectorRegistryReleases] = None - ab_internal: Optional[AirbyteInternal] = None - supportsRefreshes: Optional[bool] = False - supportsFileTransfer: Optional[bool] = False - supportsDataActivation: Optional[bool] = False - generated: Optional[GeneratedFields] = None - packageInfo: Optional[ConnectorPackageInfo] = None - language: Optional[str] = Field( - None, description="The language the connector is written in" - ) - - -ConnectorRegistryReleases.update_forward_refs() -ConnectorReleaseCandidates.update_forward_refs() -VersionReleaseCandidate.update_forward_refs() diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistrySourceDefinition.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistrySourceDefinition.py deleted file mode 100644 index 002a4aac7..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistrySourceDefinition.py +++ /dev/null @@ -1,439 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorRegistrySourceDefinition.yaml - -from __future__ import annotations - -from datetime import date, datetime -from enum import Enum -from typing import Any, Dict, List, Optional, Union -from uuid import UUID - -from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr - - -class SourceType(Enum): - api = "api" - file = "file" - database = "database" - custom = "custom" - - -class ReleaseStage(Enum): - alpha = "alpha" - beta = "beta" - generally_available = "generally_available" - custom = "custom" - - -class SupportLevel(Enum): - community = "community" - certified = "certified" - archived = "archived" - - -class ResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - cpu_request: Optional[str] = None - cpu_limit: Optional[str] = None - memory_request: Optional[str] = None - memory_limit: Optional[str] = None - - -class JobType(Enum): - get_spec = "get_spec" - check_connection = "check_connection" - discover_schema = "discover_schema" - sync = "sync" - reset_connection = "reset_connection" - connection_updater = "connection_updater" - replicate = "replicate" - - -class AllowedHosts(BaseModel): - class Config: - extra = Extra.allow - - hosts: Optional[List[str]] = Field( - None, - description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", - ) - - -class SuggestedStreams(BaseModel): - class Config: - extra = Extra.allow - - streams: Optional[List[str]] = Field( - None, - description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", - ) - - -class RolloutConfiguration(BaseModel): - class Config: - extra = Extra.forbid - - enableProgressiveRollout: Optional[bool] = Field( - False, description="Whether to enable progressive rollout for the connector." - ) - initialPercentage: Optional[conint(ge=0, le=100)] = Field( - 0, - description="The percentage of users that should receive the new version initially.", - ) - maxPercentage: Optional[conint(ge=0, le=100)] = Field( - 50, - description="The percentage of users who should receive the release candidate during the test phase before full rollout.", - ) - advanceDelayMinutes: Optional[conint(ge=10)] = Field( - 10, - description="The number of minutes to wait before advancing the rollout percentage.", - ) - - -class DeadlineAction(Enum): - auto_upgrade = "auto_upgrade" - disable = "disable" - - -class StreamBreakingChangeScope(BaseModel): - class Config: - extra = Extra.forbid - - scopeType: Any = Field("stream", const=True) - impactedScopes: List[str] = Field( - ..., - description="List of streams that are impacted by the breaking change.", - min_items=1, - ) - - -class NormalizationDestinationDefinitionConfig(BaseModel): - class Config: - extra = Extra.allow - - normalizationRepository: str = Field( - ..., - description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", - ) - normalizationTag: str = Field( - ..., - description="a field indicating the tag of the docker repository to be used for normalization.", - ) - normalizationIntegrationType: str = Field( - ..., - description="a field indicating the type of integration dialect to use for normalization.", - ) - - -class Sl(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - - -class Ql(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - integer_400 = 400 - integer_500 = 500 - integer_600 = 600 - - -class AirbyteInternal(BaseModel): - class Config: - extra = Extra.allow - - sl: Optional[Sl] = None - ql: Optional[Ql] = None - isEnterprise: Optional[bool] = False - requireVersionIncrementsInPullRequests: Optional[bool] = Field( - True, - description="When false, version increment checks will be skipped for this connector", - ) - - -class GitInfo(BaseModel): - class Config: - extra = Extra.forbid - - commit_sha: Optional[str] = Field( - None, - description="The git commit sha of the last commit that modified this file.", - ) - commit_timestamp: Optional[datetime] = Field( - None, - description="The git commit timestamp of the last commit that modified this file.", - ) - commit_author: Optional[str] = Field( - None, - description="The git commit author of the last commit that modified this file.", - ) - commit_author_email: Optional[str] = Field( - None, - description="The git commit author email of the last commit that modified this file.", - ) - - -class SourceFileInfo(BaseModel): - metadata_etag: Optional[str] = None - metadata_file_path: Optional[str] = None - metadata_bucket_name: Optional[str] = None - metadata_last_modified: Optional[str] = None - registry_entry_generated_at: Optional[str] = None - - -class ConnectorMetrics(BaseModel): - all: Optional[Any] = None - cloud: Optional[Any] = None - oss: Optional[Any] = None - - -class Usage(Enum): - low = "low" - medium = "medium" - high = "high" - - -class SyncSuccessRate(Enum): - low = "low" - medium = "medium" - high = "high" - - -class ConnectorMetric(BaseModel): - class Config: - extra = Extra.allow - - usage: Optional[Union[str, Usage]] = None - sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None - connector_version: Optional[str] = None - - -class ConnectorPackageInfo(BaseModel): - cdk_version: Optional[str] = None - - -class JobTypeResourceLimit(BaseModel): - class Config: - extra = Extra.forbid - - jobType: JobType - resourceRequirements: ResourceRequirements - - -class BreakingChangeScope(BaseModel): - __root__: StreamBreakingChangeScope = Field( - ..., - description="A scope that can be used to limit the impact of a breaking change.", - ) - - -class GeneratedFields(BaseModel): - git: Optional[GitInfo] = None - source_file_info: Optional[SourceFileInfo] = None - metrics: Optional[ConnectorMetrics] = None - sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") - - -class ActorDefinitionResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - default: Optional[ResourceRequirements] = Field( - None, - description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", - ) - jobSpecific: Optional[List[JobTypeResourceLimit]] = None - - -class VersionBreakingChange(BaseModel): - class Config: - extra = Extra.forbid - - upgradeDeadline: date = Field( - ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." - ) - deadlineAction: Optional[DeadlineAction] = Field( - None, description="Action to do when the deadline is reached." - ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", - ) - scopedImpact: Optional[List[BreakingChangeScope]] = Field( - None, - description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", - min_items=1, - ) - - -class ConnectorBreakingChanges(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( - ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", - title="ConnectorBreakingChanges", - ) - - -class ConnectorRegistrySourceDefinition(BaseModel): - class Config: - extra = Extra.allow - - sourceDefinitionId: UUID - name: str - dockerRepository: str - dockerImageTag: str - documentationUrl: str - icon: Optional[str] = None - iconUrl: Optional[str] = None - sourceType: Optional[SourceType] = None - spec: Dict[str, Any] - tombstone: Optional[bool] = Field( - False, - description="if false, the configuration is active. if true, then this configuration is permanently off.", - ) - public: Optional[bool] = Field( - False, - description="true if this connector definition is available to all workspaces", - ) - custom: Optional[bool] = Field( - False, description="whether this is a custom connector definition" - ) - releaseStage: Optional[ReleaseStage] = None - supportLevel: Optional[SupportLevel] = None - releaseDate: Optional[date] = Field( - None, - description="The date when this connector was first released, in yyyy-mm-dd format.", - ) - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None - protocolVersion: Optional[str] = Field( - None, description="the Airbyte Protocol version supported by the connector" - ) - allowedHosts: Optional[AllowedHosts] = None - suggestedStreams: Optional[SuggestedStreams] = None - maxSecondsBetweenMessages: Optional[int] = Field( - None, - description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", - ) - erdUrl: Optional[str] = Field( - None, description="The URL where you can visualize the ERD" - ) - releases: Optional[ConnectorRegistryReleases] = None - ab_internal: Optional[AirbyteInternal] = None - generated: Optional[GeneratedFields] = None - packageInfo: Optional[ConnectorPackageInfo] = None - language: Optional[str] = Field( - None, description="The language the connector is written in" - ) - supportsFileTransfer: Optional[bool] = False - supportsDataActivation: Optional[bool] = False - - -class ConnectorRegistryReleases(BaseModel): - class Config: - extra = Extra.forbid - - releaseCandidates: Optional[ConnectorReleaseCandidates] = None - rolloutConfiguration: Optional[RolloutConfiguration] = None - breakingChanges: Optional[ConnectorBreakingChanges] = None - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", - ) - - -class ConnectorReleaseCandidates(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[ - constr(regex=r"^\d+\.\d+\.\d+(-[0-9A-Za-z-.]+)?$"), VersionReleaseCandidate - ] = Field( - ..., - description="Each entry denotes a release candidate version of a connector.", - ) - - -class VersionReleaseCandidate(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Union[ - ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition - ] = Field( - ..., - description="Contains information about a release candidate version of a connector.", - ) - - -class ConnectorRegistryDestinationDefinition(BaseModel): - class Config: - extra = Extra.allow - - destinationDefinitionId: UUID - name: str - dockerRepository: str - dockerImageTag: str - documentationUrl: str - icon: Optional[str] = None - iconUrl: Optional[str] = None - spec: Dict[str, Any] - tombstone: Optional[bool] = Field( - False, - description="if false, the configuration is active. if true, then this configuration is permanently off.", - ) - public: Optional[bool] = Field( - False, - description="true if this connector definition is available to all workspaces", - ) - custom: Optional[bool] = Field( - False, description="whether this is a custom connector definition" - ) - releaseStage: Optional[ReleaseStage] = None - supportLevel: Optional[SupportLevel] = None - releaseDate: Optional[date] = Field( - None, - description="The date when this connector was first released, in yyyy-mm-dd format.", - ) - tags: Optional[List[str]] = Field( - None, - description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", - ) - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None - protocolVersion: Optional[str] = Field( - None, description="the Airbyte Protocol version supported by the connector" - ) - normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None - supportsDbt: Optional[bool] = Field( - None, - description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", - ) - allowedHosts: Optional[AllowedHosts] = None - releases: Optional[ConnectorRegistryReleases] = None - ab_internal: Optional[AirbyteInternal] = None - supportsRefreshes: Optional[bool] = False - supportsFileTransfer: Optional[bool] = False - supportsDataActivation: Optional[bool] = False - generated: Optional[GeneratedFields] = None - packageInfo: Optional[ConnectorPackageInfo] = None - language: Optional[str] = Field( - None, description="The language the connector is written in" - ) - - -ConnectorRegistrySourceDefinition.update_forward_refs() -ConnectorRegistryReleases.update_forward_refs() -ConnectorReleaseCandidates.update_forward_refs() -VersionReleaseCandidate.update_forward_refs() diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryV0.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryV0.py deleted file mode 100644 index 9fa1ac0c9..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorRegistryV0.py +++ /dev/null @@ -1,445 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorRegistryV0.yaml - -from __future__ import annotations - -from datetime import date, datetime -from enum import Enum -from typing import Any, Dict, List, Optional, Union -from uuid import UUID - -from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr - - -class ReleaseStage(Enum): - alpha = "alpha" - beta = "beta" - generally_available = "generally_available" - custom = "custom" - - -class SupportLevel(Enum): - community = "community" - certified = "certified" - archived = "archived" - - -class ResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - cpu_request: Optional[str] = None - cpu_limit: Optional[str] = None - memory_request: Optional[str] = None - memory_limit: Optional[str] = None - - -class JobType(Enum): - get_spec = "get_spec" - check_connection = "check_connection" - discover_schema = "discover_schema" - sync = "sync" - reset_connection = "reset_connection" - connection_updater = "connection_updater" - replicate = "replicate" - - -class NormalizationDestinationDefinitionConfig(BaseModel): - class Config: - extra = Extra.allow - - normalizationRepository: str = Field( - ..., - description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", - ) - normalizationTag: str = Field( - ..., - description="a field indicating the tag of the docker repository to be used for normalization.", - ) - normalizationIntegrationType: str = Field( - ..., - description="a field indicating the type of integration dialect to use for normalization.", - ) - - -class AllowedHosts(BaseModel): - class Config: - extra = Extra.allow - - hosts: Optional[List[str]] = Field( - None, - description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", - ) - - -class RolloutConfiguration(BaseModel): - class Config: - extra = Extra.forbid - - enableProgressiveRollout: Optional[bool] = Field( - False, description="Whether to enable progressive rollout for the connector." - ) - initialPercentage: Optional[conint(ge=0, le=100)] = Field( - 0, - description="The percentage of users that should receive the new version initially.", - ) - maxPercentage: Optional[conint(ge=0, le=100)] = Field( - 50, - description="The percentage of users who should receive the release candidate during the test phase before full rollout.", - ) - advanceDelayMinutes: Optional[conint(ge=10)] = Field( - 10, - description="The number of minutes to wait before advancing the rollout percentage.", - ) - - -class DeadlineAction(Enum): - auto_upgrade = "auto_upgrade" - disable = "disable" - - -class StreamBreakingChangeScope(BaseModel): - class Config: - extra = Extra.forbid - - scopeType: Any = Field("stream", const=True) - impactedScopes: List[str] = Field( - ..., - description="List of streams that are impacted by the breaking change.", - min_items=1, - ) - - -class SourceType(Enum): - api = "api" - file = "file" - database = "database" - custom = "custom" - - -class SuggestedStreams(BaseModel): - class Config: - extra = Extra.allow - - streams: Optional[List[str]] = Field( - None, - description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", - ) - - -class Sl(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - - -class Ql(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - integer_400 = 400 - integer_500 = 500 - integer_600 = 600 - - -class AirbyteInternal(BaseModel): - class Config: - extra = Extra.allow - - sl: Optional[Sl] = None - ql: Optional[Ql] = None - isEnterprise: Optional[bool] = False - requireVersionIncrementsInPullRequests: Optional[bool] = Field( - True, - description="When false, version increment checks will be skipped for this connector", - ) - - -class GitInfo(BaseModel): - class Config: - extra = Extra.forbid - - commit_sha: Optional[str] = Field( - None, - description="The git commit sha of the last commit that modified this file.", - ) - commit_timestamp: Optional[datetime] = Field( - None, - description="The git commit timestamp of the last commit that modified this file.", - ) - commit_author: Optional[str] = Field( - None, - description="The git commit author of the last commit that modified this file.", - ) - commit_author_email: Optional[str] = Field( - None, - description="The git commit author email of the last commit that modified this file.", - ) - - -class SourceFileInfo(BaseModel): - metadata_etag: Optional[str] = None - metadata_file_path: Optional[str] = None - metadata_bucket_name: Optional[str] = None - metadata_last_modified: Optional[str] = None - registry_entry_generated_at: Optional[str] = None - - -class ConnectorMetrics(BaseModel): - all: Optional[Any] = None - cloud: Optional[Any] = None - oss: Optional[Any] = None - - -class Usage(Enum): - low = "low" - medium = "medium" - high = "high" - - -class SyncSuccessRate(Enum): - low = "low" - medium = "medium" - high = "high" - - -class ConnectorMetric(BaseModel): - class Config: - extra = Extra.allow - - usage: Optional[Union[str, Usage]] = None - sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None - connector_version: Optional[str] = None - - -class ConnectorPackageInfo(BaseModel): - cdk_version: Optional[str] = None - - -class JobTypeResourceLimit(BaseModel): - class Config: - extra = Extra.forbid - - jobType: JobType - resourceRequirements: ResourceRequirements - - -class BreakingChangeScope(BaseModel): - __root__: StreamBreakingChangeScope = Field( - ..., - description="A scope that can be used to limit the impact of a breaking change.", - ) - - -class GeneratedFields(BaseModel): - git: Optional[GitInfo] = None - source_file_info: Optional[SourceFileInfo] = None - metrics: Optional[ConnectorMetrics] = None - sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") - - -class ActorDefinitionResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - default: Optional[ResourceRequirements] = Field( - None, - description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", - ) - jobSpecific: Optional[List[JobTypeResourceLimit]] = None - - -class VersionBreakingChange(BaseModel): - class Config: - extra = Extra.forbid - - upgradeDeadline: date = Field( - ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." - ) - deadlineAction: Optional[DeadlineAction] = Field( - None, description="Action to do when the deadline is reached." - ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", - ) - scopedImpact: Optional[List[BreakingChangeScope]] = Field( - None, - description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", - min_items=1, - ) - - -class ConnectorBreakingChanges(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( - ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", - title="ConnectorBreakingChanges", - ) - - -class ConnectorRegistryV0(BaseModel): - destinations: List[ConnectorRegistryDestinationDefinition] - sources: List[ConnectorRegistrySourceDefinition] - - -class ConnectorRegistryDestinationDefinition(BaseModel): - class Config: - extra = Extra.allow - - destinationDefinitionId: UUID - name: str - dockerRepository: str - dockerImageTag: str - documentationUrl: str - icon: Optional[str] = None - iconUrl: Optional[str] = None - spec: Dict[str, Any] - tombstone: Optional[bool] = Field( - False, - description="if false, the configuration is active. if true, then this configuration is permanently off.", - ) - public: Optional[bool] = Field( - False, - description="true if this connector definition is available to all workspaces", - ) - custom: Optional[bool] = Field( - False, description="whether this is a custom connector definition" - ) - releaseStage: Optional[ReleaseStage] = None - supportLevel: Optional[SupportLevel] = None - releaseDate: Optional[date] = Field( - None, - description="The date when this connector was first released, in yyyy-mm-dd format.", - ) - tags: Optional[List[str]] = Field( - None, - description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", - ) - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None - protocolVersion: Optional[str] = Field( - None, description="the Airbyte Protocol version supported by the connector" - ) - normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None - supportsDbt: Optional[bool] = Field( - None, - description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", - ) - allowedHosts: Optional[AllowedHosts] = None - releases: Optional[ConnectorRegistryReleases] = None - ab_internal: Optional[AirbyteInternal] = None - supportsRefreshes: Optional[bool] = False - supportsFileTransfer: Optional[bool] = False - supportsDataActivation: Optional[bool] = False - generated: Optional[GeneratedFields] = None - packageInfo: Optional[ConnectorPackageInfo] = None - language: Optional[str] = Field( - None, description="The language the connector is written in" - ) - - -class ConnectorRegistryReleases(BaseModel): - class Config: - extra = Extra.forbid - - releaseCandidates: Optional[ConnectorReleaseCandidates] = None - rolloutConfiguration: Optional[RolloutConfiguration] = None - breakingChanges: Optional[ConnectorBreakingChanges] = None - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", - ) - - -class ConnectorReleaseCandidates(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[ - constr(regex=r"^\d+\.\d+\.\d+(-[0-9A-Za-z-.]+)?$"), VersionReleaseCandidate - ] = Field( - ..., - description="Each entry denotes a release candidate version of a connector.", - ) - - -class VersionReleaseCandidate(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Union[ - ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition - ] = Field( - ..., - description="Contains information about a release candidate version of a connector.", - ) - - -class ConnectorRegistrySourceDefinition(BaseModel): - class Config: - extra = Extra.allow - - sourceDefinitionId: UUID - name: str - dockerRepository: str - dockerImageTag: str - documentationUrl: str - icon: Optional[str] = None - iconUrl: Optional[str] = None - sourceType: Optional[SourceType] = None - spec: Dict[str, Any] - tombstone: Optional[bool] = Field( - False, - description="if false, the configuration is active. if true, then this configuration is permanently off.", - ) - public: Optional[bool] = Field( - False, - description="true if this connector definition is available to all workspaces", - ) - custom: Optional[bool] = Field( - False, description="whether this is a custom connector definition" - ) - releaseStage: Optional[ReleaseStage] = None - supportLevel: Optional[SupportLevel] = None - releaseDate: Optional[date] = Field( - None, - description="The date when this connector was first released, in yyyy-mm-dd format.", - ) - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None - protocolVersion: Optional[str] = Field( - None, description="the Airbyte Protocol version supported by the connector" - ) - allowedHosts: Optional[AllowedHosts] = None - suggestedStreams: Optional[SuggestedStreams] = None - maxSecondsBetweenMessages: Optional[int] = Field( - None, - description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", - ) - erdUrl: Optional[str] = Field( - None, description="The URL where you can visualize the ERD" - ) - releases: Optional[ConnectorRegistryReleases] = None - ab_internal: Optional[AirbyteInternal] = None - generated: Optional[GeneratedFields] = None - packageInfo: Optional[ConnectorPackageInfo] = None - language: Optional[str] = Field( - None, description="The language the connector is written in" - ) - supportsFileTransfer: Optional[bool] = False - supportsDataActivation: Optional[bool] = False - - -ConnectorRegistryV0.update_forward_refs() -ConnectorRegistryDestinationDefinition.update_forward_refs() -ConnectorRegistryReleases.update_forward_refs() -ConnectorReleaseCandidates.update_forward_refs() -VersionReleaseCandidate.update_forward_refs() diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorReleases.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorReleases.py deleted file mode 100644 index c0498dc30..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorReleases.py +++ /dev/null @@ -1,103 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorReleases.yaml - -from __future__ import annotations - -from datetime import date -from enum import Enum -from typing import Any, Dict, List, Optional - -from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr - - -class RolloutConfiguration(BaseModel): - class Config: - extra = Extra.forbid - - enableProgressiveRollout: Optional[bool] = Field( - False, description="Whether to enable progressive rollout for the connector." - ) - initialPercentage: Optional[conint(ge=0, le=100)] = Field( - 0, - description="The percentage of users that should receive the new version initially.", - ) - maxPercentage: Optional[conint(ge=0, le=100)] = Field( - 50, - description="The percentage of users who should receive the release candidate during the test phase before full rollout.", - ) - advanceDelayMinutes: Optional[conint(ge=10)] = Field( - 10, - description="The number of minutes to wait before advancing the rollout percentage.", - ) - - -class DeadlineAction(Enum): - auto_upgrade = "auto_upgrade" - disable = "disable" - - -class StreamBreakingChangeScope(BaseModel): - class Config: - extra = Extra.forbid - - scopeType: Any = Field("stream", const=True) - impactedScopes: List[str] = Field( - ..., - description="List of streams that are impacted by the breaking change.", - min_items=1, - ) - - -class BreakingChangeScope(BaseModel): - __root__: StreamBreakingChangeScope = Field( - ..., - description="A scope that can be used to limit the impact of a breaking change.", - ) - - -class VersionBreakingChange(BaseModel): - class Config: - extra = Extra.forbid - - upgradeDeadline: date = Field( - ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." - ) - deadlineAction: Optional[DeadlineAction] = Field( - None, description="Action to do when the deadline is reached." - ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", - ) - scopedImpact: Optional[List[BreakingChangeScope]] = Field( - None, - description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", - min_items=1, - ) - - -class ConnectorBreakingChanges(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( - ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", - title="ConnectorBreakingChanges", - ) - - -class ConnectorReleases(BaseModel): - class Config: - extra = Extra.forbid - - rolloutConfiguration: Optional[RolloutConfiguration] = None - breakingChanges: Optional[ConnectorBreakingChanges] = None - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", - ) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorTestSuiteOptions.py b/airbyte_cdk/test/models/connector_metadata/generated/ConnectorTestSuiteOptions.py deleted file mode 100644 index 6f511f53c..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ConnectorTestSuiteOptions.py +++ /dev/null @@ -1,63 +0,0 @@ -# generated by datamodel-codegen: -# filename: ConnectorTestSuiteOptions.yaml - -from __future__ import annotations - -from enum import Enum -from typing import List, Literal, Optional - -from pydantic.v1 import BaseModel, Extra, Field - - -class Suite(Enum): - unitTests = "unitTests" - integrationTests = "integrationTests" - acceptanceTests = "acceptanceTests" - liveTests = "liveTests" - - -class SecretStore(BaseModel): - class Config: - extra = Extra.forbid - - alias: Optional[str] = Field( - None, - description="The alias of the secret store which can map to its actual secret address", - ) - type: Optional[Literal["GSM"]] = Field( - None, description="The type of the secret store" - ) - - -class TestConnections(BaseModel): - class Config: - extra = Extra.forbid - - name: str = Field(..., description="The connection name") - id: str = Field(..., description="The connection ID") - - -class Secret(BaseModel): - class Config: - extra = Extra.forbid - - name: str = Field(..., description="The secret name in the secret store") - fileName: Optional[str] = Field( - None, - description="The name of the file to which the secret value would be persisted", - ) - secretStore: SecretStore - - -class ConnectorTestSuiteOptions(BaseModel): - class Config: - extra = Extra.forbid - - suite: Suite = Field(..., description="Name of the configured test suite") - testSecrets: Optional[List[Secret]] = Field( - None, description="List of secrets required to run the test suite" - ) - testConnections: Optional[List[TestConnections]] = Field( - None, - description="List of sandbox cloud connections that tests can be run against", - ) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/GeneratedFields.py b/airbyte_cdk/test/models/connector_metadata/generated/GeneratedFields.py deleted file mode 100644 index f290484f4..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/GeneratedFields.py +++ /dev/null @@ -1,74 +0,0 @@ -# generated by datamodel-codegen: -# filename: GeneratedFields.yaml - -from __future__ import annotations - -from datetime import datetime -from enum import Enum -from typing import Any, Optional, Union - -from pydantic.v1 import BaseModel, Extra, Field - - -class GitInfo(BaseModel): - class Config: - extra = Extra.forbid - - commit_sha: Optional[str] = Field( - None, - description="The git commit sha of the last commit that modified this file.", - ) - commit_timestamp: Optional[datetime] = Field( - None, - description="The git commit timestamp of the last commit that modified this file.", - ) - commit_author: Optional[str] = Field( - None, - description="The git commit author of the last commit that modified this file.", - ) - commit_author_email: Optional[str] = Field( - None, - description="The git commit author email of the last commit that modified this file.", - ) - - -class SourceFileInfo(BaseModel): - metadata_etag: Optional[str] = None - metadata_file_path: Optional[str] = None - metadata_bucket_name: Optional[str] = None - metadata_last_modified: Optional[str] = None - registry_entry_generated_at: Optional[str] = None - - -class ConnectorMetrics(BaseModel): - all: Optional[Any] = None - cloud: Optional[Any] = None - oss: Optional[Any] = None - - -class Usage(Enum): - low = "low" - medium = "medium" - high = "high" - - -class SyncSuccessRate(Enum): - low = "low" - medium = "medium" - high = "high" - - -class ConnectorMetric(BaseModel): - class Config: - extra = Extra.allow - - usage: Optional[Union[str, Usage]] = None - sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None - connector_version: Optional[str] = None - - -class GeneratedFields(BaseModel): - git: Optional[GitInfo] = None - source_file_info: Optional[SourceFileInfo] = None - metrics: Optional[ConnectorMetrics] = None - sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") diff --git a/airbyte_cdk/test/models/connector_metadata/generated/GitInfo.py b/airbyte_cdk/test/models/connector_metadata/generated/GitInfo.py deleted file mode 100644 index d59317ec2..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/GitInfo.py +++ /dev/null @@ -1,31 +0,0 @@ -# generated by datamodel-codegen: -# filename: GitInfo.yaml - -from __future__ import annotations - -from datetime import datetime -from typing import Optional - -from pydantic.v1 import BaseModel, Extra, Field - - -class GitInfo(BaseModel): - class Config: - extra = Extra.forbid - - commit_sha: Optional[str] = Field( - None, - description="The git commit sha of the last commit that modified this file.", - ) - commit_timestamp: Optional[datetime] = Field( - None, - description="The git commit timestamp of the last commit that modified this file.", - ) - commit_author: Optional[str] = Field( - None, - description="The git commit author of the last commit that modified this file.", - ) - commit_author_email: Optional[str] = Field( - None, - description="The git commit author email of the last commit that modified this file.", - ) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/JobType.py b/airbyte_cdk/test/models/connector_metadata/generated/JobType.py deleted file mode 100644 index 943ff83bc..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/JobType.py +++ /dev/null @@ -1,16 +0,0 @@ -# generated by datamodel-codegen: -# filename: JobType.yaml - -from __future__ import annotations - -from enum import Enum - - -class JobType(Enum): - get_spec = "get_spec" - check_connection = "check_connection" - discover_schema = "discover_schema" - sync = "sync" - reset_connection = "reset_connection" - connection_updater = "connection_updater" - replicate = "replicate" diff --git a/airbyte_cdk/test/models/connector_metadata/generated/NormalizationDestinationDefinitionConfig.py b/airbyte_cdk/test/models/connector_metadata/generated/NormalizationDestinationDefinitionConfig.py deleted file mode 100644 index c35edf057..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/NormalizationDestinationDefinitionConfig.py +++ /dev/null @@ -1,24 +0,0 @@ -# generated by datamodel-codegen: -# filename: NormalizationDestinationDefinitionConfig.yaml - -from __future__ import annotations - -from pydantic.v1 import BaseModel, Extra, Field - - -class NormalizationDestinationDefinitionConfig(BaseModel): - class Config: - extra = Extra.allow - - normalizationRepository: str = Field( - ..., - description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", - ) - normalizationTag: str = Field( - ..., - description="a field indicating the tag of the docker repository to be used for normalization.", - ) - normalizationIntegrationType: str = Field( - ..., - description="a field indicating the type of integration dialect to use for normalization.", - ) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/RegistryOverrides.py b/airbyte_cdk/test/models/connector_metadata/generated/RegistryOverrides.py deleted file mode 100644 index c4060c877..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/RegistryOverrides.py +++ /dev/null @@ -1,105 +0,0 @@ -# generated by datamodel-codegen: -# filename: RegistryOverrides.yaml - -from __future__ import annotations - -from enum import Enum -from typing import List, Optional - -from pydantic.v1 import AnyUrl, BaseModel, Extra, Field - - -class AllowedHosts(BaseModel): - class Config: - extra = Extra.allow - - hosts: Optional[List[str]] = Field( - None, - description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", - ) - - -class NormalizationDestinationDefinitionConfig(BaseModel): - class Config: - extra = Extra.allow - - normalizationRepository: str = Field( - ..., - description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", - ) - normalizationTag: str = Field( - ..., - description="a field indicating the tag of the docker repository to be used for normalization.", - ) - normalizationIntegrationType: str = Field( - ..., - description="a field indicating the type of integration dialect to use for normalization.", - ) - - -class SuggestedStreams(BaseModel): - class Config: - extra = Extra.allow - - streams: Optional[List[str]] = Field( - None, - description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", - ) - - -class ResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - cpu_request: Optional[str] = None - cpu_limit: Optional[str] = None - memory_request: Optional[str] = None - memory_limit: Optional[str] = None - - -class JobType(Enum): - get_spec = "get_spec" - check_connection = "check_connection" - discover_schema = "discover_schema" - sync = "sync" - reset_connection = "reset_connection" - connection_updater = "connection_updater" - replicate = "replicate" - - -class JobTypeResourceLimit(BaseModel): - class Config: - extra = Extra.forbid - - jobType: JobType - resourceRequirements: ResourceRequirements - - -class ActorDefinitionResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - default: Optional[ResourceRequirements] = Field( - None, - description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", - ) - jobSpecific: Optional[List[JobTypeResourceLimit]] = None - - -class RegistryOverrides(BaseModel): - class Config: - extra = Extra.forbid - - enabled: bool - name: Optional[str] = None - dockerRepository: Optional[str] = None - dockerImageTag: Optional[str] = None - supportsDbt: Optional[bool] = None - supportsNormalization: Optional[bool] = None - license: Optional[str] = None - documentationUrl: Optional[AnyUrl] = None - connectorSubtype: Optional[str] = None - allowedHosts: Optional[AllowedHosts] = None - normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None - suggestedStreams: Optional[SuggestedStreams] = None - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ReleaseStage.py b/airbyte_cdk/test/models/connector_metadata/generated/ReleaseStage.py deleted file mode 100644 index 8ef69075c..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ReleaseStage.py +++ /dev/null @@ -1,13 +0,0 @@ -# generated by datamodel-codegen: -# filename: ReleaseStage.yaml - -from __future__ import annotations - -from enum import Enum - - -class ReleaseStage(Enum): - alpha = "alpha" - beta = "beta" - generally_available = "generally_available" - custom = "custom" diff --git a/airbyte_cdk/test/models/connector_metadata/generated/RemoteRegistries.py b/airbyte_cdk/test/models/connector_metadata/generated/RemoteRegistries.py deleted file mode 100644 index 7a587f9d3..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/RemoteRegistries.py +++ /dev/null @@ -1,23 +0,0 @@ -# generated by datamodel-codegen: -# filename: RemoteRegistries.yaml - -from __future__ import annotations - -from typing import Optional - -from pydantic.v1 import BaseModel, Extra, Field - - -class PyPi(BaseModel): - class Config: - extra = Extra.forbid - - enabled: bool - packageName: str = Field(..., description="The name of the package on PyPi.") - - -class RemoteRegistries(BaseModel): - class Config: - extra = Extra.forbid - - pypi: Optional[PyPi] = None diff --git a/airbyte_cdk/test/models/connector_metadata/generated/ResourceRequirements.py b/airbyte_cdk/test/models/connector_metadata/generated/ResourceRequirements.py deleted file mode 100644 index 8f079d99f..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/ResourceRequirements.py +++ /dev/null @@ -1,18 +0,0 @@ -# generated by datamodel-codegen: -# filename: ResourceRequirements.yaml - -from __future__ import annotations - -from typing import Optional - -from pydantic.v1 import BaseModel, Extra - - -class ResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - cpu_request: Optional[str] = None - cpu_limit: Optional[str] = None - memory_request: Optional[str] = None - memory_limit: Optional[str] = None diff --git a/airbyte_cdk/test/models/connector_metadata/generated/RolloutConfiguration.py b/airbyte_cdk/test/models/connector_metadata/generated/RolloutConfiguration.py deleted file mode 100644 index aae003a64..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/RolloutConfiguration.py +++ /dev/null @@ -1,29 +0,0 @@ -# generated by datamodel-codegen: -# filename: RolloutConfiguration.yaml - -from __future__ import annotations - -from typing import Optional - -from pydantic.v1 import BaseModel, Extra, Field, conint - - -class RolloutConfiguration(BaseModel): - class Config: - extra = Extra.forbid - - enableProgressiveRollout: Optional[bool] = Field( - False, description="Whether to enable progressive rollout for the connector." - ) - initialPercentage: Optional[conint(ge=0, le=100)] = Field( - 0, - description="The percentage of users that should receive the new version initially.", - ) - maxPercentage: Optional[conint(ge=0, le=100)] = Field( - 50, - description="The percentage of users who should receive the release candidate during the test phase before full rollout.", - ) - advanceDelayMinutes: Optional[conint(ge=10)] = Field( - 10, - description="The number of minutes to wait before advancing the rollout percentage.", - ) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/Secret.py b/airbyte_cdk/test/models/connector_metadata/generated/Secret.py deleted file mode 100644 index 298f803fc..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/Secret.py +++ /dev/null @@ -1,33 +0,0 @@ -# generated by datamodel-codegen: -# filename: Secret.yaml - -from __future__ import annotations - -from typing import Literal, Optional - -from pydantic.v1 import BaseModel, Extra, Field - - -class SecretStore(BaseModel): - class Config: - extra = Extra.forbid - - alias: Optional[str] = Field( - None, - description="The alias of the secret store which can map to its actual secret address", - ) - type: Optional[Literal["GSM"]] = Field( - None, description="The type of the secret store" - ) - - -class Secret(BaseModel): - class Config: - extra = Extra.forbid - - name: str = Field(..., description="The secret name in the secret store") - fileName: Optional[str] = Field( - None, - description="The name of the file to which the secret value would be persisted", - ) - secretStore: SecretStore diff --git a/airbyte_cdk/test/models/connector_metadata/generated/SecretStore.py b/airbyte_cdk/test/models/connector_metadata/generated/SecretStore.py deleted file mode 100644 index 5e004a9df..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/SecretStore.py +++ /dev/null @@ -1,21 +0,0 @@ -# generated by datamodel-codegen: -# filename: SecretStore.yaml - -from __future__ import annotations - -from typing import Literal, Optional - -from pydantic.v1 import BaseModel, Extra, Field - - -class SecretStore(BaseModel): - class Config: - extra = Extra.forbid - - alias: Optional[str] = Field( - None, - description="The alias of the secret store which can map to its actual secret address", - ) - type: Optional[Literal["GSM"]] = Field( - None, description="The type of the secret store" - ) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/SourceFileInfo.py b/airbyte_cdk/test/models/connector_metadata/generated/SourceFileInfo.py deleted file mode 100644 index 03b3410d3..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/SourceFileInfo.py +++ /dev/null @@ -1,16 +0,0 @@ -# generated by datamodel-codegen: -# filename: SourceFileInfo.yaml - -from __future__ import annotations - -from typing import Optional - -from pydantic.v1 import BaseModel - - -class SourceFileInfo(BaseModel): - metadata_etag: Optional[str] = None - metadata_file_path: Optional[str] = None - metadata_bucket_name: Optional[str] = None - metadata_last_modified: Optional[str] = None - registry_entry_generated_at: Optional[str] = None diff --git a/airbyte_cdk/test/models/connector_metadata/generated/SuggestedStreams.py b/airbyte_cdk/test/models/connector_metadata/generated/SuggestedStreams.py deleted file mode 100644 index 5969de281..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/SuggestedStreams.py +++ /dev/null @@ -1,18 +0,0 @@ -# generated by datamodel-codegen: -# filename: SuggestedStreams.yaml - -from __future__ import annotations - -from typing import List, Optional - -from pydantic.v1 import BaseModel, Extra, Field - - -class SuggestedStreams(BaseModel): - class Config: - extra = Extra.allow - - streams: Optional[List[str]] = Field( - None, - description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", - ) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/SupportLevel.py b/airbyte_cdk/test/models/connector_metadata/generated/SupportLevel.py deleted file mode 100644 index 5b369890d..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/SupportLevel.py +++ /dev/null @@ -1,12 +0,0 @@ -# generated by datamodel-codegen: -# filename: SupportLevel.yaml - -from __future__ import annotations - -from enum import Enum - - -class SupportLevel(Enum): - community = "community" - certified = "certified" - archived = "archived" diff --git a/airbyte_cdk/test/models/connector_metadata/generated/TestConnections.py b/airbyte_cdk/test/models/connector_metadata/generated/TestConnections.py deleted file mode 100644 index 9450a3c19..000000000 --- a/airbyte_cdk/test/models/connector_metadata/generated/TestConnections.py +++ /dev/null @@ -1,14 +0,0 @@ -# generated by datamodel-codegen: -# filename: TestConnections.yaml - -from __future__ import annotations - -from pydantic.v1 import BaseModel, Extra, Field - - -class TestConnections(BaseModel): - class Config: - extra = Extra.forbid - - name: str = Field(..., description="The connection name") - id: str = Field(..., description="The connection ID") diff --git a/airbyte_cdk/test/models/connector_metadata/generated/__init__.py b/airbyte_cdk/test/models/connector_metadata/generated/__init__.py index d026bba2a..e69de29bb 100644 --- a/airbyte_cdk/test/models/connector_metadata/generated/__init__.py +++ b/airbyte_cdk/test/models/connector_metadata/generated/__init__.py @@ -1,31 +0,0 @@ -# generated by bin/generate_component_manifest_files.py -from .ActorDefinitionResourceRequirements import * -from .AirbyteInternal import * -from .AllowedHosts import * -from .ConnectorBreakingChanges import * -from .ConnectorBuildOptions import * -from .ConnectorIPCOptions import * -from .ConnectorMetadataDefinitionV0 import * -from .ConnectorMetrics import * -from .ConnectorPackageInfo import * -from .ConnectorRegistryDestinationDefinition import * -from .ConnectorRegistryReleases import * -from .ConnectorRegistrySourceDefinition import * -from .ConnectorRegistryV0 import * -from .ConnectorReleases import * -from .ConnectorTestSuiteOptions import * -from .GeneratedFields import * -from .GitInfo import * -from .JobType import * -from .NormalizationDestinationDefinitionConfig import * -from .RegistryOverrides import * -from .ReleaseStage import * -from .RemoteRegistries import * -from .ResourceRequirements import * -from .RolloutConfiguration import * -from .Secret import * -from .SecretStore import * -from .SourceFileInfo import * -from .SuggestedStreams import * -from .SupportLevel import * -from .TestConnections import * diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index e791792bb..44e6df60e 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -1,5 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +import json +import os import re import sys import tempfile @@ -9,6 +11,7 @@ import anyio import dagger import httpx +import yaml PYTHON_IMAGE = "python:3.10" LOCAL_YAML_DIR_PATH = "airbyte_cdk/sources/declarative" @@ -40,10 +43,13 @@ def generate_init_module_content(yaml_files: list[str]) -> str: async def download_metadata_schemas(temp_dir: Path) -> list[str]: """Download metadata schema YAML files from GitHub to a temporary directory.""" + token = os.getenv("GITHUB_TOKEN") or os.getenv("GH_TOKEN") headers = { "User-Agent": "airbyte-python-cdk-build", "Accept": "application/vnd.github.v3+json", } + if token: + headers["Authorization"] = f"Bearer {token}" async with httpx.AsyncClient(headers=headers, timeout=30.0) as client: try: @@ -53,7 +59,7 @@ async def download_metadata_schemas(temp_dir: Path) -> list[str]: except httpx.HTTPStatusError as e: if e.response.status_code == 403: print( - "Warning: GitHub API rate limit exceeded. Using cached schemas if available.", + "Warning: GitHub API rate limit exceeded. Provide GITHUB_TOKEN to authenticate.", file=sys.stderr, ) raise @@ -231,6 +237,89 @@ async def generate_models_from_schemas( await codegen_container.directory("/generated").export(output_dir_path) +def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: str) -> None: + """Consolidate all YAML schemas into a single JSON schema file.""" + schemas = {} + + for yaml_file in yaml_dir_path.glob("*.yaml"): + schema_name = yaml_file.stem + with yaml_file.open('r') as f: + schema_content = yaml.safe_load(f) + schemas[schema_name] = schema_content + + # Find the main schema (ConnectorMetadataDefinitionV0) + main_schema = schemas.get("ConnectorMetadataDefinitionV0") + + if main_schema: + # Create a consolidated schema with definitions + consolidated = { + "$schema": main_schema.get("$schema", "http://json-schema.org/draft-07/schema#"), + "title": "Connector Metadata Schema", + "description": "Consolidated JSON schema for Airbyte connector metadata validation", + **main_schema, + "definitions": {} + } + + # Add all other schemas as definitions + for schema_name, schema_content in schemas.items(): + if schema_name != "ConnectorMetadataDefinitionV0": + consolidated["definitions"][schema_name] = schema_content + + Path(output_json_path).write_text(json.dumps(consolidated, indent=2)) + print(f"Generated consolidated JSON schema: {output_json_path}", file=sys.stderr) + else: + print("Warning: ConnectorMetadataDefinitionV0 not found, generating simple consolidation", file=sys.stderr) + Path(output_json_path).write_text(json.dumps(schemas, indent=2)) + + +async def generate_metadata_models_single_file( + dagger_client: dagger.Client, + yaml_dir_path: str, + output_file_path: str, +) -> None: + """Generate all metadata models into a single Python file.""" + codegen_container = ( + dagger_client.container() + .from_(PYTHON_IMAGE) + .with_exec(["mkdir", "-p", "/generated"], use_entrypoint=True) + .with_exec(["pip", "install", " ".join(PIP_DEPENDENCIES)], use_entrypoint=True) + .with_mounted_directory( + "/yaml", dagger_client.host().directory(yaml_dir_path, include=["*.yaml"]) + ) + ) + + codegen_container = codegen_container.with_exec( + [ + "datamodel-codegen", + "--input", + "/yaml", + "--output", + "/generated/models.py", + "--disable-timestamp", + "--enum-field-as-literal", + "one", + "--set-default-enum-member", + "--use-double-quotes", + "--remove-special-field-name-prefix", + "--field-extra-keys", + "deprecated", + "deprecation_message", + ], + use_entrypoint=True, + ) + + original_content = await codegen_container.file("/generated/models.py").contents() + post_processed_content = original_content.replace( + "from pydantic", "from pydantic.v1" + ) + + codegen_container = codegen_container.with_new_file( + "/generated/models_processed.py", contents=post_processed_content + ) + + await codegen_container.file("/generated/models_processed.py").export(output_file_path) + + async def main(): async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: print("Generating declarative component models...", file=sys.stderr) @@ -246,15 +335,22 @@ async def main(): print("\nGenerating metadata models...", file=sys.stderr) with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) - metadata_yaml_files = await download_metadata_schemas(temp_path) + await download_metadata_schemas(temp_path) + + output_dir = Path(LOCAL_METADATA_OUTPUT_DIR_PATH) + output_dir.mkdir(parents=True, exist_ok=True) - await generate_models_from_schemas( + print("Generating single Python file with all models...", file=sys.stderr) + output_file = str(output_dir / "models.py") + await generate_metadata_models_single_file( dagger_client=dagger_client, yaml_dir_path=str(temp_path), - output_dir_path=LOCAL_METADATA_OUTPUT_DIR_PATH, - yaml_files=metadata_yaml_files, - metadata_models=True, + output_file_path=output_file, ) + + print("Generating consolidated JSON schema...", file=sys.stderr) + json_schema_file = str(output_dir / "metadata_schema.json") + consolidate_yaml_schemas_to_json(temp_path, json_schema_file) print("\nModel generation complete!", file=sys.stderr) From 933d478d392c1c1aa6be969e76a878f17ebb9873 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 01:39:06 +0000 Subject: [PATCH 05/20] style: Apply ruff formatting to build script Co-Authored-By: AJ Steers --- bin/generate_component_manifest_files.py | 65 ++++++++++++------------ 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index 44e6df60e..fc86fa961 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -50,7 +50,7 @@ async def download_metadata_schemas(temp_dir: Path) -> list[str]: } if token: headers["Authorization"] = f"Bearer {token}" - + async with httpx.AsyncClient(headers=headers, timeout=30.0) as client: try: response = await client.get(METADATA_SCHEMAS_GITHUB_URL) @@ -64,21 +64,21 @@ async def download_metadata_schemas(temp_dir: Path) -> list[str]: ) raise raise - + yaml_files = [] for file_info in files_info: if file_info["name"].endswith(".yaml"): file_name = file_info["name"] file_url = f"{METADATA_SCHEMAS_RAW_URL_BASE}/{file_name}" - + print(f"Downloading {file_name}...", file=sys.stderr) file_response = await client.get(file_url) file_response.raise_for_status() - + file_path = temp_dir / file_name file_path.write_text(file_response.text) yaml_files.append(Path(file_name).stem) - + return yaml_files @@ -173,11 +173,9 @@ async def post_process_metadata_models(codegen_container: dagger.Container): original_content = await codegen_container.file( f"/generated/{generated_file}" ).contents() - - post_processed_content = original_content.replace( - "from pydantic", "from pydantic.v1" - ) - + + post_processed_content = original_content.replace("from pydantic", "from pydantic.v1") + codegen_container = codegen_container.with_new_file( f"/generated_post_processed/{generated_file}", contents=post_processed_content ) @@ -194,7 +192,7 @@ async def generate_models_from_schemas( ) -> None: """Generate Pydantic models from YAML schemas using datamodel-codegen.""" init_module_content = generate_init_module_content(yaml_files) - + codegen_container = ( dagger_client.container() .from_(PYTHON_IMAGE) @@ -205,7 +203,7 @@ async def generate_models_from_schemas( ) .with_new_file("/generated/__init__.py", contents=init_module_content) ) - + for yaml_file in yaml_files: codegen_container = codegen_container.with_exec( [ @@ -226,7 +224,7 @@ async def generate_models_from_schemas( ], use_entrypoint=True, ) - + if post_process: codegen_container = await post_process_codegen(codegen_container) await codegen_container.directory("/generated_post_processed").export(output_dir_path) @@ -240,16 +238,16 @@ async def generate_models_from_schemas( def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: str) -> None: """Consolidate all YAML schemas into a single JSON schema file.""" schemas = {} - + for yaml_file in yaml_dir_path.glob("*.yaml"): schema_name = yaml_file.stem - with yaml_file.open('r') as f: + with yaml_file.open("r") as f: schema_content = yaml.safe_load(f) schemas[schema_name] = schema_content - + # Find the main schema (ConnectorMetadataDefinitionV0) main_schema = schemas.get("ConnectorMetadataDefinitionV0") - + if main_schema: # Create a consolidated schema with definitions consolidated = { @@ -257,18 +255,21 @@ def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: str) "title": "Connector Metadata Schema", "description": "Consolidated JSON schema for Airbyte connector metadata validation", **main_schema, - "definitions": {} + "definitions": {}, } - + # Add all other schemas as definitions for schema_name, schema_content in schemas.items(): if schema_name != "ConnectorMetadataDefinitionV0": consolidated["definitions"][schema_name] = schema_content - + Path(output_json_path).write_text(json.dumps(consolidated, indent=2)) print(f"Generated consolidated JSON schema: {output_json_path}", file=sys.stderr) else: - print("Warning: ConnectorMetadataDefinitionV0 not found, generating simple consolidation", file=sys.stderr) + print( + "Warning: ConnectorMetadataDefinitionV0 not found, generating simple consolidation", + file=sys.stderr, + ) Path(output_json_path).write_text(json.dumps(schemas, indent=2)) @@ -287,7 +288,7 @@ async def generate_metadata_models_single_file( "/yaml", dagger_client.host().directory(yaml_dir_path, include=["*.yaml"]) ) ) - + codegen_container = codegen_container.with_exec( [ "datamodel-codegen", @@ -307,16 +308,14 @@ async def generate_metadata_models_single_file( ], use_entrypoint=True, ) - + original_content = await codegen_container.file("/generated/models.py").contents() - post_processed_content = original_content.replace( - "from pydantic", "from pydantic.v1" - ) - + post_processed_content = original_content.replace("from pydantic", "from pydantic.v1") + codegen_container = codegen_container.with_new_file( "/generated/models_processed.py", contents=post_processed_content ) - + await codegen_container.file("/generated/models_processed.py").export(output_file_path) @@ -331,15 +330,15 @@ async def main(): yaml_files=declarative_yaml_files, post_process=True, ) - + print("\nGenerating metadata models...", file=sys.stderr) with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) await download_metadata_schemas(temp_path) - + output_dir = Path(LOCAL_METADATA_OUTPUT_DIR_PATH) output_dir.mkdir(parents=True, exist_ok=True) - + print("Generating single Python file with all models...", file=sys.stderr) output_file = str(output_dir / "models.py") await generate_metadata_models_single_file( @@ -347,11 +346,11 @@ async def main(): yaml_dir_path=str(temp_path), output_file_path=output_file, ) - + print("Generating consolidated JSON schema...", file=sys.stderr) json_schema_file = str(output_dir / "metadata_schema.json") consolidate_yaml_schemas_to_json(temp_path, json_schema_file) - + print("\nModel generation complete!", file=sys.stderr) From c89faabf0d6c75d0e4d3f04d33e12393020a87fd Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:00:24 +0000 Subject: [PATCH 06/20] docs: Move metadata models documentation to CONTRIBUTING.md - Moved detailed documentation from README to CONTRIBUTING.md - Simplified README to just show usage and link to CONTRIBUTING.md - Added docstring to __init__.py pointing to CONTRIBUTING.md - Removed py.typed from connector_metadata (already exists at root) Co-Authored-By: AJ Steers --- .../test/models/connector_metadata/README.md | 85 +------------------ .../models/connector_metadata/__init__.py | 7 ++ .../test/models/connector_metadata/py.typed | 1 - docs/CONTRIBUTING.md | 27 ++++++ 4 files changed, 37 insertions(+), 83 deletions(-) delete mode 100644 airbyte_cdk/test/models/connector_metadata/py.typed diff --git a/airbyte_cdk/test/models/connector_metadata/README.md b/airbyte_cdk/test/models/connector_metadata/README.md index e254e6da1..bafffec21 100644 --- a/airbyte_cdk/test/models/connector_metadata/README.md +++ b/airbyte_cdk/test/models/connector_metadata/README.md @@ -2,94 +2,15 @@ This package contains Pydantic models for validating Airbyte connector `metadata.yaml` files. -## Overview - -The models are automatically generated from JSON Schema YAML files maintained in the [airbytehq/airbyte](https://github.com/airbytehq/airbyte) repository at: -``` -airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ -``` - -During the CDK build process (`poetry run poe build`), these schemas are downloaded from GitHub and used to generate Pydantic models via `datamodel-code-generator`. All models are generated into a single Python file for simplicity and easier imports. - ## Usage -### Validating a metadata.yaml file - ```python -from pathlib import Path -import yaml from airbyte_cdk.test.models import ConnectorMetadataDefinitionV0 +import yaml -# Load metadata.yaml -metadata_path = Path("path/to/metadata.yaml") -metadata_dict = yaml.safe_load(metadata_path.read_text()) - -# Validate using Pydantic -try: - metadata = ConnectorMetadataDefinitionV0(**metadata_dict) - print("✓ Metadata is valid!") -except Exception as e: - print(f"✗ Validation failed: {e}") -``` - -### Accessing metadata fields - -```python -from airbyte_cdk.test.models import ConnectorMetadataDefinitionV0 - -metadata = ConnectorMetadataDefinitionV0(**metadata_dict) - -# Access fields with full type safety -print(f"Connector: {metadata.data.name}") -print(f"Docker repository: {metadata.data.dockerRepository}") -print(f"Docker image tag: {metadata.data.dockerImageTag}") -print(f"Support level: {metadata.data.supportLevel}") -``` - -### Accessing other models - -All generated models are available in the `generated.models` module: - -```python -from airbyte_cdk.test.models.connector_metadata.generated.models import ( - ConnectorBreakingChanges, - ConnectorReleases, - ReleaseStage, - SupportLevel, -) +metadata = ConnectorMetadataDefinitionV0(**yaml.safe_load(metadata_yaml)) ``` -### Available models - -The main model is `ConnectorMetadataDefinitionV0`, which includes nested models for: - -- `ConnectorType` - Source or destination -- `ConnectorSubtype` - API, database, file, etc. -- `SupportLevel` - Community, certified, etc. -- `ReleaseStage` - Alpha, beta, generally_available -- `ConnectorBreakingChanges` - Breaking change definitions -- `ConnectorReleases` - Release information -- `AllowedHosts` - Network access configuration -- And many more... - ## Regenerating Models -Models are regenerated automatically when you run: - -```bash -poetry run poe build -``` - -This command: -1. Downloads the latest schema YAML files from the airbyte repository -2. Generates all Pydantic models into a single file using `datamodel-code-generator` -3. Generates a consolidated JSON schema file for external validation tools -4. Outputs to `airbyte_cdk/test/models/connector_metadata/generated/`: - - `models.py` - All Pydantic models in a single file - - `metadata_schema.json` - Consolidated JSON schema - -## Schema Source - -The authoritative schemas are maintained in the [airbyte monorepo](https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src). - -Any changes to metadata validation should be made there, and will be automatically picked up by the CDK build process. +See the [Contributing Guide](../../../docs/CONTRIBUTING.md#regenerating-connector-metadata-models) for information on regenerating these models. diff --git a/airbyte_cdk/test/models/connector_metadata/__init__.py b/airbyte_cdk/test/models/connector_metadata/__init__.py index 2828d2b8f..136eab240 100644 --- a/airbyte_cdk/test/models/connector_metadata/__init__.py +++ b/airbyte_cdk/test/models/connector_metadata/__init__.py @@ -1,3 +1,10 @@ +"""Connector metadata models for validation and testing. + +These models are auto-generated from JSON schemas in the airbytehq/airbyte repository. +For information on regenerating these models, see the Contributing Guide: +https://github.com/airbytehq/airbyte-python-cdk/blob/main/docs/CONTRIBUTING.md#regenerating-connector-metadata-models +""" + from .generated.models import ConnectorMetadataDefinitionV0, ConnectorTestSuiteOptions __all__ = [ diff --git a/airbyte_cdk/test/models/connector_metadata/py.typed b/airbyte_cdk/test/models/connector_metadata/py.typed deleted file mode 100644 index 8b1378917..000000000 --- a/airbyte_cdk/test/models/connector_metadata/py.typed +++ /dev/null @@ -1 +0,0 @@ - diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index bb7dc9b2c..359e8f1ce 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -85,6 +85,33 @@ poetry run poe build This will generate the code generator docker image and the component manifest files based on the schemas and templates. +## Regenerating Connector Metadata Models + +The CDK includes Pydantic models for validating connector `metadata.yaml` files. These models are automatically generated from JSON Schema YAML files maintained in the [airbytehq/airbyte repository](https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src). + +To regenerate the metadata models, run: + +```bash +poetry run poe build +``` + +This command: +1. Downloads the latest schema YAML files from the airbyte repository +2. Generates all Pydantic models into a single file using `datamodel-code-generator` +3. Generates a consolidated JSON schema file for external validation tools +4. Outputs to `airbyte_cdk/test/models/connector_metadata/generated/`: + - `models.py` - All Pydantic models in a single file + - `metadata_schema.json` - Consolidated JSON schema + +The models can be imported and used for validation: + +```python +from airbyte_cdk.test.models import ConnectorMetadataDefinitionV0 +import yaml + +metadata = ConnectorMetadataDefinitionV0(**yaml.safe_load(metadata_yaml)) +``` + ## Generating API Reference Docs Documentation auto-gen code lives in the `/docs` folder. Based on the doc strings of public methods, we generate API documentation using [pdoc](https://pdoc.dev). From a56208bd130682b262e0d428d3ab68b965625cbf Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Mon, 27 Oct 2025 13:08:54 -0700 Subject: [PATCH 07/20] chore: revert unrelated format changes on other generated file --- .../models/declarative_component_schema.py | 144 +++++++----------- 1 file changed, 58 insertions(+), 86 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index d175a8432..35186ef71 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -928,28 +928,24 @@ class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( - Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", - examples=[ - {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, - { - "app_id": { - "type": "string", - "path_in_connector_config": ["info", "app_id"], - } - }, - ], - title="OAuth user input", - ) + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + examples=[ + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, + { + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], + } + }, + ], + title="OAuth user input", ) - oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = ( - Field( - None, - description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', - title="DeclarativeOAuth Connector Specification", - ) + oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( + None, + description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', + title="DeclarativeOAuth Connector Specification", ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, @@ -967,9 +963,7 @@ class Config: complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", - examples=[ - {"client_id": {"type": "string"}, "client_secret": {"type": "string"}} - ], + examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( @@ -1473,9 +1467,7 @@ class CustomConfigTransformation(BaseModel): class_name: str = Field( ..., description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_..`.", - examples=[ - "source_declarative_manifest.components.MyCustomConfigTransformation" - ], + examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], ) parameters: Optional[Dict[str, Any]] = Field( None, @@ -1893,9 +1885,7 @@ class OAuthAuthenticator(BaseModel): scopes: Optional[List[str]] = Field( None, description="List of scopes that should be granted to the access token.", - examples=[ - ["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"] - ], + examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], title="Scopes", ) token_expiry_date: Optional[str] = Field( @@ -2094,9 +2084,7 @@ class RecordSelector(BaseModel): description="Responsible for filtering records to be emitted by the Source.", title="Record Filter", ) - schema_normalization: Optional[ - Union[SchemaNormalization, CustomSchemaNormalization] - ] = Field( + schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( None, description="Responsible for normalization according to the schema.", title="Schema Normalization", @@ -2138,12 +2126,10 @@ class DpathValidator(BaseModel): ], title="Field Path", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( - Field( - ..., - description="The condition that the specified config value will be evaluated against", - title="Validation Strategy", - ) + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( + ..., + description="The condition that the specified config value will be evaluated against", + title="Validation Strategy", ) @@ -2160,12 +2146,10 @@ class PredicateValidator(BaseModel): ], title="Value", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( - Field( - ..., - description="The validation strategy to apply to the value.", - title="Validation Strategy", - ) + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( + ..., + description="The validation strategy to apply to the value.", + title="Validation Strategy", ) @@ -2190,12 +2174,12 @@ class ConfigAddFields(BaseModel): class CompositeErrorHandler(BaseModel): type: Literal["CompositeErrorHandler"] - error_handlers: List[ - Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler] - ] = Field( - ..., - description="List of error handlers to iterate on to determine how to handle a failed response.", - title="Error Handlers", + error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( + Field( + ..., + description="List of error handlers to iterate on to determine how to handle a failed response.", + title="Error Handlers", + ) ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -2357,9 +2341,9 @@ class Config: type: Literal["DeclarativeSource"] check: Union[CheckStream, CheckDynamicStream] - streams: Optional[ - List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] - ] = None + streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( + None + ) dynamic_streams: List[DynamicDeclarativeStream] version: str = Field( ..., @@ -2484,20 +2468,16 @@ class Config: extra = Extra.allow type: Literal["DeclarativeStream"] - name: Optional[str] = Field( - "", description="The stream name.", example=["Users"], title="Name" - ) + name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages.", title="Retriever", ) - incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = ( - Field( - None, - description="Component used to fetch data incrementally based on a time field in the data.", - title="Incremental Sync", - ) + incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( + None, + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", ) primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") schema_loader: Optional[ @@ -2671,20 +2651,18 @@ class HttpRequester(BaseModelWithDeprecations): description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", title="Query Properties", ) - request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = ( - Field( - None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", - examples=[ - {"unit": "day"}, - { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, - ], - title="Query Parameters", - ) + request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( + None, + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + examples=[ + {"unit": "day"}, + { + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, + ], + title="Query Parameters", ) request_headers: Optional[Union[Dict[str, str], str]] = Field( None, @@ -2856,9 +2834,7 @@ class QueryProperties(BaseModel): class StateDelegatingStream(BaseModel): type: Literal["StateDelegatingStream"] - name: str = Field( - ..., description="The stream name.", example=["Users"], title="Name" - ) + name: str = Field(..., description="The stream name.", example=["Users"], title="Name") full_refresh_stream: DeclarativeStream = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", @@ -2945,17 +2921,13 @@ class AsyncRetriever(BaseModel): status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( ..., description="Responsible for fetching the actual status of the async job." ) - download_target_extractor: Optional[ - Union[DpathExtractor, CustomRecordExtractor] - ] = Field( + download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( None, description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", ) download_extractor: Optional[ Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] - ] = Field( - None, description="Responsible for fetching the records from provided urls." - ) + ] = Field(None, description="Responsible for fetching the records from provided urls.") creation_requester: Union[HttpRequester, CustomRequester] = Field( ..., description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", From 0f48425a75b5f43001ea22ac2fb04176bb577d1c Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Mon, 27 Oct 2025 13:10:36 -0700 Subject: [PATCH 08/20] Delete airbyte_cdk/test/models/connector_metadata/README.md --- .../test/models/connector_metadata/README.md | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 airbyte_cdk/test/models/connector_metadata/README.md diff --git a/airbyte_cdk/test/models/connector_metadata/README.md b/airbyte_cdk/test/models/connector_metadata/README.md deleted file mode 100644 index bafffec21..000000000 --- a/airbyte_cdk/test/models/connector_metadata/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Airbyte Connector Metadata Models - -This package contains Pydantic models for validating Airbyte connector `metadata.yaml` files. - -## Usage - -```python -from airbyte_cdk.test.models import ConnectorMetadataDefinitionV0 -import yaml - -metadata = ConnectorMetadataDefinitionV0(**yaml.safe_load(metadata_yaml)) -``` - -## Regenerating Models - -See the [Contributing Guide](../../../docs/CONTRIBUTING.md#regenerating-connector-metadata-models) for information on regenerating these models. From 07d7014349d52efc80f0870418cfa04c53d3a7f9 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Mon, 27 Oct 2025 13:13:14 -0700 Subject: [PATCH 09/20] docs: clean up docstring (merged content from `README.md`) --- .../test/models/connector_metadata/__init__.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/test/models/connector_metadata/__init__.py b/airbyte_cdk/test/models/connector_metadata/__init__.py index 136eab240..ff18779f7 100644 --- a/airbyte_cdk/test/models/connector_metadata/__init__.py +++ b/airbyte_cdk/test/models/connector_metadata/__init__.py @@ -1,4 +1,15 @@ -"""Connector metadata models for validation and testing. +"""Pydantic and JSON Schema models for `metadata.yaml` validation and testing. + +## Usage + +```python +from airbyte_cdk.test.models import ConnectorMetadataDefinitionV0 +import yaml + +metadata = ConnectorMetadataDefinitionV0(**yaml.safe_load(metadata_yaml)) +``` + +## Regenerating Models These models are auto-generated from JSON schemas in the airbytehq/airbyte repository. For information on regenerating these models, see the Contributing Guide: From c63223a79febfd1e60cf110303b78ae2f67f1d0c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:22:28 +0000 Subject: [PATCH 10/20] feat: Replace HTTP downloads with sparse git clone for metadata schemas - Switch from downloading individual YAML files via GitHub API to using sparse git clone - Eliminates rate limiting issues (60 req/hour -> no API calls) - Fix single-file model generation to properly filter out relative imports - Add multi-line import block detection and filtering - Generate and commit metadata models and consolidated JSON schema artifacts Co-Authored-By: AJ Steers --- .../models/declarative_component_schema.py | 146 +- .../generated/metadata_schema.json | 1261 +++++++++++++++++ .../connector_metadata/generated/models.py | 665 +++++++++ bin/generate_component_manifest_files.py | 171 ++- 4 files changed, 2130 insertions(+), 113 deletions(-) create mode 100644 airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json create mode 100644 airbyte_cdk/test/models/connector_metadata/generated/models.py diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 35186ef71..421c6779f 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1,5 +1,3 @@ -# Copyright (c) 2025 Airbyte, Inc., all rights reserved. - # generated by datamodel-codegen: # filename: declarative_component_schema.yaml @@ -928,24 +926,28 @@ class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", - examples=[ - {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, - { - "app_id": { - "type": "string", - "path_in_connector_config": ["info", "app_id"], - } - }, - ], - title="OAuth user input", + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( + Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + examples=[ + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, + { + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], + } + }, + ], + title="OAuth user input", + ) ) - oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( - None, - description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', - title="DeclarativeOAuth Connector Specification", + oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = ( + Field( + None, + description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', + title="DeclarativeOAuth Connector Specification", + ) ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, @@ -963,7 +965,9 @@ class Config: complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", - examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], + examples=[ + {"client_id": {"type": "string"}, "client_secret": {"type": "string"}} + ], title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( @@ -1467,7 +1471,9 @@ class CustomConfigTransformation(BaseModel): class_name: str = Field( ..., description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_..`.", - examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], + examples=[ + "source_declarative_manifest.components.MyCustomConfigTransformation" + ], ) parameters: Optional[Dict[str, Any]] = Field( None, @@ -1885,7 +1891,9 @@ class OAuthAuthenticator(BaseModel): scopes: Optional[List[str]] = Field( None, description="List of scopes that should be granted to the access token.", - examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], + examples=[ + ["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"] + ], title="Scopes", ) token_expiry_date: Optional[str] = Field( @@ -2084,7 +2092,9 @@ class RecordSelector(BaseModel): description="Responsible for filtering records to be emitted by the Source.", title="Record Filter", ) - schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( + schema_normalization: Optional[ + Union[SchemaNormalization, CustomSchemaNormalization] + ] = Field( None, description="Responsible for normalization according to the schema.", title="Schema Normalization", @@ -2126,10 +2136,12 @@ class DpathValidator(BaseModel): ], title="Field Path", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( - ..., - description="The condition that the specified config value will be evaluated against", - title="Validation Strategy", + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( + Field( + ..., + description="The condition that the specified config value will be evaluated against", + title="Validation Strategy", + ) ) @@ -2146,10 +2158,12 @@ class PredicateValidator(BaseModel): ], title="Value", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( - ..., - description="The validation strategy to apply to the value.", - title="Validation Strategy", + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( + Field( + ..., + description="The validation strategy to apply to the value.", + title="Validation Strategy", + ) ) @@ -2174,12 +2188,12 @@ class ConfigAddFields(BaseModel): class CompositeErrorHandler(BaseModel): type: Literal["CompositeErrorHandler"] - error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( - Field( - ..., - description="List of error handlers to iterate on to determine how to handle a failed response.", - title="Error Handlers", - ) + error_handlers: List[ + Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler] + ] = Field( + ..., + description="List of error handlers to iterate on to determine how to handle a failed response.", + title="Error Handlers", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -2341,9 +2355,9 @@ class Config: type: Literal["DeclarativeSource"] check: Union[CheckStream, CheckDynamicStream] - streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( - None - ) + streams: Optional[ + List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] + ] = None dynamic_streams: List[DynamicDeclarativeStream] version: str = Field( ..., @@ -2468,16 +2482,20 @@ class Config: extra = Extra.allow type: Literal["DeclarativeStream"] - name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") + name: Optional[str] = Field( + "", description="The stream name.", example=["Users"], title="Name" + ) retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages.", title="Retriever", ) - incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( - None, - description="Component used to fetch data incrementally based on a time field in the data.", - title="Incremental Sync", + incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = ( + Field( + None, + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", + ) ) primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") schema_loader: Optional[ @@ -2651,18 +2669,20 @@ class HttpRequester(BaseModelWithDeprecations): description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", title="Query Properties", ) - request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( - None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", - examples=[ - {"unit": "day"}, - { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, - ], - title="Query Parameters", + request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = ( + Field( + None, + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + examples=[ + {"unit": "day"}, + { + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, + ], + title="Query Parameters", + ) ) request_headers: Optional[Union[Dict[str, str], str]] = Field( None, @@ -2834,7 +2854,9 @@ class QueryProperties(BaseModel): class StateDelegatingStream(BaseModel): type: Literal["StateDelegatingStream"] - name: str = Field(..., description="The stream name.", example=["Users"], title="Name") + name: str = Field( + ..., description="The stream name.", example=["Users"], title="Name" + ) full_refresh_stream: DeclarativeStream = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", @@ -2921,13 +2943,17 @@ class AsyncRetriever(BaseModel): status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( ..., description="Responsible for fetching the actual status of the async job." ) - download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( + download_target_extractor: Optional[ + Union[DpathExtractor, CustomRecordExtractor] + ] = Field( None, description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", ) download_extractor: Optional[ Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] - ] = Field(None, description="Responsible for fetching the records from provided urls.") + ] = Field( + None, description="Responsible for fetching the records from provided urls." + ) creation_requester: Union[HttpRequester, CustomRequester] = Field( ..., description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", diff --git a/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json b/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json new file mode 100644 index 000000000..63e0137c2 --- /dev/null +++ b/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json @@ -0,0 +1,1261 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ConnectorMetadataDefinitionV0", + "description": "describes the metadata of a connector", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorMetadataDefinitionV0.yml", + "type": "object", + "required": [ + "metadataSpecVersion", + "data" + ], + "additionalProperties": false, + "properties": { + "metadataSpecVersion": { + "type": "string" + }, + "data": { + "type": "object", + "required": [ + "name", + "definitionId", + "connectorType", + "dockerRepository", + "dockerImageTag", + "license", + "documentationUrl", + "githubIssueLabel", + "connectorSubtype", + "releaseStage" + ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "definitionId": { + "type": "string", + "format": "uuid" + }, + "connectorBuildOptions": { + "$ref": "ConnectorBuildOptions.yaml" + }, + "connectorTestSuitesOptions": { + "type": "array", + "items": { + "$ref": "ConnectorTestSuiteOptions.yaml" + } + }, + "connectorType": { + "type": "string", + "enum": [ + "destination", + "source" + ] + }, + "dockerRepository": { + "type": "string" + }, + "dockerImageTag": { + "type": "string" + }, + "supportsDbt": { + "type": "boolean" + }, + "supportsNormalization": { + "type": "boolean" + }, + "license": { + "type": "string" + }, + "documentationUrl": { + "type": "string", + "format": "uri" + }, + "githubIssueLabel": { + "type": "string" + }, + "maxSecondsBetweenMessages": { + "description": "Maximum delay between 2 airbyte protocol messages, in second. The source will timeout if this delay is reached", + "type": "integer" + }, + "releaseDate": { + "description": "The date when this connector was first released, in yyyy-mm-dd format.", + "type": "string", + "format": "date" + }, + "protocolVersion": { + "type": "string", + "description": "the Airbyte Protocol version supported by the connector" + }, + "erdUrl": { + "type": "string", + "description": "The URL where you can visualize the ERD" + }, + "connectorSubtype": { + "type": "string", + "enum": [ + "api", + "database", + "datalake", + "file", + "custom", + "message_queue", + "unknown", + "vectorstore" + ] + }, + "releaseStage": { + "$ref": "ReleaseStage.yaml" + }, + "supportLevel": { + "$ref": "SupportLevel.yaml" + }, + "tags": { + "type": "array", + "description": "An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", + "items": { + "type": "string" + }, + "default": [] + }, + "registryOverrides": { + "anyOf": [ + { + "type": "object", + "additionalProperties": false, + "properties": { + "oss": { + "anyOf": [ + { + "$ref": "RegistryOverrides.yaml" + } + ] + }, + "cloud": { + "anyOf": [ + { + "$ref": "RegistryOverrides.yaml" + } + ] + } + } + } + ] + }, + "allowedHosts": { + "$ref": "AllowedHosts.yaml" + }, + "releases": { + "$ref": "ConnectorReleases.yaml" + }, + "normalizationConfig": { + "$ref": "NormalizationDestinationDefinitionConfig.yaml" + }, + "suggestedStreams": { + "$ref": "SuggestedStreams.yaml" + }, + "resourceRequirements": { + "$ref": "ActorDefinitionResourceRequirements.yaml" + }, + "ab_internal": { + "$ref": "AirbyteInternal.yaml" + }, + "remoteRegistries": { + "$ref": "RemoteRegistries.yaml" + }, + "supportsRefreshes": { + "type": "boolean", + "default": false + }, + "generated": { + "$ref": "GeneratedFields.yaml" + }, + "supportsFileTransfer": { + "type": "boolean", + "default": false + }, + "supportsDataActivation": { + "type": "boolean", + "default": false + }, + "connectorIPCOptions": { + "$ref": "ConnectorIPCOptions.yaml" + } + } + } + }, + "definitions": { + "TestConnections": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestConnections.yaml", + "title": "TestConnections", + "description": "List of sandbox cloud connections that tests can be run against", + "type": "object", + "required": [ + "name", + "id" + ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "The connection name" + }, + "id": { + "type": "string", + "description": "The connection ID" + } + } + }, + "SupportLevel": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/SupportLevel.yaml", + "title": "SupportLevel", + "description": "enum that describes a connector's release stage", + "type": "string", + "enum": [ + "community", + "certified", + "archived" + ] + }, + "SuggestedStreams": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SuggestedStreams.yaml", + "title": "SuggestedStreams", + "description": "A source's suggested streams. These will be suggested by default for new connections using this source. Otherwise, all streams will be selected. This is useful for when your source has a lot of streams, but the average user will only want a subset of them synced.", + "type": "object", + "additionalProperties": true, + "properties": { + "streams": { + "type": "array", + "description": "An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", + "items": { + "type": "string" + } + } + } + }, + "SourceFileInfo": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/SourceFileInfo.yaml", + "title": "SourceFileInfo", + "description": "Information about the source file that generated the registry entry", + "type": "object", + "properties": { + "metadata_etag": { + "type": "string" + }, + "metadata_file_path": { + "type": "string" + }, + "metadata_bucket_name": { + "type": "string" + }, + "metadata_last_modified": { + "type": "string" + }, + "registry_entry_generated_at": { + "type": "string" + } + } + }, + "SecretStore": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestSecret.yaml", + "title": "SecretStore", + "description": "An object describing a secret store metadata", + "type": "object", + "required": [ + "name", + "secretStore" + ], + "additionalProperties": false, + "properties": { + "alias": { + "type": "string", + "description": "The alias of the secret store which can map to its actual secret address" + }, + "type": { + "type": "string", + "description": "The type of the secret store", + "enum": [ + "GSM" + ] + } + } + }, + "Secret": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestSecret.yaml", + "title": "Secret", + "description": "An object describing a secret's metadata", + "type": "object", + "required": [ + "name", + "secretStore" + ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "The secret name in the secret store" + }, + "fileName": { + "type": "string", + "description": "The name of the file to which the secret value would be persisted" + }, + "secretStore": { + "$ref": "SecretStore.yaml" + } + } + }, + "RolloutConfiguration": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/RolloutConfiguration.yaml", + "title": "RolloutConfiguration", + "description": "configuration for the rollout of a connector", + "type": "object", + "additionalProperties": false, + "properties": { + "enableProgressiveRollout": { + "type": "boolean", + "default": false, + "description": "Whether to enable progressive rollout for the connector." + }, + "initialPercentage": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 0, + "description": "The percentage of users that should receive the new version initially." + }, + "maxPercentage": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 50, + "description": "The percentage of users who should receive the release candidate during the test phase before full rollout." + }, + "advanceDelayMinutes": { + "type": "integer", + "minimum": 10, + "default": 10, + "description": "The number of minutes to wait before advancing the rollout percentage." + } + } + }, + "ResourceRequirements": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ResourceRequirements.yaml", + "title": "ResourceRequirements", + "description": "generic configuration for pod source requirements", + "type": "object", + "additionalProperties": false, + "properties": { + "cpu_request": { + "type": "string" + }, + "cpu_limit": { + "type": "string" + }, + "memory_request": { + "type": "string" + }, + "memory_limit": { + "type": "string" + } + } + }, + "RemoteRegistries": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/RemoteRegistries.yml", + "title": "RemoteRegistries", + "description": "describes how the connector is published to remote registries", + "type": "object", + "additionalProperties": false, + "properties": { + "pypi": { + "$ref": "#/definitions/PyPi" + } + }, + "definitions": { + "PyPi": { + "title": "PyPi", + "description": "describes the PyPi publishing options", + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "packageName" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "packageName": { + "type": "string", + "description": "The name of the package on PyPi." + } + } + } + } + }, + "ReleaseStage": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte-platform/blob/main/airbyte-config/config-models/src/main/resources/types/ReleaseStage.yaml", + "title": "ReleaseStage", + "description": "enum that describes a connector's release stage", + "type": "string", + "enum": [ + "alpha", + "beta", + "generally_available", + "custom" + ] + }, + "RegistryOverrides": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/RegistryOverrides.yml", + "title": "RegistryOverrides", + "description": "describes the overrides per registry of a connector", + "type": "object", + "additionalProperties": false, + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "name": { + "type": "string" + }, + "dockerRepository": { + "type": "string" + }, + "dockerImageTag": { + "type": "string" + }, + "supportsDbt": { + "type": "boolean" + }, + "supportsNormalization": { + "type": "boolean" + }, + "license": { + "type": "string" + }, + "documentationUrl": { + "type": "string", + "format": "uri" + }, + "connectorSubtype": { + "type": "string" + }, + "allowedHosts": { + "$ref": "AllowedHosts.yaml" + }, + "normalizationConfig": { + "$ref": "NormalizationDestinationDefinitionConfig.yaml" + }, + "suggestedStreams": { + "$ref": "SuggestedStreams.yaml" + }, + "resourceRequirements": { + "$ref": "ActorDefinitionResourceRequirements.yaml" + } + } + }, + "NormalizationDestinationDefinitionConfig": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/NormalizationDestinationDefinitionConfig.yaml", + "title": "NormalizationDestinationDefinitionConfig", + "description": "describes a normalization config for destination definition", + "type": "object", + "required": [ + "normalizationRepository", + "normalizationTag", + "normalizationIntegrationType" + ], + "additionalProperties": true, + "properties": { + "normalizationRepository": { + "type": "string", + "description": "a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used." + }, + "normalizationTag": { + "type": "string", + "description": "a field indicating the tag of the docker repository to be used for normalization." + }, + "normalizationIntegrationType": { + "type": "string", + "description": "a field indicating the type of integration dialect to use for normalization." + } + } + }, + "JobType": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/JobType.yaml", + "title": "JobType", + "description": "enum that describes the different types of jobs that the platform runs.", + "type": "string", + "enum": [ + "get_spec", + "check_connection", + "discover_schema", + "sync", + "reset_connection", + "connection_updater", + "replicate" + ] + }, + "GitInfo": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/GitInfo.yaml", + "title": "GitInfo", + "description": "Information about the author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.", + "type": "object", + "additionalProperties": false, + "properties": { + "commit_sha": { + "type": "string", + "description": "The git commit sha of the last commit that modified this file." + }, + "commit_timestamp": { + "type": "string", + "format": "date-time", + "description": "The git commit timestamp of the last commit that modified this file." + }, + "commit_author": { + "type": "string", + "description": "The git commit author of the last commit that modified this file." + }, + "commit_author_email": { + "type": "string", + "description": "The git commit author email of the last commit that modified this file." + } + } + }, + "GeneratedFields": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/GeneratedFields.yaml", + "title": "GeneratedFields", + "description": "Optional schema for fields generated at metadata upload time", + "type": "object", + "properties": { + "git": { + "$ref": "GitInfo.yaml" + }, + "source_file_info": { + "$ref": "SourceFileInfo.yaml" + }, + "metrics": { + "$ref": "ConnectorMetrics.yaml" + }, + "sbomUrl": { + "type": "string", + "description": "URL to the SBOM file" + } + } + }, + "ConnectorTestSuiteOptions": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorTestOptions.yaml", + "title": "ConnectorTestSuiteOptions", + "description": "Options for a specific connector test suite.", + "type": "object", + "required": [ + "suite" + ], + "additionalProperties": false, + "properties": { + "suite": { + "description": "Name of the configured test suite", + "type": "string", + "enum": [ + "unitTests", + "integrationTests", + "acceptanceTests", + "liveTests" + ] + }, + "testSecrets": { + "description": "List of secrets required to run the test suite", + "type": "array", + "items": { + "$ref": "Secret.yaml" + } + }, + "testConnections": { + "description": "List of sandbox cloud connections that tests can be run against", + "type": "array", + "items": { + "$ref": "TestConnections.yaml" + } + } + } + }, + "ConnectorReleases": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml", + "title": "ConnectorReleases", + "description": "Contains information about different types of releases for a connector.", + "type": "object", + "additionalProperties": false, + "properties": { + "rolloutConfiguration": { + "$ref": "RolloutConfiguration.yaml" + }, + "breakingChanges": { + "$ref": "ConnectorBreakingChanges.yaml" + }, + "migrationDocumentationUrl": { + "description": "URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + "type": "string", + "format": "uri" + } + } + }, + "ConnectorRegistryV0": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte-platform/blob/main/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryV0.yaml", + "title": "ConnectorRegistryV0", + "description": "describes the collection of connectors retrieved from a registry", + "type": "object", + "required": [ + "destinations", + "sources" + ], + "properties": { + "destinations": { + "type": "array", + "items": { + "$ref": "ConnectorRegistryDestinationDefinition.yaml" + } + }, + "sources": { + "type": "array", + "items": { + "$ref": "ConnectorRegistrySourceDefinition.yaml" + } + } + } + }, + "ConnectorRegistrySourceDefinition": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte-platform/blob/main/airbyte-config/config-models/src/main/resources/types/ConnectorRegistrySourceDefinition.yaml", + "title": "ConnectorRegistrySourceDefinition", + "description": "describes a source", + "type": "object", + "required": [ + "sourceDefinitionId", + "name", + "dockerRepository", + "dockerImageTag", + "documentationUrl", + "spec" + ], + "additionalProperties": true, + "properties": { + "sourceDefinitionId": { + "type": "string", + "format": "uuid" + }, + "name": { + "type": "string" + }, + "dockerRepository": { + "type": "string" + }, + "dockerImageTag": { + "type": "string" + }, + "documentationUrl": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "iconUrl": { + "type": "string" + }, + "sourceType": { + "type": "string", + "enum": [ + "api", + "file", + "database", + "custom" + ] + }, + "spec": { + "type": "object" + }, + "tombstone": { + "description": "if false, the configuration is active. if true, then this configuration is permanently off.", + "type": "boolean", + "default": false + }, + "public": { + "description": "true if this connector definition is available to all workspaces", + "type": "boolean", + "default": false + }, + "custom": { + "description": "whether this is a custom connector definition", + "type": "boolean", + "default": false + }, + "releaseStage": { + "$ref": "ReleaseStage.yaml" + }, + "supportLevel": { + "$ref": "SupportLevel.yaml" + }, + "releaseDate": { + "description": "The date when this connector was first released, in yyyy-mm-dd format.", + "type": "string", + "format": "date" + }, + "resourceRequirements": { + "$ref": "ActorDefinitionResourceRequirements.yaml" + }, + "protocolVersion": { + "type": "string", + "description": "the Airbyte Protocol version supported by the connector" + }, + "allowedHosts": { + "$ref": "AllowedHosts.yaml" + }, + "suggestedStreams": { + "$ref": "SuggestedStreams.yaml" + }, + "maxSecondsBetweenMessages": { + "description": "Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", + "type": "integer" + }, + "erdUrl": { + "type": "string", + "description": "The URL where you can visualize the ERD" + }, + "releases": { + "$ref": "ConnectorRegistryReleases.yaml" + }, + "ab_internal": { + "$ref": "AirbyteInternal.yaml" + }, + "generated": { + "$ref": "GeneratedFields.yaml" + }, + "packageInfo": { + "$ref": "ConnectorPackageInfo.yaml" + }, + "language": { + "type": "string", + "description": "The language the connector is written in" + }, + "supportsFileTransfer": { + "type": "boolean", + "default": false + }, + "supportsDataActivation": { + "type": "boolean", + "default": false + } + } + }, + "ConnectorRegistryReleases": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryReleases.yaml", + "title": "ConnectorRegistryReleases", + "description": "Contains information about different types of releases for a connector.", + "type": "object", + "additionalProperties": false, + "properties": { + "releaseCandidates": { + "$ref": "#/definitions/ConnectorReleaseCandidates" + }, + "rolloutConfiguration": { + "$ref": "RolloutConfiguration.yaml" + }, + "breakingChanges": { + "$ref": "ConnectorBreakingChanges.yaml" + }, + "migrationDocumentationUrl": { + "description": "URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + "type": "string", + "format": "uri" + } + }, + "definitions": { + "ConnectorReleaseCandidates": { + "description": "Each entry denotes a release candidate version of a connector.", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "patternProperties": { + "^\\d+\\.\\d+\\.\\d+(-[0-9A-Za-z-.]+)?$": { + "$ref": "#/definitions/VersionReleaseCandidate" + } + } + }, + "VersionReleaseCandidate": { + "description": "Contains information about a release candidate version of a connector.", + "additionalProperties": false, + "type": "object", + "oneOf": [ + { + "$ref": "ConnectorRegistrySourceDefinition.yaml" + }, + { + "$ref": "ConnectorRegistryDestinationDefinition.yaml" + } + ] + } + } + }, + "ConnectorRegistryDestinationDefinition": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte-platform/blob/main/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryDestinationDefinition.yaml", + "title": "ConnectorRegistryDestinationDefinition", + "description": "describes a destination", + "type": "object", + "required": [ + "destinationDefinitionId", + "name", + "dockerRepository", + "dockerImageTag", + "documentationUrl", + "spec" + ], + "additionalProperties": true, + "properties": { + "destinationDefinitionId": { + "type": "string", + "format": "uuid" + }, + "name": { + "type": "string" + }, + "dockerRepository": { + "type": "string" + }, + "dockerImageTag": { + "type": "string" + }, + "documentationUrl": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "iconUrl": { + "type": "string" + }, + "spec": { + "type": "object" + }, + "tombstone": { + "description": "if false, the configuration is active. if true, then this configuration is permanently off.", + "type": "boolean", + "default": false + }, + "public": { + "description": "true if this connector definition is available to all workspaces", + "type": "boolean", + "default": false + }, + "custom": { + "description": "whether this is a custom connector definition", + "type": "boolean", + "default": false + }, + "releaseStage": { + "$ref": "ReleaseStage.yaml" + }, + "supportLevel": { + "$ref": "SupportLevel.yaml" + }, + "releaseDate": { + "description": "The date when this connector was first released, in yyyy-mm-dd format.", + "type": "string", + "format": "date" + }, + "tags": { + "type": "array", + "description": "An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", + "items": { + "type": "string" + } + }, + "resourceRequirements": { + "$ref": "ActorDefinitionResourceRequirements.yaml" + }, + "protocolVersion": { + "type": "string", + "description": "the Airbyte Protocol version supported by the connector" + }, + "normalizationConfig": { + "$ref": "NormalizationDestinationDefinitionConfig.yaml" + }, + "supportsDbt": { + "type": "boolean", + "description": "an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used." + }, + "allowedHosts": { + "$ref": "AllowedHosts.yaml" + }, + "releases": { + "$ref": "ConnectorRegistryReleases.yaml" + }, + "ab_internal": { + "$ref": "AirbyteInternal.yaml" + }, + "supportsRefreshes": { + "type": "boolean", + "default": false + }, + "supportsFileTransfer": { + "type": "boolean", + "default": false + }, + "supportsDataActivation": { + "type": "boolean", + "default": false + }, + "generated": { + "$ref": "GeneratedFields.yaml" + }, + "packageInfo": { + "$ref": "ConnectorPackageInfo.yaml" + }, + "language": { + "type": "string", + "description": "The language the connector is written in" + } + } + }, + "ConnectorPackageInfo": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorPackageInfo.yaml", + "title": "ConnectorPackageInfo", + "description": "Information about the contents of the connector image", + "type": "object", + "properties": { + "cdk_version": { + "type": "string" + } + } + }, + "ConnectorMetrics": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorMetrics.yaml", + "title": "ConnectorMetrics", + "description": "Information about the source file that generated the registry entry", + "type": "object", + "properties": { + "all": { + "type": "ConnectorMetric" + }, + "cloud": { + "type": "ConnectorMetric" + }, + "oss": { + "type": "ConnectorMetric" + } + }, + "definitions": { + "ConnectorMetric": { + "type": "object", + "properties": { + "usage": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [ + "low", + "medium", + "high" + ] + } + ] + }, + "sync_success_rate": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [ + "low", + "medium", + "high" + ] + } + ] + }, + "connector_version": { + "type": "string" + } + }, + "additionalProperties": true + } + } + }, + "ConnectorIPCOptions": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorIPCOptions.yaml", + "title": "ConnectorIPCOptions", + "type": "object", + "required": [ + "dataChannel" + ], + "additionalProperties": false, + "properties": { + "dataChannel": { + "type": "object", + "required": [ + "version", + "supportedSerialization", + "supportedTransport" + ], + "additionalProperties": false, + "properties": { + "version": { + "type": "string" + }, + "supportedSerialization": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "JSONL", + "PROTOBUF", + "FLATBUFFERS" + ] + } + }, + "supportedTransport": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "STDIO", + "SOCKET" + ] + } + } + } + } + } + }, + "ConnectorBuildOptions": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml", + "title": "ConnectorBuildOptions", + "description": "metadata specific to the build process.", + "type": "object", + "additionalProperties": false, + "properties": { + "baseImage": { + "type": "string" + } + } + }, + "ConnectorBreakingChanges": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBreakingChanges.yaml", + "title": "ConnectorBreakingChanges", + "description": "Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "patternProperties": { + "^\\d+\\.\\d+\\.\\d+$": { + "$ref": "#/definitions/VersionBreakingChange" + } + }, + "definitions": { + "VersionBreakingChange": { + "description": "Contains information about a breaking change, including the deadline to upgrade and a message detailing the change.", + "type": "object", + "additionalProperties": false, + "required": [ + "upgradeDeadline", + "message" + ], + "properties": { + "upgradeDeadline": { + "description": "The deadline by which to upgrade before the breaking change takes effect.", + "type": "string", + "format": "date" + }, + "message": { + "description": "Descriptive message detailing the breaking change.", + "type": "string" + }, + "deadlineAction": { + "description": "Action to do when the deadline is reached.", + "type": "string", + "enum": [ + "auto_upgrade", + "disable" + ] + }, + "migrationDocumentationUrl": { + "description": "URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + "type": "string", + "format": "uri" + }, + "scopedImpact": { + "description": "List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/BreakingChangeScope" + } + } + } + }, + "BreakingChangeScope": { + "description": "A scope that can be used to limit the impact of a breaking change.", + "type": "object", + "oneOf": [ + { + "$ref": "#/definitions/StreamBreakingChangeScope" + } + ] + }, + "StreamBreakingChangeScope": { + "description": "A scope that can be used to limit the impact of a breaking change to specific streams.", + "type": "object", + "additionalProperties": false, + "required": [ + "scopeType", + "impactedScopes" + ], + "properties": { + "scopeType": { + "type": "const", + "const": "stream" + }, + "impactedScopes": { + "description": "List of streams that are impacted by the breaking change.", + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + } + } + } + } + }, + "AllowedHosts": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/AllowedHosts.yaml", + "title": "AllowedHosts", + "description": "A connector's allowed hosts. If present, the platform will limit communication to only hosts which are listed in `AllowedHosts.hosts`.", + "type": "object", + "additionalProperties": true, + "properties": { + "hosts": { + "type": "array", + "description": "An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + "items": { + "type": "string" + } + } + } + }, + "AirbyteInternal": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/AirbyteInternal.yml", + "title": "AirbyteInternal", + "description": "Fields for internal use only", + "type": "object", + "additionalProperties": true, + "properties": { + "sl": { + "type": "integer", + "enum": [ + 0, + 100, + 200, + 300 + ] + }, + "ql": { + "type": "integer", + "enum": [ + 0, + 100, + 200, + 300, + 400, + 500, + 600 + ] + }, + "isEnterprise": { + "type": "boolean", + "default": false + }, + "requireVersionIncrementsInPullRequests": { + "type": "boolean", + "default": true, + "description": "When false, version increment checks will be skipped for this connector" + } + } + }, + "ActorDefinitionResourceRequirements": { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ActorDefinitionResourceRequirements.yaml", + "title": "ActorDefinitionResourceRequirements", + "description": "actor definition specific resource requirements", + "type": "object", + "additionalProperties": false, + "properties": { + "default": { + "description": "if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + "$ref": "ResourceRequirements.yaml" + }, + "jobSpecific": { + "type": "array", + "items": { + "$ref": "#/definitions/JobTypeResourceLimit" + } + } + }, + "definitions": { + "JobTypeResourceLimit": { + "description": "sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set.", + "type": "object", + "additionalProperties": false, + "required": [ + "jobType", + "resourceRequirements" + ], + "properties": { + "jobType": { + "$ref": "JobType.yaml" + }, + "resourceRequirements": { + "$ref": "ResourceRequirements.yaml" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/airbyte_cdk/test/models/connector_metadata/generated/models.py b/airbyte_cdk/test/models/connector_metadata/generated/models.py new file mode 100644 index 000000000..1d63c17f5 --- /dev/null +++ b/airbyte_cdk/test/models/connector_metadata/generated/models.py @@ -0,0 +1,665 @@ +from __future__ import annotations + +from datetime import date +from datetime import datetime +from enum import Enum +from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional +from typing import Any, Optional, Union +from typing import Dict, Optional, Union +from typing import List +from typing import List, Optional +from typing import Literal, Optional +from typing import Optional +from uuid import UUID + +from pydantic.v1 import AnyUrl, BaseModel, Extra +from pydantic.v1 import AnyUrl, BaseModel, Extra, Field +from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, constr +from pydantic.v1 import BaseModel +from pydantic.v1 import BaseModel, Extra +from pydantic.v1 import BaseModel, Extra, Field +from pydantic.v1 import BaseModel, Extra, Field, conint +from pydantic.v1 import BaseModel, Field + + +class JobTypeResourceLimit(BaseModel): + class Config: + extra = Extra.forbid + + jobType: JobType + resourceRequirements: ResourceRequirements + + +class ActorDefinitionResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + default: Optional[ResourceRequirements] = Field( + None, + description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + ) + jobSpecific: Optional[List[JobTypeResourceLimit]] = None + + +class Sl(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + + +class Ql(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + integer_400 = 400 + integer_500 = 500 + integer_600 = 600 + + +class AirbyteInternal(BaseModel): + class Config: + extra = Extra.allow + + sl: Optional[Sl] = None + ql: Optional[Ql] = None + isEnterprise: Optional[bool] = False + requireVersionIncrementsInPullRequests: Optional[bool] = Field( + True, + description="When false, version increment checks will be skipped for this connector", + ) + + +class AllowedHosts(BaseModel): + class Config: + extra = Extra.allow + + hosts: Optional[List[str]] = Field( + None, + description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + ) + + +class DeadlineAction(Enum): + auto_upgrade = "auto_upgrade" + disable = "disable" + + +class StreamBreakingChangeScope(BaseModel): + class Config: + extra = Extra.forbid + + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( + ..., + description="List of streams that are impacted by the breaking change.", + min_items=1, + ) + + +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( + ..., + description="A scope that can be used to limit the impact of a breaking change.", + ) + + +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field(..., description="Descriptive message detailing the breaking change.") + deadlineAction: Optional[DeadlineAction] = Field( + None, description="Action to do when the deadline is reached." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + title="ConnectorBreakingChanges", + ) + + +class ConnectorBuildOptions(BaseModel): + class Config: + extra = Extra.forbid + + baseImage: Optional[str] = None + + +class SupportedSerializationEnum(Enum): + JSONL = "JSONL" + PROTOBUF = "PROTOBUF" + FLATBUFFERS = "FLATBUFFERS" + + +class SupportedTransportEnum(Enum): + STDIO = "STDIO" + SOCKET = "SOCKET" + + +class DataChannel(BaseModel): + class Config: + extra = Extra.forbid + + version: str + supportedSerialization: List[SupportedSerializationEnum] + supportedTransport: List[SupportedTransportEnum] + + +class ConnectorIPCOptions(BaseModel): + class Config: + extra = Extra.forbid + + dataChannel: DataChannel + + +class ConnectorType(Enum): + destination = "destination" + source = "source" + + +class ConnectorSubtype(Enum): + api = "api" + database = "database" + datalake = "datalake" + file = "file" + custom = "custom" + message_queue = "message_queue" + unknown = "unknown" + vectorstore = "vectorstore" + + +class RegistryOverrides(BaseModel): + class Config: + extra = Extra.forbid + + oss: Optional[RegistryOverrides_1] = None + cloud: Optional[RegistryOverrides_1] = None + + +class Data(BaseModel): + class Config: + extra = Extra.forbid + + name: str + icon: Optional[str] = None + definitionId: UUID + connectorBuildOptions: Optional[ConnectorBuildOptions] = None + connectorTestSuitesOptions: Optional[List[ConnectorTestSuiteOptions]] = None + connectorType: ConnectorType + dockerRepository: str + dockerImageTag: str + supportsDbt: Optional[bool] = None + supportsNormalization: Optional[bool] = None + license: str + documentationUrl: AnyUrl + githubIssueLabel: str + maxSecondsBetweenMessages: Optional[int] = Field( + None, + description="Maximum delay between 2 airbyte protocol messages, in second. The source will timeout if this delay is reached", + ) + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + erdUrl: Optional[str] = Field(None, description="The URL where you can visualize the ERD") + connectorSubtype: ConnectorSubtype + releaseStage: ReleaseStage + supportLevel: Optional[SupportLevel] = None + tags: Optional[List[str]] = Field( + [], + description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", + ) + registryOverrides: Optional[RegistryOverrides] = None + allowedHosts: Optional[AllowedHosts] = None + releases: Optional[ConnectorReleases] = None + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + suggestedStreams: Optional[SuggestedStreams] = None + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + ab_internal: Optional[AirbyteInternal] = None + remoteRegistries: Optional[RemoteRegistries] = None + supportsRefreshes: Optional[bool] = False + generated: Optional[GeneratedFields] = None + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + connectorIPCOptions: Optional[ConnectorIPCOptions] = None + + +class ConnectorMetadataDefinitionV0(BaseModel): + class Config: + extra = Extra.forbid + + metadataSpecVersion: str + data: Data + + +class ConnectorMetrics(BaseModel): + all: Optional[Any] = None + cloud: Optional[Any] = None + oss: Optional[Any] = None + + +class Usage(Enum): + low = "low" + medium = "medium" + high = "high" + + +class SyncSuccessRate(Enum): + low = "low" + medium = "medium" + high = "high" + + +class ConnectorMetric(BaseModel): + class Config: + extra = Extra.allow + + usage: Optional[Union[str, Usage]] = None + sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None + connector_version: Optional[str] = None + + +class ConnectorPackageInfo(BaseModel): + cdk_version: Optional[str] = None + + +class ConnectorRegistryDestinationDefinition(BaseModel): + class Config: + extra = Extra.allow + + destinationDefinitionId: UUID + name: str + dockerRepository: str + dockerImageTag: str + documentationUrl: str + icon: Optional[str] = None + iconUrl: Optional[str] = None + spec: Dict[str, Any] + tombstone: Optional[bool] = Field( + False, + description="if false, the configuration is active. if true, then this configuration is permanently off.", + ) + public: Optional[bool] = Field( + False, + description="true if this connector definition is available to all workspaces", + ) + custom: Optional[bool] = Field( + False, description="whether this is a custom connector definition" + ) + releaseStage: Optional[ReleaseStage] = None + supportLevel: Optional[SupportLevel] = None + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + tags: Optional[List[str]] = Field( + None, + description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", + ) + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + supportsDbt: Optional[bool] = Field( + None, + description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", + ) + allowedHosts: Optional[AllowedHosts] = None + releases: Optional[ConnectorRegistryReleases] = None + ab_internal: Optional[AirbyteInternal] = None + supportsRefreshes: Optional[bool] = False + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + generated: Optional[GeneratedFields] = None + packageInfo: Optional[ConnectorPackageInfo] = None + language: Optional[str] = Field(None, description="The language the connector is written in") + + +ConnectorRegistryDestinationDefinition.update_forward_refs() + + +class ConnectorRegistryReleases(BaseModel): + class Config: + extra = Extra.forbid + + releaseCandidates: Optional[ConnectorReleaseCandidates] = None + rolloutConfiguration: Optional[RolloutConfiguration] = None + breakingChanges: Optional[ConnectorBreakingChanges] = None + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + ) + + +class ConnectorReleaseCandidates(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+(-[0-9A-Za-z-.]+)?$"), VersionReleaseCandidate] = ( + Field( + ..., + description="Each entry denotes a release candidate version of a connector.", + ) + ) + + +class VersionReleaseCandidate(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Union[ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition] = ( + Field( + ..., + description="Contains information about a release candidate version of a connector.", + ) + ) + + +ConnectorRegistryReleases.update_forward_refs() +ConnectorReleaseCandidates.update_forward_refs() +VersionReleaseCandidate.update_forward_refs() + + +class SourceType(Enum): + api = "api" + file = "file" + database = "database" + custom = "custom" + + +class ConnectorRegistrySourceDefinition(BaseModel): + class Config: + extra = Extra.allow + + sourceDefinitionId: UUID + name: str + dockerRepository: str + dockerImageTag: str + documentationUrl: str + icon: Optional[str] = None + iconUrl: Optional[str] = None + sourceType: Optional[SourceType] = None + spec: Dict[str, Any] + tombstone: Optional[bool] = Field( + False, + description="if false, the configuration is active. if true, then this configuration is permanently off.", + ) + public: Optional[bool] = Field( + False, + description="true if this connector definition is available to all workspaces", + ) + custom: Optional[bool] = Field( + False, description="whether this is a custom connector definition" + ) + releaseStage: Optional[ReleaseStage] = None + supportLevel: Optional[SupportLevel] = None + releaseDate: Optional[date] = Field( + None, + description="The date when this connector was first released, in yyyy-mm-dd format.", + ) + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + protocolVersion: Optional[str] = Field( + None, description="the Airbyte Protocol version supported by the connector" + ) + allowedHosts: Optional[AllowedHosts] = None + suggestedStreams: Optional[SuggestedStreams] = None + maxSecondsBetweenMessages: Optional[int] = Field( + None, + description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", + ) + erdUrl: Optional[str] = Field(None, description="The URL where you can visualize the ERD") + releases: Optional[ConnectorRegistryReleases] = None + ab_internal: Optional[AirbyteInternal] = None + generated: Optional[GeneratedFields] = None + packageInfo: Optional[ConnectorPackageInfo] = None + language: Optional[str] = Field(None, description="The language the connector is written in") + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False + + +class ConnectorRegistryV0(BaseModel): + destinations: List[ConnectorRegistryDestinationDefinition] + sources: List[ConnectorRegistrySourceDefinition] + + +class ConnectorReleases(BaseModel): + class Config: + extra = Extra.forbid + + rolloutConfiguration: Optional[RolloutConfiguration] = None + breakingChanges: Optional[ConnectorBreakingChanges] = None + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + ) + + +class Suite(Enum): + unitTests = "unitTests" + integrationTests = "integrationTests" + acceptanceTests = "acceptanceTests" + liveTests = "liveTests" + + +class ConnectorTestSuiteOptions(BaseModel): + class Config: + extra = Extra.forbid + + suite: Suite = Field(..., description="Name of the configured test suite") + testSecrets: Optional[List[Secret]] = Field( + None, description="List of secrets required to run the test suite" + ) + testConnections: Optional[List[TestConnections]] = Field( + None, + description="List of sandbox cloud connections that tests can be run against", + ) + + +class GeneratedFields(BaseModel): + git: Optional[GitInfo] = None + source_file_info: Optional[SourceFileInfo] = None + metrics: Optional[ConnectorMetrics] = None + sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") + + +class GitInfo(BaseModel): + class Config: + extra = Extra.forbid + + commit_sha: Optional[str] = Field( + None, + description="The git commit sha of the last commit that modified this file.", + ) + commit_timestamp: Optional[datetime] = Field( + None, + description="The git commit timestamp of the last commit that modified this file.", + ) + commit_author: Optional[str] = Field( + None, + description="The git commit author of the last commit that modified this file.", + ) + commit_author_email: Optional[str] = Field( + None, + description="The git commit author email of the last commit that modified this file.", + ) + + +class JobType(Enum): + get_spec = "get_spec" + check_connection = "check_connection" + discover_schema = "discover_schema" + sync = "sync" + reset_connection = "reset_connection" + connection_updater = "connection_updater" + replicate = "replicate" + + +class NormalizationDestinationDefinitionConfig(BaseModel): + class Config: + extra = Extra.allow + + normalizationRepository: str = Field( + ..., + description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", + ) + normalizationTag: str = Field( + ..., + description="a field indicating the tag of the docker repository to be used for normalization.", + ) + normalizationIntegrationType: str = Field( + ..., + description="a field indicating the type of integration dialect to use for normalization.", + ) + + +class RegistryOverrides(BaseModel): + class Config: + extra = Extra.forbid + + enabled: bool + name: Optional[str] = None + dockerRepository: Optional[str] = None + dockerImageTag: Optional[str] = None + supportsDbt: Optional[bool] = None + supportsNormalization: Optional[bool] = None + license: Optional[str] = None + documentationUrl: Optional[AnyUrl] = None + connectorSubtype: Optional[str] = None + allowedHosts: Optional[AllowedHosts] = None + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + suggestedStreams: Optional[SuggestedStreams] = None + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + + +class ReleaseStage(Enum): + alpha = "alpha" + beta = "beta" + generally_available = "generally_available" + custom = "custom" + + +class PyPi(BaseModel): + class Config: + extra = Extra.forbid + + enabled: bool + packageName: str = Field(..., description="The name of the package on PyPi.") + + +class RemoteRegistries(BaseModel): + class Config: + extra = Extra.forbid + + pypi: Optional[PyPi] = None + + +class ResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + cpu_request: Optional[str] = None + cpu_limit: Optional[str] = None + memory_request: Optional[str] = None + memory_limit: Optional[str] = None + + +class RolloutConfiguration(BaseModel): + class Config: + extra = Extra.forbid + + enableProgressiveRollout: Optional[bool] = Field( + False, description="Whether to enable progressive rollout for the connector." + ) + initialPercentage: Optional[conint(ge=0, le=100)] = Field( + 0, + description="The percentage of users that should receive the new version initially.", + ) + maxPercentage: Optional[conint(ge=0, le=100)] = Field( + 50, + description="The percentage of users who should receive the release candidate during the test phase before full rollout.", + ) + advanceDelayMinutes: Optional[conint(ge=10)] = Field( + 10, + description="The number of minutes to wait before advancing the rollout percentage.", + ) + + +class Secret(BaseModel): + class Config: + extra = Extra.forbid + + name: str = Field(..., description="The secret name in the secret store") + fileName: Optional[str] = Field( + None, + description="The name of the file to which the secret value would be persisted", + ) + secretStore: SecretStore + + +class SecretStore(BaseModel): + class Config: + extra = Extra.forbid + + alias: Optional[str] = Field( + None, + description="The alias of the secret store which can map to its actual secret address", + ) + type: Optional[Literal["GSM"]] = Field(None, description="The type of the secret store") + + +class SourceFileInfo(BaseModel): + metadata_etag: Optional[str] = None + metadata_file_path: Optional[str] = None + metadata_bucket_name: Optional[str] = None + metadata_last_modified: Optional[str] = None + registry_entry_generated_at: Optional[str] = None + + +class SuggestedStreams(BaseModel): + class Config: + extra = Extra.allow + + streams: Optional[List[str]] = Field( + None, + description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", + ) + + +class SupportLevel(Enum): + community = "community" + certified = "certified" + archived = "archived" + + +class TestConnections(BaseModel): + class Config: + extra = Extra.forbid + + name: str = Field(..., description="The connection name") + id: str = Field(..., description="The connection ID") diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index fc86fa961..ecabf3b89 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -1,8 +1,8 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. import json -import os import re +import subprocess import sys import tempfile from glob import glob @@ -10,15 +10,11 @@ import anyio import dagger -import httpx import yaml PYTHON_IMAGE = "python:3.10" LOCAL_YAML_DIR_PATH = "airbyte_cdk/sources/declarative" LOCAL_OUTPUT_DIR_PATH = "airbyte_cdk/sources/declarative/models" - -METADATA_SCHEMAS_GITHUB_URL = "https://api.github.com/repos/airbytehq/airbyte/contents/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" -METADATA_SCHEMAS_RAW_URL_BASE = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" LOCAL_METADATA_OUTPUT_DIR_PATH = "airbyte_cdk/test/models/connector_metadata/generated" PIP_DEPENDENCIES = [ @@ -41,45 +37,40 @@ def generate_init_module_content(yaml_files: list[str]) -> str: return header -async def download_metadata_schemas(temp_dir: Path) -> list[str]: - """Download metadata schema YAML files from GitHub to a temporary directory.""" - token = os.getenv("GITHUB_TOKEN") or os.getenv("GH_TOKEN") - headers = { - "User-Agent": "airbyte-python-cdk-build", - "Accept": "application/vnd.github.v3+json", - } - if token: - headers["Authorization"] = f"Bearer {token}" - - async with httpx.AsyncClient(headers=headers, timeout=30.0) as client: - try: - response = await client.get(METADATA_SCHEMAS_GITHUB_URL) - response.raise_for_status() - files_info = response.json() - except httpx.HTTPStatusError as e: - if e.response.status_code == 403: - print( - "Warning: GitHub API rate limit exceeded. Provide GITHUB_TOKEN to authenticate.", - file=sys.stderr, - ) - raise - raise - - yaml_files = [] - for file_info in files_info: - if file_info["name"].endswith(".yaml"): - file_name = file_info["name"] - file_url = f"{METADATA_SCHEMAS_RAW_URL_BASE}/{file_name}" - - print(f"Downloading {file_name}...", file=sys.stderr) - file_response = await client.get(file_url) - file_response.raise_for_status() - - file_path = temp_dir / file_name - file_path.write_text(file_response.text) - yaml_files.append(Path(file_name).stem) - - return yaml_files +def clone_metadata_schemas(temp_dir: Path) -> Path: + """Clone metadata schema YAML files from GitHub using sparse checkout.""" + repo_url = "https://github.com/airbytehq/airbyte.git" + schema_path = "airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" + + clone_dir = temp_dir / "airbyte" + + print("Cloning metadata schemas from airbyte repo...", file=sys.stderr) + + subprocess.run( + [ + "git", + "clone", + "--depth", + "1", + "--filter=blob:none", + "--sparse", + repo_url, + str(clone_dir), + ], + check=True, + capture_output=True, + ) + + subprocess.run( + ["git", "-C", str(clone_dir), "sparse-checkout", "set", schema_path], + check=True, + capture_output=True, + ) + + schemas_dir = clone_dir / schema_path + print(f"Cloned schemas to {schemas_dir}", file=sys.stderr) + + return schemas_dir def replace_base_model_for_classes_with_deprecated_fields(post_processed_content: str) -> str: @@ -282,7 +273,7 @@ async def generate_metadata_models_single_file( codegen_container = ( dagger_client.container() .from_(PYTHON_IMAGE) - .with_exec(["mkdir", "-p", "/generated"], use_entrypoint=True) + .with_exec(["mkdir", "-p", "/generated_temp"], use_entrypoint=True) .with_exec(["pip", "install", " ".join(PIP_DEPENDENCIES)], use_entrypoint=True) .with_mounted_directory( "/yaml", dagger_client.host().directory(yaml_dir_path, include=["*.yaml"]) @@ -295,7 +286,7 @@ async def generate_metadata_models_single_file( "--input", "/yaml", "--output", - "/generated/models.py", + "/generated_temp", "--disable-timestamp", "--enum-field-as-literal", "one", @@ -309,14 +300,88 @@ async def generate_metadata_models_single_file( use_entrypoint=True, ) - original_content = await codegen_container.file("/generated/models.py").contents() - post_processed_content = original_content.replace("from pydantic", "from pydantic.v1") + generated_files = await codegen_container.directory("/generated_temp").entries() + + future_imports = set() + stdlib_imports = set() + third_party_imports = set() + classes_and_updates = [] + + for file_name in sorted(generated_files): + if file_name.endswith(".py") and file_name != "__init__.py": + content = await codegen_container.file(f"/generated_temp/{file_name}").contents() + + lines = content.split("\n") + in_imports = True + in_relative_import_block = False + class_content = [] + + for line in lines: + if in_imports: + if line.startswith("from __future__"): + future_imports.add(line) + elif ( + line.startswith("from datetime") + or line.startswith("from enum") + or line.startswith("from typing") + or line.startswith("from uuid") + ): + stdlib_imports.add(line) + elif line.startswith("from pydantic") or line.startswith("import "): + third_party_imports.add(line) + elif line.startswith("from ."): + in_relative_import_block = True + if not line.rstrip().endswith(",") and not line.rstrip().endswith("("): + in_relative_import_block = False + elif in_relative_import_block: + if line.strip().endswith(")"): + in_relative_import_block = False + elif line.strip() and not line.startswith("#"): + in_imports = False + class_content.append(line) + else: + class_content.append(line) + + if class_content: + classes_and_updates.append("\n".join(class_content)) + + import_sections = [] + if future_imports: + import_sections.append("\n".join(sorted(future_imports))) + if stdlib_imports: + import_sections.append("\n".join(sorted(stdlib_imports))) + if third_party_imports: + import_sections.append("\n".join(sorted(third_party_imports))) + + final_content = "\n\n".join(import_sections) + "\n\n\n" + "\n\n\n".join(classes_and_updates) + + post_processed_content = final_content.replace("from pydantic", "from pydantic.v1") + + lines = post_processed_content.split("\n") + filtered_lines = [] + in_relative_import = False + + for line in lines: + if line.strip().startswith("from . import"): + in_relative_import = True + if not line.rstrip().endswith(",") and not line.rstrip().endswith("("): + in_relative_import = False + continue + + if in_relative_import: + if line.strip().endswith(")"): + in_relative_import = False + continue + + filtered_lines.append(line) + + post_processed_content = "\n".join(filtered_lines) codegen_container = codegen_container.with_new_file( - "/generated/models_processed.py", contents=post_processed_content + "/generated/models.py", contents=post_processed_content ) - await codegen_container.file("/generated/models_processed.py").export(output_file_path) + await codegen_container.file("/generated/models.py").export(output_file_path) async def main(): @@ -334,7 +399,7 @@ async def main(): print("\nGenerating metadata models...", file=sys.stderr) with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) - await download_metadata_schemas(temp_path) + schemas_dir = clone_metadata_schemas(temp_path) output_dir = Path(LOCAL_METADATA_OUTPUT_DIR_PATH) output_dir.mkdir(parents=True, exist_ok=True) @@ -343,13 +408,13 @@ async def main(): output_file = str(output_dir / "models.py") await generate_metadata_models_single_file( dagger_client=dagger_client, - yaml_dir_path=str(temp_path), + yaml_dir_path=str(schemas_dir), output_file_path=output_file, ) print("Generating consolidated JSON schema...", file=sys.stderr) json_schema_file = str(output_dir / "metadata_schema.json") - consolidate_yaml_schemas_to_json(temp_path, json_schema_file) + consolidate_yaml_schemas_to_json(schemas_dir, json_schema_file) print("\nModel generation complete!", file=sys.stderr) From da4371f4094ab88bca04bae942dce7edb1185b87 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:26:41 +0000 Subject: [PATCH 11/20] Revert accidental formatting of generated file declarative_component_schema.py Co-Authored-By: AJ Steers --- .../models/declarative_component_schema.py | 146 +++++++----------- 1 file changed, 60 insertions(+), 86 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 421c6779f..35186ef71 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. + # generated by datamodel-codegen: # filename: declarative_component_schema.yaml @@ -926,28 +928,24 @@ class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( - Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", - examples=[ - {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, - { - "app_id": { - "type": "string", - "path_in_connector_config": ["info", "app_id"], - } - }, - ], - title="OAuth user input", - ) + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + examples=[ + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, + { + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], + } + }, + ], + title="OAuth user input", ) - oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = ( - Field( - None, - description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', - title="DeclarativeOAuth Connector Specification", - ) + oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( + None, + description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', + title="DeclarativeOAuth Connector Specification", ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, @@ -965,9 +963,7 @@ class Config: complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", - examples=[ - {"client_id": {"type": "string"}, "client_secret": {"type": "string"}} - ], + examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( @@ -1471,9 +1467,7 @@ class CustomConfigTransformation(BaseModel): class_name: str = Field( ..., description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_..`.", - examples=[ - "source_declarative_manifest.components.MyCustomConfigTransformation" - ], + examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], ) parameters: Optional[Dict[str, Any]] = Field( None, @@ -1891,9 +1885,7 @@ class OAuthAuthenticator(BaseModel): scopes: Optional[List[str]] = Field( None, description="List of scopes that should be granted to the access token.", - examples=[ - ["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"] - ], + examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], title="Scopes", ) token_expiry_date: Optional[str] = Field( @@ -2092,9 +2084,7 @@ class RecordSelector(BaseModel): description="Responsible for filtering records to be emitted by the Source.", title="Record Filter", ) - schema_normalization: Optional[ - Union[SchemaNormalization, CustomSchemaNormalization] - ] = Field( + schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( None, description="Responsible for normalization according to the schema.", title="Schema Normalization", @@ -2136,12 +2126,10 @@ class DpathValidator(BaseModel): ], title="Field Path", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( - Field( - ..., - description="The condition that the specified config value will be evaluated against", - title="Validation Strategy", - ) + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( + ..., + description="The condition that the specified config value will be evaluated against", + title="Validation Strategy", ) @@ -2158,12 +2146,10 @@ class PredicateValidator(BaseModel): ], title="Value", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( - Field( - ..., - description="The validation strategy to apply to the value.", - title="Validation Strategy", - ) + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( + ..., + description="The validation strategy to apply to the value.", + title="Validation Strategy", ) @@ -2188,12 +2174,12 @@ class ConfigAddFields(BaseModel): class CompositeErrorHandler(BaseModel): type: Literal["CompositeErrorHandler"] - error_handlers: List[ - Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler] - ] = Field( - ..., - description="List of error handlers to iterate on to determine how to handle a failed response.", - title="Error Handlers", + error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( + Field( + ..., + description="List of error handlers to iterate on to determine how to handle a failed response.", + title="Error Handlers", + ) ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -2355,9 +2341,9 @@ class Config: type: Literal["DeclarativeSource"] check: Union[CheckStream, CheckDynamicStream] - streams: Optional[ - List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] - ] = None + streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( + None + ) dynamic_streams: List[DynamicDeclarativeStream] version: str = Field( ..., @@ -2482,20 +2468,16 @@ class Config: extra = Extra.allow type: Literal["DeclarativeStream"] - name: Optional[str] = Field( - "", description="The stream name.", example=["Users"], title="Name" - ) + name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages.", title="Retriever", ) - incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = ( - Field( - None, - description="Component used to fetch data incrementally based on a time field in the data.", - title="Incremental Sync", - ) + incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( + None, + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", ) primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") schema_loader: Optional[ @@ -2669,20 +2651,18 @@ class HttpRequester(BaseModelWithDeprecations): description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", title="Query Properties", ) - request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = ( - Field( - None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", - examples=[ - {"unit": "day"}, - { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, - ], - title="Query Parameters", - ) + request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( + None, + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + examples=[ + {"unit": "day"}, + { + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, + ], + title="Query Parameters", ) request_headers: Optional[Union[Dict[str, str], str]] = Field( None, @@ -2854,9 +2834,7 @@ class QueryProperties(BaseModel): class StateDelegatingStream(BaseModel): type: Literal["StateDelegatingStream"] - name: str = Field( - ..., description="The stream name.", example=["Users"], title="Name" - ) + name: str = Field(..., description="The stream name.", example=["Users"], title="Name") full_refresh_stream: DeclarativeStream = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", @@ -2943,17 +2921,13 @@ class AsyncRetriever(BaseModel): status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( ..., description="Responsible for fetching the actual status of the async job." ) - download_target_extractor: Optional[ - Union[DpathExtractor, CustomRecordExtractor] - ] = Field( + download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( None, description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", ) download_extractor: Optional[ Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] - ] = Field( - None, description="Responsible for fetching the records from provided urls." - ) + ] = Field(None, description="Responsible for fetching the records from provided urls.") creation_requester: Union[HttpRequester, CustomRequester] = Field( ..., description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", From 5373480da8d0780e647db0f868137c0811291996 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:29:15 +0000 Subject: [PATCH 12/20] Add exclusions for auto-generated files in ruff and pre-commit configs - Exclude declarative_component_schema.py from formatting/linting - Exclude connector_metadata generated models from formatting/linting - Prevents accidental reformatting of auto-generated code Co-Authored-By: AJ Steers --- .pre-commit-config.yaml | 5 ++++- ruff.toml | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8414c5fe8..c5789dea9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,10 @@ exclude: | ^.*?/build/.*$| ^.*?/dist/.*$| ^.*?/\.coverage$| - ^.*?/coverage\.xml$ + ^.*?/coverage\.xml$| + + ^airbyte_cdk/sources/declarative/models/declarative_component_schema\.py$| + ^airbyte_cdk/test/models/connector_metadata/generated/models\.py$ ) repos: diff --git a/ruff.toml b/ruff.toml index 5ed2f45e2..fca658f77 100644 --- a/ruff.toml +++ b/ruff.toml @@ -2,5 +2,10 @@ target-version = "py310" line-length = 100 +exclude = [ + "airbyte_cdk/sources/declarative/models/declarative_component_schema.py", + "airbyte_cdk/test/models/connector_metadata/generated/models.py", +] + [lint] select = ["I"] From 7e4e3f41eb9724e3520e3403533b57128b52b14b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:42:35 +0000 Subject: [PATCH 13/20] Fix JSON schema consolidation to properly resolve references - Remove $id fields from definitions to ensure refs resolve to root - Fix .yaml refs to use #/definitions/ format - Replace custom type references with $ref - Skip invalid 'type: const' entries - Merge nested definitions into top-level definitions Co-Authored-By: AJ Steers --- .../generated/metadata_schema.json | 490 ++++++++---------- .../connector_metadata/generated/models.py | 73 ++- bin/generate_component_manifest_files.py | 43 +- 3 files changed, 323 insertions(+), 283 deletions(-) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json b/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json index 63e0137c2..2e55f8f7f 100644 --- a/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json +++ b/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json @@ -40,12 +40,12 @@ "format": "uuid" }, "connectorBuildOptions": { - "$ref": "ConnectorBuildOptions.yaml" + "$ref": "#/definitions/ConnectorBuildOptions" }, "connectorTestSuitesOptions": { "type": "array", "items": { - "$ref": "ConnectorTestSuiteOptions.yaml" + "$ref": "#/definitions/ConnectorTestSuiteOptions" } }, "connectorType": { @@ -108,10 +108,10 @@ ] }, "releaseStage": { - "$ref": "ReleaseStage.yaml" + "$ref": "#/definitions/ReleaseStage" }, "supportLevel": { - "$ref": "SupportLevel.yaml" + "$ref": "#/definitions/SupportLevel" }, "tags": { "type": "array", @@ -130,14 +130,14 @@ "oss": { "anyOf": [ { - "$ref": "RegistryOverrides.yaml" + "$ref": "#/definitions/RegistryOverrides" } ] }, "cloud": { "anyOf": [ { - "$ref": "RegistryOverrides.yaml" + "$ref": "#/definitions/RegistryOverrides" } ] } @@ -146,32 +146,32 @@ ] }, "allowedHosts": { - "$ref": "AllowedHosts.yaml" + "$ref": "#/definitions/AllowedHosts" }, "releases": { - "$ref": "ConnectorReleases.yaml" + "$ref": "#/definitions/ConnectorReleases" }, "normalizationConfig": { - "$ref": "NormalizationDestinationDefinitionConfig.yaml" + "$ref": "#/definitions/NormalizationDestinationDefinitionConfig" }, "suggestedStreams": { - "$ref": "SuggestedStreams.yaml" + "$ref": "#/definitions/SuggestedStreams" }, "resourceRequirements": { - "$ref": "ActorDefinitionResourceRequirements.yaml" + "$ref": "#/definitions/ActorDefinitionResourceRequirements" }, "ab_internal": { - "$ref": "AirbyteInternal.yaml" + "$ref": "#/definitions/AirbyteInternal" }, "remoteRegistries": { - "$ref": "RemoteRegistries.yaml" + "$ref": "#/definitions/RemoteRegistries" }, "supportsRefreshes": { "type": "boolean", "default": false }, "generated": { - "$ref": "GeneratedFields.yaml" + "$ref": "#/definitions/GeneratedFields" }, "supportsFileTransfer": { "type": "boolean", @@ -182,7 +182,7 @@ "default": false }, "connectorIPCOptions": { - "$ref": "ConnectorIPCOptions.yaml" + "$ref": "#/definitions/ConnectorIPCOptions" } } } @@ -190,7 +190,6 @@ "definitions": { "TestConnections": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestConnections.yaml", "title": "TestConnections", "description": "List of sandbox cloud connections that tests can be run against", "type": "object", @@ -212,7 +211,6 @@ }, "SupportLevel": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/SupportLevel.yaml", "title": "SupportLevel", "description": "enum that describes a connector's release stage", "type": "string", @@ -224,7 +222,6 @@ }, "SuggestedStreams": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/SuggestedStreams.yaml", "title": "SuggestedStreams", "description": "A source's suggested streams. These will be suggested by default for new connections using this source. Otherwise, all streams will be selected. This is useful for when your source has a lot of streams, but the average user will only want a subset of them synced.", "type": "object", @@ -241,7 +238,6 @@ }, "SourceFileInfo": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/SourceFileInfo.yaml", "title": "SourceFileInfo", "description": "Information about the source file that generated the registry entry", "type": "object", @@ -265,7 +261,6 @@ }, "SecretStore": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestSecret.yaml", "title": "SecretStore", "description": "An object describing a secret store metadata", "type": "object", @@ -290,7 +285,6 @@ }, "Secret": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/TestSecret.yaml", "title": "Secret", "description": "An object describing a secret's metadata", "type": "object", @@ -309,13 +303,12 @@ "description": "The name of the file to which the secret value would be persisted" }, "secretStore": { - "$ref": "SecretStore.yaml" + "$ref": "#/definitions/SecretStore" } } }, "RolloutConfiguration": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/RolloutConfiguration.yaml", "title": "RolloutConfiguration", "description": "configuration for the rollout of a connector", "type": "object", @@ -350,7 +343,6 @@ }, "ResourceRequirements": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ResourceRequirements.yaml", "title": "ResourceRequirements", "description": "generic configuration for pod source requirements", "type": "object", @@ -370,9 +362,27 @@ } } }, + "PyPi": { + "title": "PyPi", + "description": "describes the PyPi publishing options", + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "packageName" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "packageName": { + "type": "string", + "description": "The name of the package on PyPi." + } + } + }, "RemoteRegistries": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/RemoteRegistries.yml", "title": "RemoteRegistries", "description": "describes how the connector is published to remote registries", "type": "object", @@ -381,32 +391,10 @@ "pypi": { "$ref": "#/definitions/PyPi" } - }, - "definitions": { - "PyPi": { - "title": "PyPi", - "description": "describes the PyPi publishing options", - "type": "object", - "additionalProperties": false, - "required": [ - "enabled", - "packageName" - ], - "properties": { - "enabled": { - "type": "boolean" - }, - "packageName": { - "type": "string", - "description": "The name of the package on PyPi." - } - } - } } }, "ReleaseStage": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte-platform/blob/main/airbyte-config/config-models/src/main/resources/types/ReleaseStage.yaml", "title": "ReleaseStage", "description": "enum that describes a connector's release stage", "type": "string", @@ -419,7 +407,6 @@ }, "RegistryOverrides": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/RegistryOverrides.yml", "title": "RegistryOverrides", "description": "describes the overrides per registry of a connector", "type": "object", @@ -458,22 +445,21 @@ "type": "string" }, "allowedHosts": { - "$ref": "AllowedHosts.yaml" + "$ref": "#/definitions/AllowedHosts" }, "normalizationConfig": { - "$ref": "NormalizationDestinationDefinitionConfig.yaml" + "$ref": "#/definitions/NormalizationDestinationDefinitionConfig" }, "suggestedStreams": { - "$ref": "SuggestedStreams.yaml" + "$ref": "#/definitions/SuggestedStreams" }, "resourceRequirements": { - "$ref": "ActorDefinitionResourceRequirements.yaml" + "$ref": "#/definitions/ActorDefinitionResourceRequirements" } } }, "NormalizationDestinationDefinitionConfig": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/NormalizationDestinationDefinitionConfig.yaml", "title": "NormalizationDestinationDefinitionConfig", "description": "describes a normalization config for destination definition", "type": "object", @@ -500,7 +486,6 @@ }, "JobType": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/JobType.yaml", "title": "JobType", "description": "enum that describes the different types of jobs that the platform runs.", "type": "string", @@ -516,7 +501,6 @@ }, "GitInfo": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/GitInfo.yaml", "title": "GitInfo", "description": "Information about the author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.", "type": "object", @@ -543,19 +527,18 @@ }, "GeneratedFields": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/GeneratedFields.yaml", "title": "GeneratedFields", "description": "Optional schema for fields generated at metadata upload time", "type": "object", "properties": { "git": { - "$ref": "GitInfo.yaml" + "$ref": "#/definitions/GitInfo" }, "source_file_info": { - "$ref": "SourceFileInfo.yaml" + "$ref": "#/definitions/SourceFileInfo" }, "metrics": { - "$ref": "ConnectorMetrics.yaml" + "$ref": "#/definitions/ConnectorMetrics" }, "sbomUrl": { "type": "string", @@ -565,7 +548,6 @@ }, "ConnectorTestSuiteOptions": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorTestOptions.yaml", "title": "ConnectorTestSuiteOptions", "description": "Options for a specific connector test suite.", "type": "object", @@ -588,31 +570,30 @@ "description": "List of secrets required to run the test suite", "type": "array", "items": { - "$ref": "Secret.yaml" + "$ref": "#/definitions/Secret" } }, "testConnections": { "description": "List of sandbox cloud connections that tests can be run against", "type": "array", "items": { - "$ref": "TestConnections.yaml" + "$ref": "#/definitions/TestConnections" } } } }, "ConnectorReleases": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml", "title": "ConnectorReleases", "description": "Contains information about different types of releases for a connector.", "type": "object", "additionalProperties": false, "properties": { "rolloutConfiguration": { - "$ref": "RolloutConfiguration.yaml" + "$ref": "#/definitions/RolloutConfiguration" }, "breakingChanges": { - "$ref": "ConnectorBreakingChanges.yaml" + "$ref": "#/definitions/ConnectorBreakingChanges" }, "migrationDocumentationUrl": { "description": "URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", @@ -623,7 +604,6 @@ }, "ConnectorRegistryV0": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte-platform/blob/main/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryV0.yaml", "title": "ConnectorRegistryV0", "description": "describes the collection of connectors retrieved from a registry", "type": "object", @@ -635,20 +615,19 @@ "destinations": { "type": "array", "items": { - "$ref": "ConnectorRegistryDestinationDefinition.yaml" + "$ref": "#/definitions/ConnectorRegistryDestinationDefinition" } }, "sources": { "type": "array", "items": { - "$ref": "ConnectorRegistrySourceDefinition.yaml" + "$ref": "#/definitions/ConnectorRegistrySourceDefinition" } } } }, "ConnectorRegistrySourceDefinition": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte-platform/blob/main/airbyte-config/config-models/src/main/resources/types/ConnectorRegistrySourceDefinition.yaml", "title": "ConnectorRegistrySourceDefinition", "description": "describes a source", "type": "object", @@ -712,10 +691,10 @@ "default": false }, "releaseStage": { - "$ref": "ReleaseStage.yaml" + "$ref": "#/definitions/ReleaseStage" }, "supportLevel": { - "$ref": "SupportLevel.yaml" + "$ref": "#/definitions/SupportLevel" }, "releaseDate": { "description": "The date when this connector was first released, in yyyy-mm-dd format.", @@ -723,17 +702,17 @@ "format": "date" }, "resourceRequirements": { - "$ref": "ActorDefinitionResourceRequirements.yaml" + "$ref": "#/definitions/ActorDefinitionResourceRequirements" }, "protocolVersion": { "type": "string", "description": "the Airbyte Protocol version supported by the connector" }, "allowedHosts": { - "$ref": "AllowedHosts.yaml" + "$ref": "#/definitions/AllowedHosts" }, "suggestedStreams": { - "$ref": "SuggestedStreams.yaml" + "$ref": "#/definitions/SuggestedStreams" }, "maxSecondsBetweenMessages": { "description": "Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", @@ -744,16 +723,16 @@ "description": "The URL where you can visualize the ERD" }, "releases": { - "$ref": "ConnectorRegistryReleases.yaml" + "$ref": "#/definitions/ConnectorRegistryReleases" }, "ab_internal": { - "$ref": "AirbyteInternal.yaml" + "$ref": "#/definitions/AirbyteInternal" }, "generated": { - "$ref": "GeneratedFields.yaml" + "$ref": "#/definitions/GeneratedFields" }, "packageInfo": { - "$ref": "ConnectorPackageInfo.yaml" + "$ref": "#/definitions/ConnectorPackageInfo" }, "language": { "type": "string", @@ -769,9 +748,33 @@ } } }, + "ConnectorReleaseCandidates": { + "description": "Each entry denotes a release candidate version of a connector.", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "patternProperties": { + "^\\d+\\.\\d+\\.\\d+(-[0-9A-Za-z-.]+)?$": { + "$ref": "#/definitions/VersionReleaseCandidate" + } + } + }, + "VersionReleaseCandidate": { + "description": "Contains information about a release candidate version of a connector.", + "additionalProperties": false, + "type": "object", + "oneOf": [ + { + "$ref": "#/definitions/ConnectorRegistrySourceDefinition" + }, + { + "$ref": "#/definitions/ConnectorRegistryDestinationDefinition" + } + ] + }, "ConnectorRegistryReleases": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryReleases.yaml", "title": "ConnectorRegistryReleases", "description": "Contains information about different types of releases for a connector.", "type": "object", @@ -781,48 +784,20 @@ "$ref": "#/definitions/ConnectorReleaseCandidates" }, "rolloutConfiguration": { - "$ref": "RolloutConfiguration.yaml" + "$ref": "#/definitions/RolloutConfiguration" }, "breakingChanges": { - "$ref": "ConnectorBreakingChanges.yaml" + "$ref": "#/definitions/ConnectorBreakingChanges" }, "migrationDocumentationUrl": { "description": "URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", "type": "string", "format": "uri" } - }, - "definitions": { - "ConnectorReleaseCandidates": { - "description": "Each entry denotes a release candidate version of a connector.", - "type": "object", - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "patternProperties": { - "^\\d+\\.\\d+\\.\\d+(-[0-9A-Za-z-.]+)?$": { - "$ref": "#/definitions/VersionReleaseCandidate" - } - } - }, - "VersionReleaseCandidate": { - "description": "Contains information about a release candidate version of a connector.", - "additionalProperties": false, - "type": "object", - "oneOf": [ - { - "$ref": "ConnectorRegistrySourceDefinition.yaml" - }, - { - "$ref": "ConnectorRegistryDestinationDefinition.yaml" - } - ] - } } }, "ConnectorRegistryDestinationDefinition": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte-platform/blob/main/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryDestinationDefinition.yaml", "title": "ConnectorRegistryDestinationDefinition", "description": "describes a destination", "type": "object", @@ -877,10 +852,10 @@ "default": false }, "releaseStage": { - "$ref": "ReleaseStage.yaml" + "$ref": "#/definitions/ReleaseStage" }, "supportLevel": { - "$ref": "SupportLevel.yaml" + "$ref": "#/definitions/SupportLevel" }, "releaseDate": { "description": "The date when this connector was first released, in yyyy-mm-dd format.", @@ -895,27 +870,27 @@ } }, "resourceRequirements": { - "$ref": "ActorDefinitionResourceRequirements.yaml" + "$ref": "#/definitions/ActorDefinitionResourceRequirements" }, "protocolVersion": { "type": "string", "description": "the Airbyte Protocol version supported by the connector" }, "normalizationConfig": { - "$ref": "NormalizationDestinationDefinitionConfig.yaml" + "$ref": "#/definitions/NormalizationDestinationDefinitionConfig" }, "supportsDbt": { "type": "boolean", "description": "an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used." }, "allowedHosts": { - "$ref": "AllowedHosts.yaml" + "$ref": "#/definitions/AllowedHosts" }, "releases": { - "$ref": "ConnectorRegistryReleases.yaml" + "$ref": "#/definitions/ConnectorRegistryReleases" }, "ab_internal": { - "$ref": "AirbyteInternal.yaml" + "$ref": "#/definitions/AirbyteInternal" }, "supportsRefreshes": { "type": "boolean", @@ -930,10 +905,10 @@ "default": false }, "generated": { - "$ref": "GeneratedFields.yaml" + "$ref": "#/definitions/GeneratedFields" }, "packageInfo": { - "$ref": "ConnectorPackageInfo.yaml" + "$ref": "#/definitions/ConnectorPackageInfo" }, "language": { "type": "string", @@ -943,7 +918,6 @@ }, "ConnectorPackageInfo": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorPackageInfo.yaml", "title": "ConnectorPackageInfo", "description": "Information about the contents of the connector image", "type": "object", @@ -953,68 +927,64 @@ } } }, + "ConnectorMetric": { + "type": "object", + "properties": { + "usage": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [ + "low", + "medium", + "high" + ] + } + ] + }, + "sync_success_rate": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "string", + "enum": [ + "low", + "medium", + "high" + ] + } + ] + }, + "connector_version": { + "type": "string" + } + }, + "additionalProperties": true + }, "ConnectorMetrics": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorMetrics.yaml", "title": "ConnectorMetrics", "description": "Information about the source file that generated the registry entry", "type": "object", "properties": { "all": { - "type": "ConnectorMetric" + "$ref": "#/definitions/ConnectorMetric" }, "cloud": { - "type": "ConnectorMetric" + "$ref": "#/definitions/ConnectorMetric" }, "oss": { - "type": "ConnectorMetric" - } - }, - "definitions": { - "ConnectorMetric": { - "type": "object", - "properties": { - "usage": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "string", - "enum": [ - "low", - "medium", - "high" - ] - } - ] - }, - "sync_success_rate": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "string", - "enum": [ - "low", - "medium", - "high" - ] - } - ] - }, - "connector_version": { - "type": "string" - } - }, - "additionalProperties": true + "$ref": "#/definitions/ConnectorMetric" } } }, "ConnectorIPCOptions": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorIPCOptions.yaml", "title": "ConnectorIPCOptions", "type": "object", "required": [ @@ -1061,7 +1031,6 @@ }, "ConnectorBuildOptions": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml", "title": "ConnectorBuildOptions", "description": "metadata specific to the build process.", "type": "object", @@ -1072,9 +1041,80 @@ } } }, + "VersionBreakingChange": { + "description": "Contains information about a breaking change, including the deadline to upgrade and a message detailing the change.", + "type": "object", + "additionalProperties": false, + "required": [ + "upgradeDeadline", + "message" + ], + "properties": { + "upgradeDeadline": { + "description": "The deadline by which to upgrade before the breaking change takes effect.", + "type": "string", + "format": "date" + }, + "message": { + "description": "Descriptive message detailing the breaking change.", + "type": "string" + }, + "deadlineAction": { + "description": "Action to do when the deadline is reached.", + "type": "string", + "enum": [ + "auto_upgrade", + "disable" + ] + }, + "migrationDocumentationUrl": { + "description": "URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + "type": "string", + "format": "uri" + }, + "scopedImpact": { + "description": "List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/BreakingChangeScope" + } + } + } + }, + "BreakingChangeScope": { + "description": "A scope that can be used to limit the impact of a breaking change.", + "type": "object", + "oneOf": [ + { + "$ref": "#/definitions/StreamBreakingChangeScope" + } + ] + }, + "StreamBreakingChangeScope": { + "description": "A scope that can be used to limit the impact of a breaking change to specific streams.", + "type": "object", + "additionalProperties": false, + "required": [ + "scopeType", + "impactedScopes" + ], + "properties": { + "scopeType": { + "const": "stream" + }, + "impactedScopes": { + "description": "List of streams that are impacted by the breaking change.", + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + } + } + }, "ConnectorBreakingChanges": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBreakingChanges.yaml", "title": "ConnectorBreakingChanges", "description": "Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", "type": "object", @@ -1084,86 +1124,10 @@ "^\\d+\\.\\d+\\.\\d+$": { "$ref": "#/definitions/VersionBreakingChange" } - }, - "definitions": { - "VersionBreakingChange": { - "description": "Contains information about a breaking change, including the deadline to upgrade and a message detailing the change.", - "type": "object", - "additionalProperties": false, - "required": [ - "upgradeDeadline", - "message" - ], - "properties": { - "upgradeDeadline": { - "description": "The deadline by which to upgrade before the breaking change takes effect.", - "type": "string", - "format": "date" - }, - "message": { - "description": "Descriptive message detailing the breaking change.", - "type": "string" - }, - "deadlineAction": { - "description": "Action to do when the deadline is reached.", - "type": "string", - "enum": [ - "auto_upgrade", - "disable" - ] - }, - "migrationDocumentationUrl": { - "description": "URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", - "type": "string", - "format": "uri" - }, - "scopedImpact": { - "description": "List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/BreakingChangeScope" - } - } - } - }, - "BreakingChangeScope": { - "description": "A scope that can be used to limit the impact of a breaking change.", - "type": "object", - "oneOf": [ - { - "$ref": "#/definitions/StreamBreakingChangeScope" - } - ] - }, - "StreamBreakingChangeScope": { - "description": "A scope that can be used to limit the impact of a breaking change to specific streams.", - "type": "object", - "additionalProperties": false, - "required": [ - "scopeType", - "impactedScopes" - ], - "properties": { - "scopeType": { - "type": "const", - "const": "stream" - }, - "impactedScopes": { - "description": "List of streams that are impacted by the breaking change.", - "type": "array", - "minItems": 1, - "items": { - "type": "string" - } - } - } - } } }, "AllowedHosts": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/AllowedHosts.yaml", "title": "AllowedHosts", "description": "A connector's allowed hosts. If present, the platform will limit communication to only hosts which are listed in `AllowedHosts.hosts`.", "type": "object", @@ -1180,7 +1144,6 @@ }, "AirbyteInternal": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/AirbyteInternal.yml", "title": "AirbyteInternal", "description": "Fields for internal use only", "type": "object", @@ -1218,9 +1181,25 @@ } } }, + "JobTypeResourceLimit": { + "description": "sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set.", + "type": "object", + "additionalProperties": false, + "required": [ + "jobType", + "resourceRequirements" + ], + "properties": { + "jobType": { + "$ref": "#/definitions/JobType" + }, + "resourceRequirements": { + "$ref": "#/definitions/ResourceRequirements" + } + } + }, "ActorDefinitionResourceRequirements": { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ActorDefinitionResourceRequirements.yaml", "title": "ActorDefinitionResourceRequirements", "description": "actor definition specific resource requirements", "type": "object", @@ -1228,7 +1207,7 @@ "properties": { "default": { "description": "if set, these are the requirements that should be set for ALL jobs run for this actor definition.", - "$ref": "ResourceRequirements.yaml" + "$ref": "#/definitions/ResourceRequirements" }, "jobSpecific": { "type": "array", @@ -1236,25 +1215,6 @@ "$ref": "#/definitions/JobTypeResourceLimit" } } - }, - "definitions": { - "JobTypeResourceLimit": { - "description": "sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set.", - "type": "object", - "additionalProperties": false, - "required": [ - "jobType", - "resourceRequirements" - ], - "properties": { - "jobType": { - "$ref": "JobType.yaml" - }, - "resourceRequirements": { - "$ref": "ResourceRequirements.yaml" - } - } - } } } } diff --git a/airbyte_cdk/test/models/connector_metadata/generated/models.py b/airbyte_cdk/test/models/connector_metadata/generated/models.py index 1d63c17f5..ab6cfd60e 100644 --- a/airbyte_cdk/test/models/connector_metadata/generated/models.py +++ b/airbyte_cdk/test/models/connector_metadata/generated/models.py @@ -42,6 +42,7 @@ class Config: jobSpecific: Optional[List[JobTypeResourceLimit]] = None + class Sl(Enum): integer_0 = 0 integer_100 = 100 @@ -72,6 +73,7 @@ class Config: ) + class AllowedHosts(BaseModel): class Config: extra = Extra.allow @@ -82,6 +84,7 @@ class Config: ) + class DeadlineAction(Enum): auto_upgrade = "auto_upgrade" disable = "disable" @@ -114,7 +117,9 @@ class Config: ..., description="The deadline by which to upgrade before the breaking change takes effect.", ) - message: str = Field(..., description="Descriptive message detailing the breaking change.") + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) deadlineAction: Optional[DeadlineAction] = Field( None, description="Action to do when the deadline is reached." ) @@ -140,6 +145,7 @@ class Config: ) + class ConnectorBuildOptions(BaseModel): class Config: extra = Extra.forbid @@ -147,6 +153,7 @@ class Config: baseImage: Optional[str] = None + class SupportedSerializationEnum(Enum): JSONL = "JSONL" PROTOBUF = "PROTOBUF" @@ -174,6 +181,7 @@ class Config: dataChannel: DataChannel + class ConnectorType(Enum): destination = "destination" source = "source" @@ -226,7 +234,9 @@ class Config: protocolVersion: Optional[str] = Field( None, description="the Airbyte Protocol version supported by the connector" ) - erdUrl: Optional[str] = Field(None, description="The URL where you can visualize the ERD") + erdUrl: Optional[str] = Field( + None, description="The URL where you can visualize the ERD" + ) connectorSubtype: ConnectorSubtype releaseStage: ReleaseStage supportLevel: Optional[SupportLevel] = None @@ -257,6 +267,7 @@ class Config: data: Data + class ConnectorMetrics(BaseModel): all: Optional[Any] = None cloud: Optional[Any] = None @@ -284,10 +295,12 @@ class Config: connector_version: Optional[str] = None + class ConnectorPackageInfo(BaseModel): cdk_version: Optional[str] = None + class ConnectorRegistryDestinationDefinition(BaseModel): class Config: extra = Extra.allow @@ -338,12 +351,15 @@ class Config: supportsDataActivation: Optional[bool] = False generated: Optional[GeneratedFields] = None packageInfo: Optional[ConnectorPackageInfo] = None - language: Optional[str] = Field(None, description="The language the connector is written in") + language: Optional[str] = Field( + None, description="The language the connector is written in" + ) ConnectorRegistryDestinationDefinition.update_forward_refs() + class ConnectorRegistryReleases(BaseModel): class Config: extra = Extra.forbid @@ -361,11 +377,11 @@ class ConnectorReleaseCandidates(BaseModel): class Config: extra = Extra.forbid - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+(-[0-9A-Za-z-.]+)?$"), VersionReleaseCandidate] = ( - Field( - ..., - description="Each entry denotes a release candidate version of a connector.", - ) + __root__: Dict[ + constr(regex=r"^\d+\.\d+\.\d+(-[0-9A-Za-z-.]+)?$"), VersionReleaseCandidate + ] = Field( + ..., + description="Each entry denotes a release candidate version of a connector.", ) @@ -373,11 +389,11 @@ class VersionReleaseCandidate(BaseModel): class Config: extra = Extra.forbid - __root__: Union[ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition] = ( - Field( - ..., - description="Contains information about a release candidate version of a connector.", - ) + __root__: Union[ + ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition + ] = Field( + ..., + description="Contains information about a release candidate version of a connector.", ) @@ -386,6 +402,7 @@ class Config: VersionReleaseCandidate.update_forward_refs() + class SourceType(Enum): api = "api" file = "file" @@ -433,21 +450,27 @@ class Config: None, description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", ) - erdUrl: Optional[str] = Field(None, description="The URL where you can visualize the ERD") + erdUrl: Optional[str] = Field( + None, description="The URL where you can visualize the ERD" + ) releases: Optional[ConnectorRegistryReleases] = None ab_internal: Optional[AirbyteInternal] = None generated: Optional[GeneratedFields] = None packageInfo: Optional[ConnectorPackageInfo] = None - language: Optional[str] = Field(None, description="The language the connector is written in") + language: Optional[str] = Field( + None, description="The language the connector is written in" + ) supportsFileTransfer: Optional[bool] = False supportsDataActivation: Optional[bool] = False + class ConnectorRegistryV0(BaseModel): destinations: List[ConnectorRegistryDestinationDefinition] sources: List[ConnectorRegistrySourceDefinition] + class ConnectorReleases(BaseModel): class Config: extra = Extra.forbid @@ -460,6 +483,7 @@ class Config: ) + class Suite(Enum): unitTests = "unitTests" integrationTests = "integrationTests" @@ -481,6 +505,7 @@ class Config: ) + class GeneratedFields(BaseModel): git: Optional[GitInfo] = None source_file_info: Optional[SourceFileInfo] = None @@ -488,6 +513,7 @@ class GeneratedFields(BaseModel): sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") + class GitInfo(BaseModel): class Config: extra = Extra.forbid @@ -510,6 +536,7 @@ class Config: ) + class JobType(Enum): get_spec = "get_spec" check_connection = "check_connection" @@ -520,6 +547,7 @@ class JobType(Enum): replicate = "replicate" + class NormalizationDestinationDefinitionConfig(BaseModel): class Config: extra = Extra.allow @@ -538,6 +566,7 @@ class Config: ) + class RegistryOverrides(BaseModel): class Config: extra = Extra.forbid @@ -557,6 +586,7 @@ class Config: resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + class ReleaseStage(Enum): alpha = "alpha" beta = "beta" @@ -564,6 +594,7 @@ class ReleaseStage(Enum): custom = "custom" + class PyPi(BaseModel): class Config: extra = Extra.forbid @@ -579,6 +610,7 @@ class Config: pypi: Optional[PyPi] = None + class ResourceRequirements(BaseModel): class Config: extra = Extra.forbid @@ -589,6 +621,7 @@ class Config: memory_limit: Optional[str] = None + class RolloutConfiguration(BaseModel): class Config: extra = Extra.forbid @@ -610,6 +643,7 @@ class Config: ) + class Secret(BaseModel): class Config: extra = Extra.forbid @@ -622,6 +656,7 @@ class Config: secretStore: SecretStore + class SecretStore(BaseModel): class Config: extra = Extra.forbid @@ -630,7 +665,10 @@ class Config: None, description="The alias of the secret store which can map to its actual secret address", ) - type: Optional[Literal["GSM"]] = Field(None, description="The type of the secret store") + type: Optional[Literal["GSM"]] = Field( + None, description="The type of the secret store" + ) + class SourceFileInfo(BaseModel): @@ -641,6 +679,7 @@ class SourceFileInfo(BaseModel): registry_entry_generated_at: Optional[str] = None + class SuggestedStreams(BaseModel): class Config: extra = Extra.allow @@ -651,12 +690,14 @@ class Config: ) + class SupportLevel(Enum): community = "community" certified = "certified" archived = "archived" + class TestConnections(BaseModel): class Config: extra = Extra.forbid diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index ecabf3b89..bd8e8e7bd 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -236,6 +236,37 @@ def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: str) schema_content = yaml.safe_load(f) schemas[schema_name] = schema_content + all_schema_names = set(schemas.keys()) + + for schema_content in schemas.values(): + if isinstance(schema_content, dict) and "definitions" in schema_content: + all_schema_names.update(schema_content["definitions"].keys()) + + def fix_refs(obj, in_definition=False): + """Recursively fix $ref and type references in schema objects.""" + if isinstance(obj, dict): + new_obj = {} + for key, value in obj.items(): + if key == "$id" and in_definition: + continue + elif key == "$ref" and isinstance(value, str): + if value.endswith(".yaml"): + schema_name = value.replace(".yaml", "") + new_obj[key] = f"#/definitions/{schema_name}" + else: + new_obj[key] = value + elif key == "type" and isinstance(value, str) and value in all_schema_names: + new_obj["$ref"] = f"#/definitions/{value}" + elif key == "type" and value == "const": + pass + else: + new_obj[key] = fix_refs(value, in_definition=in_definition) + return new_obj + elif isinstance(obj, list): + return [fix_refs(item, in_definition=in_definition) for item in obj] + else: + return obj + # Find the main schema (ConnectorMetadataDefinitionV0) main_schema = schemas.get("ConnectorMetadataDefinitionV0") @@ -249,10 +280,18 @@ def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: str) "definitions": {}, } - # Add all other schemas as definitions + # Add all schemas (including their internal definitions) as top-level definitions for schema_name, schema_content in schemas.items(): if schema_name != "ConnectorMetadataDefinitionV0": - consolidated["definitions"][schema_name] = schema_content + if isinstance(schema_content, dict) and "definitions" in schema_content: + for def_name, def_content in schema_content["definitions"].items(): + consolidated["definitions"][def_name] = fix_refs(def_content, in_definition=True) + schema_without_defs = {k: v for k, v in schema_content.items() if k != "definitions"} + consolidated["definitions"][schema_name] = fix_refs(schema_without_defs, in_definition=True) + else: + consolidated["definitions"][schema_name] = fix_refs(schema_content, in_definition=True) + + consolidated = fix_refs(consolidated, in_definition=False) Path(output_json_path).write_text(json.dumps(consolidated, indent=2)) print(f"Generated consolidated JSON schema: {output_json_path}", file=sys.stderr) From 015a60ee5a2e757a972e1a24fd11d5b90cc47adc Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:45:06 +0000 Subject: [PATCH 14/20] Remove $schema and $id from definitions to fix IDE validation - IDE validators expect definitions to not have $schema or $id fields - These fields should only exist at the root level - Fixes validation errors in IDEs when using the consolidated schema Co-Authored-By: AJ Steers --- .../models/declarative_component_schema.py | 146 +++++++++++------- .../generated/metadata_schema.json | 29 ---- bin/generate_component_manifest_files.py | 2 +- mypy.ini | 7 +- 4 files changed, 93 insertions(+), 91 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 35186ef71..421c6779f 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1,5 +1,3 @@ -# Copyright (c) 2025 Airbyte, Inc., all rights reserved. - # generated by datamodel-codegen: # filename: declarative_component_schema.yaml @@ -928,24 +926,28 @@ class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", - examples=[ - {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, - { - "app_id": { - "type": "string", - "path_in_connector_config": ["info", "app_id"], - } - }, - ], - title="OAuth user input", + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( + Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + examples=[ + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, + { + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], + } + }, + ], + title="OAuth user input", + ) ) - oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( - None, - description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', - title="DeclarativeOAuth Connector Specification", + oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = ( + Field( + None, + description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', + title="DeclarativeOAuth Connector Specification", + ) ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, @@ -963,7 +965,9 @@ class Config: complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", - examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], + examples=[ + {"client_id": {"type": "string"}, "client_secret": {"type": "string"}} + ], title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( @@ -1467,7 +1471,9 @@ class CustomConfigTransformation(BaseModel): class_name: str = Field( ..., description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_..`.", - examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], + examples=[ + "source_declarative_manifest.components.MyCustomConfigTransformation" + ], ) parameters: Optional[Dict[str, Any]] = Field( None, @@ -1885,7 +1891,9 @@ class OAuthAuthenticator(BaseModel): scopes: Optional[List[str]] = Field( None, description="List of scopes that should be granted to the access token.", - examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], + examples=[ + ["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"] + ], title="Scopes", ) token_expiry_date: Optional[str] = Field( @@ -2084,7 +2092,9 @@ class RecordSelector(BaseModel): description="Responsible for filtering records to be emitted by the Source.", title="Record Filter", ) - schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( + schema_normalization: Optional[ + Union[SchemaNormalization, CustomSchemaNormalization] + ] = Field( None, description="Responsible for normalization according to the schema.", title="Schema Normalization", @@ -2126,10 +2136,12 @@ class DpathValidator(BaseModel): ], title="Field Path", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( - ..., - description="The condition that the specified config value will be evaluated against", - title="Validation Strategy", + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( + Field( + ..., + description="The condition that the specified config value will be evaluated against", + title="Validation Strategy", + ) ) @@ -2146,10 +2158,12 @@ class PredicateValidator(BaseModel): ], title="Value", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( - ..., - description="The validation strategy to apply to the value.", - title="Validation Strategy", + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( + Field( + ..., + description="The validation strategy to apply to the value.", + title="Validation Strategy", + ) ) @@ -2174,12 +2188,12 @@ class ConfigAddFields(BaseModel): class CompositeErrorHandler(BaseModel): type: Literal["CompositeErrorHandler"] - error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( - Field( - ..., - description="List of error handlers to iterate on to determine how to handle a failed response.", - title="Error Handlers", - ) + error_handlers: List[ + Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler] + ] = Field( + ..., + description="List of error handlers to iterate on to determine how to handle a failed response.", + title="Error Handlers", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -2341,9 +2355,9 @@ class Config: type: Literal["DeclarativeSource"] check: Union[CheckStream, CheckDynamicStream] - streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( - None - ) + streams: Optional[ + List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] + ] = None dynamic_streams: List[DynamicDeclarativeStream] version: str = Field( ..., @@ -2468,16 +2482,20 @@ class Config: extra = Extra.allow type: Literal["DeclarativeStream"] - name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") + name: Optional[str] = Field( + "", description="The stream name.", example=["Users"], title="Name" + ) retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages.", title="Retriever", ) - incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( - None, - description="Component used to fetch data incrementally based on a time field in the data.", - title="Incremental Sync", + incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = ( + Field( + None, + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", + ) ) primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") schema_loader: Optional[ @@ -2651,18 +2669,20 @@ class HttpRequester(BaseModelWithDeprecations): description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", title="Query Properties", ) - request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( - None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", - examples=[ - {"unit": "day"}, - { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, - ], - title="Query Parameters", + request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = ( + Field( + None, + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + examples=[ + {"unit": "day"}, + { + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, + ], + title="Query Parameters", + ) ) request_headers: Optional[Union[Dict[str, str], str]] = Field( None, @@ -2834,7 +2854,9 @@ class QueryProperties(BaseModel): class StateDelegatingStream(BaseModel): type: Literal["StateDelegatingStream"] - name: str = Field(..., description="The stream name.", example=["Users"], title="Name") + name: str = Field( + ..., description="The stream name.", example=["Users"], title="Name" + ) full_refresh_stream: DeclarativeStream = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", @@ -2921,13 +2943,17 @@ class AsyncRetriever(BaseModel): status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( ..., description="Responsible for fetching the actual status of the async job." ) - download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( + download_target_extractor: Optional[ + Union[DpathExtractor, CustomRecordExtractor] + ] = Field( None, description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", ) download_extractor: Optional[ Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] - ] = Field(None, description="Responsible for fetching the records from provided urls.") + ] = Field( + None, description="Responsible for fetching the records from provided urls." + ) creation_requester: Union[HttpRequester, CustomRequester] = Field( ..., description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", diff --git a/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json b/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json index 2e55f8f7f..a0ea84026 100644 --- a/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json +++ b/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json @@ -189,7 +189,6 @@ }, "definitions": { "TestConnections": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "TestConnections", "description": "List of sandbox cloud connections that tests can be run against", "type": "object", @@ -210,7 +209,6 @@ } }, "SupportLevel": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "SupportLevel", "description": "enum that describes a connector's release stage", "type": "string", @@ -221,7 +219,6 @@ ] }, "SuggestedStreams": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "SuggestedStreams", "description": "A source's suggested streams. These will be suggested by default for new connections using this source. Otherwise, all streams will be selected. This is useful for when your source has a lot of streams, but the average user will only want a subset of them synced.", "type": "object", @@ -237,7 +234,6 @@ } }, "SourceFileInfo": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "SourceFileInfo", "description": "Information about the source file that generated the registry entry", "type": "object", @@ -260,7 +256,6 @@ } }, "SecretStore": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "SecretStore", "description": "An object describing a secret store metadata", "type": "object", @@ -284,7 +279,6 @@ } }, "Secret": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "Secret", "description": "An object describing a secret's metadata", "type": "object", @@ -308,7 +302,6 @@ } }, "RolloutConfiguration": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "RolloutConfiguration", "description": "configuration for the rollout of a connector", "type": "object", @@ -342,7 +335,6 @@ } }, "ResourceRequirements": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ResourceRequirements", "description": "generic configuration for pod source requirements", "type": "object", @@ -382,7 +374,6 @@ } }, "RemoteRegistries": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "RemoteRegistries", "description": "describes how the connector is published to remote registries", "type": "object", @@ -394,7 +385,6 @@ } }, "ReleaseStage": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ReleaseStage", "description": "enum that describes a connector's release stage", "type": "string", @@ -406,7 +396,6 @@ ] }, "RegistryOverrides": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "RegistryOverrides", "description": "describes the overrides per registry of a connector", "type": "object", @@ -459,7 +448,6 @@ } }, "NormalizationDestinationDefinitionConfig": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "NormalizationDestinationDefinitionConfig", "description": "describes a normalization config for destination definition", "type": "object", @@ -485,7 +473,6 @@ } }, "JobType": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "JobType", "description": "enum that describes the different types of jobs that the platform runs.", "type": "string", @@ -500,7 +487,6 @@ ] }, "GitInfo": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "GitInfo", "description": "Information about the author of the last commit that modified this file. DO NOT DEFINE THIS FIELD MANUALLY. It will be overwritten by the CI.", "type": "object", @@ -526,7 +512,6 @@ } }, "GeneratedFields": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "GeneratedFields", "description": "Optional schema for fields generated at metadata upload time", "type": "object", @@ -547,7 +532,6 @@ } }, "ConnectorTestSuiteOptions": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ConnectorTestSuiteOptions", "description": "Options for a specific connector test suite.", "type": "object", @@ -583,7 +567,6 @@ } }, "ConnectorReleases": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ConnectorReleases", "description": "Contains information about different types of releases for a connector.", "type": "object", @@ -603,7 +586,6 @@ } }, "ConnectorRegistryV0": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ConnectorRegistryV0", "description": "describes the collection of connectors retrieved from a registry", "type": "object", @@ -627,7 +609,6 @@ } }, "ConnectorRegistrySourceDefinition": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ConnectorRegistrySourceDefinition", "description": "describes a source", "type": "object", @@ -774,7 +755,6 @@ ] }, "ConnectorRegistryReleases": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ConnectorRegistryReleases", "description": "Contains information about different types of releases for a connector.", "type": "object", @@ -797,7 +777,6 @@ } }, "ConnectorRegistryDestinationDefinition": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ConnectorRegistryDestinationDefinition", "description": "describes a destination", "type": "object", @@ -917,7 +896,6 @@ } }, "ConnectorPackageInfo": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ConnectorPackageInfo", "description": "Information about the contents of the connector image", "type": "object", @@ -967,7 +945,6 @@ "additionalProperties": true }, "ConnectorMetrics": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ConnectorMetrics", "description": "Information about the source file that generated the registry entry", "type": "object", @@ -984,7 +961,6 @@ } }, "ConnectorIPCOptions": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ConnectorIPCOptions", "type": "object", "required": [ @@ -1030,7 +1006,6 @@ } }, "ConnectorBuildOptions": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ConnectorBuildOptions", "description": "metadata specific to the build process.", "type": "object", @@ -1114,7 +1089,6 @@ } }, "ConnectorBreakingChanges": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ConnectorBreakingChanges", "description": "Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", "type": "object", @@ -1127,7 +1101,6 @@ } }, "AllowedHosts": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "AllowedHosts", "description": "A connector's allowed hosts. If present, the platform will limit communication to only hosts which are listed in `AllowedHosts.hosts`.", "type": "object", @@ -1143,7 +1116,6 @@ } }, "AirbyteInternal": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "AirbyteInternal", "description": "Fields for internal use only", "type": "object", @@ -1199,7 +1171,6 @@ } }, "ActorDefinitionResourceRequirements": { - "$schema": "http://json-schema.org/draft-07/schema#", "title": "ActorDefinitionResourceRequirements", "description": "actor definition specific resource requirements", "type": "object", diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index bd8e8e7bd..1dbb35001 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -247,7 +247,7 @@ def fix_refs(obj, in_definition=False): if isinstance(obj, dict): new_obj = {} for key, value in obj.items(): - if key == "$id" and in_definition: + if (key == "$id" or key == "$schema") and in_definition: continue elif key == "$ref" and isinstance(value, str): if value.endswith(".yaml"): diff --git a/mypy.ini b/mypy.ini index ff616f462..57612bc59 100644 --- a/mypy.ini +++ b/mypy.ini @@ -12,7 +12,8 @@ disallow_untyped_calls = True disallow_incomplete_defs = True disallow_untyped_defs = True warn_return_any = True -exclude = unit_tests/ +# Exclude tests and auto-generated files from type checking +exclude = (unit_tests/|airbyte_cdk/sources/declarative/models/declarative_component_schema\.py|airbyte_cdk/test/models/connector_metadata/generated/) # Only alert on the files we want to check follow_imports = silent @@ -25,3 +26,7 @@ plugins = ["pydantic.mypy", "pytest-mypy-plugins"] [mypy-airbyte_cdk.models] ignore_errors = True +[mypy-airbyte_cdk.test.models.connector_metadata.generated.*] +ignore_errors = True +[mypy-airbyte_cdk.sources.declarative.models.declarative_component_schema] +ignore_errors = True From fe4b9cc3ef862bba5e677ac07d84f8064a172c44 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:56:25 +0000 Subject: [PATCH 15/20] Refactor: Extract metadata generation into separate script - Created new bin/generate_connector_metadata_files.py for metadata models - Reverted bin/generate_component_manifest_files.py to main (no changes) - Updated build script to call both generators - Keeps concerns separated for easier review and maintenance Co-Authored-By: AJ Steers --- bin/generate-component-manifest-dagger.sh | 1 + bin/generate_component_manifest_files.py | 385 +++------------------- bin/generate_connector_metadata_files.py | 291 ++++++++++++++++ 3 files changed, 333 insertions(+), 344 deletions(-) create mode 100755 bin/generate_connector_metadata_files.py diff --git a/bin/generate-component-manifest-dagger.sh b/bin/generate-component-manifest-dagger.sh index 908f92a6c..9e760e1b8 100755 --- a/bin/generate-component-manifest-dagger.sh +++ b/bin/generate-component-manifest-dagger.sh @@ -8,3 +8,4 @@ set -e python bin/generate_component_manifest_files.py +python bin/generate_connector_metadata_files.py diff --git a/bin/generate_component_manifest_files.py b/bin/generate_component_manifest_files.py index 1dbb35001..51b3d8efb 100755 --- a/bin/generate_component_manifest_files.py +++ b/bin/generate_component_manifest_files.py @@ -1,21 +1,17 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. -import json import re -import subprocess import sys -import tempfile from glob import glob from pathlib import Path import anyio import dagger -import yaml PYTHON_IMAGE = "python:3.10" LOCAL_YAML_DIR_PATH = "airbyte_cdk/sources/declarative" LOCAL_OUTPUT_DIR_PATH = "airbyte_cdk/sources/declarative/models" -LOCAL_METADATA_OUTPUT_DIR_PATH = "airbyte_cdk/test/models/connector_metadata/generated" + PIP_DEPENDENCIES = [ "datamodel_code_generator==0.26.3", @@ -26,53 +22,13 @@ def get_all_yaml_files_without_ext() -> list[str]: return [Path(f).stem for f in glob(f"{LOCAL_YAML_DIR_PATH}/*.yaml")] -def get_all_yaml_files_from_dir(directory: str) -> list[str]: - return [Path(f).stem for f in glob(f"{directory}/*.yaml")] - - -def generate_init_module_content(yaml_files: list[str]) -> str: +def generate_init_module_content() -> str: header = "# generated by bin/generate_component_manifest_files.py\n" - for module_name in yaml_files: + for module_name in get_all_yaml_files_without_ext(): header += f"from .{module_name} import *\n" return header -def clone_metadata_schemas(temp_dir: Path) -> Path: - """Clone metadata schema YAML files from GitHub using sparse checkout.""" - repo_url = "https://github.com/airbytehq/airbyte.git" - schema_path = "airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" - - clone_dir = temp_dir / "airbyte" - - print("Cloning metadata schemas from airbyte repo...", file=sys.stderr) - - subprocess.run( - [ - "git", - "clone", - "--depth", - "1", - "--filter=blob:none", - "--sparse", - repo_url, - str(clone_dir), - ], - check=True, - capture_output=True, - ) - - subprocess.run( - ["git", "-C", str(clone_dir), "sparse-checkout", "set", schema_path], - check=True, - capture_output=True, - ) - - schemas_dir = clone_dir / schema_path - print(f"Cloned schemas to {schemas_dir}", file=sys.stderr) - - return schemas_dir - - def replace_base_model_for_classes_with_deprecated_fields(post_processed_content: str) -> str: """ Replace the base model for classes with deprecated fields. @@ -154,308 +110,49 @@ async def post_process_codegen(codegen_container: dagger.Container): return codegen_container -async def post_process_metadata_models(codegen_container: dagger.Container): - """Post-process metadata models to use pydantic.v1 compatibility layer.""" - codegen_container = codegen_container.with_exec( - ["mkdir", "/generated_post_processed"], use_entrypoint=True - ) - for generated_file in await codegen_container.directory("/generated").entries(): - if generated_file.endswith(".py"): - original_content = await codegen_container.file( - f"/generated/{generated_file}" - ).contents() - - post_processed_content = original_content.replace("from pydantic", "from pydantic.v1") - - codegen_container = codegen_container.with_new_file( - f"/generated_post_processed/{generated_file}", contents=post_processed_content - ) - return codegen_container - - -async def generate_models_from_schemas( - dagger_client: dagger.Client, - yaml_dir_path: str, - output_dir_path: str, - yaml_files: list[str], - post_process: bool = False, - metadata_models: bool = False, -) -> None: - """Generate Pydantic models from YAML schemas using datamodel-codegen.""" - init_module_content = generate_init_module_content(yaml_files) - - codegen_container = ( - dagger_client.container() - .from_(PYTHON_IMAGE) - .with_exec(["mkdir", "/generated"], use_entrypoint=True) - .with_exec(["pip", "install", " ".join(PIP_DEPENDENCIES)], use_entrypoint=True) - .with_mounted_directory( - "/yaml", dagger_client.host().directory(yaml_dir_path, include=["*.yaml"]) - ) - .with_new_file("/generated/__init__.py", contents=init_module_content) - ) - - for yaml_file in yaml_files: - codegen_container = codegen_container.with_exec( - [ - "datamodel-codegen", - "--input", - f"/yaml/{yaml_file}.yaml", - "--output", - f"/generated/{yaml_file}.py", - "--disable-timestamp", - "--enum-field-as-literal", - "one", - "--set-default-enum-member", - "--use-double-quotes", - "--remove-special-field-name-prefix", - "--field-extra-keys", - "deprecated", - "deprecation_message", - ], - use_entrypoint=True, - ) - - if post_process: - codegen_container = await post_process_codegen(codegen_container) - await codegen_container.directory("/generated_post_processed").export(output_dir_path) - elif metadata_models: - codegen_container = await post_process_metadata_models(codegen_container) - await codegen_container.directory("/generated_post_processed").export(output_dir_path) - else: - await codegen_container.directory("/generated").export(output_dir_path) - - -def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: str) -> None: - """Consolidate all YAML schemas into a single JSON schema file.""" - schemas = {} - - for yaml_file in yaml_dir_path.glob("*.yaml"): - schema_name = yaml_file.stem - with yaml_file.open("r") as f: - schema_content = yaml.safe_load(f) - schemas[schema_name] = schema_content - - all_schema_names = set(schemas.keys()) - - for schema_content in schemas.values(): - if isinstance(schema_content, dict) and "definitions" in schema_content: - all_schema_names.update(schema_content["definitions"].keys()) - - def fix_refs(obj, in_definition=False): - """Recursively fix $ref and type references in schema objects.""" - if isinstance(obj, dict): - new_obj = {} - for key, value in obj.items(): - if (key == "$id" or key == "$schema") and in_definition: - continue - elif key == "$ref" and isinstance(value, str): - if value.endswith(".yaml"): - schema_name = value.replace(".yaml", "") - new_obj[key] = f"#/definitions/{schema_name}" - else: - new_obj[key] = value - elif key == "type" and isinstance(value, str) and value in all_schema_names: - new_obj["$ref"] = f"#/definitions/{value}" - elif key == "type" and value == "const": - pass - else: - new_obj[key] = fix_refs(value, in_definition=in_definition) - return new_obj - elif isinstance(obj, list): - return [fix_refs(item, in_definition=in_definition) for item in obj] - else: - return obj - - # Find the main schema (ConnectorMetadataDefinitionV0) - main_schema = schemas.get("ConnectorMetadataDefinitionV0") - - if main_schema: - # Create a consolidated schema with definitions - consolidated = { - "$schema": main_schema.get("$schema", "http://json-schema.org/draft-07/schema#"), - "title": "Connector Metadata Schema", - "description": "Consolidated JSON schema for Airbyte connector metadata validation", - **main_schema, - "definitions": {}, - } - - # Add all schemas (including their internal definitions) as top-level definitions - for schema_name, schema_content in schemas.items(): - if schema_name != "ConnectorMetadataDefinitionV0": - if isinstance(schema_content, dict) and "definitions" in schema_content: - for def_name, def_content in schema_content["definitions"].items(): - consolidated["definitions"][def_name] = fix_refs(def_content, in_definition=True) - schema_without_defs = {k: v for k, v in schema_content.items() if k != "definitions"} - consolidated["definitions"][schema_name] = fix_refs(schema_without_defs, in_definition=True) - else: - consolidated["definitions"][schema_name] = fix_refs(schema_content, in_definition=True) - - consolidated = fix_refs(consolidated, in_definition=False) - - Path(output_json_path).write_text(json.dumps(consolidated, indent=2)) - print(f"Generated consolidated JSON schema: {output_json_path}", file=sys.stderr) - else: - print( - "Warning: ConnectorMetadataDefinitionV0 not found, generating simple consolidation", - file=sys.stderr, - ) - Path(output_json_path).write_text(json.dumps(schemas, indent=2)) - - -async def generate_metadata_models_single_file( - dagger_client: dagger.Client, - yaml_dir_path: str, - output_file_path: str, -) -> None: - """Generate all metadata models into a single Python file.""" - codegen_container = ( - dagger_client.container() - .from_(PYTHON_IMAGE) - .with_exec(["mkdir", "-p", "/generated_temp"], use_entrypoint=True) - .with_exec(["pip", "install", " ".join(PIP_DEPENDENCIES)], use_entrypoint=True) - .with_mounted_directory( - "/yaml", dagger_client.host().directory(yaml_dir_path, include=["*.yaml"]) - ) - ) - - codegen_container = codegen_container.with_exec( - [ - "datamodel-codegen", - "--input", - "/yaml", - "--output", - "/generated_temp", - "--disable-timestamp", - "--enum-field-as-literal", - "one", - "--set-default-enum-member", - "--use-double-quotes", - "--remove-special-field-name-prefix", - "--field-extra-keys", - "deprecated", - "deprecation_message", - ], - use_entrypoint=True, - ) - - generated_files = await codegen_container.directory("/generated_temp").entries() - - future_imports = set() - stdlib_imports = set() - third_party_imports = set() - classes_and_updates = [] - - for file_name in sorted(generated_files): - if file_name.endswith(".py") and file_name != "__init__.py": - content = await codegen_container.file(f"/generated_temp/{file_name}").contents() - - lines = content.split("\n") - in_imports = True - in_relative_import_block = False - class_content = [] - - for line in lines: - if in_imports: - if line.startswith("from __future__"): - future_imports.add(line) - elif ( - line.startswith("from datetime") - or line.startswith("from enum") - or line.startswith("from typing") - or line.startswith("from uuid") - ): - stdlib_imports.add(line) - elif line.startswith("from pydantic") or line.startswith("import "): - third_party_imports.add(line) - elif line.startswith("from ."): - in_relative_import_block = True - if not line.rstrip().endswith(",") and not line.rstrip().endswith("("): - in_relative_import_block = False - elif in_relative_import_block: - if line.strip().endswith(")"): - in_relative_import_block = False - elif line.strip() and not line.startswith("#"): - in_imports = False - class_content.append(line) - else: - class_content.append(line) - - if class_content: - classes_and_updates.append("\n".join(class_content)) - - import_sections = [] - if future_imports: - import_sections.append("\n".join(sorted(future_imports))) - if stdlib_imports: - import_sections.append("\n".join(sorted(stdlib_imports))) - if third_party_imports: - import_sections.append("\n".join(sorted(third_party_imports))) - - final_content = "\n\n".join(import_sections) + "\n\n\n" + "\n\n\n".join(classes_and_updates) - - post_processed_content = final_content.replace("from pydantic", "from pydantic.v1") - - lines = post_processed_content.split("\n") - filtered_lines = [] - in_relative_import = False - - for line in lines: - if line.strip().startswith("from . import"): - in_relative_import = True - if not line.rstrip().endswith(",") and not line.rstrip().endswith("("): - in_relative_import = False - continue - - if in_relative_import: - if line.strip().endswith(")"): - in_relative_import = False - continue - - filtered_lines.append(line) - - post_processed_content = "\n".join(filtered_lines) - - codegen_container = codegen_container.with_new_file( - "/generated/models.py", contents=post_processed_content - ) - - await codegen_container.file("/generated/models.py").export(output_file_path) - - async def main(): + init_module_content = generate_init_module_content() + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - print("Generating declarative component models...", file=sys.stderr) - declarative_yaml_files = get_all_yaml_files_without_ext() - await generate_models_from_schemas( - dagger_client=dagger_client, - yaml_dir_path=LOCAL_YAML_DIR_PATH, - output_dir_path=LOCAL_OUTPUT_DIR_PATH, - yaml_files=declarative_yaml_files, - post_process=True, + codegen_container = ( + dagger_client.container() + .from_(PYTHON_IMAGE) + .with_exec(["mkdir", "/generated"], use_entrypoint=True) + .with_exec(["pip", "install", " ".join(PIP_DEPENDENCIES)], use_entrypoint=True) + .with_mounted_directory( + "/yaml", dagger_client.host().directory(LOCAL_YAML_DIR_PATH, include=["*.yaml"]) + ) + .with_new_file("/generated/__init__.py", contents=init_module_content) ) - - print("\nGenerating metadata models...", file=sys.stderr) - with tempfile.TemporaryDirectory() as temp_dir: - temp_path = Path(temp_dir) - schemas_dir = clone_metadata_schemas(temp_path) - - output_dir = Path(LOCAL_METADATA_OUTPUT_DIR_PATH) - output_dir.mkdir(parents=True, exist_ok=True) - - print("Generating single Python file with all models...", file=sys.stderr) - output_file = str(output_dir / "models.py") - await generate_metadata_models_single_file( - dagger_client=dagger_client, - yaml_dir_path=str(schemas_dir), - output_file_path=output_file, + for yaml_file in get_all_yaml_files_without_ext(): + codegen_container = codegen_container.with_exec( + [ + "datamodel-codegen", + "--input", + f"/yaml/{yaml_file}.yaml", + "--output", + f"/generated/{yaml_file}.py", + "--disable-timestamp", + "--enum-field-as-literal", + "one", + "--set-default-enum-member", + "--use-double-quotes", + "--remove-special-field-name-prefix", + # allow usage of the extra key such as `deprecated`, etc. + "--field-extra-keys", + # account the `deprecated` flag provided for the field. + "deprecated", + # account the `deprecation_message` provided for the field. + "deprecation_message", + ], + use_entrypoint=True, ) - print("Generating consolidated JSON schema...", file=sys.stderr) - json_schema_file = str(output_dir / "metadata_schema.json") - consolidate_yaml_schemas_to_json(schemas_dir, json_schema_file) - - print("\nModel generation complete!", file=sys.stderr) + await ( + (await post_process_codegen(codegen_container)) + .directory("/generated_post_processed") + .export(LOCAL_OUTPUT_DIR_PATH) + ) anyio.run(main) diff --git a/bin/generate_connector_metadata_files.py b/bin/generate_connector_metadata_files.py new file mode 100755 index 000000000..4ae26aaa1 --- /dev/null +++ b/bin/generate_connector_metadata_files.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +""" +Generate Pydantic models and JSON schema for connector metadata validation. + +This script downloads metadata schema YAML files from the airbyte monorepo and generates: +1. A single Python file with all Pydantic models (models.py) +2. A consolidated JSON schema file (metadata_schema.json) + +The generated files are used for validating connector metadata.yaml files. +""" + +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +import anyio +import dagger +import yaml + +PYTHON_IMAGE = "python:3.10" +OUTPUT_DIR_PATH = "airbyte_cdk/test/models/connector_metadata/generated" +AIRBYTE_REPO_URL = "https://github.com/airbytehq/airbyte.git" +SCHEMA_PATH = "airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" + +PIP_DEPENDENCIES = [ + "datamodel_code_generator==0.26.3", +] + + +def clone_schemas_from_github(temp_dir: Path) -> Path: + """Clone metadata schema YAML files from GitHub using sparse checkout.""" + clone_dir = temp_dir / "airbyte" + + print("Cloning metadata schemas from airbyte repo...", file=sys.stderr) + + subprocess.run( + [ + "git", + "clone", + "--depth", + "1", + "--filter=blob:none", + "--sparse", + AIRBYTE_REPO_URL, + str(clone_dir), + ], + check=True, + capture_output=True, + ) + + subprocess.run( + ["git", "-C", str(clone_dir), "sparse-checkout", "set", SCHEMA_PATH], + check=True, + capture_output=True, + ) + + schemas_dir = clone_dir / SCHEMA_PATH + print(f"Cloned schemas to {schemas_dir}", file=sys.stderr) + + return schemas_dir + + +async def generate_models_single_file( + dagger_client: dagger.Client, + yaml_dir_path: str, + output_file_path: str, +) -> None: + """Generate all metadata models into a single Python file using datamodel-codegen.""" + codegen_container = ( + dagger_client.container() + .from_(PYTHON_IMAGE) + .with_exec(["mkdir", "-p", "/generated_temp"], use_entrypoint=True) + .with_exec(["pip", "install", " ".join(PIP_DEPENDENCIES)], use_entrypoint=True) + .with_mounted_directory( + "/yaml", dagger_client.host().directory(yaml_dir_path, include=["*.yaml"]) + ) + ) + + codegen_container = codegen_container.with_exec( + [ + "datamodel-codegen", + "--input", + "/yaml", + "--output", + "/generated_temp", + "--disable-timestamp", + "--enum-field-as-literal", + "one", + "--set-default-enum-member", + "--use-double-quotes", + "--remove-special-field-name-prefix", + "--field-extra-keys", + "deprecated", + "deprecation_message", + ], + use_entrypoint=True, + ) + + generated_files = await codegen_container.directory("/generated_temp").entries() + + future_imports = set() + stdlib_imports = set() + third_party_imports = set() + classes_and_updates = [] + + for file_name in sorted(generated_files): + if file_name.endswith(".py") and file_name != "__init__.py": + content = await codegen_container.file(f"/generated_temp/{file_name}").contents() + + lines = content.split("\n") + in_imports = True + in_relative_import_block = False + class_content = [] + + for line in lines: + if in_imports: + if line.startswith("from __future__"): + future_imports.add(line) + elif ( + line.startswith("from datetime") + or line.startswith("from enum") + or line.startswith("from typing") + or line.startswith("from uuid") + ): + stdlib_imports.add(line) + elif line.startswith("from pydantic") or line.startswith("import "): + third_party_imports.add(line) + elif line.startswith("from ."): + in_relative_import_block = True + if not line.rstrip().endswith(",") and not line.rstrip().endswith("("): + in_relative_import_block = False + elif in_relative_import_block: + if line.strip().endswith(")"): + in_relative_import_block = False + elif line.strip() and not line.startswith("#"): + in_imports = False + class_content.append(line) + else: + class_content.append(line) + + if class_content: + classes_and_updates.append("\n".join(class_content)) + + import_sections = [] + if future_imports: + import_sections.append("\n".join(sorted(future_imports))) + if stdlib_imports: + import_sections.append("\n".join(sorted(stdlib_imports))) + if third_party_imports: + import_sections.append("\n".join(sorted(third_party_imports))) + + final_content = "\n\n".join(import_sections) + "\n\n\n" + "\n\n\n".join(classes_and_updates) + + post_processed_content = final_content.replace("from pydantic", "from pydantic.v1") + + lines = post_processed_content.split("\n") + filtered_lines = [] + in_relative_import = False + + for line in lines: + if line.strip().startswith("from . import"): + in_relative_import = True + if not line.rstrip().endswith(",") and not line.rstrip().endswith("("): + in_relative_import = False + continue + + if in_relative_import: + if line.strip().endswith(")"): + in_relative_import = False + continue + + filtered_lines.append(line) + + post_processed_content = "\n".join(filtered_lines) + + codegen_container = codegen_container.with_new_file( + "/generated/models.py", contents=post_processed_content + ) + + await codegen_container.file("/generated/models.py").export(output_file_path) + + +def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: str) -> None: + """Consolidate all YAML schemas into a single JSON schema file.""" + schemas = {} + + for yaml_file in yaml_dir_path.glob("*.yaml"): + schema_name = yaml_file.stem + with yaml_file.open("r") as f: + schema_content = yaml.safe_load(f) + schemas[schema_name] = schema_content + + all_schema_names = set(schemas.keys()) + + for schema_content in schemas.values(): + if isinstance(schema_content, dict) and "definitions" in schema_content: + all_schema_names.update(schema_content["definitions"].keys()) + + def fix_refs(obj, in_definition=False): + """Recursively fix $ref and type references in schema objects.""" + if isinstance(obj, dict): + new_obj = {} + for key, value in obj.items(): + if (key == "$id" or key == "$schema") and in_definition: + continue + elif key == "$ref" and isinstance(value, str): + if value.endswith(".yaml"): + schema_name = value.replace(".yaml", "") + new_obj[key] = f"#/definitions/{schema_name}" + else: + new_obj[key] = value + elif key == "type" and isinstance(value, str) and value in all_schema_names: + new_obj["$ref"] = f"#/definitions/{value}" + elif key == "type" and value == "const": + pass + else: + new_obj[key] = fix_refs(value, in_definition=in_definition) + return new_obj + elif isinstance(obj, list): + return [fix_refs(item, in_definition=in_definition) for item in obj] + else: + return obj + + # Find the main schema (ConnectorMetadataDefinitionV0) + main_schema = schemas.get("ConnectorMetadataDefinitionV0") + + if main_schema: + # Create a consolidated schema with definitions + consolidated = { + "$schema": main_schema.get("$schema", "http://json-schema.org/draft-07/schema#"), + "title": "Connector Metadata Schema", + "description": "Consolidated JSON schema for Airbyte connector metadata validation", + **main_schema, + "definitions": {}, + } + + # Add all schemas (including their internal definitions) as top-level definitions + for schema_name, schema_content in schemas.items(): + if schema_name != "ConnectorMetadataDefinitionV0": + if isinstance(schema_content, dict) and "definitions" in schema_content: + for def_name, def_content in schema_content["definitions"].items(): + consolidated["definitions"][def_name] = fix_refs(def_content, in_definition=True) + schema_without_defs = {k: v for k, v in schema_content.items() if k != "definitions"} + consolidated["definitions"][schema_name] = fix_refs(schema_without_defs, in_definition=True) + else: + consolidated["definitions"][schema_name] = fix_refs(schema_content, in_definition=True) + + consolidated = fix_refs(consolidated, in_definition=False) + + Path(output_json_path).write_text(json.dumps(consolidated, indent=2)) + print(f"Generated consolidated JSON schema: {output_json_path}", file=sys.stderr) + else: + print( + "Warning: ConnectorMetadataDefinitionV0 not found, generating simple consolidation", + file=sys.stderr, + ) + Path(output_json_path).write_text(json.dumps(schemas, indent=2)) + + +async def main(): + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: + print("Generating connector metadata models...", file=sys.stderr) + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + schemas_dir = clone_schemas_from_github(temp_path) + + output_dir = Path(OUTPUT_DIR_PATH) + output_dir.mkdir(parents=True, exist_ok=True) + + print("Generating single Python file with all models...", file=sys.stderr) + output_file = str(output_dir / "models.py") + await generate_models_single_file( + dagger_client=dagger_client, + yaml_dir_path=str(schemas_dir), + output_file_path=output_file, + ) + + print("Generating consolidated JSON schema...", file=sys.stderr) + json_schema_file = str(output_dir / "metadata_schema.json") + consolidate_yaml_schemas_to_json(schemas_dir, json_schema_file) + + print("Connector metadata model generation complete!", file=sys.stderr) + + +if __name__ == "__main__": + anyio.run(main) From 66d4eeb212cb09b9ec49c1a43beee85776f36962 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:01:01 +0000 Subject: [PATCH 16/20] Move metadata generation to poe tasks instead of shell script - Added assemble-declarative and assemble-metadata poe tasks - Reverted shell script to main (no changes) - Updated assemble task to call both generators - Addresses PR feedback to move script call upstream Co-Authored-By: AJ Steers --- bin/generate-component-manifest-dagger.sh | 1 - pyproject.toml | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/generate-component-manifest-dagger.sh b/bin/generate-component-manifest-dagger.sh index 9e760e1b8..908f92a6c 100755 --- a/bin/generate-component-manifest-dagger.sh +++ b/bin/generate-component-manifest-dagger.sh @@ -8,4 +8,3 @@ set -e python bin/generate_component_manifest_files.py -python bin/generate_connector_metadata_files.py diff --git a/pyproject.toml b/pyproject.toml index a1fb961e5..71b02bcb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,7 +144,9 @@ lock = { shell = "poetry lock", help = "Lock all dependencies." } pre-commit = {cmd = "poetry run pre-commit run --all-files", help = "Run all pre-commit hooks on all files."} # Build tasks -assemble = {cmd = "bin/generate-component-manifest-dagger.sh", help = "Generate component manifest files."} +assemble-declarative = {cmd = "python bin/generate_component_manifest_files.py", help = "Generate declarative component manifest files."} +assemble-metadata = {cmd = "python bin/generate_connector_metadata_files.py", help = "Generate connector metadata models."} +assemble = {sequence = ["assemble-declarative", "assemble-metadata"], help = "Generate all manifest files."} build-package = {cmd = "poetry build", help = "Build the python package: source and wheels archives."} build = {sequence = ["assemble", "openapi-generate", "build-package", "ruff-fix"], help = "Run all tasks to build the package."} From 23837ebb61f3680176b5a1df82413e32757f609c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:13:44 +0000 Subject: [PATCH 17/20] Replace Dagger with uvx in metadata generation script - Removed Dagger dependency from new script - Use uvx to run datamodel-codegen directly - Simplified script significantly (no async, no container orchestration) - Script is now ~280 lines vs ~290 lines with Dagger - Addresses PR feedback to avoid Dagger in new code Co-Authored-By: AJ Steers --- bin/generate_connector_metadata_files.py | 176 +++++++++++------------ 1 file changed, 83 insertions(+), 93 deletions(-) diff --git a/bin/generate_connector_metadata_files.py b/bin/generate_connector_metadata_files.py index 4ae26aaa1..dfc9b0dea 100755 --- a/bin/generate_connector_metadata_files.py +++ b/bin/generate_connector_metadata_files.py @@ -17,18 +17,16 @@ import tempfile from pathlib import Path -import anyio -import dagger -import yaml +try: + import yaml +except ImportError: + print("Error: pyyaml is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) -PYTHON_IMAGE = "python:3.10" OUTPUT_DIR_PATH = "airbyte_cdk/test/models/connector_metadata/generated" AIRBYTE_REPO_URL = "https://github.com/airbytehq/airbyte.git" SCHEMA_PATH = "airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src" - -PIP_DEPENDENCIES = [ - "datamodel_code_generator==0.26.3", -] +DATAMODEL_CODEGEN_VERSION = "0.26.3" def clone_schemas_from_github(temp_dir: Path) -> Path: @@ -64,29 +62,27 @@ def clone_schemas_from_github(temp_dir: Path) -> Path: return schemas_dir -async def generate_models_single_file( - dagger_client: dagger.Client, - yaml_dir_path: str, - output_file_path: str, +def generate_models_single_file( + yaml_dir_path: Path, + output_file_path: Path, + temp_dir: Path, ) -> None: """Generate all metadata models into a single Python file using datamodel-codegen.""" - codegen_container = ( - dagger_client.container() - .from_(PYTHON_IMAGE) - .with_exec(["mkdir", "-p", "/generated_temp"], use_entrypoint=True) - .with_exec(["pip", "install", " ".join(PIP_DEPENDENCIES)], use_entrypoint=True) - .with_mounted_directory( - "/yaml", dagger_client.host().directory(yaml_dir_path, include=["*.yaml"]) - ) - ) + generated_temp = temp_dir / "generated_temp" + generated_temp.mkdir(parents=True, exist_ok=True) + + print("Running datamodel-codegen via uvx...", file=sys.stderr) - codegen_container = codegen_container.with_exec( + subprocess.run( [ + "uvx", + "--from", + f"datamodel-code-generator=={DATAMODEL_CODEGEN_VERSION}", "datamodel-codegen", "--input", - "/yaml", + str(yaml_dir_path), "--output", - "/generated_temp", + str(generated_temp), "--disable-timestamp", "--enum-field-as-literal", "one", @@ -97,53 +93,52 @@ async def generate_models_single_file( "deprecated", "deprecation_message", ], - use_entrypoint=True, + check=True, ) - generated_files = await codegen_container.directory("/generated_temp").entries() - future_imports = set() stdlib_imports = set() third_party_imports = set() classes_and_updates = [] - for file_name in sorted(generated_files): - if file_name.endswith(".py") and file_name != "__init__.py": - content = await codegen_container.file(f"/generated_temp/{file_name}").contents() - - lines = content.split("\n") - in_imports = True - in_relative_import_block = False - class_content = [] - - for line in lines: - if in_imports: - if line.startswith("from __future__"): - future_imports.add(line) - elif ( - line.startswith("from datetime") - or line.startswith("from enum") - or line.startswith("from typing") - or line.startswith("from uuid") - ): - stdlib_imports.add(line) - elif line.startswith("from pydantic") or line.startswith("import "): - third_party_imports.add(line) - elif line.startswith("from ."): - in_relative_import_block = True - if not line.rstrip().endswith(",") and not line.rstrip().endswith("("): - in_relative_import_block = False - elif in_relative_import_block: - if line.strip().endswith(")"): - in_relative_import_block = False - elif line.strip() and not line.startswith("#"): - in_imports = False - class_content.append(line) - else: + for py_file in sorted(generated_temp.glob("*.py")): + if py_file.name == "__init__.py": + continue + + content = py_file.read_text() + lines = content.split("\n") + in_imports = True + in_relative_import_block = False + class_content = [] + + for line in lines: + if in_imports: + if line.startswith("from __future__"): + future_imports.add(line) + elif ( + line.startswith("from datetime") + or line.startswith("from enum") + or line.startswith("from typing") + or line.startswith("from uuid") + ): + stdlib_imports.add(line) + elif line.startswith("from pydantic") or line.startswith("import "): + third_party_imports.add(line) + elif line.startswith("from ."): + in_relative_import_block = True + if not line.rstrip().endswith(",") and not line.rstrip().endswith("("): + in_relative_import_block = False + elif in_relative_import_block: + if line.strip().endswith(")"): + in_relative_import_block = False + elif line.strip() and not line.startswith("#"): + in_imports = False class_content.append(line) + else: + class_content.append(line) - if class_content: - classes_and_updates.append("\n".join(class_content)) + if class_content: + classes_and_updates.append("\n".join(class_content)) import_sections = [] if future_imports: @@ -177,22 +172,18 @@ async def generate_models_single_file( post_processed_content = "\n".join(filtered_lines) - codegen_container = codegen_container.with_new_file( - "/generated/models.py", contents=post_processed_content - ) - - await codegen_container.file("/generated/models.py").export(output_file_path) + output_file_path.write_text(post_processed_content) + print(f"Generated models: {output_file_path}", file=sys.stderr) -def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: str) -> None: +def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: Path) -> None: """Consolidate all YAML schemas into a single JSON schema file.""" schemas = {} for yaml_file in yaml_dir_path.glob("*.yaml"): schema_name = yaml_file.stem - with yaml_file.open("r") as f: - schema_content = yaml.safe_load(f) - schemas[schema_name] = schema_content + schema_content = yaml.safe_load(yaml_file.read_text()) + schemas[schema_name] = schema_content all_schema_names = set(schemas.keys()) @@ -251,41 +242,40 @@ def fix_refs(obj, in_definition=False): consolidated = fix_refs(consolidated, in_definition=False) - Path(output_json_path).write_text(json.dumps(consolidated, indent=2)) + output_json_path.write_text(json.dumps(consolidated, indent=2)) print(f"Generated consolidated JSON schema: {output_json_path}", file=sys.stderr) else: print( "Warning: ConnectorMetadataDefinitionV0 not found, generating simple consolidation", file=sys.stderr, ) - Path(output_json_path).write_text(json.dumps(schemas, indent=2)) + output_json_path.write_text(json.dumps(schemas, indent=2)) -async def main(): - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - print("Generating connector metadata models...", file=sys.stderr) +def main(): + print("Generating connector metadata models...", file=sys.stderr) - with tempfile.TemporaryDirectory() as temp_dir: - temp_path = Path(temp_dir) - schemas_dir = clone_schemas_from_github(temp_path) + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + schemas_dir = clone_schemas_from_github(temp_path) - output_dir = Path(OUTPUT_DIR_PATH) - output_dir.mkdir(parents=True, exist_ok=True) + output_dir = Path(OUTPUT_DIR_PATH) + output_dir.mkdir(parents=True, exist_ok=True) - print("Generating single Python file with all models...", file=sys.stderr) - output_file = str(output_dir / "models.py") - await generate_models_single_file( - dagger_client=dagger_client, - yaml_dir_path=str(schemas_dir), - output_file_path=output_file, - ) + print("Generating single Python file with all models...", file=sys.stderr) + output_file = output_dir / "models.py" + generate_models_single_file( + yaml_dir_path=schemas_dir, + output_file_path=output_file, + temp_dir=temp_path, + ) - print("Generating consolidated JSON schema...", file=sys.stderr) - json_schema_file = str(output_dir / "metadata_schema.json") - consolidate_yaml_schemas_to_json(schemas_dir, json_schema_file) + print("Generating consolidated JSON schema...", file=sys.stderr) + json_schema_file = output_dir / "metadata_schema.json" + consolidate_yaml_schemas_to_json(schemas_dir, json_schema_file) - print("Connector metadata model generation complete!", file=sys.stderr) + print("Connector metadata model generation complete!", file=sys.stderr) if __name__ == "__main__": - anyio.run(main) + main() From c6865749b6fc46e0b8e1cc35d95245693418e5d4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:17:56 +0000 Subject: [PATCH 18/20] Simplify metadata generation: generate Python from JSON schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Changed order of operations: YAML → JSON → Python (instead of YAML → Python → merge) - Removed all manual Python file merging logic (~100 lines of code removed) - datamodel-codegen now generates single file directly from consolidated JSON - Much simpler and more maintainable - Addresses AJ's feedback to avoid manual Python code manipulation Co-Authored-By: AJ Steers --- .../connector_metadata/generated/models.py | 811 +++++++++--------- bin/generate_connector_metadata_files.py | 168 +--- 2 files changed, 427 insertions(+), 552 deletions(-) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/models.py b/airbyte_cdk/test/models/connector_metadata/generated/models.py index ab6cfd60e..d81cc1902 100644 --- a/airbyte_cdk/test/models/connector_metadata/generated/models.py +++ b/airbyte_cdk/test/models/connector_metadata/generated/models.py @@ -1,157 +1,249 @@ +# generated by datamodel-codegen: +# filename: metadata_schema.json + from __future__ import annotations -from datetime import date -from datetime import datetime +from datetime import date, datetime from enum import Enum -from typing import Any, Dict, List, Optional -from typing import Any, Dict, Optional -from typing import Any, Optional, Union -from typing import Dict, Optional, Union -from typing import List -from typing import List, Optional -from typing import Literal, Optional -from typing import Optional +from typing import Any, Dict, List, Literal, Optional, Union from uuid import UUID -from pydantic.v1 import AnyUrl, BaseModel, Extra -from pydantic.v1 import AnyUrl, BaseModel, Extra, Field -from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, constr -from pydantic.v1 import BaseModel -from pydantic.v1 import BaseModel, Extra -from pydantic.v1 import BaseModel, Extra, Field -from pydantic.v1 import BaseModel, Extra, Field, conint -from pydantic.v1 import BaseModel, Field +from pydantic.v1 import AnyUrl, BaseModel, Extra, Field, conint, constr -class JobTypeResourceLimit(BaseModel): +class ConnectorType(Enum): + destination = "destination" + source = "source" + + +class ConnectorSubtype(Enum): + api = "api" + database = "database" + datalake = "datalake" + file = "file" + custom = "custom" + message_queue = "message_queue" + unknown = "unknown" + vectorstore = "vectorstore" + + +class TestConnections(BaseModel): class Config: extra = Extra.forbid - jobType: JobType - resourceRequirements: ResourceRequirements + name: str = Field(..., description="The connection name") + id: str = Field(..., description="The connection ID") -class ActorDefinitionResourceRequirements(BaseModel): +class SupportLevel(Enum): + community = "community" + certified = "certified" + archived = "archived" + + +class SuggestedStreams(BaseModel): class Config: - extra = Extra.forbid + extra = Extra.allow - default: Optional[ResourceRequirements] = Field( + streams: Optional[List[str]] = Field( None, - description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", ) - jobSpecific: Optional[List[JobTypeResourceLimit]] = None +class SourceFileInfo(BaseModel): + metadata_etag: Optional[str] = None + metadata_file_path: Optional[str] = None + metadata_bucket_name: Optional[str] = None + metadata_last_modified: Optional[str] = None + registry_entry_generated_at: Optional[str] = None -class Sl(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 +class SecretStore(BaseModel): + class Config: + extra = Extra.forbid -class Ql(Enum): - integer_0 = 0 - integer_100 = 100 - integer_200 = 200 - integer_300 = 300 - integer_400 = 400 - integer_500 = 500 - integer_600 = 600 + alias: Optional[str] = Field( + None, + description="The alias of the secret store which can map to its actual secret address", + ) + type: Optional[Literal["GSM"]] = Field( + None, description="The type of the secret store" + ) -class AirbyteInternal(BaseModel): +class Secret(BaseModel): class Config: - extra = Extra.allow + extra = Extra.forbid - sl: Optional[Sl] = None - ql: Optional[Ql] = None - isEnterprise: Optional[bool] = False - requireVersionIncrementsInPullRequests: Optional[bool] = Field( - True, - description="When false, version increment checks will be skipped for this connector", + name: str = Field(..., description="The secret name in the secret store") + fileName: Optional[str] = Field( + None, + description="The name of the file to which the secret value would be persisted", ) + secretStore: SecretStore - -class AllowedHosts(BaseModel): +class RolloutConfiguration(BaseModel): class Config: - extra = Extra.allow + extra = Extra.forbid - hosts: Optional[List[str]] = Field( - None, - description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + enableProgressiveRollout: Optional[bool] = Field( + False, description="Whether to enable progressive rollout for the connector." + ) + initialPercentage: Optional[conint(ge=0, le=100)] = Field( + 0, + description="The percentage of users that should receive the new version initially.", + ) + maxPercentage: Optional[conint(ge=0, le=100)] = Field( + 50, + description="The percentage of users who should receive the release candidate during the test phase before full rollout.", + ) + advanceDelayMinutes: Optional[conint(ge=10)] = Field( + 10, + description="The number of minutes to wait before advancing the rollout percentage.", ) +class ResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid -class DeadlineAction(Enum): - auto_upgrade = "auto_upgrade" - disable = "disable" + cpu_request: Optional[str] = None + cpu_limit: Optional[str] = None + memory_request: Optional[str] = None + memory_limit: Optional[str] = None -class StreamBreakingChangeScope(BaseModel): +class PyPi(BaseModel): class Config: extra = Extra.forbid - scopeType: Any = Field("stream", const=True) - impactedScopes: List[str] = Field( - ..., - description="List of streams that are impacted by the breaking change.", - min_items=1, - ) + enabled: bool + packageName: str = Field(..., description="The name of the package on PyPi.") -class BreakingChangeScope(BaseModel): - __root__: StreamBreakingChangeScope = Field( +class RemoteRegistries(BaseModel): + class Config: + extra = Extra.forbid + + pypi: Optional[PyPi] = None + + +class ReleaseStage(Enum): + alpha = "alpha" + beta = "beta" + generally_available = "generally_available" + custom = "custom" + + +class NormalizationDestinationDefinitionConfig(BaseModel): + class Config: + extra = Extra.allow + + normalizationRepository: str = Field( ..., - description="A scope that can be used to limit the impact of a breaking change.", + description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", + ) + normalizationTag: str = Field( + ..., + description="a field indicating the tag of the docker repository to be used for normalization.", + ) + normalizationIntegrationType: str = Field( + ..., + description="a field indicating the type of integration dialect to use for normalization.", ) -class VersionBreakingChange(BaseModel): +class JobType(Enum): + get_spec = "get_spec" + check_connection = "check_connection" + discover_schema = "discover_schema" + sync = "sync" + reset_connection = "reset_connection" + connection_updater = "connection_updater" + replicate = "replicate" + + +class GitInfo(BaseModel): class Config: extra = Extra.forbid - upgradeDeadline: date = Field( - ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." + commit_sha: Optional[str] = Field( + None, + description="The git commit sha of the last commit that modified this file.", ) - deadlineAction: Optional[DeadlineAction] = Field( - None, description="Action to do when the deadline is reached." + commit_timestamp: Optional[datetime] = Field( + None, + description="The git commit timestamp of the last commit that modified this file.", ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( + commit_author: Optional[str] = Field( None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + description="The git commit author of the last commit that modified this file.", ) - scopedImpact: Optional[List[BreakingChangeScope]] = Field( + commit_author_email: Optional[str] = Field( None, - description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", - min_items=1, + description="The git commit author email of the last commit that modified this file.", ) -class ConnectorBreakingChanges(BaseModel): +class Suite(Enum): + unitTests = "unitTests" + integrationTests = "integrationTests" + acceptanceTests = "acceptanceTests" + liveTests = "liveTests" + + +class ConnectorTestSuiteOptions(BaseModel): class Config: extra = Extra.forbid - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( - ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", - title="ConnectorBreakingChanges", + suite: Suite = Field(..., description="Name of the configured test suite") + testSecrets: Optional[List[Secret]] = Field( + None, description="List of secrets required to run the test suite" + ) + testConnections: Optional[List[TestConnections]] = Field( + None, + description="List of sandbox cloud connections that tests can be run against", ) +class SourceType(Enum): + api = "api" + file = "file" + database = "database" + custom = "custom" + -class ConnectorBuildOptions(BaseModel): +class ConnectorPackageInfo(BaseModel): + cdk_version: Optional[str] = None + + +class Usage(Enum): + low = "low" + medium = "medium" + high = "high" + + +class SyncSuccessRate(Enum): + low = "low" + medium = "medium" + high = "high" + + +class ConnectorMetric(BaseModel): class Config: - extra = Extra.forbid + extra = Extra.allow + + usage: Optional[Union[str, Usage]] = None + sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None + connector_version: Optional[str] = None - baseImage: Optional[str] = None +class ConnectorMetrics(BaseModel): + all: Optional[ConnectorMetric] = None + cloud: Optional[ConnectorMetric] = None + oss: Optional[ConnectorMetric] = None class SupportedSerializationEnum(Enum): @@ -181,29 +273,176 @@ class Config: dataChannel: DataChannel +class ConnectorBuildOptions(BaseModel): + class Config: + extra = Extra.forbid -class ConnectorType(Enum): - destination = "destination" - source = "source" + baseImage: Optional[str] = None -class ConnectorSubtype(Enum): - api = "api" - database = "database" - datalake = "datalake" - file = "file" - custom = "custom" - message_queue = "message_queue" - unknown = "unknown" - vectorstore = "vectorstore" +class DeadlineAction(Enum): + auto_upgrade = "auto_upgrade" + disable = "disable" + + +class StreamBreakingChangeScope(BaseModel): + class Config: + extra = Extra.forbid + + scopeType: str = Field("stream", const=True) + impactedScopes: List[str] = Field( + ..., + description="List of streams that are impacted by the breaking change.", + min_items=1, + ) + + +class AllowedHosts(BaseModel): + class Config: + extra = Extra.allow + + hosts: Optional[List[str]] = Field( + None, + description="An array of hosts that this connector can connect to. AllowedHosts not being present for the source or destination means that access to all hosts is allowed. An empty list here means that no network access is granted.", + ) + + +class Sl(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + + +class Ql(Enum): + integer_0 = 0 + integer_100 = 100 + integer_200 = 200 + integer_300 = 300 + integer_400 = 400 + integer_500 = 500 + integer_600 = 600 + + +class AirbyteInternal(BaseModel): + class Config: + extra = Extra.allow + + sl: Optional[Sl] = None + ql: Optional[Ql] = None + isEnterprise: Optional[bool] = False + requireVersionIncrementsInPullRequests: Optional[bool] = Field( + True, + description="When false, version increment checks will be skipped for this connector", + ) + + +class JobTypeResourceLimit(BaseModel): + class Config: + extra = Extra.forbid + + jobType: JobType + resourceRequirements: ResourceRequirements + + +class ActorDefinitionResourceRequirements(BaseModel): + class Config: + extra = Extra.forbid + + default: Optional[ResourceRequirements] = Field( + None, + description="if set, these are the requirements that should be set for ALL jobs run for this actor definition.", + ) + jobSpecific: Optional[List[JobTypeResourceLimit]] = None class RegistryOverrides(BaseModel): class Config: extra = Extra.forbid - oss: Optional[RegistryOverrides_1] = None - cloud: Optional[RegistryOverrides_1] = None + enabled: bool + name: Optional[str] = None + dockerRepository: Optional[str] = None + dockerImageTag: Optional[str] = None + supportsDbt: Optional[bool] = None + supportsNormalization: Optional[bool] = None + license: Optional[str] = None + documentationUrl: Optional[AnyUrl] = None + connectorSubtype: Optional[str] = None + allowedHosts: Optional[AllowedHosts] = None + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + suggestedStreams: Optional[SuggestedStreams] = None + resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None + + +class GeneratedFields(BaseModel): + git: Optional[GitInfo] = None + source_file_info: Optional[SourceFileInfo] = None + metrics: Optional[ConnectorMetrics] = None + sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") + + +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( + ..., + description="A scope that can be used to limit the impact of a breaking change.", + ) + + +class RegistryOverridesModel(BaseModel): + class Config: + extra = Extra.forbid + + oss: Optional[RegistryOverrides] = None + cloud: Optional[RegistryOverrides] = None + + +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + deadlineAction: Optional[DeadlineAction] = Field( + None, description="Action to do when the deadline is reached." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + title="ConnectorBreakingChanges", + ) + + +class ConnectorReleases(BaseModel): + class Config: + extra = Extra.forbid + + rolloutConfiguration: Optional[RolloutConfiguration] = None + breakingChanges: Optional[ConnectorBreakingChanges] = None + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + ) class Data(BaseModel): @@ -244,7 +483,7 @@ class Config: [], description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", ) - registryOverrides: Optional[RegistryOverrides] = None + registryOverrides: Optional[RegistryOverridesModel] = None allowedHosts: Optional[AllowedHosts] = None releases: Optional[ConnectorReleases] = None normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None @@ -267,51 +506,23 @@ class Config: data: Data - -class ConnectorMetrics(BaseModel): - all: Optional[Any] = None - cloud: Optional[Any] = None - oss: Optional[Any] = None - - -class Usage(Enum): - low = "low" - medium = "medium" - high = "high" - - -class SyncSuccessRate(Enum): - low = "low" - medium = "medium" - high = "high" - - -class ConnectorMetric(BaseModel): - class Config: - extra = Extra.allow - - usage: Optional[Union[str, Usage]] = None - sync_success_rate: Optional[Union[str, SyncSuccessRate]] = None - connector_version: Optional[str] = None - - - -class ConnectorPackageInfo(BaseModel): - cdk_version: Optional[str] = None - +class ConnectorRegistryV0(BaseModel): + destinations: List[ConnectorRegistryDestinationDefinition] + sources: List[ConnectorRegistrySourceDefinition] -class ConnectorRegistryDestinationDefinition(BaseModel): +class ConnectorRegistrySourceDefinition(BaseModel): class Config: extra = Extra.allow - destinationDefinitionId: UUID + sourceDefinitionId: UUID name: str dockerRepository: str dockerImageTag: str documentationUrl: str icon: Optional[str] = None iconUrl: Optional[str] = None + sourceType: Optional[SourceType] = None spec: Dict[str, Any] tombstone: Optional[bool] = Field( False, @@ -330,47 +541,28 @@ class Config: None, description="The date when this connector was first released, in yyyy-mm-dd format.", ) - tags: Optional[List[str]] = Field( - None, - description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", - ) resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None protocolVersion: Optional[str] = Field( None, description="the Airbyte Protocol version supported by the connector" ) - normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None - supportsDbt: Optional[bool] = Field( + allowedHosts: Optional[AllowedHosts] = None + suggestedStreams: Optional[SuggestedStreams] = None + maxSecondsBetweenMessages: Optional[int] = Field( None, - description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", + description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", + ) + erdUrl: Optional[str] = Field( + None, description="The URL where you can visualize the ERD" ) - allowedHosts: Optional[AllowedHosts] = None releases: Optional[ConnectorRegistryReleases] = None ab_internal: Optional[AirbyteInternal] = None - supportsRefreshes: Optional[bool] = False - supportsFileTransfer: Optional[bool] = False - supportsDataActivation: Optional[bool] = False generated: Optional[GeneratedFields] = None packageInfo: Optional[ConnectorPackageInfo] = None language: Optional[str] = Field( None, description="The language the connector is written in" ) - - -ConnectorRegistryDestinationDefinition.update_forward_refs() - - - -class ConnectorRegistryReleases(BaseModel): - class Config: - extra = Extra.forbid - - releaseCandidates: Optional[ConnectorReleaseCandidates] = None - rolloutConfiguration: Optional[RolloutConfiguration] = None - breakingChanges: Optional[ConnectorBreakingChanges] = None - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", - ) + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False class ConnectorReleaseCandidates(BaseModel): @@ -397,31 +589,30 @@ class Config: ) -ConnectorRegistryReleases.update_forward_refs() -ConnectorReleaseCandidates.update_forward_refs() -VersionReleaseCandidate.update_forward_refs() - - +class ConnectorRegistryReleases(BaseModel): + class Config: + extra = Extra.forbid -class SourceType(Enum): - api = "api" - file = "file" - database = "database" - custom = "custom" + releaseCandidates: Optional[ConnectorReleaseCandidates] = None + rolloutConfiguration: Optional[RolloutConfiguration] = None + breakingChanges: Optional[ConnectorBreakingChanges] = None + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + ) -class ConnectorRegistrySourceDefinition(BaseModel): +class ConnectorRegistryDestinationDefinition(BaseModel): class Config: extra = Extra.allow - sourceDefinitionId: UUID + destinationDefinitionId: UUID name: str dockerRepository: str dockerImageTag: str documentationUrl: str icon: Optional[str] = None iconUrl: Optional[str] = None - sourceType: Optional[SourceType] = None spec: Dict[str, Any] tombstone: Optional[bool] = Field( False, @@ -440,267 +631,33 @@ class Config: None, description="The date when this connector was first released, in yyyy-mm-dd format.", ) + tags: Optional[List[str]] = Field( + None, + description="An array of tags that describe the connector. E.g: language:python, keyword:rds, etc.", + ) resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None protocolVersion: Optional[str] = Field( None, description="the Airbyte Protocol version supported by the connector" ) - allowedHosts: Optional[AllowedHosts] = None - suggestedStreams: Optional[SuggestedStreams] = None - maxSecondsBetweenMessages: Optional[int] = Field( + normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None + supportsDbt: Optional[bool] = Field( None, - description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach", - ) - erdUrl: Optional[str] = Field( - None, description="The URL where you can visualize the ERD" + description="an optional flag indicating whether DBT is used in the normalization. If the flag value is NULL - DBT is not used.", ) + allowedHosts: Optional[AllowedHosts] = None releases: Optional[ConnectorRegistryReleases] = None ab_internal: Optional[AirbyteInternal] = None + supportsRefreshes: Optional[bool] = False + supportsFileTransfer: Optional[bool] = False + supportsDataActivation: Optional[bool] = False generated: Optional[GeneratedFields] = None packageInfo: Optional[ConnectorPackageInfo] = None language: Optional[str] = Field( None, description="The language the connector is written in" ) - supportsFileTransfer: Optional[bool] = False - supportsDataActivation: Optional[bool] = False - - - -class ConnectorRegistryV0(BaseModel): - destinations: List[ConnectorRegistryDestinationDefinition] - sources: List[ConnectorRegistrySourceDefinition] - - - -class ConnectorReleases(BaseModel): - class Config: - extra = Extra.forbid - - rolloutConfiguration: Optional[RolloutConfiguration] = None - breakingChanges: Optional[ConnectorBreakingChanges] = None - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", - ) - - - -class Suite(Enum): - unitTests = "unitTests" - integrationTests = "integrationTests" - acceptanceTests = "acceptanceTests" - liveTests = "liveTests" - - -class ConnectorTestSuiteOptions(BaseModel): - class Config: - extra = Extra.forbid - - suite: Suite = Field(..., description="Name of the configured test suite") - testSecrets: Optional[List[Secret]] = Field( - None, description="List of secrets required to run the test suite" - ) - testConnections: Optional[List[TestConnections]] = Field( - None, - description="List of sandbox cloud connections that tests can be run against", - ) - - - -class GeneratedFields(BaseModel): - git: Optional[GitInfo] = None - source_file_info: Optional[SourceFileInfo] = None - metrics: Optional[ConnectorMetrics] = None - sbomUrl: Optional[str] = Field(None, description="URL to the SBOM file") - -class GitInfo(BaseModel): - class Config: - extra = Extra.forbid - - commit_sha: Optional[str] = Field( - None, - description="The git commit sha of the last commit that modified this file.", - ) - commit_timestamp: Optional[datetime] = Field( - None, - description="The git commit timestamp of the last commit that modified this file.", - ) - commit_author: Optional[str] = Field( - None, - description="The git commit author of the last commit that modified this file.", - ) - commit_author_email: Optional[str] = Field( - None, - description="The git commit author email of the last commit that modified this file.", - ) - - - -class JobType(Enum): - get_spec = "get_spec" - check_connection = "check_connection" - discover_schema = "discover_schema" - sync = "sync" - reset_connection = "reset_connection" - connection_updater = "connection_updater" - replicate = "replicate" - - - -class NormalizationDestinationDefinitionConfig(BaseModel): - class Config: - extra = Extra.allow - - normalizationRepository: str = Field( - ..., - description="a field indicating the name of the repository to be used for normalization. If the value of the flag is NULL - normalization is not used.", - ) - normalizationTag: str = Field( - ..., - description="a field indicating the tag of the docker repository to be used for normalization.", - ) - normalizationIntegrationType: str = Field( - ..., - description="a field indicating the type of integration dialect to use for normalization.", - ) - - - -class RegistryOverrides(BaseModel): - class Config: - extra = Extra.forbid - - enabled: bool - name: Optional[str] = None - dockerRepository: Optional[str] = None - dockerImageTag: Optional[str] = None - supportsDbt: Optional[bool] = None - supportsNormalization: Optional[bool] = None - license: Optional[str] = None - documentationUrl: Optional[AnyUrl] = None - connectorSubtype: Optional[str] = None - allowedHosts: Optional[AllowedHosts] = None - normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None - suggestedStreams: Optional[SuggestedStreams] = None - resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None - - - -class ReleaseStage(Enum): - alpha = "alpha" - beta = "beta" - generally_available = "generally_available" - custom = "custom" - - - -class PyPi(BaseModel): - class Config: - extra = Extra.forbid - - enabled: bool - packageName: str = Field(..., description="The name of the package on PyPi.") - - -class RemoteRegistries(BaseModel): - class Config: - extra = Extra.forbid - - pypi: Optional[PyPi] = None - - - -class ResourceRequirements(BaseModel): - class Config: - extra = Extra.forbid - - cpu_request: Optional[str] = None - cpu_limit: Optional[str] = None - memory_request: Optional[str] = None - memory_limit: Optional[str] = None - - - -class RolloutConfiguration(BaseModel): - class Config: - extra = Extra.forbid - - enableProgressiveRollout: Optional[bool] = Field( - False, description="Whether to enable progressive rollout for the connector." - ) - initialPercentage: Optional[conint(ge=0, le=100)] = Field( - 0, - description="The percentage of users that should receive the new version initially.", - ) - maxPercentage: Optional[conint(ge=0, le=100)] = Field( - 50, - description="The percentage of users who should receive the release candidate during the test phase before full rollout.", - ) - advanceDelayMinutes: Optional[conint(ge=10)] = Field( - 10, - description="The number of minutes to wait before advancing the rollout percentage.", - ) - - - -class Secret(BaseModel): - class Config: - extra = Extra.forbid - - name: str = Field(..., description="The secret name in the secret store") - fileName: Optional[str] = Field( - None, - description="The name of the file to which the secret value would be persisted", - ) - secretStore: SecretStore - - - -class SecretStore(BaseModel): - class Config: - extra = Extra.forbid - - alias: Optional[str] = Field( - None, - description="The alias of the secret store which can map to its actual secret address", - ) - type: Optional[Literal["GSM"]] = Field( - None, description="The type of the secret store" - ) - - - -class SourceFileInfo(BaseModel): - metadata_etag: Optional[str] = None - metadata_file_path: Optional[str] = None - metadata_bucket_name: Optional[str] = None - metadata_last_modified: Optional[str] = None - registry_entry_generated_at: Optional[str] = None - - - -class SuggestedStreams(BaseModel): - class Config: - extra = Extra.allow - - streams: Optional[List[str]] = Field( - None, - description="An array of streams that this connector suggests the average user will want. SuggestedStreams not being present for the source means that all streams are suggested. An empty list here means that no streams are suggested.", - ) - - - -class SupportLevel(Enum): - community = "community" - certified = "certified" - archived = "archived" - - - -class TestConnections(BaseModel): - class Config: - extra = Extra.forbid - - name: str = Field(..., description="The connection name") - id: str = Field(..., description="The connection ID") +ConnectorRegistryV0.update_forward_refs() +ConnectorRegistrySourceDefinition.update_forward_refs() +ConnectorReleaseCandidates.update_forward_refs() +VersionReleaseCandidate.update_forward_refs() diff --git a/bin/generate_connector_metadata_files.py b/bin/generate_connector_metadata_files.py index dfc9b0dea..4e3924306 100755 --- a/bin/generate_connector_metadata_files.py +++ b/bin/generate_connector_metadata_files.py @@ -5,8 +5,8 @@ Generate Pydantic models and JSON schema for connector metadata validation. This script downloads metadata schema YAML files from the airbyte monorepo and generates: -1. A single Python file with all Pydantic models (models.py) -2. A consolidated JSON schema file (metadata_schema.json) +1. A consolidated JSON schema file (metadata_schema.json) +2. A single Python file with all Pydantic models (models.py) generated from the JSON schema The generated files are used for validating connector metadata.yaml files. """ @@ -62,120 +62,6 @@ def clone_schemas_from_github(temp_dir: Path) -> Path: return schemas_dir -def generate_models_single_file( - yaml_dir_path: Path, - output_file_path: Path, - temp_dir: Path, -) -> None: - """Generate all metadata models into a single Python file using datamodel-codegen.""" - generated_temp = temp_dir / "generated_temp" - generated_temp.mkdir(parents=True, exist_ok=True) - - print("Running datamodel-codegen via uvx...", file=sys.stderr) - - subprocess.run( - [ - "uvx", - "--from", - f"datamodel-code-generator=={DATAMODEL_CODEGEN_VERSION}", - "datamodel-codegen", - "--input", - str(yaml_dir_path), - "--output", - str(generated_temp), - "--disable-timestamp", - "--enum-field-as-literal", - "one", - "--set-default-enum-member", - "--use-double-quotes", - "--remove-special-field-name-prefix", - "--field-extra-keys", - "deprecated", - "deprecation_message", - ], - check=True, - ) - - future_imports = set() - stdlib_imports = set() - third_party_imports = set() - classes_and_updates = [] - - for py_file in sorted(generated_temp.glob("*.py")): - if py_file.name == "__init__.py": - continue - - content = py_file.read_text() - lines = content.split("\n") - in_imports = True - in_relative_import_block = False - class_content = [] - - for line in lines: - if in_imports: - if line.startswith("from __future__"): - future_imports.add(line) - elif ( - line.startswith("from datetime") - or line.startswith("from enum") - or line.startswith("from typing") - or line.startswith("from uuid") - ): - stdlib_imports.add(line) - elif line.startswith("from pydantic") or line.startswith("import "): - third_party_imports.add(line) - elif line.startswith("from ."): - in_relative_import_block = True - if not line.rstrip().endswith(",") and not line.rstrip().endswith("("): - in_relative_import_block = False - elif in_relative_import_block: - if line.strip().endswith(")"): - in_relative_import_block = False - elif line.strip() and not line.startswith("#"): - in_imports = False - class_content.append(line) - else: - class_content.append(line) - - if class_content: - classes_and_updates.append("\n".join(class_content)) - - import_sections = [] - if future_imports: - import_sections.append("\n".join(sorted(future_imports))) - if stdlib_imports: - import_sections.append("\n".join(sorted(stdlib_imports))) - if third_party_imports: - import_sections.append("\n".join(sorted(third_party_imports))) - - final_content = "\n\n".join(import_sections) + "\n\n\n" + "\n\n\n".join(classes_and_updates) - - post_processed_content = final_content.replace("from pydantic", "from pydantic.v1") - - lines = post_processed_content.split("\n") - filtered_lines = [] - in_relative_import = False - - for line in lines: - if line.strip().startswith("from . import"): - in_relative_import = True - if not line.rstrip().endswith(",") and not line.rstrip().endswith("("): - in_relative_import = False - continue - - if in_relative_import: - if line.strip().endswith(")"): - in_relative_import = False - continue - - filtered_lines.append(line) - - post_processed_content = "\n".join(filtered_lines) - - output_file_path.write_text(post_processed_content) - print(f"Generated models: {output_file_path}", file=sys.stderr) - - def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: Path) -> None: """Consolidate all YAML schemas into a single JSON schema file.""" schemas = {} @@ -252,6 +138,42 @@ def fix_refs(obj, in_definition=False): output_json_path.write_text(json.dumps(schemas, indent=2)) +def generate_models_from_json_schema(json_schema_path: Path, output_file_path: Path) -> None: + """Generate Pydantic models from consolidated JSON schema.""" + print("Running datamodel-codegen via uvx...", file=sys.stderr) + + subprocess.run( + [ + "uvx", + "--from", + f"datamodel-code-generator=={DATAMODEL_CODEGEN_VERSION}", + "datamodel-codegen", + "--input", + str(json_schema_path), + "--output", + str(output_file_path), + "--input-file-type", + "jsonschema", + "--disable-timestamp", + "--enum-field-as-literal", + "one", + "--set-default-enum-member", + "--use-double-quotes", + "--remove-special-field-name-prefix", + "--field-extra-keys", + "deprecated", + "deprecation_message", + ], + check=True, + ) + + content = output_file_path.read_text() + content = content.replace("from pydantic", "from pydantic.v1") + output_file_path.write_text(content) + + print(f"Generated models: {output_file_path}", file=sys.stderr) + + def main(): print("Generating connector metadata models...", file=sys.stderr) @@ -262,18 +184,14 @@ def main(): output_dir = Path(OUTPUT_DIR_PATH) output_dir.mkdir(parents=True, exist_ok=True) - print("Generating single Python file with all models...", file=sys.stderr) - output_file = output_dir / "models.py" - generate_models_single_file( - yaml_dir_path=schemas_dir, - output_file_path=output_file, - temp_dir=temp_path, - ) - - print("Generating consolidated JSON schema...", file=sys.stderr) + print("Consolidating YAML schemas into JSON...", file=sys.stderr) json_schema_file = output_dir / "metadata_schema.json" consolidate_yaml_schemas_to_json(schemas_dir, json_schema_file) + print("Generating Python models from JSON schema...", file=sys.stderr) + output_file = output_dir / "models.py" + generate_models_from_json_schema(json_schema_file, output_file) + print("Connector metadata model generation complete!", file=sys.stderr) From ba912fe463bc1578cfa0c719f3ace4c8307d3955 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:20:15 +0000 Subject: [PATCH 19/20] Fix schema consolidation per CodeRabbit feedback - Preserve main schema's internal definitions (don't overwrite them) - Only convert custom types to $ref, not JSON primitives (string, object, etc.) - Support YAML refs with fragments (e.g., Foo.yaml#/Bar) - Use regex to properly parse .yaml references with optional fragments Addresses CodeRabbit's critical feedback on bin/generate_connector_metadata_files.py Co-Authored-By: AJ Steers --- .../generated/metadata_schema.json | 3 +- .../connector_metadata/generated/models.py | 2 +- bin/generate_connector_metadata_files.py | 39 +++++++++++-------- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json b/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json index a0ea84026..aef8ce028 100644 --- a/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json +++ b/airbyte_cdk/test/models/connector_metadata/generated/metadata_schema.json @@ -1,8 +1,8 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorMetadataDefinitionV0.yml", "title": "ConnectorMetadataDefinitionV0", "description": "describes the metadata of a connector", - "$id": "https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorMetadataDefinitionV0.yml", "type": "object", "required": [ "metadataSpecVersion", @@ -1076,6 +1076,7 @@ ], "properties": { "scopeType": { + "type": "const", "const": "stream" }, "impactedScopes": { diff --git a/airbyte_cdk/test/models/connector_metadata/generated/models.py b/airbyte_cdk/test/models/connector_metadata/generated/models.py index d81cc1902..51227677c 100644 --- a/airbyte_cdk/test/models/connector_metadata/generated/models.py +++ b/airbyte_cdk/test/models/connector_metadata/generated/models.py @@ -289,7 +289,7 @@ class StreamBreakingChangeScope(BaseModel): class Config: extra = Extra.forbid - scopeType: str = Field("stream", const=True) + scopeType: Any = Field("stream", const=True) impactedScopes: List[str] = Field( ..., description="List of streams that are impacted by the breaking change.", diff --git a/bin/generate_connector_metadata_files.py b/bin/generate_connector_metadata_files.py index 4e3924306..8763d50f0 100755 --- a/bin/generate_connector_metadata_files.py +++ b/bin/generate_connector_metadata_files.py @@ -12,6 +12,7 @@ """ import json +import re import subprocess import sys import tempfile @@ -72,6 +73,7 @@ def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: Path schemas[schema_name] = schema_content all_schema_names = set(schemas.keys()) + json_primitives = {"string", "number", "integer", "boolean", "object", "array", "null"} for schema_content in schemas.values(): if isinstance(schema_content, dict) and "definitions" in schema_content: @@ -85,13 +87,18 @@ def fix_refs(obj, in_definition=False): if (key == "$id" or key == "$schema") and in_definition: continue elif key == "$ref" and isinstance(value, str): - if value.endswith(".yaml"): - schema_name = value.replace(".yaml", "") - new_obj[key] = f"#/definitions/{schema_name}" + m = re.match(r"(?:.*/)?(?P[^/#]+)\.yaml(?P#.*)?$", value) + if m: + schema_name = m.group("name") + frag = m.group("frag") or "" + new_obj[key] = f"#/definitions/{schema_name}{frag}" + else: + new_obj[key] = value + elif key == "type" and isinstance(value, str): + if value in all_schema_names and value not in json_primitives: + new_obj["$ref"] = f"#/definitions/{value}" else: new_obj[key] = value - elif key == "type" and isinstance(value, str) and value in all_schema_names: - new_obj["$ref"] = f"#/definitions/{value}" elif key == "type" and value == "const": pass else: @@ -106,26 +113,26 @@ def fix_refs(obj, in_definition=False): main_schema = schemas.get("ConnectorMetadataDefinitionV0") if main_schema: - # Create a consolidated schema with definitions - consolidated = { - "$schema": main_schema.get("$schema", "http://json-schema.org/draft-07/schema#"), - "title": "Connector Metadata Schema", - "description": "Consolidated JSON schema for Airbyte connector metadata validation", - **main_schema, - "definitions": {}, - } + # Create a consolidated schema preserving main schema structure + consolidated = dict(main_schema) # shallow copy + consolidated.setdefault("$schema", "http://json-schema.org/draft-07/schema#") + consolidated.setdefault("title", "Connector Metadata Schema") + consolidated.setdefault("description", "Consolidated JSON schema for Airbyte connector metadata validation") + + consolidated_definitions = dict(consolidated.get("definitions", {})) # Add all schemas (including their internal definitions) as top-level definitions for schema_name, schema_content in schemas.items(): if schema_name != "ConnectorMetadataDefinitionV0": if isinstance(schema_content, dict) and "definitions" in schema_content: for def_name, def_content in schema_content["definitions"].items(): - consolidated["definitions"][def_name] = fix_refs(def_content, in_definition=True) + consolidated_definitions[def_name] = fix_refs(def_content, in_definition=True) schema_without_defs = {k: v for k, v in schema_content.items() if k != "definitions"} - consolidated["definitions"][schema_name] = fix_refs(schema_without_defs, in_definition=True) + consolidated_definitions[schema_name] = fix_refs(schema_without_defs, in_definition=True) else: - consolidated["definitions"][schema_name] = fix_refs(schema_content, in_definition=True) + consolidated_definitions[schema_name] = fix_refs(schema_content, in_definition=True) + consolidated["definitions"] = consolidated_definitions consolidated = fix_refs(consolidated, in_definition=False) output_json_path.write_text(json.dumps(consolidated, indent=2)) From 3c2a4f878767df458f4362fa28335a0cd4a380fe Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:31:34 +0000 Subject: [PATCH 20/20] Add type annotations and fix formatting - Added type hints to fix_refs function and main function - Fixed ruff formatting issues - All mypy checks now pass for the new script Co-Authored-By: AJ Steers --- bin/generate_connector_metadata_files.py | 27 +++++++++++++++++------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/bin/generate_connector_metadata_files.py b/bin/generate_connector_metadata_files.py index 8763d50f0..3d24f0b52 100755 --- a/bin/generate_connector_metadata_files.py +++ b/bin/generate_connector_metadata_files.py @@ -17,6 +17,7 @@ import sys import tempfile from pathlib import Path +from typing import Any try: import yaml @@ -79,7 +80,7 @@ def consolidate_yaml_schemas_to_json(yaml_dir_path: Path, output_json_path: Path if isinstance(schema_content, dict) and "definitions" in schema_content: all_schema_names.update(schema_content["definitions"].keys()) - def fix_refs(obj, in_definition=False): + def fix_refs(obj: Any, in_definition: bool = False) -> Any: """Recursively fix $ref and type references in schema objects.""" if isinstance(obj, dict): new_obj = {} @@ -117,8 +118,10 @@ def fix_refs(obj, in_definition=False): consolidated = dict(main_schema) # shallow copy consolidated.setdefault("$schema", "http://json-schema.org/draft-07/schema#") consolidated.setdefault("title", "Connector Metadata Schema") - consolidated.setdefault("description", "Consolidated JSON schema for Airbyte connector metadata validation") - + consolidated.setdefault( + "description", "Consolidated JSON schema for Airbyte connector metadata validation" + ) + consolidated_definitions = dict(consolidated.get("definitions", {})) # Add all schemas (including their internal definitions) as top-level definitions @@ -126,11 +129,19 @@ def fix_refs(obj, in_definition=False): if schema_name != "ConnectorMetadataDefinitionV0": if isinstance(schema_content, dict) and "definitions" in schema_content: for def_name, def_content in schema_content["definitions"].items(): - consolidated_definitions[def_name] = fix_refs(def_content, in_definition=True) - schema_without_defs = {k: v for k, v in schema_content.items() if k != "definitions"} - consolidated_definitions[schema_name] = fix_refs(schema_without_defs, in_definition=True) + consolidated_definitions[def_name] = fix_refs( + def_content, in_definition=True + ) + schema_without_defs = { + k: v for k, v in schema_content.items() if k != "definitions" + } + consolidated_definitions[schema_name] = fix_refs( + schema_without_defs, in_definition=True + ) else: - consolidated_definitions[schema_name] = fix_refs(schema_content, in_definition=True) + consolidated_definitions[schema_name] = fix_refs( + schema_content, in_definition=True + ) consolidated["definitions"] = consolidated_definitions consolidated = fix_refs(consolidated, in_definition=False) @@ -181,7 +192,7 @@ def generate_models_from_json_schema(json_schema_path: Path, output_file_path: P print(f"Generated models: {output_file_path}", file=sys.stderr) -def main(): +def main() -> None: print("Generating connector metadata models...", file=sys.stderr) with tempfile.TemporaryDirectory() as temp_dir: