Merge branch 'main' into main

Author: tobymao

.circleci/continue_config.yml

@@ -93,9 +93,13 @@ jobs:
       - run:
           name: Run linters and code style checks
           command: make py-style
-      - run:
-          name: Exercise the benchmarks
-          command: make benchmark-ci
+      - unless:
+          condition:
+            equal: ["3.9", << parameters.python_version >>]
+          steps:
+            - run:
+                name: Exercise the benchmarks
+                command: make benchmark-ci
       - run:
           name: Run cicd tests
           command: make cicd-test
@@ -144,8 +148,11 @@ jobs:
       - halt_unless_core
       - checkout
       - run:
-          name: Run the migration test
-          command: ./.circleci/test_migration.sh
+          name: Run the migration test - sushi
+          command: ./.circleci/test_migration.sh sushi "--gateway duckdb_persistent"
+      - run:
+          name: Run the migration test - sushi_dbt
+          command: ./.circleci/test_migration.sh sushi_dbt "--config migration_test_config"
 
   ui_style:
     docker:
@@ -172,7 +179,7 @@ jobs:
 
   ui_test:
     docker:
-      - image: mcr.microsoft.com/playwright:v1.52.0-jammy
+      - image: mcr.microsoft.com/playwright:v1.54.1-jammy
     resource_class: medium
     steps:
       - halt_unless_client
@@ -201,39 +208,6 @@ jobs:
           name: Run tests
           command: npm --prefix web/client run test
 
-  trigger_private_tests:
-    docker:
-      - image: cimg/python:3.12.0
-    resource_class: small
-    steps:
-      - checkout
-      - run:
-          name: Install setuptools scm
-          command: pip install setuptools_scm
-      - run:
-          name: Trigger private tests
-          command: |
-            export COMMIT_MESSAGE="$(git log --format=%s -n 1 $CIRCLE_SHA1)"
-            export FORMATTED_COMMIT_MESSAGE="${COMMIT_MESSAGE//\"/\\\"}"
-            # returns a version string like 0.1.0.dev11
-            export PACKAGE_VERSION="$(python ./.circleci/get_scm_version.py)"
-            curl --request POST \
-              --url $TOBIKO_PRIVATE_CIRCLECI_URL \
-              --header "Circle-Token: $TOBIKO_PRIVATE_CIRCLECI_KEY" \
-              --header "content-type: application/json" \
-              --data '{
-                "branch":"main",
-                "parameters":{
-                  "run_main_pr":false,
-                  "run_sqlmesh_commit":true,
-                  "sqlmesh_branch":"'$CIRCLE_BRANCH'",
-                  "sqlmesh_commit_author":"'$CIRCLE_USERNAME'",
-                  "sqlmesh_commit_hash":"'$CIRCLE_SHA1'",
-                  "sqlmesh_commit_message":"'"$FORMATTED_COMMIT_MESSAGE"'",
-                  "sqlmesh_package_version":"'$PACKAGE_VERSION'"
-                  }
-              }'
-
   engine_tests_docker:
     parameters:
       engine:
@@ -270,6 +244,9 @@ jobs:
     steps:
       - halt_unless_core
       - checkout
+      - run:
+          name: Install OS-level dependencies
+          command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>"
       - run:
           name: Generate database name
           command: |
@@ -279,6 +256,8 @@ jobs:
             echo "export SNOWFLAKE_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
             echo "export DATABRICKS_CATALOG='$TEST_DB_NAME'" >> "$BASH_ENV"
             echo "export REDSHIFT_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export GCP_POSTGRES_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export FABRIC_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
       - run:
           name: Create test database
           command: ./.circleci/manage-test-db.sh << parameters.engine >> "$TEST_DB_NAME" up
@@ -306,6 +285,7 @@ workflows:
                 - "3.10"
                 - "3.11"
                 - "3.12"
+                - "3.13"
       - cicd_tests_windows
       - engine_tests_docker:
           name: engine_<< matrix.engine >>
@@ -329,19 +309,14 @@ workflows:
           matrix:
             parameters:
               engine:
-                - snowflake
+                #- snowflake
                 - databricks
                 - redshift
                 - bigquery
                 - clickhouse-cloud
                 - athena
-          filters:
-            branches:
-              only:
-                - main
-      - trigger_private_tests:
-          requires:
-            - style_and_cicd_tests
+                - fabric
+                - gcp-postgres
           filters:
             branches:
               only:

.circleci/install-prerequisites.sh

@@ -12,17 +12,31 @@ fi
 
 ENGINE="$1"
 
-COMMON_DEPENDENCIES="libpq-dev netcat-traditional"
+COMMON_DEPENDENCIES="libpq-dev netcat-traditional unixodbc-dev"
 ENGINE_DEPENDENCIES=""
 
 if [ "$ENGINE" == "spark" ]; then
     ENGINE_DEPENDENCIES="default-jdk"
+elif [ "$ENGINE" == "fabric" ]; then
+    echo "Installing Microsoft package repository"
+
+    # ref: https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server
+    curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb
+    sudo dpkg -i packages-microsoft-prod.deb
+    rm packages-microsoft-prod.deb
+
+    ENGINE_DEPENDENCIES="msodbcsql18"    
 fi
 
 ALL_DEPENDENCIES="$COMMON_DEPENDENCIES $ENGINE_DEPENDENCIES"
 
 echo "Installing OS-level dependencies: $ALL_DEPENDENCIES"
 
-sudo apt-get clean && sudo apt-get -y update && sudo apt-get -y install $ALL_DEPENDENCIES
+sudo apt-get clean && sudo apt-get -y update && sudo ACCEPT_EULA='Y' apt-get -y install $ALL_DEPENDENCIES
+
+if [ "$ENGINE" == "spark" ]; then
+    echo "Using Java version for spark:"
+    java -version
+fi
 
 echo "All done"

.circleci/manage-test-db.sh

@@ -51,6 +51,8 @@ databricks_init() {
 
     # Note: the cluster doesnt need to be running to create / drop catalogs, but it does need to be running to run the integration tests
     echo "Ensuring cluster is running"
+    # the || true is to prevent the following error from causing an abort:
+    # > Error: is in unexpected state Running.
     databricks clusters start $CLUSTER_ID || true
 }
 
@@ -80,7 +82,11 @@ redshift_down() {
     EXIT_CODE=1
     ATTEMPTS=0
     while [ $EXIT_CODE -ne 0 ] && [ $ATTEMPTS -lt 5 ]; do
-        redshift_exec "select pg_terminate_backend(procpid) from pg_stat_activity where datname = '$1'"
+        # note: sometimes this pg_terminate_backend() call can randomly fail with: ERROR:  Insufficient privileges 
+        # if it does, let's proceed with the drop anyway rather than aborting and never attempting the drop
+        redshift_exec "select pg_terminate_backend(procpid) from pg_stat_activity where datname = '$1'" || true
+        
+        # perform drop
         redshift_exec "drop database $1;" && EXIT_CODE=$? || EXIT_CODE=$?
         if [ $EXIT_CODE -ne 0 ]; then
             echo "Unable to drop database; retrying..."
@@ -109,6 +115,51 @@ clickhouse-cloud_init() {
     echo "Clickhouse Cloud instance $CLICKHOUSE_CLOUD_HOST is up and running"
 }
 
+# GCP Postgres
+gcp-postgres_init() {
+    # Download and start Cloud SQL Proxy
+    curl -fsSL -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.18.0/cloud-sql-proxy.linux.amd64
+    chmod +x cloud-sql-proxy
+    echo "$GCP_POSTGRES_KEYFILE_JSON" > /tmp/keyfile.json
+    ./cloud-sql-proxy --credentials-file /tmp/keyfile.json $GCP_POSTGRES_INSTANCE_CONNECTION_STRING &
+
+    # Wait for proxy to start
+    sleep 5
+}
+
+gcp-postgres_exec() {
+    PGPASSWORD=$GCP_POSTGRES_PASSWORD psql -h 127.0.0.1 -U $GCP_POSTGRES_USER -c "$1" postgres
+}
+
+gcp-postgres_up() {
+    gcp-postgres_exec "create database $1"
+}
+
+gcp-postgres_down() {
+    gcp-postgres_exec "drop database $1"
+}
+
+# Fabric
+fabric_init() {    
+    python --version #note: as at 2025-08-20, ms-fabric-cli is pinned to Python >= 3.10, <3.13
+    pip install ms-fabric-cli
+    
+    # to prevent the '[EncryptionFailed] An error occurred with the encrypted cache.' error
+    # ref: https://microsoft.github.io/fabric-cli/#switch-to-interactive-mode-optional
+    fab config set encryption_fallback_enabled true 
+
+    echo "Logging in to Fabric"
+    fab auth login -u $FABRIC_CLIENT_ID -p $FABRIC_CLIENT_SECRET --tenant $FABRIC_TENANT_ID
+}
+
+fabric_up() {
+    fab create "SQLMesh CircleCI.Workspace/$1.Warehouse"
+}
+
+fabric_down() {
+    fab rm -f "SQLMesh CircleCI.Workspace/$1.Warehouse" || true
+}
+
 INIT_FUNC="${ENGINE}_init"
 UP_FUNC="${ENGINE}_up"
 DOWN_FUNC="${ENGINE}_down"

.circleci/test_migration.sh

@@ -1,11 +1,6 @@
 #!/usr/bin/env bash
 set -ex
 
-GATEWAY_NAME="duckdb_persistent"
-TMP_DIR=$(mktemp -d)
-SUSHI_DIR="$TMP_DIR/sushi"
-
-
 if [[ -z $(git tag --points-at HEAD) ]]; then
     # If the current commit is not tagged, we need to find the last tag
     LAST_TAG=$(git describe --tags --abbrev=0)
@@ -14,28 +9,48 @@ else
     LAST_TAG=$(git tag --sort=-creatordate | head -n 2 | tail -n 1)
 fi
 
+if [ "$1" == "" ]; then
+    echo "Usage: $0 <example name> <sqlmesh opts>"
+    echo "eg $0 sushi '--gateway duckdb_persistent'"
+    exit 1
+fi
+
+
+TMP_DIR=$(mktemp -d)
+EXAMPLE_NAME="$1"
+SQLMESH_OPTS="$2"
+EXAMPLE_DIR="./examples/$EXAMPLE_NAME"
+TEST_DIR="$TMP_DIR/$EXAMPLE_NAME"
+
+echo "Running migration test for '$EXAMPLE_NAME' in '$TEST_DIR' for example project '$EXAMPLE_DIR' using options '$SQLMESH_OPTS'"
+
 git checkout $LAST_TAG
 
 # Install dependencies from the previous release.
 make install-dev
 
-cp -r ./examples/sushi $TMP_DIR
+cp -r $EXAMPLE_DIR $TEST_DIR
+
+# this is only needed temporarily until the released tag for $LAST_TAG includes this config
+if [ "$EXAMPLE_NAME" == "sushi_dbt" ]; then
+    echo 'migration_test_config = sqlmesh_config(Path(__file__).parent, dbt_target_name="duckdb")' >> $TEST_DIR/config.py
+fi    
 
 # Run initial plan
-pushd $SUSHI_DIR
+pushd $TEST_DIR
 rm -rf ./data/*
-sqlmesh --gateway $GATEWAY_NAME plan --no-prompts --auto-apply
+sqlmesh $SQLMESH_OPTS plan --no-prompts --auto-apply
 rm -rf .cache
 popd
 
-# Switch back to the starting state of the repository
+# Switch back to the starting state of the repository    
 git checkout -
 
 # Install updated dependencies.
 make install-dev
 
 # Migrate and make sure the diff is empty
-pushd $SUSHI_DIR
-sqlmesh --gateway $GATEWAY_NAME migrate
-sqlmesh --gateway $GATEWAY_NAME diff prod
-popd
+pushd $TEST_DIR
+sqlmesh $SQLMESH_OPTS migrate
+sqlmesh $SQLMESH_OPTS diff prod
+popd

.claude/agents/code-reviewer.md

@@ -0,0 +1,73 @@
+---
+name: code-reviewer
+description: Use this agent PROACTIVELY when you need expert code review after writing or modifying code. This agent should be called after completing any coding task to ensure quality, architectural compliance, and catch potential issues. Examples: <example>Context: The user has just implemented a new feature for processing SQLMesh snapshots. user: 'I just added a new method to handle snapshot fingerprinting in the Context class' assistant: 'Let me use the code-reviewer agent to analyze this implementation for potential issues and architectural compliance' <commentary>Since code was just written, use the code-reviewer agent to review the implementation for quality, edge cases, and adherence to SQLMesh patterns.</commentary></example> <example>Context: An agent just generated a database migration script. user: 'Here's the migration I created for adding a new state table' assistant: 'Now I'll have the code-reviewer agent examine this migration for safety and best practices' <commentary>Since a migration was created, use the code-reviewer agent to ensure it follows SQLMesh migration patterns and handles edge cases safely.</commentary></example>
+tools: Glob, Grep, LS, Read, NotebookRead, WebFetch, TodoWrite, WebSearch, Bash
+model: sonnet
+color: blue
+---
+
+You are an Expert Code Reviewer, a senior software engineer with deep expertise in code quality, architecture, and best practices. You NEVER write code yourself - your sole focus is providing thorough, insightful code reviews that catch issues other engineers might miss.
+
+Your core responsibilities:
+
+## Analysis Approach
+
+- Examine code for architectural alignment with established patterns and principles
+- Identify potential edge cases, race conditions, and error scenarios
+- Evaluate performance implications and scalability concerns
+- Check for security vulnerabilities and data safety issues
+- Assess maintainability, readability, and documentation quality
+- Verify adherence to project-specific coding standards and conventions
+
+## Review Methodology
+
+- **Architectural Review**: Does the code follow established patterns? Does it fit well within the existing codebase structure?
+- **Logic Analysis**: Are there logical flaws, edge cases, or scenarios that could cause failures?
+- **Error Handling**: Is error handling comprehensive and appropriate? Are failure modes considered?
+- **Performance Review**: Are there performance bottlenecks, inefficient algorithms, or resource leaks?
+- **Security Assessment**: Are there potential security vulnerabilities or data exposure risks?
+- **Maintainability Check**: Is the code readable, well-structured, and properly documented?
+
+### Standard Code Review Checklist
+
+- Code is simple and readable
+- Functions, classes, and variables are well-named
+- No duplicated code
+- Proper error handling with specific error types
+- No exposed secrets, API keys, or credentials
+- Input validation and sanitization implemented
+- Good test coverage including edge cases
+- Performance considerations addressed
+- Security best practices followed
+- Documentation updated for significant changes
+
+## Feedback Structure
+
+Organize your reviews into clear categories:
+
+- **Critical Issues**: Problems that could cause failures, security issues, or data corruption
+- **Architectural Concerns**: Deviations from established patterns or design principles
+- **Edge Cases**: Scenarios that might not be handled properly
+- **Performance Considerations**: Potential bottlenecks or inefficiencies
+- **Maintainability Improvements**: Suggestions for better code organization or documentation
+- **Documentation**: Suggestions to update documentation for significant changes
+
+## Communication Style
+
+- Be constructive and specific in your feedback
+- Explain the 'why' behind your suggestions, not just the 'what'
+- Prioritize issues by severity and impact
+- Acknowledge good practices when you see them
+- Provide context for your recommendations
+- Ask clarifying questions when code intent is unclear
+
+## Important Constraints
+
+- You NEVER write, modify, or suggest specific code implementations
+- You focus purely on analysis and high-level guidance
+- You always consider the broader system context and existing codebase patterns
+- You escalate concerns about fundamental architectural decisions
+- You validate that solutions align with project requirements and constraints
+
+When reviewing code, assume you're looking at recently written code unless explicitly told otherwise. Focus on providing actionable insights that help improve code quality while respecting the existing architectural decisions and project constraints.
+