stackql · general-kroll-4-life · Dec 31, 2024
diff --git a/.github/workflows/scenario.yaml b/.github/workflows/scenario.yaml
@@ -55,6 +55,9 @@ jobs:
           pip3 install -r cicd/requirements.txt
 
       - name: Run Walkthrough Scenarios
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RO_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RO_AWS_SECRET_ACCESS_KEY }}
         run: |
           python3 test/python/markdown_testing/markdown_testing.py 2>&1 | tee cicd/log/markdown-testing-results.log 
 

diff --git a/.vscode/.gitignore b/.vscode/.gitignore
@@ -1,4 +1,5 @@
 *
 !.gitignore
 !launch.json
-!settings.json
+!settings.json
+!example.env
diff --git a/.vscode/example.env b/.vscode/example.env
@@ -0,0 +1,4 @@
+DUMMY_DIGITALOCEAN_USERNAME=myusername
+DUMMY_DIGITALOCEAN_PASSWORD=mypassword
+DD_API_KEY=myusername
+DD_APPLICATION_KEY=mypassword
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -155,7 +155,10 @@
         "select JSON_EXTRACT(saml.samlIdentity, '$.username') as saml_username from github.scim.saml_ids saml where saml.org = 'dummyorg';",
         "select kind, name, maximumCardsPerInstance from google.compute.acceleratorTypes where project = 'defective-response-content-project' and zone = 'australia-southeast1-a' order by name desc;",
         "registry pull google;",
-        "create materialized view nv as select BackupId, BackupState  from aws.cloudhsm.backups where region = 'ap-southeast-2' order by BackupId;"
+        "create materialized view nv as select BackupId, BackupState  from aws.cloudhsm.backups where region = 'ap-southeast-2' order by BackupId;",
+        "SELECT instance_id FROM aws.ec2.instances WHERE region IN ('us-east-1', 'ap-southeast-2');",
+        "SELECT instance_id FROM aws.ec2_solid_gold.instances WHERE region IN ('us-east-1', 'ap-southeast-2');",
+        "SELECT region FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id;",
       ],
       "default": "show providers;"
     },
@@ -182,12 +185,13 @@
       "type": "pickString",
       "id": "authString",
       "description": "Auth Input arg String",
-      "default": "{ \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" },  \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\"  }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
+      "default": "{}",
       "options": [
         "{ \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" },  \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\"  }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
         "{ \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" },s \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/aws/functional-test-dummy-aws-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/netlify/netlify-token.txt\"  }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/k8s/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/sumologic/sumologic-token.txt\", \"type\": \"basic\" } }",
         "{ \"pgi\": { \"type\": \"sql_data_source::postgres\", \"sqlDataSource\": { \"dsn\": \"postgres://stackql:stackql@127.0.0.1:8432\" } }, \"azure\": { \"type\": \"azure_default\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\"  }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }",
-        "{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" } }"
+        "{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" } }",
+        "{}"
       ]
     },
     {
@@ -381,12 +385,7 @@
     {
       "name": "generic exec",
       "type": "go",
-      "env": {
-        "DUMMY_DIGITALOCEAN_USERNAME": "myusername",
-        "DUMMY_DIGITALOCEAN_PASSWORD": "mypassword",
-        "DD_API_KEY": "myusername",
-        "DD_APPLICATION_KEY": "mypassword"
-      },
+      "envFile": "${workspaceFolder}/.vscode/.env",
       "request": "launch",
       "mode": "debug",
       "program": "${workspaceFolder}/stackql",

diff --git a/cicd/requirements.txt b/cicd/requirements.txt
@@ -1,8 +1,10 @@
 Flask==3.0.3
+Jinja2==3.1.4
 mistune==3.0.2
 psycopg2-binary>=2.9.9
 psycopg[binary]>=3.1.16
 PyYaml>=6.0.1
 requests==2.32.3
 robotframework==6.1.1
 sqlalchemy==1.4.44
+tabulate==0.9.0
diff --git a/docs/developer_guide.md b/docs/developer_guide.md
@@ -15,6 +15,7 @@ The short of things is that for basic build and unit testing, these are needed:
 
 - Install `golang` on your system **if you do not already have version >= 1.21**, per [the `golang` doco](https://go.dev/doc/install).
 - Install `python` on your system **if you do not already have version >= 3.11**, available from [the `python` website](https://www.python.org/downloads/) and numerous package managers.
+- Using a `venv` or otherwise, install the requisite python packages, eg: (system permitting) from the repository root: `pip install -r cicd/requirements.txt`.
 
 Then, each of these should be run from the repository root:
 
@@ -25,7 +26,6 @@ Then, each of these should be run from the repository root:
 
 For serious development, simulated integration tests are essential.  So, there are more dependencies:
 
-- Install the python dependencies (including `robot` framework).  Simplest way, system permitting, is `pip install -r cicd/requirements.txt`.
 - Install `psql`.  On some systems, this can be done as client only and/or with various package managers; fallback is to just [install postgres manually](https://www.postgresql.org/download/).
 
 Having installed all dependencies, the `robot` tests should be run from the repository root directory (this relies upon the executable in `./build/stackql`, built above):
@@ -88,9 +88,15 @@ robot --variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true -d test/robot/functional
 
 ### Manually Testing
 
-Please see [the mock testing doco](/test/mockserver/README.md#manually-testing-mocks).
+Please see [the mock testing doco](/test/python/flask/README.md).
 
 
+## Debuggers
+
+The `vscode` tooling configuration is mostly ready to use, as seen in the `.vscode` directory.  You will need to create a file at the `.gitignore`d location `.vscode/.env`.  Simplest thing just copy the example to get going: `cp .vscode/example.env .vscode/.env`.
+
+The debugger config is pretty messy, and probably with time we will slim it down.  That said, it is far from useless as an example.
+
 ## Provider development
 
 Keen to expose some new functionality though `stackql`?  We are very keen on this!  

diff --git a/docs/walkthroughs/README.md b/docs/walkthroughs/README.md
@@ -16,7 +16,13 @@ in order to setup, run, verify and tear down testing scenarios.  The tests *can*
 
 ## Running from CI
 
-The canonical, **ruleset-protected** tag form is `scenario-<run_number>-<anything>`.  At this stage, `run_number` must refer to a `stackql` run for which a `linux` `amd64` stackql binary archive is present at the time the tag is run.  
+The canonical, **ruleset-protected** tag form is `scenario-<<run_number>>-<<anything>>`.  At this stage, `run_number` must refer to a `stackql` run for which a `linux` `amd64` stackql binary archive is present at the time the tag is run.  
 
 
+## Plumbing
+
+These walkthroughs are runnable using CI.  This is built upon:
+
+- `jinja2` templates, with `<<` and `>>` as delimiters.
+
 
diff --git a/docs/walkthroughs/list-aws-instances.md b/docs/walkthroughs/list-aws-instances.md
@@ -0,0 +1,47 @@
+
+## Setup
+
+First, for whichever AWS user you would like to use, grant read only privileges on EC2 (eg: using `arn:aws:iam::aws:policy/ReadOnlyAccess`).  Then, create a set of AWS CLI credentials per [the AWS documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-authentication-user.html#cli-authentication-user-get), and store them in the appropriate environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
+
+Then, do this in bash:
+
+```bash setup stackql-shell app_root_path=./test/tmp/.list-aws-instances.stackql
+
+stackql shell --approot=<<app_root_path>>  --registry="{ \"url\": \"file://$(pwd)/test/registry\", \"localDocRoot\": \"$(pwd)/test/registry\", \"verifyConfig\": { \"nopVerify\": true } }"
+```
+
+## Method
+
+Do this in the `stackql` shell, replacing the tuple of regions with whichever AWS regions hold interest for you (these are not templated in the example):
+
+```sql stackql-shell
+
+
+SELECT instance_id, region
+FROM aws.ec2_nextgen.instances
+WHERE region IN ('us-east-1', 'ap-southeast-2', 'eu-west-1');
+
+```
+
+## Result
+
+
+Assuming you have chosen regions wisely, you will see something like this included in the output:
+
+```sql stackql stdout expectation stdout-table-contains-data
+|---------------------|----------------|
+|     instance_id     |     region     |
+|---------------------|----------------|
+| i-some-silly-id-011 | us-east-1      |
+|---------------------|----------------|
+| i-some-other-id-011 | ap-southeast-2 |
+|---------------------|----------------|
+```
+
+## Cleanup
+
+```bash teardown best-effort app_root_path=./test/tmp/.list-aws-instances.stackql
+
+rm -rf <<app_root_path>>
+
+```
diff --git a/...kthroughs/get-google-accelerator-types.md → ...throughs/list-google-accelerator-types.md b/...kthroughs/get-google-accelerator-types.md → ...throughs/list-google-accelerator-types.md
@@ -1,20 +1,20 @@
 
 ## Setup
 
-First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete).  Grant the service account at least `Viewer` role equivalent privileges, per [the GCP dumentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles).
+First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete).  Grant the service account at least `Viewer` role equivalent privileges, per [the GCP documentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles).
 
 Then, do this in bash:
 
-```bash setup stackql-shell credentials-path=cicd/keys/testing/google-ro-credentials.json app-root-path=./test/tmp/.get-google-accel.stackql
+```bash setup stackql-shell credentials_path=cicd/keys/testing/google-ro-credentials.json app_root_path=./test/tmp/.get-google-accel.stackql
 
-export GOOGLE_CREDENTIALS="$(cat <credentials-path>)";
+export GOOGLE_CREDENTIALS="$(cat <<credentials_path>>)";
 
-stackql shell --approot=<app-root-path>
+stackql shell --approot=<<app_root_path>>
 ```
 
 ## Method
 
-Do this in the `stackql` shell, replacing `<project>` with your GCP project name, and `<zone>` as desired, eg: `australia-southeast1-a`:
+Do this in the `stackql` shell, replacing `<<project>>` with your GCP project name, and `<<zone>>` as desired, eg: `australia-southeast1-a`:
 
 ```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a
 
@@ -25,8 +25,8 @@ select
   kind 
 FROM google.compute.accelerator_types 
 WHERE 
-  project = '<project>' 
-  AND zone = '<zone>'
+  project = '<<project>>' 
+  AND zone = '<<zone>>'
 ORDER BY
   name desc
 ;
@@ -36,7 +36,7 @@ ORDER BY
 ## Result
 
 
-You will see something very much like this included in the output, presuming you have one VM (if you have zero, only the headers should appper, more VMs means more rows):
+You will see exactly this included in the output:
 
 ```sql expectation stdout-contains-all
 |---------------------|-------------------------|
@@ -52,19 +52,10 @@ You will see something very much like this included in the output, presuming you
 |---------------------|-------------------------|
 ```
 
-<!---  EXPECTATION
-google\ provider,\ version\ 'v24.11.00274'\ successfully\ installed
-goodbye
--->
-
-<x-expectation style="display: none;">
-<stdout-contains-nonempty-table></stdout-contains-nonempty-table>
-</x-expectation>
-
 ## Cleanup
 
-```bash teardown best-effort app-root-path=./test/tmp/.get-google-accel.stackql
+```bash teardown best-effort app_root_path=./test/tmp/.get-google-accel.stackql
 
-rm -rf <app-root-path>
+rm -rf <<app_root_path>>
 
 ```
diff --git a/docs/walkthroughs/get-google-vms.md → docs/walkthroughs/list-google-vms.md b/docs/walkthroughs/get-google-vms.md → docs/walkthroughs/list-google-vms.md
@@ -1,20 +1,20 @@
 
 ## Setup
 
-First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete).  Grant the service account at least `Viewer` role equivalent privileges, per [the GCP dumentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles).
+First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete).  Grant the service account at least `Viewer` role equivalent privileges, per [the GCP documentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles).
 
 Then, do this in bash:
 
-```bash setup stackql-shell credentials-path=cicd/keys/testing/google-ro-credentials.json app-root-path=./test/tmp/.get-google-vms.stackql
+```bash setup stackql-shell credentials_path=cicd/keys/testing/google-ro-credentials.json app_root_path=./test/tmp/.get-google-vms.stackql
 
-export GOOGLE_CREDENTIALS="$(cat <credentials-path>)";
+export GOOGLE_CREDENTIALS="$(cat <<credentials_path>>)";
 
-stackql shell --approot=<app-root-path>
+stackql shell --approot=<<app_root_path>>
 ```
 
 ## Method
 
-Do this in the `stackql` shell, replacing `<project>` with your GCP project name, and `<zone>` as desired, eg: `australia-southeast1-a`:
+Do this in the `stackql` shell, replacing `<<project>>` with your GCP project name, and `<<zone>>` as desired, eg: `australia-southeast1-a`:
 
 ```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a
 
@@ -25,8 +25,8 @@ select
   id 
 FROM google.compute.instances 
 WHERE 
-  project = '<project>' 
-  AND zone = '<zone>'
+  project = '<<project>>' 
+  AND zone = '<<zone>>'
 ;
 
 ```
@@ -55,8 +55,8 @@ goodbye
 
 ## Cleanup
 
-```bash teardown best-effort app-root-path=./test/tmp/.get-google-vms.stackql
+```bash teardown best-effort app_root_path=./test/tmp/.get-google-vms.stackql
 
-rm -rf <app-root-path>
+rm -rf <<app_root_path>>
 
 ```