From 816bf13f48c7f8a6d7515bbe4cabba0e0cac9931 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Sun, 29 Dec 2024 15:45:17 +1100 Subject: [PATCH 01/10] view-tuple-overwrite Summary: - Support for tuple overwrite in views. - Updated version of `aws.cloud_control` incorporated into test suite. - Added robot test `View Tuple Replacement Working As Exemplified by AWS EC2 Instances List and Detail`. --- .vscode/launch.json | 17 +- docs/walkthroughs/README.md | 8 +- docs/walkthroughs/list-aws-instances.md | 53 + ...es.md => list-google-accelerator-types.md} | 29 +- .../{get-google-vms.md => list-google-vms.md} | 18 +- .../dependencyplanner/dependencyplanner.go | 41 + .../stackql/drm/prepared_statement_ctx.go | 12 + .../sql_system/sql/sqlite/sqlengine-setup.ddl | 4 +- ...parameter-pushed-and-response-filtered.txt | 4 +- ...parameter-pushed-and-response-filtered.txt | 11 +- test/python/flask/aws/app.py | 7 +- test/python/flask/aws/root_path_cfg.json | 48 + ...oud_control_ec2_instance_detail.jinja.json | 7 + ...loud_control_ec2_instances_list.jinja.json | 24 + .../markdown_testing/markdown_testing.py | 2 +- test/registry/src/aws/v0.1.0/provider.yaml | 17 + .../src/aws/v0.1.0/services/acmpca.yaml | 2 +- .../aws/v0.1.0/services/cloud_control.yaml | 103 +- .../v0.1.0/services/cloud_control_legacy.yaml | 1014 +++++++++++++++++ .../src/aws/v0.1.0/services/ec2_nextgen.yaml | 4 +- .../src/aws/v0.1.0/services/pseudo_s3.yaml | 16 +- .../stackql_mocked_from_cmd_line.robot | 38 +- test/robot/lib/stackql_context.py | 8 +- 23 files changed, 1388 insertions(+), 99 deletions(-) create mode 100644 docs/walkthroughs/list-aws-instances.md rename docs/walkthroughs/{get-google-accelerator-types.md => list-google-accelerator-types.md} (52%) rename docs/walkthroughs/{get-google-vms.md => list-google-vms.md} (67%) create mode 100644 test/python/flask/aws/templates/cloud_control_ec2_instance_detail.jinja.json create mode 100644 test/python/flask/aws/templates/cloud_control_ec2_instances_list.jinja.json create mode 100644 test/registry/src/aws/v0.1.0/services/cloud_control_legacy.yaml diff --git a/.vscode/launch.json b/.vscode/launch.json index 70ea776e..c6972e7b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -155,7 +155,10 @@ "select JSON_EXTRACT(saml.samlIdentity, '$.username') as saml_username from github.scim.saml_ids saml where saml.org = 'dummyorg';", "select kind, name, maximumCardsPerInstance from google.compute.acceleratorTypes where project = 'defective-response-content-project' and zone = 'australia-southeast1-a' order by name desc;", "registry pull google;", - "create materialized view nv as select BackupId, BackupState from aws.cloudhsm.backups where region = 'ap-southeast-2' order by BackupId;" + "create materialized view nv as select BackupId, BackupState from aws.cloudhsm.backups where region = 'ap-southeast-2' order by BackupId;", + "SELECT instance_id FROM aws.ec2.instances WHERE region IN ('us-east-1', 'ap-southeast-2');", + "SELECT instance_id FROM aws.ec2_solid_gold.instances WHERE region IN ('us-east-1', 'ap-southeast-2');", + "SELECT region FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id;", ], "default": "show providers;" }, @@ -182,12 +185,13 @@ "type": "pickString", "id": "authString", "description": "Auth Input arg String", - "default": "{ \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }", + "default": "{}", "options": [ "{ \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }", "{ \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" },s \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/aws/functional-test-dummy-aws-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/netlify/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/k8s/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/sumologic/sumologic-token.txt\", \"type\": \"basic\" } }", "{ \"pgi\": { \"type\": \"sql_data_source::postgres\", \"sqlDataSource\": { \"dsn\": \"postgres://stackql:stackql@127.0.0.1:8432\" } }, \"azure\": { \"type\": \"azure_default\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }", - "{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" } }" + "{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" } }", + "{}" ] }, { @@ -381,12 +385,7 @@ { "name": "generic exec", "type": "go", - "env": { - "DUMMY_DIGITALOCEAN_USERNAME": "myusername", - "DUMMY_DIGITALOCEAN_PASSWORD": "mypassword", - "DD_API_KEY": "myusername", - "DD_APPLICATION_KEY": "mypassword" - }, + "envFile": "${workspaceFolder}/.vscode/.env", "request": "launch", "mode": "debug", "program": "${workspaceFolder}/stackql", diff --git a/docs/walkthroughs/README.md b/docs/walkthroughs/README.md index 436e2639..1b10d333 100644 --- a/docs/walkthroughs/README.md +++ b/docs/walkthroughs/README.md @@ -16,7 +16,13 @@ in order to setup, run, verify and tear down testing scenarios. The tests *can* ## Running from CI -The canonical, **ruleset-protected** tag form is `scenario--`. At this stage, `run_number` must refer to a `stackql` run for which a `linux` `amd64` stackql binary archive is present at the time the tag is run. +The canonical, **ruleset-protected** tag form is `scenario-<>-<>`. At this stage, `run_number` must refer to a `stackql` run for which a `linux` `amd64` stackql binary archive is present at the time the tag is run. +## Plumbing + +These walkthroughs are runnable using CI. This is built upon: + +- `jinja2` templates, with `<<` and `>>` as delimiters. + diff --git a/docs/walkthroughs/list-aws-instances.md b/docs/walkthroughs/list-aws-instances.md new file mode 100644 index 00000000..40a5935b --- /dev/null +++ b/docs/walkthroughs/list-aws-instances.md @@ -0,0 +1,53 @@ + +## Setup + +First, create a set of AWS CLI credentials per [the AWS documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-authentication-user.html#cli-authentication-user-get), and store them in the appropriate environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. + +Then, do this in bash: + +```bash setup stackql-shell credentials_path=cicd/keys/testing/google-ro-credentials.json app_root_path=./test/tmp/.list-aws-instances.stackql + + +stackql shell --approot=<> +``` + +## Method + +Do this in the `stackql` shell, replacing `<>` with your GCP project name, and `<>` as desired, eg: `australia-southeast1-a`: + +```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a + +registry pull aws; + +SELECT instance_id, region +FROM aws.ec2.instances +WHERE region IN ('us-east-1', 'eu-west-1'); + +``` + +## Result + + +You will see exactly this included in the output: + +```sql expectation stdout-contains-all +|---------------------|-------------------------| +| name | kind | +|---------------------|-------------------------| +| nvidia-tesla-t4-vws | compute#acceleratorType | +|---------------------|-------------------------| +| nvidia-tesla-t4 | compute#acceleratorType | +|---------------------|-------------------------| +| nvidia-tesla-p4-vws | compute#acceleratorType | +|---------------------|-------------------------| +| nvidia-tesla-p4 | compute#acceleratorType | +|---------------------|-------------------------| +``` + +## Cleanup + +```bash teardown best-effort app_root_path=./test/tmp/.list-aws-instances.stackql + +rm -rf <> + +``` \ No newline at end of file diff --git a/docs/walkthroughs/get-google-accelerator-types.md b/docs/walkthroughs/list-google-accelerator-types.md similarity index 52% rename from docs/walkthroughs/get-google-accelerator-types.md rename to docs/walkthroughs/list-google-accelerator-types.md index 42f84752..3de08e4f 100644 --- a/docs/walkthroughs/get-google-accelerator-types.md +++ b/docs/walkthroughs/list-google-accelerator-types.md @@ -1,20 +1,20 @@ ## Setup -First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP dumentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles). +First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP documentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles). Then, do this in bash: -```bash setup stackql-shell credentials-path=cicd/keys/testing/google-ro-credentials.json app-root-path=./test/tmp/.get-google-accel.stackql +```bash setup stackql-shell credentials_path=cicd/keys/testing/google-ro-credentials.json app_root_path=./test/tmp/.get-google-accel.stackql -export GOOGLE_CREDENTIALS="$(cat )"; +export GOOGLE_CREDENTIALS="$(cat <>)"; -stackql shell --approot= +stackql shell --approot=<> ``` ## Method -Do this in the `stackql` shell, replacing `` with your GCP project name, and `` as desired, eg: `australia-southeast1-a`: +Do this in the `stackql` shell, replacing `<>` with your GCP project name, and `<>` as desired, eg: `australia-southeast1-a`: ```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a @@ -25,8 +25,8 @@ select kind FROM google.compute.accelerator_types WHERE - project = '' - AND zone = '' + project = '<>' + AND zone = '<>' ORDER BY name desc ; @@ -36,7 +36,7 @@ ORDER BY ## Result -You will see something very much like this included in the output, presuming you have one VM (if you have zero, only the headers should appper, more VMs means more rows): +You will see exactly this included in the output: ```sql expectation stdout-contains-all |---------------------|-------------------------| @@ -52,19 +52,10 @@ You will see something very much like this included in the output, presuming you |---------------------|-------------------------| ``` - - - - - - ## Cleanup -```bash teardown best-effort app-root-path=./test/tmp/.get-google-accel.stackql +```bash teardown best-effort app_root_path=./test/tmp/.get-google-accel.stackql -rm -rf +rm -rf <> ``` \ No newline at end of file diff --git a/docs/walkthroughs/get-google-vms.md b/docs/walkthroughs/list-google-vms.md similarity index 67% rename from docs/walkthroughs/get-google-vms.md rename to docs/walkthroughs/list-google-vms.md index f2f70059..27e5b511 100644 --- a/docs/walkthroughs/get-google-vms.md +++ b/docs/walkthroughs/list-google-vms.md @@ -1,20 +1,20 @@ ## Setup -First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP dumentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles). +First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP documentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles). Then, do this in bash: -```bash setup stackql-shell credentials-path=cicd/keys/testing/google-ro-credentials.json app-root-path=./test/tmp/.get-google-vms.stackql +```bash setup stackql-shell credentials_path=cicd/keys/testing/google-ro-credentials.json app_root_path=./test/tmp/.get-google-vms.stackql -export GOOGLE_CREDENTIALS="$(cat )"; +export GOOGLE_CREDENTIALS="$(cat <>)"; -stackql shell --approot= +stackql shell --approot=<> ``` ## Method -Do this in the `stackql` shell, replacing `` with your GCP project name, and `` as desired, eg: `australia-southeast1-a`: +Do this in the `stackql` shell, replacing `<>` with your GCP project name, and `<>` as desired, eg: `australia-southeast1-a`: ```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a @@ -25,8 +25,8 @@ select id FROM google.compute.instances WHERE - project = '' - AND zone = '' + project = '<>' + AND zone = '<>' ; ``` @@ -55,8 +55,8 @@ goodbye ## Cleanup -```bash teardown best-effort app-root-path=./test/tmp/.get-google-vms.stackql +```bash teardown best-effort app_root_path=./test/tmp/.get-google-vms.stackql -rm -rf +rm -rf <> ``` \ No newline at end of file diff --git a/internal/stackql/dependencyplanner/dependencyplanner.go b/internal/stackql/dependencyplanner/dependencyplanner.go index 26dd8772..fe32f974 100644 --- a/internal/stackql/dependencyplanner/dependencyplanner.go +++ b/internal/stackql/dependencyplanner/dependencyplanner.go @@ -586,6 +586,43 @@ func (dp *standardDependencyPlanner) processAcquire( return anTab, dp.tcc, nil } +func (dp *standardDependencyPlanner) getScalarParam(param interface{}) (interface{}, bool) { + paramMeta, isParamMeta := param.(parserutil.ParameterMetadata) + if isParamMeta { + val := paramMeta.GetVal() + _, valIsSQLVal := val.(*sqlparser.SQLVal) + if valIsSQLVal { + return val, true + } + } + return nil, false +} + +func (dp *standardDependencyPlanner) isVectorParam(param interface{}) bool { + paramMeta, isParamMeta := param.(parserutil.ParameterMetadata) + if isParamMeta { + val := paramMeta.GetVal() + _, valIsSQLVal := val.(sqlparser.ValTuple) + if valIsSQLVal { + return true + } + } + return false +} + +func (dp *standardDependencyPlanner) extractStaticSourceParam(e dataflow.Edge, paramKey string) (interface{}, bool) { + if e.GetSource() == nil || e. + GetSource().GetAnnotation() == nil || e. + GetSource().GetAnnotation().GetParameters() == nil { + return nil, false + } + v, ok := e.GetSource().GetAnnotation().GetParameters()[paramKey] + if !ok { + return nil, false + } + return dp.getScalarParam(v) +} + //nolint:gocognit,nestif // live with it func (dp *standardDependencyPlanner) getStreamFromEdge( e dataflow.Edge, @@ -639,9 +676,13 @@ func (dp *standardDependencyPlanner) getStreamFromEdge( params := toAc.GetParameters() staticParams := make(map[string]interface{}) for k, v := range params { + existingSourceParam, isExistingFromSource := dp.extractStaticSourceParam(e, k) + // isVector := dp.isVectorParam(v) if _, ok := incomingCols[k]; !ok { staticParams[k] = v incomingCols[k] = struct{}{} + } else if isExistingFromSource && false { + staticParams[k] = existingSourceParam } } if len(staticParams) > 0 { diff --git a/internal/stackql/drm/prepared_statement_ctx.go b/internal/stackql/drm/prepared_statement_ctx.go index ee23e3c0..6fd0da19 100644 --- a/internal/stackql/drm/prepared_statement_ctx.go +++ b/internal/stackql/drm/prepared_statement_ctx.go @@ -53,9 +53,21 @@ type standardPreparedStatementCtx struct { func (ps *standardPreparedStatementCtx) GetOrderedTccs() []internaldto.TxnControlCounters { var rv []internaldto.TxnControlCounters + // absolute hack + // TODO: fix all the counter logic + keysRemaining := make(map[string]struct{}) + for k := range ps.aliasToTccMap { + keysRemaining[k] = struct{}{} + } for _, alias := range ps.aliasOrdering { + delete(keysRemaining, alias) rv = append(rv, ps.aliasToTccMap[alias]...) } + // if len(keysRemaining) == 1 { + // for k := range keysRemaining { + // rv = append(rv, ps.aliasToTccMap[k]...) + // } + // } return rv } diff --git a/internal/stackql/sql_system/sql/sqlite/sqlengine-setup.ddl b/internal/stackql/sql_system/sql/sqlite/sqlengine-setup.ddl index 867a1dcf..4a96bb22 100644 --- a/internal/stackql/sql_system/sql/sqlite/sqlengine-setup.ddl +++ b/internal/stackql/sql_system/sql/sqlite/sqlengine-setup.ddl @@ -267,7 +267,7 @@ VALUES ( IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.BlockPublicAcls'') = 0, ''false'', ''true'') as BlockPublicAcls, IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.IgnorePublicAcls'') = 0, ''false'', ''true'') as IgnorePublicAcls, JSON_EXTRACT(Properties, ''$.Tags'') as Tags - FROM aws.cloud_control.resources WHERE region = ''ap-southeast-2'' and data__TypeName = ''AWS::S3::Bucket'' and data__Identifier = ''stackql-trial-bucket-01'' + FROM aws.cloud_control.resource WHERE region = ''ap-southeast-2'' and data__TypeName = ''AWS::S3::Bucket'' and data__Identifier = ''stackql-trial-bucket-01'' ;' ); @@ -291,7 +291,7 @@ VALUES ( IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.BlockPublicAcls'') = 0, ''false'', ''true'') as BlockPublicAcls, IIF(JSON_EXTRACT(Properties, ''$.PublicAccessBlockConfiguration.IgnorePublicAcls'') = 0, ''false'', ''true'') as IgnorePublicAcls, JSON_EXTRACT(Properties, ''$.Tags'') as Tags - FROM aws.cloud_control.resources WHERE region = ''ap-southeast-2'' and data__TypeName = ''AWS::S3::Bucket'' + FROM aws.cloud_control_legacy.resources WHERE region = ''ap-southeast-2'' and data__TypeName = ''AWS::S3::Bucket'' ;' ); diff --git a/test/assets/expected/aws/cloud_control/select-projection-bucket-view-parameter-pushed-and-response-filtered.txt b/test/assets/expected/aws/cloud_control/select-projection-bucket-view-parameter-pushed-and-response-filtered.txt index 5e1b5908..08a1503b 100644 --- a/test/assets/expected/aws/cloud_control/select-projection-bucket-view-parameter-pushed-and-response-filtered.txt +++ b/test/assets/expected/aws/cloud_control/select-projection-bucket-view-parameter-pushed-and-response-filtered.txt @@ -1,5 +1,5 @@ |-------------------------|------------------------------------------| -| BucketName | DomainName | +| bucket_name | domain_name | |-------------------------|------------------------------------------| | stackql-trial-bucket-01 | stackql-trial-bucket-01.s3.amazonaws.com | -|-------------------------|------------------------------------------| +|-------------------------|------------------------------------------| \ No newline at end of file diff --git a/test/assets/expected/aws/cloud_control/select-star-bucket-view-parameter-pushed-and-response-filtered.txt b/test/assets/expected/aws/cloud_control/select-star-bucket-view-parameter-pushed-and-response-filtered.txt index 3fb770a9..db5dcc74 100644 --- a/test/assets/expected/aws/cloud_control/select-star-bucket-view-parameter-pushed-and-response-filtered.txt +++ b/test/assets/expected/aws/cloud_control/select-star-bucket-view-parameter-pushed-and-response-filtered.txt @@ -1,6 +1,5 @@ -|--------------------------------------|-------------------------|------------------------------------------|---------------------------------------------------------|-------------------------------------------------------------------|------------------------------------------------------------------------|---------------------|-----------------------|-------------------|-----------------|------------------|------------------------------------------------| -| Arn | BucketName | DomainName | RegionalDomainName | DualStackDomainName | WebsiteURL | ObjectOwnership | RestrictPublicBuckets | BlockPublicPolicy | BlockPublicAcls | IgnorePublicAcls | Tags | -|--------------------------------------|-------------------------|------------------------------------------|---------------------------------------------------------|-------------------------------------------------------------------|------------------------------------------------------------------------|---------------------|-----------------------|-------------------|-----------------|------------------|------------------------------------------------| -| arn:aws:s3:::stackql-trial-bucket-01 | stackql-trial-bucket-01 | stackql-trial-bucket-01.s3.amazonaws.com | stackql-trial-bucket-01.s3.ap-southeast-1.amazonaws.com | stackql-trial-bucket-01.s3.dualstack.ap-southeast-1.amazonaws.com | http://stackql-trial-bucket-01.s3-website-ap-southeast-1.amazonaws.com | BucketOwnerEnforced | true | true | true | true | [{"Value":"first-ever-bucket","Key":"sundry"}] | -|--------------------------------------|-------------------------|------------------------------------------|---------------------------------------------------------|-------------------------------------------------------------------|------------------------------------------------------------------------|---------------------|-----------------------|-------------------|-----------------|------------------|------------------------------------------------| - +|----------------|-------------------------|--------------------------|----------------|--------------------------|-------------------|-------------------------|--------------------|------------------------------------|--------------------------|-------------------------|-----------------------|------------------------|----------------------------|---------------------------|---------------------|-------------------------------------------------------|--------------------------------------------------------------------------------------------------------|---------------------------|------------------------------------------------|--------------------------|-----------------------|--------------------------------------|------------------------------------------|-------------------------------------------------------------------|---------------------------------------------------------|------------------------------------------------------------------------| +| region | data__Identifier | accelerate_configuration | access_control | analytics_configurations | bucket_encryption | bucket_name | cors_configuration | intelligent_tiering_configurations | inventory_configurations | lifecycle_configuration | logging_configuration | metrics_configurations | notification_configuration | object_lock_configuration | object_lock_enabled | ownership_controls | public_access_block_configuration | replication_configuration | tags | versioning_configuration | website_configuration | arn | domain_name | dual_stack_domain_name | regional_domain_name | website_url | +|----------------|-------------------------|--------------------------|----------------|--------------------------|-------------------|-------------------------|--------------------|------------------------------------|--------------------------|-------------------------|-----------------------|------------------------|----------------------------|---------------------------|---------------------|-------------------------------------------------------|--------------------------------------------------------------------------------------------------------|---------------------------|------------------------------------------------|--------------------------|-----------------------|--------------------------------------|------------------------------------------|-------------------------------------------------------------------|---------------------------------------------------------|------------------------------------------------------------------------| +| ap-southeast-2 | stackql-trial-bucket-01 | null | null | null | null | stackql-trial-bucket-01 | null | null | null | null | null | null | null | null | null | {"Rules":[{"ObjectOwnership":"BucketOwnerEnforced"}]} | {"RestrictPublicBuckets":true,"BlockPublicPolicy":true,"BlockPublicAcls":true,"IgnorePublicAcls":true} | null | [{"Value":"first-ever-bucket","Key":"sundry"}] | null | null | arn:aws:s3:::stackql-trial-bucket-01 | stackql-trial-bucket-01.s3.amazonaws.com | stackql-trial-bucket-01.s3.dualstack.ap-southeast-1.amazonaws.com | stackql-trial-bucket-01.s3.ap-southeast-1.amazonaws.com | http://stackql-trial-bucket-01.s3-website-ap-southeast-1.amazonaws.com | +|----------------|-------------------------|--------------------------|----------------|--------------------------|-------------------|-------------------------|--------------------|------------------------------------|--------------------------|-------------------------|-----------------------|------------------------|----------------------------|---------------------------|---------------------|-------------------------------------------------------|--------------------------------------------------------------------------------------------------------|---------------------------|------------------------------------------------|--------------------------|-----------------------|--------------------------------------|------------------------------------------|-------------------------------------------------------------------|---------------------------------------------------------|------------------------------------------------------------------------| diff --git a/test/python/flask/aws/app.py b/test/python/flask/aws/app.py index 90418c9b..73446342 100644 --- a/test/python/flask/aws/app.py +++ b/test/python/flask/aws/app.py @@ -207,6 +207,10 @@ def match_route(self, req: Request) -> dict: config_path = os.path.join(os.path.dirname(__file__), "root_path_cfg.json") cfg_obj: GetMatcherConfig = GetMatcherConfig() +def _extract_request_region(request: Request) -> str: + auth_header = request.headers.get("Authorization", "") + return '' if len(auth_header.split('/')) < 3 else auth_header.split('/')[2] + # Routes generated from mockserver configuration @app.route('/', methods=['POST', "GET"]) def handle_root_requests(): @@ -228,7 +232,8 @@ def generic_handler(request: Request): return jsonify({'error': f'Missing template for route: {request}'}), 500 logger.info(f"routing to template: {route_cfg['template']}") twelve_days_ago = (datetime.datetime.now() - datetime.timedelta(days=12)).strftime("%Y-%m-%d") - response = make_response(render_template(route_cfg["template"], request=request, twelve_days_ago=twelve_days_ago)) + region = _extract_request_region(request) + response = make_response(render_template(route_cfg["template"], request=request, region=region, twelve_days_ago=twelve_days_ago)) response.headers.update(route_cfg.get("response_headers", {})) response.status_code = route_cfg.get("status", 200) return response diff --git a/test/python/flask/aws/root_path_cfg.json b/test/python/flask/aws/root_path_cfg.json index d5d73739..0ba8cae7 100644 --- a/test/python/flask/aws/root_path_cfg.json +++ b/test/python/flask/aws/root_path_cfg.json @@ -442,6 +442,54 @@ "Content-Type": ["application/x-amz-json-1.0"] } }, + "POST:/:newkey": { + "method": "POST", + "path": "/", + "headers": { + "Authorization": [ + "^.*SignedHeaders=accept;content-type;host;x-amz-date;x-amz-target.*$" + ], + "X-Amz-Target": [ + "CloudApiService.ListResources" + ] + }, + "body_conditions": { + "type": "JSON", + "json": { + "TypeName": "AWS::EC2::Instance" + }, + "matchType": "ONLY_MATCHING_FIELDS" + }, + "template": "cloud_control_ec2_instances_list.jinja.json", + "status": 200, + "response_headers": { + "Content-Type": ["application/x-amz-json-1.0"] + } + }, + "POST:/:newkey2": { + "method": "POST", + "path": "/", + "headers": { + "Authorization": [ + "^.*SignedHeaders=.*content-type;host;x-amz-date;x-amz-target.*$" + ], + "X-Amz-Target": [ + "CloudApiService.GetResource" + ] + }, + "body_conditions": { + "type": "JSON", + "json": { + "TypeName": "AWS::EC2::Instance" + }, + "matchType": "ONLY_MATCHING_FIELDS" + }, + "template": "cloud_control_ec2_instance_detail.jinja.json", + "status": 200, + "response_headers": { + "Content-Type": ["application/x-amz-json-1.0"] + } + }, "POST:/:20": { "method": "POST", "path": "/", diff --git a/test/python/flask/aws/templates/cloud_control_ec2_instance_detail.jinja.json b/test/python/flask/aws/templates/cloud_control_ec2_instance_detail.jinja.json new file mode 100644 index 00000000..8899ea57 --- /dev/null +++ b/test/python/flask/aws/templates/cloud_control_ec2_instance_detail.jinja.json @@ -0,0 +1,7 @@ +{ + "ResourceDescription": { + "Identifier": "{{ request.json['Identifier'] }}", + "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"aws-stack-dev-web-sg\"],\"PrivateDnsName\":\"ip-10-2-1-156.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"10.2.1.156\",\"UserData\":\"IyEvYmluL2Jhc2gKeXVtIHVwZGF0ZSAteQp5dW0gaW5zdGFsbCAteSBodHRwZApzeXN0ZW1jdGwgc3RhcnQgaHR0cGQKc3lzdGVtY3RsIGVuYWJsZSBodHRwZAplY2hvICc8IURPQ1RZUEUgaHRtbD48aHRtbCBsYW5nPSJlbiI+PGhlYWQ+PG1ldGEgY2hhcnNldD0iVVRGLTgiPjxtZXRhIG5hbWU9InZpZXdwb3J0IiBjb250ZW50PSJ3aWR0aD1kZXZpY2Utd2lkdGgsIGluaXRpYWwtc2NhbGU9MS4wIj48dGl0bGU+U3RhY2tRTCBvbiBBV1M8L3RpdGxlPjxzdHlsZT5ib2R5IHtmb250LWZhbWlseTogVGFob21hLCBzYW5zLXNlcmlmOyBkaXNwbGF5OiBmbGV4OyBqdXN0aWZ5LWNvbnRlbnQ6IGNlbnRlcjsgYWxpZ24taXRlbXM6IGNlbnRlcjsgaGVpZ2h0OiAxMDB2aDsgbWFyZ2luOiAwOyBiYWNrZ3JvdW5kLWNvbG9yOiAjZjBmMGYwOyB0ZXh0LWFsaWduOiBjZW50ZXI7fSBpbWcge2hlaWdodDogYXV0bzt9IGNvZGUge2JhY2tncm91bmQtY29sb3I6ICNlOGU4ZTg7IHBhZGRpbmc6IDJweCA2cHg7IGJvcmRlci1yYWRpdXM6IDNweDsgZm9udC13ZWlnaHQ6IGJvbGQ7fSBwIHtmb250LXNpemU6IDEuNWVtOyBmb250LXdlaWdodDogYm9sZDt9PC9zdHlsZT48L2hlYWQ+JyA+IC92YXIvd3d3L2h0bWwvaW5kZXguaHRtbAplY2hvICc8Ym9keT48ZGl2PjxhIGhyZWY9Imh0dHBzOi8vZ2l0aHViLmNvbS9zdGFja3FsL3N0YWNrcWwiPjxpbWcgc3JjPSJodHRwczovL3N0YWNrcWwuaW8vaW1nL3N0YWNrcWwtbG9nby1ib2xkLnBuZyIgYWx0PSJTdGFja1FMIExvZ28iPjwvYT48cD5IZWxsbywgPGEgaHJlZj0iaHR0cHM6Ly9weXBpLm9yZy9wcm9qZWN0L3N0YWNrcWwtZGVwbG95LyI+PGNvZGU+c3RhY2txbC1kZXBsb3k8L2NvZGU+PC9hPiBvbiBBV1MhPC9wPjwvZGl2PjwvYm9keT48L2h0bWw+JyA+PiAvdmFyL3d3dy9odG1sL2luZGV4Lmh0bWw=\",\"BlockDeviceMappings\":[{\"Ebs\":{\"SnapshotId\":\"snap-099392264f95c4223\",\"VolumeType\":\"gp3\",\"Encrypted\":false,\"Iops\":3000,\"VolumeSize\":8,\"DeleteOnTermination\":true},\"DeviceName\":\"/dev/xvda\"}],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-0e4cb97c9e4ab3e9a\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"10.2.1.156\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"10.2.1.156\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"10.2.1.156\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-08d705c9b6ecfff45\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-050a905d286cb29b5\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-030a5acd7c996ef60\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"aws-stack\",\"Key\":\"StackName\"},{\"Value\":\"stackql\",\"Key\":\"Provisioner\"},{\"Value\":\"dev\",\"Key\":\"StackEnv\"},{\"Value\":\"aws-stack-dev-instance\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"InstanceId\":\"i-0cb58b995b19153a8\",\"PublicIp\":\"13.211.134.69\",\"InstanceInitiatedShutdownBehavior\":\"stop\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-13-211-134-69.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-08d705c9b6ecfff45\"],\"DisableApiTermination\":false,\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-00e086ac8c9504aec\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"},\"CreditSpecification\":{\"CPUCredits\":\"standard\"}}" + }, + "TypeName": "AWS::EC2::Instance" + } \ No newline at end of file diff --git a/test/python/flask/aws/templates/cloud_control_ec2_instances_list.jinja.json b/test/python/flask/aws/templates/cloud_control_ec2_instances_list.jinja.json new file mode 100644 index 00000000..d4deea39 --- /dev/null +++ b/test/python/flask/aws/templates/cloud_control_ec2_instances_list.jinja.json @@ -0,0 +1,24 @@ +{ + "ResourceDescriptions": [ + {% if region == 'ap-southeast-2' %} + { + "Identifier": "i-0fc989cf4efcd8b88", + "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"launch-wizard\"],\"PrivateDnsName\":\"ip-172-31-6-142.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"172.31.6.142\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-0a06b014d09e72b0d\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"172.31.6.142\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"172.31.6.142\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"172.31.6.142\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-07d3754577c36eeac\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-07c9c942f637281b9\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-0035ee596a0a12a7b\",\"InstanceType\":\"t2.nano\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"Fred\",\"Key\":\"Test\"},{\"Value\":\"test\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-0fc989cf4efcd8b88\",\"PublicIp\":\"13.211.203.69\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-13-211-203-69.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-07d3754577c36eeac\"],\"KeyName\":\"stackql-test\",\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-0343d19bfd304188e\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" + }, + { + "Identifier": "i-0d1be1a3ab2a09a88", + "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"launch-wizard\"],\"PrivateDnsName\":\"ip-172-31-10-204.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"172.31.10.204\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-09bbae22caf877d1e\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"172.31.10.204\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"172.31.10.204\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"172.31.10.204\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-07d3754577c36eeac\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-08cab5f9ca9ea8b9d\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-0ec0514235185af79\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"test1\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-0d1be1a3ab2a09a88\",\"PublicIp\":\"54.66.216.138\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":true,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-54-66-216-138.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-07d3754577c36eeac\"],\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-0343d19bfd304188e\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" + }, + { + "Identifier": "i-0cb58b995b19153a8", + "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"aws-stack-dev-web-sg\"],\"PrivateDnsName\":\"ip-10-2-1-156.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"10.2.1.156\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-0e4cb97c9e4ab3e9a\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"10.2.1.156\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"10.2.1.156\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"10.2.1.156\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-08d705c9b6ecfff45\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-050a905d286cb29b5\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-030a5acd7c996ef60\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"aws-stack\",\"Key\":\"StackName\"},{\"Value\":\"stackql\",\"Key\":\"Provisioner\"},{\"Value\":\"dev\",\"Key\":\"StackEnv\"},{\"Value\":\"aws-stack-dev-instance\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-0cb58b995b19153a8\",\"PublicIp\":\"13.211.134.69\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-13-211-134-69.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-08d705c9b6ecfff45\"],\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-00e086ac8c9504aec\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" + } + {% elif region == 'us-east-1' %} + { + "Identifier": "i-05259c370b23083bf", + "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"aws-stack-dev-web-sg\"],\"PrivateDnsName\":\"ip-10-2-1-199.ec2.internal\",\"PrivateIpAddress\":\"10.2.1.199\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-0e16ac880cb33918f\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-055d48a8b1a6259bf\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"10.2.1.199\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"10.2.1.199\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"10.2.1.199\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-073ac14cc1a4ddfb0\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-0e16ac880cb33918f\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-04ad5c232cc4c4a2e\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-012967cc5a8c9f891\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"aws-stack-dev-instance\",\"Key\":\"Name\"},{\"Value\":\"aws-stack\",\"Key\":\"StackName\"},{\"Value\":\"dev\",\"Key\":\"StackEnv\"},{\"Value\":\"stackql\",\"Key\":\"Provisioner\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-05259c370b23083bf\",\"PublicIp\":\"18.234.253.185\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"us-east-1e\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-18-234-253-185.compute-1.amazonaws.com\",\"SecurityGroupIds\":[\"sg-073ac14cc1a4ddfb0\"],\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-0df79c273d7be8cce\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" + } + {% endif %} + ], + "TypeName": "AWS::EC2::Instance" +} \ No newline at end of file diff --git a/test/python/markdown_testing/markdown_testing.py b/test/python/markdown_testing/markdown_testing.py index 5ce4e1da..61512848 100644 --- a/test/python/markdown_testing/markdown_testing.py +++ b/test/python/markdown_testing/markdown_testing.py @@ -83,7 +83,7 @@ def get_execution_language(self) -> str: return self.node.get('lang', '') def expand(self) -> str: - return self.get_text().replace("<", "{").replace(">", "}").format(**self._local_vars) + return self.get_text().replace("<<", "{").replace(">>", "}").format(**self._local_vars) def __str__(self): return json.dumps(self.node, indent=2) diff --git a/test/registry/src/aws/v0.1.0/provider.yaml b/test/registry/src/aws/v0.1.0/provider.yaml index 3a9cd694..4ff8dee9 100644 --- a/test/registry/src/aws/v0.1.0/provider.yaml +++ b/test/registry/src/aws/v0.1.0/provider.yaml @@ -20,6 +20,15 @@ providerServices: $ref: aws/v0.1.0/services/cloud_control.yaml title: Cloud Control API version: v0.1.0 + cloud_control_legacy: + description: cloud_control_legacy + id: cloud_control_legacy:v0.1.0 + name: cloud_control_legacy + preferred: true + service: + $ref: aws/v0.1.0/services/cloud_control_legacy.yaml + title: Cloud Control Legacy API + version: v0.1.0 cloudhsm: description: cloud_hsm id: cloud_hsm:v2.0.0 @@ -55,6 +64,14 @@ providerServices: service: $ref: aws/v0.1.0/services/ec2_nextgen.yaml title: EC2 NextGen + ec2_solid_gold: + description: ec2_solid_gold + id: ec2_solid_gold:v0.1.0 + name: ec2_solid_gold + preferred: true + service: + $ref: aws/v0.1.0/services/ec2_solid_gold.yaml + title: EC2 SolidGold version: v0.1.0 iam: description: iam diff --git a/test/registry/src/aws/v0.1.0/services/acmpca.yaml b/test/registry/src/aws/v0.1.0/services/acmpca.yaml index 0dbcbbac..1acc3a2b 100644 --- a/test/registry/src/aws/v0.1.0/services/acmpca.yaml +++ b/test/registry/src/aws/v0.1.0/services/acmpca.yaml @@ -1486,7 +1486,7 @@ components: JSON_EXTRACT(Properties, '$.CertificateChain') as certificate_chain, JSON_EXTRACT(Properties, '$.Status') as status, JSON_EXTRACT(Properties, '$.CompleteCertificateChain') as complete_certificate_chain - FROM aws.cloud_control.resources WHERE data__TypeName = 'AWS::ACMPCA::CertificateAuthorityActivation' + FROM aws.cloud_control.resource WHERE data__TypeName = 'AWS::ACMPCA::CertificateAuthorityActivation' AND data__Identifier = '' AND region = 'us-east-1' fallback: diff --git a/test/registry/src/aws/v0.1.0/services/cloud_control.yaml b/test/registry/src/aws/v0.1.0/services/cloud_control.yaml index a0d2c2ac..42b82b1e 100644 --- a/test/registry/src/aws/v0.1.0/services/cloud_control.yaml +++ b/test/registry/src/aws/v0.1.0/services/cloud_control.yaml @@ -2,7 +2,7 @@ openapi: 3.0.0 info: version: '2021-09-30' x-release: v4 - title: AWS Cloud Control API + title: AWS Cloud Control api description: 'For more information about Amazon Web Services Cloud Control API, see the Amazon Web Services Cloud Control API User Guide.' x-logo: url: 'https://twitter.com/awscloud/profile_image?size=original' @@ -430,9 +430,10 @@ paths: schema: properties: MaxResults: + # $ref: '#/components/schemas/MaxResults' type: integer maximum: 100 - minimum: 1 + minimum: 1 NextToken: $ref: '#/components/schemas/NextToken' ResourceModel: @@ -522,8 +523,57 @@ paths: description: Success components: x-stackQL-resources: + resource_raw: + name: resource + x-cfn-schema-name: ResourceDescription + x-example-where-clause: WHERE region = 'us-east-1' AND data__TypeName = '' AND data__Identifier = '' + x-type: native + methods: + get_resource_raw: + operation: + $ref: '#/paths/~1?Action=GetResource&Version=2021-09-30/post' + request: + mediaType: application/x-amz-json-1.0 + response: + mediaType: application/json + openAPIDocKey: '200' + id: aws.cloud_control.resource_raw + sqlVerbs: + delete: [] + insert: [] + select: + - $ref: '#/components/x-stackQL-resources/resource_raw/methods/get_resource_raw' + update: [] + title: resource_raw + resource: + name: resource + x-cfn-schema-name: ResourceDescription + x-example-where-clause: WHERE region = 'us-east-1' AND data__TypeName = '' AND data__Identifier = '' + x-type: native + methods: + get_resource: + operation: + $ref: '#/paths/~1?Action=GetResource&Version=2021-09-30/post' + request: + mediaType: application/x-amz-json-1.0 + response: + mediaType: application/json + objectKey: '$.ResourceDescription' + openAPIDocKey: '200' + id: aws.cloud_control.resources + sqlVerbs: + delete: [] + insert: [] + select: + - $ref: '#/components/x-stackQL-resources/resource/methods/get_resource' + update: [] + title: resource resources: name: resources + x-cfn-schema-name: ResourceDescriptions + x-example-where-clause: WHERE region = 'us-east-1' AND data__TypeName = '' + x-description: 'For more information about Amazon Web Services Cloud Control API, see the Amazon Web Services Cloud Control API User Guide.' + x-type: native methods: list_resources: operation: @@ -550,15 +600,6 @@ components: response: mediaType: application/json openAPIDocKey: '200' - get_resource: - operation: - $ref: '#/paths/~1?Action=GetResource&Version=2021-09-30/post' - request: - mediaType: application/x-amz-json-1.0 - response: - mediaType: application/json - objectKey: '$.ResourceDescription' - openAPIDocKey: '200' update_resource: operation: $ref: '#/paths/~1?Action=UpdateResource&Version=2021-09-30/post' @@ -574,30 +615,40 @@ components: insert: - $ref: '#/components/x-stackQL-resources/resources/methods/create_resource' select: - - $ref: '#/components/x-stackQL-resources/resources/methods/get_resource' - $ref: '#/components/x-stackQL-resources/resources/methods/list_resources' update: - $ref: '#/components/x-stackQL-resources/resources/methods/update_resource' title: resources - resource: - name: resource + resource_request: + name: resource_request + x-cfn-schema-name: ProgressEvent + x-example-where-clause: WHERE region = 'us-east-1' AND data__RequestToken = '' + x-description: 'For more information about Amazon Web Services Cloud Control API, see the Amazon Web Services Cloud Control API User Guide.' + x-type: native methods: - get_resource: + get_resource_request: operation: - $ref: '#/paths/~1?Action=GetResource&Version=2021-09-30/post' + $ref: '#/paths/~1?Action=GetResourceRequestStatus&Version=2021-09-30/post' request: mediaType: application/x-amz-json-1.0 response: - mediaType: application/json - objectKey: '$.ResourceDescription' + mediaType: application/x-amz-json-1.0 + objectKey: '$.ProgressEvent' openAPIDocKey: '200' - id: aws.cloud_control.resource + id: aws.cloud_control.resource_request sqlVerbs: + delete: [] + insert: [] select: - - $ref: '#/components/x-stackQL-resources/resources/methods/get_resource' - title: resource + - $ref: '#/components/x-stackQL-resources/resource_request/methods/get_resource_request' + update: [] + title: resource_request resource_requests: name: resource_requests + x-cfn-schema-name: ResourceRequestStatusSummaries + x-example-where-clause: WHERE region = 'us-east-1' AND data__ResourceRequestStatusFilter = '' + x-description: 'For more information about Amazon Web Services Cloud Control API, see the Amazon Web Services Cloud Control API User Guide.' + x-type: native methods: list_resource_requests: operation: @@ -608,15 +659,6 @@ components: mediaType: application/x-amz-json-1.0 objectKey: '$.ResourceRequestStatusSummaries' openAPIDocKey: '200' - get_resource_request: - operation: - $ref: '#/paths/~1?Action=GetResourceRequestStatus&Version=2021-09-30/post' - request: - mediaType: application/x-amz-json-1.0 - response: - mediaType: application/x-amz-json-1.0 - objectKey: '$.ProgressEvent' - openAPIDocKey: '200' cancel_resource_request: operation: $ref: '#/paths/~1?Action=CancelResourceRequest&Version=2021-09-30/post' @@ -631,7 +673,6 @@ components: - $ref: '#/components/x-stackQL-resources/resource_requests/methods/cancel_resource_request' insert: [] select: - - $ref: '#/components/x-stackQL-resources/resource_requests/methods/get_resource_request' - $ref: '#/components/x-stackQL-resources/resource_requests/methods/list_resource_requests' update: [] title: resource_requests diff --git a/test/registry/src/aws/v0.1.0/services/cloud_control_legacy.yaml b/test/registry/src/aws/v0.1.0/services/cloud_control_legacy.yaml new file mode 100644 index 00000000..a0d2c2ac --- /dev/null +++ b/test/registry/src/aws/v0.1.0/services/cloud_control_legacy.yaml @@ -0,0 +1,1014 @@ +openapi: 3.0.0 +info: + version: '2021-09-30' + x-release: v4 + title: AWS Cloud Control API + description: 'For more information about Amazon Web Services Cloud Control API, see the Amazon Web Services Cloud Control API User Guide.' + x-logo: + url: 'https://twitter.com/awscloud/profile_image?size=original' + backgroundColor: '#FFFFFF' + termsOfService: 'https://aws.amazon.com/service-terms/' + contact: + name: Mike Ralphson + email: mike.ralphson@gmail.com + url: 'https://github.com/mermade/aws2openapi' + x-twitter: PermittedSoc + license: + name: Apache 2.0 License + url: 'http://www.apache.org/licenses/' + x-providerName: amazonaws.com + x-serviceName: cloudcontrolapi + x-aws-signingName: cloudcontrolapi + x-origin: + - contentType: application/json + url: 'https://raw.githubusercontent.com/aws/aws-sdk-js/master/apis/cloudcontrol-2021-09-30.normal.json' + converter: + url: 'https://github.com/mermade/aws2openapi' + version: 1.0.0 + x-apisguru-driver: external + x-apiClientRegistration: + url: 'https://portal.aws.amazon.com/gp/aws/developer/registration/index.html?nc2=h_ct' + x-apisguru-categories: + - cloud + x-preferred: true +externalDocs: + description: Amazon Web Services documentation + url: 'https://docs.aws.amazon.com/cloudcontrolapi/' +servers: + - url: 'https://cloudcontrolapi.{region}.amazonaws.com' + variables: + region: + description: The AWS region + enum: + - us-east-1 + - us-east-2 + - us-west-1 + - us-west-2 + - us-gov-west-1 + - us-gov-east-1 + - ca-central-1 + - eu-north-1 + - eu-west-1 + - eu-west-2 + - eu-west-3 + - eu-central-1 + - eu-south-1 + - af-south-1 + - ap-northeast-1 + - ap-northeast-2 + - ap-northeast-3 + - ap-southeast-1 + - ap-southeast-2 + - ap-east-1 + - ap-south-1 + - sa-east-1 + - me-south-1 + default: us-east-1 + description: The CloudControlApi multi-region endpoint + - url: 'https://cloudcontrolapi.{region}.amazonaws.com.cn' + variables: + region: + description: The AWS region + enum: + - cn-north-1 + - cn-northwest-1 + default: cn-north-1 + description: The CloudControlApi endpoint for China (Beijing) and China (Ningxia) +paths: + /?Action=CancelResourceRequest&Version=2021-09-30: + parameters: + - $ref: '#/components/parameters/X-Amz-Content-Sha256' + - $ref: '#/components/parameters/X-Amz-Date' + - $ref: '#/components/parameters/X-Amz-Algorithm' + - $ref: '#/components/parameters/X-Amz-Credential' + - $ref: '#/components/parameters/X-Amz-Security-Token' + - $ref: '#/components/parameters/X-Amz-Signature' + - $ref: '#/components/parameters/X-Amz-SignedHeaders' + post: + operationId: CancelResourceRequest + parameters: + - description: Action Header + in: header + name: X-Amz-Target + required: false + schema: + default: CloudApiService.CancelResourceRequest + enum: + - CloudApiService.CancelResourceRequest + type: string + - in: header + name: Content-Type + required: false + schema: + default: application/x-amz-json-1.0 + enum: + - application/x-amz-json-1.0 + type: string + requestBody: + content: + application/x-amz-json-1.0: + schema: + $ref: '#/components/schemas/CancelResourceRequestInput' + required: true + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/CancelResourceRequestOutput' + description: Success + /?Action=CreateResource&Version=2021-09-30: + parameters: + - $ref: '#/components/parameters/X-Amz-Content-Sha256' + - $ref: '#/components/parameters/X-Amz-Date' + - $ref: '#/components/parameters/X-Amz-Algorithm' + - $ref: '#/components/parameters/X-Amz-Credential' + - $ref: '#/components/parameters/X-Amz-Security-Token' + - $ref: '#/components/parameters/X-Amz-Signature' + - $ref: '#/components/parameters/X-Amz-SignedHeaders' + post: + operationId: CreateResource + parameters: + - description: Action Header + in: header + name: X-Amz-Target + required: false + schema: + default: CloudApiService.CreateResource + enum: + - CloudApiService.CreateResource + type: string + - in: header + name: Content-Type + required: false + schema: + default: application/x-amz-json-1.0 + enum: + - application/x-amz-json-1.0 + type: string + requestBody: + content: + application/x-amz-json-1.0: + schema: + $ref: '#/components/schemas/CreateResourceInput' + required: true + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/ProgressEvent' + description: Success + /?Action=DeleteResource&Version=2021-09-30: + parameters: + - $ref: '#/components/parameters/X-Amz-Content-Sha256' + - $ref: '#/components/parameters/X-Amz-Date' + - $ref: '#/components/parameters/X-Amz-Algorithm' + - $ref: '#/components/parameters/X-Amz-Credential' + - $ref: '#/components/parameters/X-Amz-Security-Token' + - $ref: '#/components/parameters/X-Amz-Signature' + - $ref: '#/components/parameters/X-Amz-SignedHeaders' + post: + operationId: DeleteResource + parameters: + - description: Action Header + in: header + name: X-Amz-Target + required: false + schema: + default: CloudApiService.DeleteResource + enum: + - CloudApiService.DeleteResource + type: string + - in: header + name: Content-Type + required: false + schema: + default: application/x-amz-json-1.0 + enum: + - application/x-amz-json-1.0 + type: string + requestBody: + content: + application/x-amz-json-1.0: + schema: + $ref: '#/components/schemas/DeleteResourceInput' + required: true + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteResourceOutput' + description: Success + /?Action=GetResource&Version=2021-09-30: + parameters: + - $ref: '#/components/parameters/X-Amz-Content-Sha256' + - $ref: '#/components/parameters/X-Amz-Date' + - $ref: '#/components/parameters/X-Amz-Algorithm' + - $ref: '#/components/parameters/X-Amz-Credential' + - $ref: '#/components/parameters/X-Amz-Security-Token' + - $ref: '#/components/parameters/X-Amz-Signature' + - $ref: '#/components/parameters/X-Amz-SignedHeaders' + post: + operationId: GetResource + parameters: + - description: Action Header + in: header + name: X-Amz-Target + required: false + schema: + default: CloudApiService.GetResource + enum: + - CloudApiService.GetResource + type: string + - in: header + name: Content-Type + required: false + schema: + default: application/x-amz-json-1.0 + enum: + - application/x-amz-json-1.0 + type: string + requestBody: + content: + application/x-amz-json-1.0: + schema: + $ref: '#/components/schemas/GetResourceInput' + required: true + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/GetResourceOutput' + description: Success + /?Action=GetResourceRequestStatus&Version=2021-09-30: + parameters: + - $ref: '#/components/parameters/X-Amz-Content-Sha256' + - $ref: '#/components/parameters/X-Amz-Date' + - $ref: '#/components/parameters/X-Amz-Algorithm' + - $ref: '#/components/parameters/X-Amz-Credential' + - $ref: '#/components/parameters/X-Amz-Security-Token' + - $ref: '#/components/parameters/X-Amz-Signature' + - $ref: '#/components/parameters/X-Amz-SignedHeaders' + post: + operationId: GetResourceRequestStatus + parameters: + - description: Action Header + in: header + name: X-Amz-Target + required: false + schema: + default: CloudApiService.GetResourceRequestStatus + enum: + - CloudApiService.GetResourceRequestStatus + type: string + - in: header + name: Content-Type + required: false + schema: + default: application/x-amz-json-1.0 + enum: + - application/x-amz-json-1.0 + type: string + requestBody: + content: + application/x-amz-json-1.0: + schema: + $ref: '#/components/schemas/GetResourceRequestStatusInput' + required: true + responses: + '200': + content: + application/x-amz-json-1.0: + schema: + $ref: '#/components/schemas/GetResourceRequestStatusOutput' + description: Success + /?Action=ListResourceRequests&Version=2021-09-30: + parameters: + - $ref: '#/components/parameters/X-Amz-Content-Sha256' + - $ref: '#/components/parameters/X-Amz-Date' + - $ref: '#/components/parameters/X-Amz-Algorithm' + - $ref: '#/components/parameters/X-Amz-Credential' + - $ref: '#/components/parameters/X-Amz-Security-Token' + - $ref: '#/components/parameters/X-Amz-Signature' + - $ref: '#/components/parameters/X-Amz-SignedHeaders' + post: + operationId: ListResourceRequests + parameters: + - description: Pagination limit + in: query + name: MaxResults + required: false + schema: + type: string + - description: Pagination token + in: query + name: NextToken + required: false + schema: + type: string + - description: Action Header + in: header + name: X-Amz-Target + required: false + schema: + default: CloudApiService.ListResourceRequests + enum: + - CloudApiService.ListResourceRequests + type: string + - in: header + name: Content-Type + required: false + schema: + default: application/x-amz-json-1.0 + enum: + - application/x-amz-json-1.0 + type: string + - in: header + name: Accept + required: false + schema: + default: application/x-amz-json-1.0 + enum: + - application/x-amz-json-1.0 + type: string + requestBody: + content: + application/x-amz-json-1.0: + schema: + properties: + ResourceRequestStatusFilter: + properties: + OperationStatuses: + enum: + - PENDING + - IN_PROGRESS + - SUCCESS + - FAILED + - CANCEL_IN_PROGRESS + - CANCEL_COMPLETE + type: string + Operations: + enum: + - CREATE + - DELETE + - UPDATE + type: string + type: object + type: object + required: + - ResourceRequestStatusFilter + required: true + responses: + '200': + content: + application/x-amz-json-1.0: + schema: + properties: + NextToken: + type: string + ResourceRequestStatusSummaries: + items: + $ref: '#/components/schemas/ProgressEvent' + type: array + type: object + description: Success + /?Action=ListResources&Version=2021-09-30: + parameters: + - $ref: '#/components/parameters/X-Amz-Content-Sha256' + - $ref: '#/components/parameters/X-Amz-Date' + - $ref: '#/components/parameters/X-Amz-Algorithm' + - $ref: '#/components/parameters/X-Amz-Credential' + - $ref: '#/components/parameters/X-Amz-Security-Token' + - $ref: '#/components/parameters/X-Amz-Signature' + - $ref: '#/components/parameters/X-Amz-SignedHeaders' + post: + operationId: ListResources + parameters: + - description: Pagination limit + in: query + name: MaxResults + required: false + schema: + type: string + - description: Pagination token + in: query + name: NextToken + required: false + schema: + type: string + - description: Action Header + in: header + name: X-Amz-Target + required: false + schema: + default: CloudApiService.ListResources + enum: + - CloudApiService.ListResources + type: string + - in: header + name: Content-Type + required: false + schema: + default: application/x-amz-json-1.0 + enum: + - application/x-amz-json-1.0 + type: string + - in: header + name: Accept + required: false + schema: + default: application/x-amz-json-1.0 + enum: + - application/x-amz-json-1.0 + type: string + requestBody: + content: + application/x-amz-json-1.0: + schema: + properties: + MaxResults: + type: integer + maximum: 100 + minimum: 1 + NextToken: + $ref: '#/components/schemas/NextToken' + ResourceModel: + type: string + RoleArn: + $ref: '#/components/schemas/RoleArn' + TypeName: + $ref: '#/components/schemas/TypeName' + TypeVersionId: + $ref: '#/components/schemas/TypeVersionId' + required: + - TypeName + type: object + required: true + responses: + '200': + content: + application/x-amz-json-1.0: + schema: + properties: + RequestToken: + type: string + ResourceDescriptions: + items: + $ref: '#/components/schemas/ResourceDescription' + type: array + type: object + description: Success + /?Action=UpdateResource&Version=2021-09-30: + parameters: + - $ref: '#/components/parameters/X-Amz-Content-Sha256' + - $ref: '#/components/parameters/X-Amz-Date' + - $ref: '#/components/parameters/X-Amz-Algorithm' + - $ref: '#/components/parameters/X-Amz-Credential' + - $ref: '#/components/parameters/X-Amz-Security-Token' + - $ref: '#/components/parameters/X-Amz-Signature' + - $ref: '#/components/parameters/X-Amz-SignedHeaders' + post: + operationId: UpdateResource + parameters: + - description: Action Header + in: header + name: X-Amz-Target + required: false + schema: + default: CloudApiService.UpdateResource + enum: + - CloudApiService.UpdateResource + type: string + - in: header + name: Content-Type + required: false + schema: + default: application/x-amz-json-1.0 + enum: + - application/x-amz-json-1.0 + type: string + requestBody: + content: + application/x-amz-json-1.0: + schema: + properties: + ClientName: + type: string + Identifier: + $ref: '#/components/schemas/Identifier' + PatchDocument: + type: string + RoleArn: + $ref: '#/components/schemas/RoleArn' + TypeName: + $ref: '#/components/schemas/TypeName' + TypeVersionId: + $ref: '#/components/schemas/TypeVersionId' + required: + - Identifier + - PatchDocument + - TypeName + type: object + required: true + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateResourceOutput' + description: Success +components: + x-stackQL-resources: + resources: + name: resources + methods: + list_resources: + operation: + $ref: '#/paths/~1?Action=ListResources&Version=2021-09-30/post' + request: + mediaType: application/x-amz-json-1.0 + response: + mediaType: application/x-amz-json-1.0 + objectKey: '$.ResourceDescriptions' + openAPIDocKey: '200' + create_resource: + operation: + $ref: '#/paths/~1?Action=CreateResource&Version=2021-09-30/post' + request: + mediaType: application/x-amz-json-1.0 + response: + mediaType: application/json + openAPIDocKey: '200' + delete_resource: + operation: + $ref: '#/paths/~1?Action=DeleteResource&Version=2021-09-30/post' + request: + mediaType: application/x-amz-json-1.0 + response: + mediaType: application/json + openAPIDocKey: '200' + get_resource: + operation: + $ref: '#/paths/~1?Action=GetResource&Version=2021-09-30/post' + request: + mediaType: application/x-amz-json-1.0 + response: + mediaType: application/json + objectKey: '$.ResourceDescription' + openAPIDocKey: '200' + update_resource: + operation: + $ref: '#/paths/~1?Action=UpdateResource&Version=2021-09-30/post' + request: + mediaType: application/x-amz-json-1.0 + response: + mediaType: application/json + openAPIDocKey: '200' + id: aws.cloud_control.resources + sqlVerbs: + delete: + - $ref: '#/components/x-stackQL-resources/resources/methods/delete_resource' + insert: + - $ref: '#/components/x-stackQL-resources/resources/methods/create_resource' + select: + - $ref: '#/components/x-stackQL-resources/resources/methods/get_resource' + - $ref: '#/components/x-stackQL-resources/resources/methods/list_resources' + update: + - $ref: '#/components/x-stackQL-resources/resources/methods/update_resource' + title: resources + resource: + name: resource + methods: + get_resource: + operation: + $ref: '#/paths/~1?Action=GetResource&Version=2021-09-30/post' + request: + mediaType: application/x-amz-json-1.0 + response: + mediaType: application/json + objectKey: '$.ResourceDescription' + openAPIDocKey: '200' + id: aws.cloud_control.resource + sqlVerbs: + select: + - $ref: '#/components/x-stackQL-resources/resources/methods/get_resource' + title: resource + resource_requests: + name: resource_requests + methods: + list_resource_requests: + operation: + $ref: '#/paths/~1?Action=ListResourceRequests&Version=2021-09-30/post' + request: + mediaType: application/x-amz-json-1.0 + response: + mediaType: application/x-amz-json-1.0 + objectKey: '$.ResourceRequestStatusSummaries' + openAPIDocKey: '200' + get_resource_request: + operation: + $ref: '#/paths/~1?Action=GetResourceRequestStatus&Version=2021-09-30/post' + request: + mediaType: application/x-amz-json-1.0 + response: + mediaType: application/x-amz-json-1.0 + objectKey: '$.ProgressEvent' + openAPIDocKey: '200' + cancel_resource_request: + operation: + $ref: '#/paths/~1?Action=CancelResourceRequest&Version=2021-09-30/post' + request: + mediaType: application/x-amz-json-1.0 + response: + mediaType: application/x-amz-json-1.0 + openAPIDocKey: '200' + id: aws.cloud_control.resource_requests + sqlVerbs: + delete: + - $ref: '#/components/x-stackQL-resources/resource_requests/methods/cancel_resource_request' + insert: [] + select: + - $ref: '#/components/x-stackQL-resources/resource_requests/methods/get_resource_request' + - $ref: '#/components/x-stackQL-resources/resource_requests/methods/list_resource_requests' + update: [] + title: resource_requests + parameters: + X-Amz-Content-Sha256: + name: X-Amz-Content-Sha256 + in: header + schema: + type: string + required: false + X-Amz-Date: + name: X-Amz-Date + in: header + schema: + type: string + required: false + X-Amz-Algorithm: + name: X-Amz-Algorithm + in: header + schema: + type: string + required: false + X-Amz-Credential: + name: X-Amz-Credential + in: header + schema: + type: string + required: false + X-Amz-Security-Token: + name: X-Amz-Security-Token + in: header + schema: + type: string + required: false + X-Amz-Signature: + name: X-Amz-Signature + in: header + schema: + type: string + required: false + X-Amz-SignedHeaders: + name: X-Amz-SignedHeaders + in: header + schema: + type: string + required: false + securitySchemes: + hmac: + type: apiKey + name: Authorization + in: header + description: Amazon Signature authorization v4 + x-amazon-apigateway-authtype: awsSigv4 + schemas: + AlreadyExistsException: {} + CancelResourceRequestInput: + properties: + RequestToken: + $ref: '#/components/schemas/RequestToken' + required: + - RequestToken + title: CancelResourceRequestInput + type: object + CancelResourceRequestOutput: + properties: + ProgressEvent: + $ref: '#/components/schemas/ProgressEvent' + type: object + ClientToken: + maxLength: 128 + minLength: 1 + pattern: '[-A-Za-z0-9+/=]+' + type: string + ClientTokenConflictException: {} + ConcurrentModificationException: {} + ConcurrentOperationException: {} + CreateResourceInput: + properties: + ClientToken: + type: string + DesiredState: + x-stackQL-stringOnly: true + allOf: + - $ref: '#/components/schemas/Properties' + - description:

Structured data format representing the desired state + of the resource, consisting of that resource's properties and their + desired values.

Cloud Control API currently supports JSON + as a structured data format.

 
 <p>Specify
+              the desired state as one of the following:</p> <ul> <li>
+              <p>A JSON blob</p> </li> <li> <p>A local
+              path containing the desired state in JSON data format</p> </li>
+              </ul> <p>For more information, see <a href="https://docs.aws.amazon.com/cloudcontrolapi/latest/userguide/resource-operations-create.html#resource-operations-create-desiredstate">Composing
+              the desired state of the resource</a> in the <i>Amazon Web
+              Services Cloud Control API User Guide</i>.</p> <p>For
+              more information about the properties of a specific resource, refer
+              to the related topic for the resource in the <a href="https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-template-resource-type-ref.html">Resource
+              and property types reference</a> in the <i>CloudFormation
+              Users Guide</i>.</p>
+ RoleArn: + type: string + TypeName: + type: string + TypeVersionId: + type: string + required: + - TypeName + - DesiredState + title: CreateResourceInput + type: object + CreateResourceOutput: + properties: + ProgressEvent: + $ref: '#/components/schemas/ProgressEvent' + type: object + DeleteResourceInput: + properties: + ClientToken: + type: string + Identifier: + $ref: '#/components/schemas/Identifier' + RoleArn: + type: string + TypeName: + type: string + TypeVersionId: + type: string + required: + - TypeName + - Identifier + title: DeleteResourceInput + type: object + DeleteResourceOutput: + properties: + ProgressEvent: + $ref: '#/components/schemas/ProgressEvent' + type: object + GeneralServiceException: {} + GetResourceInput: + properties: + TypeName: + $ref: '#/components/schemas/TypeName' + Identifier: + $ref: '#/components/schemas/Identifier' + TypeVersionId: + $ref: '#/components/schemas/TypeVersionId' + RoleArn: + $ref: '#/components/schemas/RoleArn' + required: + - TypeName + - Identifier + title: GetResourceInput + type: object + GetResourceOutput: + properties: + ResourceDescription: + $ref: '#/components/schemas/ResourceDescription' + TypeName: + type: string + type: object + GetResourceRequestStatusInput: + properties: + RequestToken: + $ref: '#/components/schemas/RequestToken' + required: + - RequestToken + title: GetResourceRequestStatusInput + type: object + GetResourceRequestStatusOutput: + properties: + ProgressEvent: + $ref: '#/components/schemas/ProgressEvent' + type: object + HandlerErrorCode: + enum: + - NotUpdatable + - InvalidRequest + - AccessDenied + - InvalidCredentials + - AlreadyExists + - NotFound + - ResourceConflict + - Throttling + - ServiceLimitExceeded + - NotStabilized + - GeneralServiceException + - ServiceInternalError + - ServiceTimeout + - NetworkFailure + - InternalFailure + type: string + HandlerFailureException: {} + HandlerInternalFailureException: {} + HandlerNextToken: + maxLength: 2048 + minLength: 1 + pattern: .+ + type: string + Identifier: + maxLength: 1024 + minLength: 1 + pattern: .+ + type: string + InvalidCredentialsException: {} + InvalidRequestException: {} + MaxResults: + maximum: 100 + minimum: 1 + type: integer + NetworkFailureException: {} + NextToken: + maxLength: 2048 + minLength: 1 + pattern: '[-A-Za-z0-9+/=]+' + type: string + NotStabilizedException: {} + NotUpdatableException: {} + Operation: + enum: + - CREATE + - DELETE + - UPDATE + type: string + OperationStatus: + enum: + - PENDING + - IN_PROGRESS + - SUCCESS + - FAILED + - CANCEL_IN_PROGRESS + - CANCEL_COMPLETE + type: string + OperationStatuses: + items: + $ref: '#/components/schemas/OperationStatus' + type: array + Operations: + items: + $ref: '#/components/schemas/Operation' + type: array + PatchDocument: + format: password + maxLength: 65536 + minLength: 1 + pattern: '[\s\S]*' + type: string + PrivateTypeException: {} + ProgressEvent: + example: + ErrorCode: string + EventTime: number + Identifier: string + Operation: string + OperationStatus: string + RequestToken: string + ResourceModel: string + RetryAfter: number + StatusMessage: string + TypeName: string + properties: + ErrorCode: + type: string + EventTime: + type: number + Identifier: + type: string + Operation: + type: string + OperationStatus: + type: string + RequestToken: + type: string + ResourceModel: + type: string + RetryAfter: + type: number + StatusMessage: + type: string + TypeName: + type: string + type: object + Properties: + format: password + maxLength: 65536 + minLength: 1 + pattern: '[\s\S]*' + type: string + RequestToken: + maxLength: 128 + minLength: 1 + pattern: '[-A-Za-z0-9+/=]+' + type: string + RequestTokenNotFoundException: {} + ResourceConflictException: {} + ResourceDescription: + description: Represents information about a provisioned resource. + properties: + Identifier: + type: string + Properties: + type: string + type: object + ResourceDescriptions: + items: + $ref: '#/components/schemas/ResourceDescription' + type: array + ResourceNotFoundException: {} + ResourceRequestStatusFilter: + description: The filter criteria to use in determining the requests returned. + properties: + undefined: + allOf: + - $ref: '#/components/schemas/OperationStatuses' + - description: '

The operation statuses to include in the filter.

+

  • PENDING: The operation has been requested, + but not yet initiated.

  • IN_PROGRESS: + The operation is in progress.

  • SUCCESS: + The operation completed.

  • FAILED: The + operation failed.

  • CANCEL_IN_PROGRESS: + The operation is in the process of being canceled.

  • + CANCEL_COMPLETE: The operation has been canceled.

    • +
' + type: object + ResourceRequestStatusSummaries: + items: + $ref: '#/components/schemas/ProgressEvent' + type: array + RoleArn: + maxLength: 2048 + minLength: 20 + pattern: arn:.+:iam::[0-9]{12}:role/.+ + type: string + ServiceInternalErrorException: {} + ServiceLimitExceededException: {} + StatusMessage: + maxLength: 1024 + minLength: 0 + pattern: '[\s\S]*' + type: string + ThrottlingException: {} + Timestamp: + format: date-time + type: string + TypeName: + maxLength: 196 + minLength: 10 + pattern: '[A-Za-z0-9]{2,64}::[A-Za-z0-9]{2,64}::[A-Za-z0-9]{2,64}' + type: string + TypeNotFoundException: {} + TypeVersionId: + maxLength: 128 + minLength: 1 + pattern: '[A-Za-z0-9-]+' + type: string + UnsupportedActionException: {} + UpdateResourceInput: + properties: + undefined: + allOf: + - $ref: '#/components/schemas/PatchDocument' + required: + - TypeName + - Identifier + - PatchDocument + title: UpdateResourceInput + type: object + UpdateResourceOutput: + properties: + ProgressEvent: + $ref: '#/components/schemas/ProgressEvent' + type: object +security: + - hmac: [] +x-stackQL-config: + pagination: + requestToken: + key: NextToken + location: body + responseToken: + key: NextToken + location: body diff --git a/test/registry/src/aws/v0.1.0/services/ec2_nextgen.yaml b/test/registry/src/aws/v0.1.0/services/ec2_nextgen.yaml index 8f49afcb..8ae05ba6 100644 --- a/test/registry/src/aws/v0.1.0/services/ec2_nextgen.yaml +++ b/test/registry/src/aws/v0.1.0/services/ec2_nextgen.yaml @@ -16757,12 +16757,12 @@ components: JSON_EXTRACT(detail.Properties, '$.Affinity') as affinity, JSON_EXTRACT(detail.Properties, '$.CreditSpecification') as credit_specification FROM aws.cloud_control.resources listing - LEFT OUTER JOIN aws.cloud_control.resource detail + INNER JOIN aws.cloud_control.resource detail ON detail.data__Identifier = listing.Identifier AND detail.region = listing.region WHERE listing.data__TypeName = 'AWS::EC2::Instance' AND detail.data__TypeName = 'AWS::EC2::Instance' - AND listing.region = 'us-east-1' + AND listing.region IN ('us-west-1', 'ap-southeast-1') fallback: predicate: sqlDialect == "postgres" && requiredParams == [ data__Identifier ] ddl: |- diff --git a/test/registry/src/aws/v0.1.0/services/pseudo_s3.yaml b/test/registry/src/aws/v0.1.0/services/pseudo_s3.yaml index 6747450d..7bf5c120 100644 --- a/test/registry/src/aws/v0.1.0/services/pseudo_s3.yaml +++ b/test/registry/src/aws/v0.1.0/services/pseudo_s3.yaml @@ -476,7 +476,7 @@ components: IIF(JSON_EXTRACT(Properties, '$.PublicAccessBlockConfiguration.BlockPublicAcls') = 0, 'false', 'true') as BlockPublicAcls, IIF(JSON_EXTRACT(Properties, '$.PublicAccessBlockConfiguration.IgnorePublicAcls') = 0, 'false', 'true') as IgnorePublicAcls, JSON_EXTRACT(Properties, '$.Tags') as Tags - FROM aws.cloud_control.resources WHERE region = 'ap-southeast-1' and data__TypeName = 'AWS::S3::Bucket' + FROM aws.cloud_control_legacy.resources WHERE region = 'ap-southeast-1' and data__TypeName = 'AWS::S3::Bucket' ; fallback: predicate: sqlDialect == "postgres" @@ -494,7 +494,7 @@ components: CASE WHEN json_extract_path_text(Properties, 'PublicAccessBlockConfiguration', 'BlockPublicAcls') = '0' THEN 'false' ELSE 'true' END as BlockPublicAcls, CASE WHEN json_extract_path_text(Properties, 'PublicAccessBlockConfiguration', 'IgnorePublicAcls') = '0' THEN 'false' ELSE 'true' END as IgnorePublicAcls, json_extract_path_text(Properties, 'Tags') as Tags - FROM aws.cloud_control.resources WHERE region = 'ap-southeast-1' and data__TypeName = 'AWS::S3::Bucket' + FROM aws.cloud_controllegacy.resources WHERE region = 'ap-southeast-1' and data__TypeName = 'AWS::S3::Bucket' ; s3_bucket_detail: name: s3_bucket_detail @@ -548,7 +548,7 @@ components: JSON_EXTRACT(Properties, '$.DualStackDomainName') as dual_stack_domain_name, JSON_EXTRACT(Properties, '$.RegionalDomainName') as regional_domain_name, JSON_EXTRACT(Properties, '$.WebsiteURL') as website_url - FROM aws.cloud_control.resources WHERE data__TypeName = 'AWS::S3::Bucket' + FROM aws.cloud_control.resource WHERE data__TypeName = 'AWS::S3::Bucket' AND data__Identifier = '' AND region = 'us-east-1' ; @@ -583,7 +583,7 @@ components: json_extract_path_text(Properties, 'DualStackDomainName') as dual_stack_domain_name, json_extract_path_text(Properties, 'RegionalDomainName') as regional_domain_name, json_extract_path_text(Properties, 'WebsiteURL') as website_url - FROM aws.cloud_control.resources WHERE data__TypeName = 'AWS::S3::Bucket' + FROM aws.cloud_control.resource WHERE data__TypeName = 'AWS::S3::Bucket' AND data__Identifier = '' AND region = 'us-east-1' ; @@ -640,7 +640,7 @@ components: JSON_EXTRACT(Properties, '$.DualStackDomainName') as dual_stack_domain_name, JSON_EXTRACT(Properties, '$.RegionalDomainName') as regional_domain_name, JSON_EXTRACT(Properties, '$.WebsiteURL') as website_url - FROM aws.cloud_control.resources WHERE data__TypeName = 'AWS::S3::Bucket' + FROM aws.cloud_control.resource WHERE data__TypeName = 'AWS::S3::Bucket' AND data__Identifier = '' AND region = 'us-east-1' ; @@ -675,7 +675,7 @@ components: json_extract_path_text(Properties, 'DualStackDomainName') as dual_stack_domain_name, json_extract_path_text(Properties, 'RegionalDomainName') as regional_domain_name, json_extract_path_text(Properties, 'WebsiteURL') as website_url - FROM aws.cloud_control.resources WHERE data__TypeName = 'AWS::S3::Bucket' + FROM aws.cloud_control.resource WHERE data__TypeName = 'AWS::S3::Bucket' AND data__Identifier = '' AND region = 'us-east-1' ; @@ -937,7 +937,7 @@ components: JSON_EXTRACT(Properties, '$.DualStackDomainName') as dual_stack_domain_name, JSON_EXTRACT(Properties, '$.RegionalDomainName') as regional_domain_name, JSON_EXTRACT(Properties, '$.WebsiteURL') as website_url - FROM aws.cloud_control.resources WHERE data__TypeName = 'AWS::S3::Bucket' + FROM aws.cloud_control.resource WHERE data__TypeName = 'AWS::S3::Bucket' AND data__Identifier = '' AND region = 'us-east-1' ; @@ -990,7 +990,7 @@ components: json_extract_path_text(Properties, 'DualStackDomainName') as dual_stack_domain_name, json_extract_path_text(Properties, 'RegionalDomainName') as regional_domain_name, json_extract_path_text(Properties, 'WebsiteURL') as website_url - FROM aws.cloud_control.resources WHERE data__TypeName = 'AWS::S3::Bucket' + FROM aws.cloud_control.resource WHERE data__TypeName = 'AWS::S3::Bucket' AND data__Identifier = '' AND region = 'us-east-1' ; diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index e6be88bd..35d60443 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -4115,7 +4115,7 @@ Filtered and Parameterised Projection Resource Level View of Cloud Control Resou ... ${REGISTRY_NO_VERIFY_CFG_STR} ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} - ... select BucketName, DomainName from aws.pseudo_s3.s3_bucket_listing where data__Identifier = 'stackql\-trial\-bucket\-01' and region \= 'ap\-southeast\-2' and BucketName \= 'stackql\-trial\-bucket\-01'; + ... select bucket_name, domain_name from aws.pseudo_s3.s3_bucket_polymorphic where data__Identifier = 'stackql\-trial\-bucket\-01' and region \= 'ap\-southeast\-2' and bucket_name \= 'stackql\-trial\-bucket\-01'; ... ${AWS_CC_VIEW_SELECT_PROJECTION_BUCKET_COMPLEX_EXPECTED} ... ${CURDIR}/tmp/Filtered-and-Parameterised-Projection-Resource-Level-View-of-Cloud-Control-Resource-Returns-Expected-Result.tmp @@ -4128,7 +4128,7 @@ Filtered and Parameterised Star Resource Level View of Cloud Control Resource Re ... ${REGISTRY_NO_VERIFY_CFG_STR} ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} - ... select * from aws.pseudo_s3.s3_bucket_listing where data__Identifier \= 'stackql\-trial\-bucket\-01' and region \= 'ap\-southeast\-2' and BucketName \= 'stackql\-trial\-bucket\-01'; + ... select * from aws.pseudo_s3.s3_bucket_polymorphic where data__Identifier \= 'stackql\-trial\-bucket\-01' and region \= 'ap\-southeast\-2' and bucket_name \= 'stackql\-trial\-bucket\-01'; ... ${AWS_CC_VIEW_SELECT_STAR_BUCKET_COMPLEX_EXPECTED} ... ${CURDIR}/tmp/Filtered-and-Parameterised-Star-Resource-Level-View-of-Cloud-Control-Resource-Returns-Expected-Result.tmp @@ -7352,4 +7352,36 @@ Alternate App Root Persists All Temp Materials in Alotted Directory ... stdout=${CURDIR}/tmp/Alternate-App-Root-Persists-All-Temp-Materials-in-Alotted-Directory.tmp ... stderr=${CURDIR}/tmp/Alternate-App-Root-Persists-All-Temp-Materials-in-Alotted-Directory-stderr.tmp Directory Should Exist ${TEST_TMP_EXEC_APP_ROOT_NATIVE}${/}readline - Directory Should Exist ${TEST_TMP_EXEC_APP_ROOT_NATIVE}${/}src \ No newline at end of file + Directory Should Exist ${TEST_TMP_EXEC_APP_ROOT_NATIVE}${/}src + +View Tuple Replacement Working As Exemplified by AWS EC2 Instances List and Detail + ${inputStr} = Catenate + ... SELECT region, instance_id, tenancy, security_groups + ... FROM aws.ec2_nextgen.instances + ... WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; + ${outputStr} = Catenate SEPARATOR=\n + ... |----------------|---------------------|---------|--------------------------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}instance_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}tenancy${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}security_groups${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| + ... |----------------|---------------------|---------|--------------------------| + ... |${SPACE}ap-southeast-2${SPACE}|${SPACE}i-0cb58b995b19153a8${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| + ... |----------------|---------------------|---------|--------------------------| + ... |${SPACE}ap-southeast-2${SPACE}|${SPACE}i-0cb58b995b19153a8${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| + ... |----------------|---------------------|---------|--------------------------| + ... |${SPACE}ap-southeast-2${SPACE}|${SPACE}i-0cb58b995b19153a8${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| + ... |----------------|---------------------|---------|--------------------------| + ... |${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}i-0cb58b995b19153a8${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| + ... |----------------|---------------------|---------|--------------------------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_DEFECTIVE_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stackql_approot=${TEST_TMP_EXEC_APP_ROOT} + ... stdout=${CURDIR}/tmp/View-Tuple-Replacement-Working-As-Exemplified-by-AWS-EC2-Instances-List-and-Detail.tmp + ... stderr=${CURDIR}/tmp/View-Tuple-Replacement-Working-As-Exemplified-by-AWS-EC2-Instances-List-and-Detail-stderr.tmp diff --git a/test/robot/lib/stackql_context.py b/test/robot/lib/stackql_context.py index 6a09e5e9..3656a188 100644 --- a/test/robot/lib/stackql_context.py +++ b/test/robot/lib/stackql_context.py @@ -691,12 +691,12 @@ def generate_password() -> str: SELECT_AWS_VOLUMES = "select volumeId, encrypted, size from aws.ec2.volumes where region = 'ap-southeast-1' order by volumeId asc;" SELECT_AWS_IAM_USERS_ASC = "select UserName, Arn from aws.iam.users WHERE region = 'us-east-1' order by UserName ASC;" CREATE_AWS_VOLUME = """insert into aws.ec2.volumes(AvailabilityZone, Size, region, TagSpecification) select 'ap-southeast-1a', JSON(10), 'ap-southeast-1', JSON('[ { "ResourceType": "volume", "Tag": [ { "Key": "stack", "Value": "production" }, { "Key": "name", "Value": "multi-tag-volume" } ] } ]');""" -SELECT_AWS_CLOUD_CONTROL_VPCS_DESC = "select Identifier, Properties from aws.cloud_control.resources where region = 'ap-southeast-1' and data__TypeName = 'AWS::EC2::VPC' order by Identifier desc;" -SELECT_AWS_CLOUD_CONTROL_BUCKET_PROJECTION = "SELECT JSON_EXTRACT(Properties, '$.Arn') as Arn FROM aws.cloud_control.resources WHERE region = 'ap-southeast-2' and data__TypeName = 'AWS::S3::Bucket' and data__Identifier = 'stackql-trial-bucket-01';" +SELECT_AWS_CLOUD_CONTROL_VPCS_DESC = "select Identifier, Properties from aws.cloud_control_legacy.resources where region = 'ap-southeast-1' and data__TypeName = 'AWS::EC2::VPC' order by Identifier desc;" +SELECT_AWS_CLOUD_CONTROL_BUCKET_PROJECTION = "SELECT JSON_EXTRACT(Properties, '$.Arn') as Arn FROM aws.cloud_control_legacy.resources WHERE region = 'ap-southeast-2' and data__TypeName = 'AWS::S3::Bucket' and data__Identifier = 'stackql-trial-bucket-01';" SELECT_AWS_CLOUD_CONTROL_BUCKET_VIEW_PROJECTION = "select Arn from aws.pseudo_s3.s3_bucket_listing where data__Identifier = 'stackql-trial-bucket-01' ;" SELECT_AWS_CLOUD_CONTROL_BUCKET_VIEW_STAR = "select * from aws.pseudo_s3.s3_bucket_listing where data__Identifier = 'stackql-trial-bucket-01' ;" -SELECT_AWS_CLOUD_CONTROL_BUCKET_PROJECTION_DEFECTIVE = "SELECT JSON_EXTRACT(Arn, '$.Properties') as Arn FROM aws.cloud_control.resources WHERE region = 'ap-southeast-2' and data__TypeName = 'AWS::S3::Bucket' and data__Identifier = 'stackql-trial-bucket-01';" -GET_AWS_CLOUD_CONTROL_VPCS_DESC = "select Identifier, Properties from aws.cloud_control.resources where region = 'ap-southeast-1' and data__TypeName = 'AWS::EC2::VPC' and data__Identifier = 'CloudControlExample';" +SELECT_AWS_CLOUD_CONTROL_BUCKET_PROJECTION_DEFECTIVE = "SELECT JSON_EXTRACT(Arn, '$.Properties') as Arn FROM aws.cloud_control_legacy.resources WHERE region = 'ap-southeast-2' and data__TypeName = 'AWS::S3::Bucket' and data__Identifier = 'stackql-trial-bucket-01';" +GET_AWS_CLOUD_CONTROL_VPCS_DESC = "select Identifier, Properties from aws.cloud_control_legacy.resources where region = 'ap-southeast-1' and data__TypeName = 'AWS::EC2::VPC' and data__Identifier = 'CloudControlExample';" GET_AWS_CLOUD_CONTROL_REQUEST_LOG_GROUP = """select TypeName, OperationStatus, StatusMessage, Identifier, RequestToken from aws.cloud_control.resource_requests where data__RequestToken = 'abc001' and region = 'ap-southeast-1';""" SELECT_AWS_CLOUD_CONTROL_OPERATIONS_DESC = "select TypeName, OperationStatus, StatusMessage, Identifier, RequestToken from aws.cloud_control.resource_requests where data__ResourceRequestStatusFilter='{}' and region = 'ap-southeast-1' order by RequestToken desc;" UPDATE_AWS_EC2_VOLUME = "update aws.ec2.volumes set Size = 12 WHERE region = 'ap-southeast-1' AND VolumeId = 'vol-000000000000001';" From f9658c8ab331f90cc68a093c00905480978ac26a Mon Sep 17 00:00:00 2001 From: General Kroll Date: Mon, 30 Dec 2024 01:24:55 +1100 Subject: [PATCH 02/10] - All working locally. --- internal/stackql/dependencyplanner/dependencyplanner.go | 4 ++-- internal/stackql/drm/drm_cfg.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/stackql/dependencyplanner/dependencyplanner.go b/internal/stackql/dependencyplanner/dependencyplanner.go index fe32f974..18b9e4a8 100644 --- a/internal/stackql/dependencyplanner/dependencyplanner.go +++ b/internal/stackql/dependencyplanner/dependencyplanner.go @@ -677,8 +677,8 @@ func (dp *standardDependencyPlanner) getStreamFromEdge( staticParams := make(map[string]interface{}) for k, v := range params { existingSourceParam, isExistingFromSource := dp.extractStaticSourceParam(e, k) - // isVector := dp.isVectorParam(v) - if _, ok := incomingCols[k]; !ok { + isVector := dp.isVectorParam(v) + if _, ok := incomingCols[k]; !ok && !isVector { staticParams[k] = v incomingCols[k] = struct{}{} } else if isExistingFromSource && false { diff --git a/internal/stackql/drm/drm_cfg.go b/internal/stackql/drm/drm_cfg.go index 219c9594..06a5c65f 100644 --- a/internal/stackql/drm/drm_cfg.go +++ b/internal/stackql/drm/drm_cfg.go @@ -558,7 +558,7 @@ func (dc *staticDRMConfig) GenerateInsertDML( } priorParameters := tabAnnotated.GetParameters() // TODO: fix this for dependent table where dependency has `IN` clause!!! - transformedParams, paramErr := util.TransformSQLRawParameters(priorParameters, false) + transformedParams, paramErr := util.TransformSQLRawParameters(priorParameters, true) if paramErr != nil { return nil, paramErr } From b1944ce66a56a2b35c5227d9a368f84cd9992e2c Mon Sep 17 00:00:00 2001 From: General Kroll Date: Mon, 30 Dec 2024 09:36:51 +1100 Subject: [PATCH 03/10] - All working locally with repeats. --- internal/stackql/drm/drm_cfg.go | 2 +- internal/stackql/router/parameter_router.go | 38 +++++++++++++++++-- .../stackql_mocked_from_cmd_line.robot | 2 +- test/robot/lib/StackQLInterfaces.py | 28 +++++++------- 4 files changed, 51 insertions(+), 19 deletions(-) diff --git a/internal/stackql/drm/drm_cfg.go b/internal/stackql/drm/drm_cfg.go index 06a5c65f..219c9594 100644 --- a/internal/stackql/drm/drm_cfg.go +++ b/internal/stackql/drm/drm_cfg.go @@ -558,7 +558,7 @@ func (dc *staticDRMConfig) GenerateInsertDML( } priorParameters := tabAnnotated.GetParameters() // TODO: fix this for dependent table where dependency has `IN` clause!!! - transformedParams, paramErr := util.TransformSQLRawParameters(priorParameters, true) + transformedParams, paramErr := util.TransformSQLRawParameters(priorParameters, false) if paramErr != nil { return nil, paramErr } diff --git a/internal/stackql/router/parameter_router.go b/internal/stackql/router/parameter_router.go index 53416f25..6bc93582 100644 --- a/internal/stackql/router/parameter_router.go +++ b/internal/stackql/router/parameter_router.go @@ -250,11 +250,12 @@ func (sp *standardParamSplitter) assembleSplitParams( combinations := combinationComposerObj.getCombinations() _, isAnythingSplit := len(combinations), combinationComposerObj.getIsAnythingSplit() for _, paramCombination := range combinations { + com := paramCombination splitAnnotationCtx := taxonomy.NewStaticStandardAnnotationCtx( rawAnnotationCtx.GetSchema(), rawAnnotationCtx.GetHIDs(), rawAnnotationCtx.GetTableMeta().Clone(), - paramCombination, + com, ) sp.splitAnnotationContextMap.Put(rawAnnotationCtx, splitAnnotationCtx) // TODO: this has gotta replace the original and also be duplicated @@ -310,6 +311,8 @@ func (pr *standardParameterRouter) GetOnConditionDataFlows() (dataflow.Collectio } var dependencyTable sqlparser.TableExpr var dependencies []taxonomy.AnnotationCtx + var destinations []taxonomy.AnnotationCtx + var destColumn *sqlparser.ColName var srcExpr sqlparser.Expr switch l := k.Left.(type) { @@ -329,6 +332,12 @@ func (pr *standardParameterRouter) GetOnConditionDataFlows() (dataflow.Collectio } else { dependencies = append(dependencies, lhr) } + splitDestinations, isDestinationSplit := splitAnnotationContextMap.Get(destHierarchy) + if isDestinationSplit { + destinations = append(destinations, splitDestinations...) + } else { + destinations = append(destinations, destHierarchy) + } dependencyTable = candidateTable srcExpr = k.Left } @@ -343,6 +352,12 @@ func (pr *standardParameterRouter) GetOnConditionDataFlows() (dataflow.Collectio } else { dependencies = append(dependencies, annCtx) } + splitDestinations, isDestinationSplit := splitAnnotationContextMap.Get(destHierarchy) + if isDestinationSplit { + destinations = append(destinations, splitDestinations...) + } else { + destinations = append(destinations, destHierarchy) + } dependencyTable = te } switch r := k.Right.(type) { @@ -368,6 +383,12 @@ func (pr *standardParameterRouter) GetOnConditionDataFlows() (dataflow.Collectio } else { dependencies = append(dependencies, rhr) } + splitDestinations, isDestinationSplit := splitAnnotationContextMap.Get(destHierarchy) + if isDestinationSplit { + destinations = append(destinations, splitDestinations...) + } else { + destinations = append(destinations, destHierarchy) + } dependencyTable = candidateTable } case *sqlparser.FuncExpr: @@ -381,16 +402,25 @@ func (pr *standardParameterRouter) GetOnConditionDataFlows() (dataflow.Collectio } else { dependencies = append(dependencies, annCtx) } + splitDestinations, isDestinationSplit := splitAnnotationContextMap.Get(destHierarchy) + if isDestinationSplit { + destinations = append(destinations, splitDestinations...) + } else { + destinations = append(destinations, destHierarchy) + } dependencyTable = te } if !selfTableCited { return nil, fmt.Errorf("table join ON comparison '%s' referencing incomplete", sqlparser.String(k)) } - // rv[dependencies] = destHierarchy - for _, dependency := range dependencies { + for i, dependency := range dependencies { srcVertex := rv.UpsertStandardDataFlowVertex(dependency, dependencyTable) - destVertex := rv.UpsertStandardDataFlowVertex(destHierarchy, destinationTable) + destination := destHierarchy + if i < len(destinations) { + destination = destinations[i] + } + destVertex := rv.UpsertStandardDataFlowVertex(destination, destinationTable) err := rv.AddOrUpdateEdge( srcVertex, diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 35d60443..fed2e867 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -7382,6 +7382,6 @@ View Tuple Replacement Working As Exemplified by AWS EC2 Instances List and Deta ... ${inputStr} ... ${outputStr} ... ${EMPTY} - ... stackql_approot=${TEST_TMP_EXEC_APP_ROOT} ... stdout=${CURDIR}/tmp/View-Tuple-Replacement-Working-As-Exemplified-by-AWS-EC2-Instances-List-and-Detail.tmp ... stderr=${CURDIR}/tmp/View-Tuple-Replacement-Working-As-Exemplified-by-AWS-EC2-Instances-List-and-Detail-stderr.tmp + ... repeat_count=20 diff --git a/test/robot/lib/StackQLInterfaces.py b/test/robot/lib/StackQLInterfaces.py index ee78893d..cb11d38f 100644 --- a/test/robot/lib/StackQLInterfaces.py +++ b/test/robot/lib/StackQLInterfaces.py @@ -822,19 +822,21 @@ def should_stackql_exec_inline_equal_both_streams( *args, **cfg ): - result = self._run_stackql_exec_command( - stackql_exe, - okta_secret_str, - github_secret_str, - k8s_secret_str, - registry_cfg, - auth_cfg_str, - sql_backend_cfg_str, - query, - *args, - **cfg - ) - return self._verify_both_streams(result, expected_output, expected_stderr_output) + repeat_count = int(cfg.pop('repeat_count', 1)) + for _ in range(repeat_count): + result = self._run_stackql_exec_command( + stackql_exe, + okta_secret_str, + github_secret_str, + k8s_secret_str, + registry_cfg, + auth_cfg_str, + sql_backend_cfg_str, + query, + *args, + **cfg + ) + return self._verify_both_streams(result, expected_output, expected_stderr_output) @keyword From b999a20028c8ce0d17d43d3c235140c6e44c521c Mon Sep 17 00:00:00 2001 From: General Kroll Date: Mon, 30 Dec 2024 09:52:58 +1100 Subject: [PATCH 04/10] - Lint issues fixed. --- internal/stackql/router/parameter_router.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/stackql/router/parameter_router.go b/internal/stackql/router/parameter_router.go index 6bc93582..aed38638 100644 --- a/internal/stackql/router/parameter_router.go +++ b/internal/stackql/router/parameter_router.go @@ -289,7 +289,7 @@ func (sp *standardParamSplitter) splitSingleParam( return rv, isSplit } -//nolint:funlen,gocognit // inherently complex functionality +//nolint:funlen,gocognit,nestif // inherently complex functionality func (pr *standardParameterRouter) GetOnConditionDataFlows() (dataflow.Collection, error) { paramSplitterObj := newParamSplitter(pr.tableToAnnotationCtx, pr.dataFlowCfg) isInitiallySplit, splitErr := paramSplitterObj.split() From 93b5431ce48f9b6033be3dc2915a02171707ea8f Mon Sep 17 00:00:00 2001 From: General Kroll Date: Mon, 30 Dec 2024 10:21:47 +1100 Subject: [PATCH 05/10] - Pared back. --- .vscode/launch.json | 17 ++--- docs/walkthroughs/README.md | 8 +-- .../get-google-accelerator-types.md | 70 +++++++++++++++++++ docs/walkthroughs/get-google-vms.md | 62 ++++++++++++++++ docs/walkthroughs/list-aws-instances.md | 53 -------------- .../stackql/drm/prepared_statement_ctx.go | 12 ---- ...parameter-pushed-and-response-filtered.txt | 4 +- ...parameter-pushed-and-response-filtered.txt | 11 +-- test/registry/src/aws/v0.1.0/provider.yaml | 9 --- .../src/aws/v0.1.0/services/pseudo_s3.yaml | 2 +- .../stackql_mocked_from_cmd_line.robot | 4 +- 11 files changed, 153 insertions(+), 99 deletions(-) create mode 100644 docs/walkthroughs/get-google-accelerator-types.md create mode 100644 docs/walkthroughs/get-google-vms.md delete mode 100644 docs/walkthroughs/list-aws-instances.md diff --git a/.vscode/launch.json b/.vscode/launch.json index c6972e7b..70ea776e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -155,10 +155,7 @@ "select JSON_EXTRACT(saml.samlIdentity, '$.username') as saml_username from github.scim.saml_ids saml where saml.org = 'dummyorg';", "select kind, name, maximumCardsPerInstance from google.compute.acceleratorTypes where project = 'defective-response-content-project' and zone = 'australia-southeast1-a' order by name desc;", "registry pull google;", - "create materialized view nv as select BackupId, BackupState from aws.cloudhsm.backups where region = 'ap-southeast-2' order by BackupId;", - "SELECT instance_id FROM aws.ec2.instances WHERE region IN ('us-east-1', 'ap-southeast-2');", - "SELECT instance_id FROM aws.ec2_solid_gold.instances WHERE region IN ('us-east-1', 'ap-southeast-2');", - "SELECT region FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id; SELECT region, instance_id, tenancy, security_groups FROM aws.ec2_nextgen.instances WHERE region IN ('us-east-1', 'ap-southeast-2', 'ap-southeast-1') order by region, instance_id;", + "create materialized view nv as select BackupId, BackupState from aws.cloudhsm.backups where region = 'ap-southeast-2' order by BackupId;" ], "default": "show providers;" }, @@ -185,13 +182,12 @@ "type": "pickString", "id": "authString", "description": "Auth Input arg String", - "default": "{}", + "default": "{ \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }", "options": [ "{ \"azure\": { \"type\": \"azure_default\" }, \"digitalocean\": { \"type\": \"bearer\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/digitalocean-key.txt\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/ryuk-it-query.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }", "{ \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" }, \"googleadmin\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json\" },s \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/okta/api-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/aws/functional-test-dummy-aws-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/netlify/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/k8s/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/test/assets/credentials/dummy/sumologic/sumologic-token.txt\", \"type\": \"basic\" } }", "{ \"pgi\": { \"type\": \"sql_data_source::postgres\", \"sqlDataSource\": { \"dsn\": \"postgres://stackql:stackql@127.0.0.1:8432\" } }, \"azure\": { \"type\": \"azure_default\" }, \"google\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/stackql-security-reviewer.json\" }, \"okta\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/okta-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"SSWS \" }, \"github\": { \"type\": \"basic\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/github-key.txt\" }, \"aws\": { \"type\": \"aws_signing_v4\", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/aws-secret-key.txt\", \"keyID\": \"AKIA376P4FQSS2ONB2NS\" }, \"netlify\": { \"type\": \"api_key\", \"valuePrefix\": \"Bearer \", \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/netlify-token.txt\" }, \"k8s\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/k8s-token.txt\", \"type\": \"api_key\", \"valuePrefix\": \"Bearer \" }, \"sumologic\": { \"credentialsfilepath\": \"${workspaceFolder}/cicd/keys/integration/sumologic-token.txt\", \"type\": \"basic\" } }", - "{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" } }", - "{}" + "{ \"digitalocean\": { \"username_var\": \"DUMMY_DIGITALOCEAN_USERNAME\", \"password_var\": \"DUMMY_DIGITALOCEAN_PASSWORD\", \"type\": \"bearer\" } }" ] }, { @@ -385,7 +381,12 @@ { "name": "generic exec", "type": "go", - "envFile": "${workspaceFolder}/.vscode/.env", + "env": { + "DUMMY_DIGITALOCEAN_USERNAME": "myusername", + "DUMMY_DIGITALOCEAN_PASSWORD": "mypassword", + "DD_API_KEY": "myusername", + "DD_APPLICATION_KEY": "mypassword" + }, "request": "launch", "mode": "debug", "program": "${workspaceFolder}/stackql", diff --git a/docs/walkthroughs/README.md b/docs/walkthroughs/README.md index 1b10d333..436e2639 100644 --- a/docs/walkthroughs/README.md +++ b/docs/walkthroughs/README.md @@ -16,13 +16,7 @@ in order to setup, run, verify and tear down testing scenarios. The tests *can* ## Running from CI -The canonical, **ruleset-protected** tag form is `scenario-<>-<>`. At this stage, `run_number` must refer to a `stackql` run for which a `linux` `amd64` stackql binary archive is present at the time the tag is run. +The canonical, **ruleset-protected** tag form is `scenario--`. At this stage, `run_number` must refer to a `stackql` run for which a `linux` `amd64` stackql binary archive is present at the time the tag is run. -## Plumbing - -These walkthroughs are runnable using CI. This is built upon: - -- `jinja2` templates, with `<<` and `>>` as delimiters. - diff --git a/docs/walkthroughs/get-google-accelerator-types.md b/docs/walkthroughs/get-google-accelerator-types.md new file mode 100644 index 00000000..42f84752 --- /dev/null +++ b/docs/walkthroughs/get-google-accelerator-types.md @@ -0,0 +1,70 @@ + +## Setup + +First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP dumentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles). + +Then, do this in bash: + +```bash setup stackql-shell credentials-path=cicd/keys/testing/google-ro-credentials.json app-root-path=./test/tmp/.get-google-accel.stackql + +export GOOGLE_CREDENTIALS="$(cat )"; + +stackql shell --approot= +``` + +## Method + +Do this in the `stackql` shell, replacing `` with your GCP project name, and `` as desired, eg: `australia-southeast1-a`: + +```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a + +registry pull google; + +select + name, + kind +FROM google.compute.accelerator_types +WHERE + project = '' + AND zone = '' +ORDER BY + name desc +; + +``` + +## Result + + +You will see something very much like this included in the output, presuming you have one VM (if you have zero, only the headers should appper, more VMs means more rows): + +```sql expectation stdout-contains-all +|---------------------|-------------------------| +| name | kind | +|---------------------|-------------------------| +| nvidia-tesla-t4-vws | compute#acceleratorType | +|---------------------|-------------------------| +| nvidia-tesla-t4 | compute#acceleratorType | +|---------------------|-------------------------| +| nvidia-tesla-p4-vws | compute#acceleratorType | +|---------------------|-------------------------| +| nvidia-tesla-p4 | compute#acceleratorType | +|---------------------|-------------------------| +``` + + + + + + + +## Cleanup + +```bash teardown best-effort app-root-path=./test/tmp/.get-google-accel.stackql + +rm -rf + +``` \ No newline at end of file diff --git a/docs/walkthroughs/get-google-vms.md b/docs/walkthroughs/get-google-vms.md new file mode 100644 index 00000000..f2f70059 --- /dev/null +++ b/docs/walkthroughs/get-google-vms.md @@ -0,0 +1,62 @@ + +## Setup + +First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP dumentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles). + +Then, do this in bash: + +```bash setup stackql-shell credentials-path=cicd/keys/testing/google-ro-credentials.json app-root-path=./test/tmp/.get-google-vms.stackql + +export GOOGLE_CREDENTIALS="$(cat )"; + +stackql shell --approot= +``` + +## Method + +Do this in the `stackql` shell, replacing `` with your GCP project name, and `` as desired, eg: `australia-southeast1-a`: + +```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a + +registry pull google; + +select + name, + id +FROM google.compute.instances +WHERE + project = '' + AND zone = '' +; + +``` + +## Result + + +You will see something very much like this included in the output, presuming you have one VM (if you have zero, only the headers should appper, more VMs means more rows): + +```sql stackql stdout expectation stdout-table-contains-data +|--------------------------------------------------|---------------------| +| name | id | +|--------------------------------------------------|---------------------| +| any-compute-cluster-1-default-abcd-00000001-0001 | 1000000000000000001 | +|--------------------------------------------------|---------------------| +``` + + + + + + + +## Cleanup + +```bash teardown best-effort app-root-path=./test/tmp/.get-google-vms.stackql + +rm -rf + +``` \ No newline at end of file diff --git a/docs/walkthroughs/list-aws-instances.md b/docs/walkthroughs/list-aws-instances.md deleted file mode 100644 index 40a5935b..00000000 --- a/docs/walkthroughs/list-aws-instances.md +++ /dev/null @@ -1,53 +0,0 @@ - -## Setup - -First, create a set of AWS CLI credentials per [the AWS documentation](https://docs.aws.amazon.com/cli/latest/userguide/cli-authentication-user.html#cli-authentication-user-get), and store them in the appropriate environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. - -Then, do this in bash: - -```bash setup stackql-shell credentials_path=cicd/keys/testing/google-ro-credentials.json app_root_path=./test/tmp/.list-aws-instances.stackql - - -stackql shell --approot=<> -``` - -## Method - -Do this in the `stackql` shell, replacing `<>` with your GCP project name, and `<>` as desired, eg: `australia-southeast1-a`: - -```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a - -registry pull aws; - -SELECT instance_id, region -FROM aws.ec2.instances -WHERE region IN ('us-east-1', 'eu-west-1'); - -``` - -## Result - - -You will see exactly this included in the output: - -```sql expectation stdout-contains-all -|---------------------|-------------------------| -| name | kind | -|---------------------|-------------------------| -| nvidia-tesla-t4-vws | compute#acceleratorType | -|---------------------|-------------------------| -| nvidia-tesla-t4 | compute#acceleratorType | -|---------------------|-------------------------| -| nvidia-tesla-p4-vws | compute#acceleratorType | -|---------------------|-------------------------| -| nvidia-tesla-p4 | compute#acceleratorType | -|---------------------|-------------------------| -``` - -## Cleanup - -```bash teardown best-effort app_root_path=./test/tmp/.list-aws-instances.stackql - -rm -rf <> - -``` \ No newline at end of file diff --git a/internal/stackql/drm/prepared_statement_ctx.go b/internal/stackql/drm/prepared_statement_ctx.go index 6fd0da19..ee23e3c0 100644 --- a/internal/stackql/drm/prepared_statement_ctx.go +++ b/internal/stackql/drm/prepared_statement_ctx.go @@ -53,21 +53,9 @@ type standardPreparedStatementCtx struct { func (ps *standardPreparedStatementCtx) GetOrderedTccs() []internaldto.TxnControlCounters { var rv []internaldto.TxnControlCounters - // absolute hack - // TODO: fix all the counter logic - keysRemaining := make(map[string]struct{}) - for k := range ps.aliasToTccMap { - keysRemaining[k] = struct{}{} - } for _, alias := range ps.aliasOrdering { - delete(keysRemaining, alias) rv = append(rv, ps.aliasToTccMap[alias]...) } - // if len(keysRemaining) == 1 { - // for k := range keysRemaining { - // rv = append(rv, ps.aliasToTccMap[k]...) - // } - // } return rv } diff --git a/test/assets/expected/aws/cloud_control/select-projection-bucket-view-parameter-pushed-and-response-filtered.txt b/test/assets/expected/aws/cloud_control/select-projection-bucket-view-parameter-pushed-and-response-filtered.txt index 08a1503b..5e1b5908 100644 --- a/test/assets/expected/aws/cloud_control/select-projection-bucket-view-parameter-pushed-and-response-filtered.txt +++ b/test/assets/expected/aws/cloud_control/select-projection-bucket-view-parameter-pushed-and-response-filtered.txt @@ -1,5 +1,5 @@ |-------------------------|------------------------------------------| -| bucket_name | domain_name | +| BucketName | DomainName | |-------------------------|------------------------------------------| | stackql-trial-bucket-01 | stackql-trial-bucket-01.s3.amazonaws.com | -|-------------------------|------------------------------------------| \ No newline at end of file +|-------------------------|------------------------------------------| diff --git a/test/assets/expected/aws/cloud_control/select-star-bucket-view-parameter-pushed-and-response-filtered.txt b/test/assets/expected/aws/cloud_control/select-star-bucket-view-parameter-pushed-and-response-filtered.txt index db5dcc74..3fb770a9 100644 --- a/test/assets/expected/aws/cloud_control/select-star-bucket-view-parameter-pushed-and-response-filtered.txt +++ b/test/assets/expected/aws/cloud_control/select-star-bucket-view-parameter-pushed-and-response-filtered.txt @@ -1,5 +1,6 @@ -|----------------|-------------------------|--------------------------|----------------|--------------------------|-------------------|-------------------------|--------------------|------------------------------------|--------------------------|-------------------------|-----------------------|------------------------|----------------------------|---------------------------|---------------------|-------------------------------------------------------|--------------------------------------------------------------------------------------------------------|---------------------------|------------------------------------------------|--------------------------|-----------------------|--------------------------------------|------------------------------------------|-------------------------------------------------------------------|---------------------------------------------------------|------------------------------------------------------------------------| -| region | data__Identifier | accelerate_configuration | access_control | analytics_configurations | bucket_encryption | bucket_name | cors_configuration | intelligent_tiering_configurations | inventory_configurations | lifecycle_configuration | logging_configuration | metrics_configurations | notification_configuration | object_lock_configuration | object_lock_enabled | ownership_controls | public_access_block_configuration | replication_configuration | tags | versioning_configuration | website_configuration | arn | domain_name | dual_stack_domain_name | regional_domain_name | website_url | -|----------------|-------------------------|--------------------------|----------------|--------------------------|-------------------|-------------------------|--------------------|------------------------------------|--------------------------|-------------------------|-----------------------|------------------------|----------------------------|---------------------------|---------------------|-------------------------------------------------------|--------------------------------------------------------------------------------------------------------|---------------------------|------------------------------------------------|--------------------------|-----------------------|--------------------------------------|------------------------------------------|-------------------------------------------------------------------|---------------------------------------------------------|------------------------------------------------------------------------| -| ap-southeast-2 | stackql-trial-bucket-01 | null | null | null | null | stackql-trial-bucket-01 | null | null | null | null | null | null | null | null | null | {"Rules":[{"ObjectOwnership":"BucketOwnerEnforced"}]} | {"RestrictPublicBuckets":true,"BlockPublicPolicy":true,"BlockPublicAcls":true,"IgnorePublicAcls":true} | null | [{"Value":"first-ever-bucket","Key":"sundry"}] | null | null | arn:aws:s3:::stackql-trial-bucket-01 | stackql-trial-bucket-01.s3.amazonaws.com | stackql-trial-bucket-01.s3.dualstack.ap-southeast-1.amazonaws.com | stackql-trial-bucket-01.s3.ap-southeast-1.amazonaws.com | http://stackql-trial-bucket-01.s3-website-ap-southeast-1.amazonaws.com | -|----------------|-------------------------|--------------------------|----------------|--------------------------|-------------------|-------------------------|--------------------|------------------------------------|--------------------------|-------------------------|-----------------------|------------------------|----------------------------|---------------------------|---------------------|-------------------------------------------------------|--------------------------------------------------------------------------------------------------------|---------------------------|------------------------------------------------|--------------------------|-----------------------|--------------------------------------|------------------------------------------|-------------------------------------------------------------------|---------------------------------------------------------|------------------------------------------------------------------------| +|--------------------------------------|-------------------------|------------------------------------------|---------------------------------------------------------|-------------------------------------------------------------------|------------------------------------------------------------------------|---------------------|-----------------------|-------------------|-----------------|------------------|------------------------------------------------| +| Arn | BucketName | DomainName | RegionalDomainName | DualStackDomainName | WebsiteURL | ObjectOwnership | RestrictPublicBuckets | BlockPublicPolicy | BlockPublicAcls | IgnorePublicAcls | Tags | +|--------------------------------------|-------------------------|------------------------------------------|---------------------------------------------------------|-------------------------------------------------------------------|------------------------------------------------------------------------|---------------------|-----------------------|-------------------|-----------------|------------------|------------------------------------------------| +| arn:aws:s3:::stackql-trial-bucket-01 | stackql-trial-bucket-01 | stackql-trial-bucket-01.s3.amazonaws.com | stackql-trial-bucket-01.s3.ap-southeast-1.amazonaws.com | stackql-trial-bucket-01.s3.dualstack.ap-southeast-1.amazonaws.com | http://stackql-trial-bucket-01.s3-website-ap-southeast-1.amazonaws.com | BucketOwnerEnforced | true | true | true | true | [{"Value":"first-ever-bucket","Key":"sundry"}] | +|--------------------------------------|-------------------------|------------------------------------------|---------------------------------------------------------|-------------------------------------------------------------------|------------------------------------------------------------------------|---------------------|-----------------------|-------------------|-----------------|------------------|------------------------------------------------| + diff --git a/test/registry/src/aws/v0.1.0/provider.yaml b/test/registry/src/aws/v0.1.0/provider.yaml index 4ff8dee9..49fe6df4 100644 --- a/test/registry/src/aws/v0.1.0/provider.yaml +++ b/test/registry/src/aws/v0.1.0/provider.yaml @@ -64,15 +64,6 @@ providerServices: service: $ref: aws/v0.1.0/services/ec2_nextgen.yaml title: EC2 NextGen - ec2_solid_gold: - description: ec2_solid_gold - id: ec2_solid_gold:v0.1.0 - name: ec2_solid_gold - preferred: true - service: - $ref: aws/v0.1.0/services/ec2_solid_gold.yaml - title: EC2 SolidGold - version: v0.1.0 iam: description: iam id: iam:v0.1.0 diff --git a/test/registry/src/aws/v0.1.0/services/pseudo_s3.yaml b/test/registry/src/aws/v0.1.0/services/pseudo_s3.yaml index 7bf5c120..f4916d64 100644 --- a/test/registry/src/aws/v0.1.0/services/pseudo_s3.yaml +++ b/test/registry/src/aws/v0.1.0/services/pseudo_s3.yaml @@ -494,7 +494,7 @@ components: CASE WHEN json_extract_path_text(Properties, 'PublicAccessBlockConfiguration', 'BlockPublicAcls') = '0' THEN 'false' ELSE 'true' END as BlockPublicAcls, CASE WHEN json_extract_path_text(Properties, 'PublicAccessBlockConfiguration', 'IgnorePublicAcls') = '0' THEN 'false' ELSE 'true' END as IgnorePublicAcls, json_extract_path_text(Properties, 'Tags') as Tags - FROM aws.cloud_controllegacy.resources WHERE region = 'ap-southeast-1' and data__TypeName = 'AWS::S3::Bucket' + FROM aws.cloud_control_legacy.resources WHERE region = 'ap-southeast-1' and data__TypeName = 'AWS::S3::Bucket' ; s3_bucket_detail: name: s3_bucket_detail diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index fed2e867..c1470b6b 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -4115,7 +4115,7 @@ Filtered and Parameterised Projection Resource Level View of Cloud Control Resou ... ${REGISTRY_NO_VERIFY_CFG_STR} ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} - ... select bucket_name, domain_name from aws.pseudo_s3.s3_bucket_polymorphic where data__Identifier = 'stackql\-trial\-bucket\-01' and region \= 'ap\-southeast\-2' and bucket_name \= 'stackql\-trial\-bucket\-01'; + ... select BucketName, DomainName from aws.pseudo_s3.s3_bucket_listing where data__Identifier = 'stackql\-trial\-bucket\-01' and region \= 'ap\-southeast\-2' and BucketName \= 'stackql\-trial\-bucket\-01'; ... ${AWS_CC_VIEW_SELECT_PROJECTION_BUCKET_COMPLEX_EXPECTED} ... ${CURDIR}/tmp/Filtered-and-Parameterised-Projection-Resource-Level-View-of-Cloud-Control-Resource-Returns-Expected-Result.tmp @@ -4128,7 +4128,7 @@ Filtered and Parameterised Star Resource Level View of Cloud Control Resource Re ... ${REGISTRY_NO_VERIFY_CFG_STR} ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} - ... select * from aws.pseudo_s3.s3_bucket_polymorphic where data__Identifier \= 'stackql\-trial\-bucket\-01' and region \= 'ap\-southeast\-2' and bucket_name \= 'stackql\-trial\-bucket\-01'; + ... select * from aws.pseudo_s3.s3_bucket_listing where data__Identifier \= 'stackql\-trial\-bucket\-01' and region \= 'ap\-southeast\-2' and BucketName \= 'stackql\-trial\-bucket\-01'; ... ${AWS_CC_VIEW_SELECT_STAR_BUCKET_COMPLEX_EXPECTED} ... ${CURDIR}/tmp/Filtered-and-Parameterised-Star-Resource-Level-View-of-Cloud-Control-Resource-Returns-Expected-Result.tmp From c2923613b55ec2eec67050d09016e94949a16c35 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Mon, 30 Dec 2024 11:41:09 +1100 Subject: [PATCH 06/10] - Postgres working locally. --- test/registry/src/aws/v0.1.0/services/ec2_nextgen.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/registry/src/aws/v0.1.0/services/ec2_nextgen.yaml b/test/registry/src/aws/v0.1.0/services/ec2_nextgen.yaml index 8ae05ba6..7dcda1df 100644 --- a/test/registry/src/aws/v0.1.0/services/ec2_nextgen.yaml +++ b/test/registry/src/aws/v0.1.0/services/ec2_nextgen.yaml @@ -16872,12 +16872,12 @@ components: json_extract_path_text(detail.Properties, 'Affinity') as affinity, json_extract_path_text(detail.Properties, 'CreditSpecification') as credit_specification FROM aws.cloud_control.resources listing - LEFT OUTER JOIN aws.cloud_control.resource detail + INNER JOIN aws.cloud_control.resource detail ON detail.data__Identifier = listing.Identifier AND detail.region = listing.region WHERE listing.data__TypeName = 'AWS::EC2::Instance' AND detail.data__TypeName = 'AWS::EC2::Instance' - AND listing.region = 'us-east-1' + AND listing.region IN ('us-west-1', 'ap-southeast-1') instances_list_only: name: instances_list_only id: aws.ec2_nextgen.instances_list_only From 59bec1909ec7685fa55e26a4fb81135fa74ba7af Mon Sep 17 00:00:00 2001 From: General Kroll Date: Mon, 30 Dec 2024 12:13:30 +1100 Subject: [PATCH 07/10] - Progress. --- .../list-google-accelerator-types.md | 61 ------------------ docs/walkthroughs/list-google-vms.md | 62 ------------------- .../dependencyplanner/dependencyplanner.go | 16 ----- 3 files changed, 139 deletions(-) delete mode 100644 docs/walkthroughs/list-google-accelerator-types.md delete mode 100644 docs/walkthroughs/list-google-vms.md diff --git a/docs/walkthroughs/list-google-accelerator-types.md b/docs/walkthroughs/list-google-accelerator-types.md deleted file mode 100644 index 3de08e4f..00000000 --- a/docs/walkthroughs/list-google-accelerator-types.md +++ /dev/null @@ -1,61 +0,0 @@ - -## Setup - -First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP documentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles). - -Then, do this in bash: - -```bash setup stackql-shell credentials_path=cicd/keys/testing/google-ro-credentials.json app_root_path=./test/tmp/.get-google-accel.stackql - -export GOOGLE_CREDENTIALS="$(cat <>)"; - -stackql shell --approot=<> -``` - -## Method - -Do this in the `stackql` shell, replacing `<>` with your GCP project name, and `<>` as desired, eg: `australia-southeast1-a`: - -```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a - -registry pull google; - -select - name, - kind -FROM google.compute.accelerator_types -WHERE - project = '<>' - AND zone = '<>' -ORDER BY - name desc -; - -``` - -## Result - - -You will see exactly this included in the output: - -```sql expectation stdout-contains-all -|---------------------|-------------------------| -| name | kind | -|---------------------|-------------------------| -| nvidia-tesla-t4-vws | compute#acceleratorType | -|---------------------|-------------------------| -| nvidia-tesla-t4 | compute#acceleratorType | -|---------------------|-------------------------| -| nvidia-tesla-p4-vws | compute#acceleratorType | -|---------------------|-------------------------| -| nvidia-tesla-p4 | compute#acceleratorType | -|---------------------|-------------------------| -``` - -## Cleanup - -```bash teardown best-effort app_root_path=./test/tmp/.get-google-accel.stackql - -rm -rf <> - -``` \ No newline at end of file diff --git a/docs/walkthroughs/list-google-vms.md b/docs/walkthroughs/list-google-vms.md deleted file mode 100644 index 27e5b511..00000000 --- a/docs/walkthroughs/list-google-vms.md +++ /dev/null @@ -1,62 +0,0 @@ - -## Setup - -First, create a google service account key using the GCP Console, per [the GCP documentation](https://cloud.google.com/iam/docs/keys-create-delete). Grant the service account at least `Viewer` role equivalent privileges, per [the GCP documentation](https://cloud.google.com/iam/docs/create-service-agents#grant-roles). - -Then, do this in bash: - -```bash setup stackql-shell credentials_path=cicd/keys/testing/google-ro-credentials.json app_root_path=./test/tmp/.get-google-vms.stackql - -export GOOGLE_CREDENTIALS="$(cat <>)"; - -stackql shell --approot=<> -``` - -## Method - -Do this in the `stackql` shell, replacing `<>` with your GCP project name, and `<>` as desired, eg: `australia-southeast1-a`: - -```sql stackql-shell input required project=stackql-demo zone=australia-southeast1-a - -registry pull google; - -select - name, - id -FROM google.compute.instances -WHERE - project = '<>' - AND zone = '<>' -; - -``` - -## Result - - -You will see something very much like this included in the output, presuming you have one VM (if you have zero, only the headers should appper, more VMs means more rows): - -```sql stackql stdout expectation stdout-table-contains-data -|--------------------------------------------------|---------------------| -| name | id | -|--------------------------------------------------|---------------------| -| any-compute-cluster-1-default-abcd-00000001-0001 | 1000000000000000001 | -|--------------------------------------------------|---------------------| -``` - - - - - - - -## Cleanup - -```bash teardown best-effort app_root_path=./test/tmp/.get-google-vms.stackql - -rm -rf <> - -``` \ No newline at end of file diff --git a/internal/stackql/dependencyplanner/dependencyplanner.go b/internal/stackql/dependencyplanner/dependencyplanner.go index 18b9e4a8..cf690277 100644 --- a/internal/stackql/dependencyplanner/dependencyplanner.go +++ b/internal/stackql/dependencyplanner/dependencyplanner.go @@ -610,19 +610,6 @@ func (dp *standardDependencyPlanner) isVectorParam(param interface{}) bool { return false } -func (dp *standardDependencyPlanner) extractStaticSourceParam(e dataflow.Edge, paramKey string) (interface{}, bool) { - if e.GetSource() == nil || e. - GetSource().GetAnnotation() == nil || e. - GetSource().GetAnnotation().GetParameters() == nil { - return nil, false - } - v, ok := e.GetSource().GetAnnotation().GetParameters()[paramKey] - if !ok { - return nil, false - } - return dp.getScalarParam(v) -} - //nolint:gocognit,nestif // live with it func (dp *standardDependencyPlanner) getStreamFromEdge( e dataflow.Edge, @@ -676,13 +663,10 @@ func (dp *standardDependencyPlanner) getStreamFromEdge( params := toAc.GetParameters() staticParams := make(map[string]interface{}) for k, v := range params { - existingSourceParam, isExistingFromSource := dp.extractStaticSourceParam(e, k) isVector := dp.isVectorParam(v) if _, ok := incomingCols[k]; !ok && !isVector { staticParams[k] = v incomingCols[k] = struct{}{} - } else if isExistingFromSource && false { - staticParams[k] = existingSourceParam } } if len(staticParams) > 0 { From a5225e7423adfa7d6aa8a50f39427e60b2ece3e3 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Mon, 30 Dec 2024 12:14:39 +1100 Subject: [PATCH 08/10] - Progress. --- .../stackql/dependencyplanner/dependencyplanner.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/internal/stackql/dependencyplanner/dependencyplanner.go b/internal/stackql/dependencyplanner/dependencyplanner.go index cf690277..6673c13f 100644 --- a/internal/stackql/dependencyplanner/dependencyplanner.go +++ b/internal/stackql/dependencyplanner/dependencyplanner.go @@ -586,18 +586,6 @@ func (dp *standardDependencyPlanner) processAcquire( return anTab, dp.tcc, nil } -func (dp *standardDependencyPlanner) getScalarParam(param interface{}) (interface{}, bool) { - paramMeta, isParamMeta := param.(parserutil.ParameterMetadata) - if isParamMeta { - val := paramMeta.GetVal() - _, valIsSQLVal := val.(*sqlparser.SQLVal) - if valIsSQLVal { - return val, true - } - } - return nil, false -} - func (dp *standardDependencyPlanner) isVectorParam(param interface{}) bool { paramMeta, isParamMeta := param.(parserutil.ParameterMetadata) if isParamMeta { From 2e9aa92edb68cb1d6fee1e9a4eea5148e6061224 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Mon, 30 Dec 2024 12:16:58 +1100 Subject: [PATCH 09/10] - Progress. --- test/python/markdown_testing/markdown_testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/python/markdown_testing/markdown_testing.py b/test/python/markdown_testing/markdown_testing.py index 61512848..5ce4e1da 100644 --- a/test/python/markdown_testing/markdown_testing.py +++ b/test/python/markdown_testing/markdown_testing.py @@ -83,7 +83,7 @@ def get_execution_language(self) -> str: return self.node.get('lang', '') def expand(self) -> str: - return self.get_text().replace("<<", "{").replace(">>", "}").format(**self._local_vars) + return self.get_text().replace("<", "{").replace(">", "}").format(**self._local_vars) def __str__(self): return json.dumps(self.node, indent=2) From 77288c91307a48a832cab1b04d40d393e2fa2fc9 Mon Sep 17 00:00:00 2001 From: General Kroll Date: Mon, 30 Dec 2024 12:27:46 +1100 Subject: [PATCH 10/10] - Tidy. --- .../cloud_control_ec2_instance_detail.jinja.json | 2 +- .../cloud_control_ec2_instances_list.jinja.json | 16 ++++++++-------- .../stackql_mocked_from_cmd_line.robot | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/test/python/flask/aws/templates/cloud_control_ec2_instance_detail.jinja.json b/test/python/flask/aws/templates/cloud_control_ec2_instance_detail.jinja.json index 8899ea57..0106831e 100644 --- a/test/python/flask/aws/templates/cloud_control_ec2_instance_detail.jinja.json +++ b/test/python/flask/aws/templates/cloud_control_ec2_instance_detail.jinja.json @@ -1,7 +1,7 @@ { "ResourceDescription": { "Identifier": "{{ request.json['Identifier'] }}", - "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"aws-stack-dev-web-sg\"],\"PrivateDnsName\":\"ip-10-2-1-156.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"10.2.1.156\",\"UserData\":\"IyEvYmluL2Jhc2gKeXVtIHVwZGF0ZSAteQp5dW0gaW5zdGFsbCAteSBodHRwZApzeXN0ZW1jdGwgc3RhcnQgaHR0cGQKc3lzdGVtY3RsIGVuYWJsZSBodHRwZAplY2hvICc8IURPQ1RZUEUgaHRtbD48aHRtbCBsYW5nPSJlbiI+PGhlYWQ+PG1ldGEgY2hhcnNldD0iVVRGLTgiPjxtZXRhIG5hbWU9InZpZXdwb3J0IiBjb250ZW50PSJ3aWR0aD1kZXZpY2Utd2lkdGgsIGluaXRpYWwtc2NhbGU9MS4wIj48dGl0bGU+U3RhY2tRTCBvbiBBV1M8L3RpdGxlPjxzdHlsZT5ib2R5IHtmb250LWZhbWlseTogVGFob21hLCBzYW5zLXNlcmlmOyBkaXNwbGF5OiBmbGV4OyBqdXN0aWZ5LWNvbnRlbnQ6IGNlbnRlcjsgYWxpZ24taXRlbXM6IGNlbnRlcjsgaGVpZ2h0OiAxMDB2aDsgbWFyZ2luOiAwOyBiYWNrZ3JvdW5kLWNvbG9yOiAjZjBmMGYwOyB0ZXh0LWFsaWduOiBjZW50ZXI7fSBpbWcge2hlaWdodDogYXV0bzt9IGNvZGUge2JhY2tncm91bmQtY29sb3I6ICNlOGU4ZTg7IHBhZGRpbmc6IDJweCA2cHg7IGJvcmRlci1yYWRpdXM6IDNweDsgZm9udC13ZWlnaHQ6IGJvbGQ7fSBwIHtmb250LXNpemU6IDEuNWVtOyBmb250LXdlaWdodDogYm9sZDt9PC9zdHlsZT48L2hlYWQ+JyA+IC92YXIvd3d3L2h0bWwvaW5kZXguaHRtbAplY2hvICc8Ym9keT48ZGl2PjxhIGhyZWY9Imh0dHBzOi8vZ2l0aHViLmNvbS9zdGFja3FsL3N0YWNrcWwiPjxpbWcgc3JjPSJodHRwczovL3N0YWNrcWwuaW8vaW1nL3N0YWNrcWwtbG9nby1ib2xkLnBuZyIgYWx0PSJTdGFja1FMIExvZ28iPjwvYT48cD5IZWxsbywgPGEgaHJlZj0iaHR0cHM6Ly9weXBpLm9yZy9wcm9qZWN0L3N0YWNrcWwtZGVwbG95LyI+PGNvZGU+c3RhY2txbC1kZXBsb3k8L2NvZGU+PC9hPiBvbiBBV1MhPC9wPjwvZGl2PjwvYm9keT48L2h0bWw+JyA+PiAvdmFyL3d3dy9odG1sL2luZGV4Lmh0bWw=\",\"BlockDeviceMappings\":[{\"Ebs\":{\"SnapshotId\":\"snap-099392264f95c4223\",\"VolumeType\":\"gp3\",\"Encrypted\":false,\"Iops\":3000,\"VolumeSize\":8,\"DeleteOnTermination\":true},\"DeviceName\":\"/dev/xvda\"}],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-0e4cb97c9e4ab3e9a\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"10.2.1.156\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"10.2.1.156\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"10.2.1.156\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-08d705c9b6ecfff45\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-050a905d286cb29b5\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-030a5acd7c996ef60\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"aws-stack\",\"Key\":\"StackName\"},{\"Value\":\"stackql\",\"Key\":\"Provisioner\"},{\"Value\":\"dev\",\"Key\":\"StackEnv\"},{\"Value\":\"aws-stack-dev-instance\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"InstanceId\":\"i-0cb58b995b19153a8\",\"PublicIp\":\"13.211.134.69\",\"InstanceInitiatedShutdownBehavior\":\"stop\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-13-211-134-69.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-08d705c9b6ecfff45\"],\"DisableApiTermination\":false,\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-00e086ac8c9504aec\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"},\"CreditSpecification\":{\"CPUCredits\":\"standard\"}}" + "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"aws-stack-dev-web-sg\"],\"PrivateDnsName\":\"ip-10-2-1-156.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"10.2.1.156\",\"UserData\":\"\",\"BlockDeviceMappings\":[{\"Ebs\":{\"SnapshotId\":\"snap-0000000000000007\",\"VolumeType\":\"gp3\",\"Encrypted\":false,\"Iops\":3000,\"VolumeSize\":8,\"DeleteOnTermination\":true},\"DeviceName\":\"/dev/xvda\"}],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-0000000000000008\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"10.2.1.156\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"10.2.1.156\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"10.2.1.156\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-000000000000005\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-00000000000000006\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-030a5acd7c996ef60\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"aws-stack\",\"Key\":\"StackName\"},{\"Value\":\"stackql\",\"Key\":\"Provisioner\"},{\"Value\":\"dev\",\"Key\":\"StackEnv\"},{\"Value\":\"aws-stack-dev-instance\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"InstanceId\":\"i-00000000000000003\",\"PublicIp\":\"13.211.134.69\",\"InstanceInitiatedShutdownBehavior\":\"stop\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-13-211-134-69.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-000000000000005\"],\"DisableApiTermination\":false,\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-00e086ac8c9504aec\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"},\"CreditSpecification\":{\"CPUCredits\":\"standard\"}}" }, "TypeName": "AWS::EC2::Instance" } \ No newline at end of file diff --git a/test/python/flask/aws/templates/cloud_control_ec2_instances_list.jinja.json b/test/python/flask/aws/templates/cloud_control_ec2_instances_list.jinja.json index d4deea39..8d7c0809 100644 --- a/test/python/flask/aws/templates/cloud_control_ec2_instances_list.jinja.json +++ b/test/python/flask/aws/templates/cloud_control_ec2_instances_list.jinja.json @@ -2,21 +2,21 @@ "ResourceDescriptions": [ {% if region == 'ap-southeast-2' %} { - "Identifier": "i-0fc989cf4efcd8b88", - "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"launch-wizard\"],\"PrivateDnsName\":\"ip-172-31-6-142.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"172.31.6.142\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-0a06b014d09e72b0d\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"172.31.6.142\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"172.31.6.142\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"172.31.6.142\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-07d3754577c36eeac\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-07c9c942f637281b9\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-0035ee596a0a12a7b\",\"InstanceType\":\"t2.nano\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"Fred\",\"Key\":\"Test\"},{\"Value\":\"test\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-0fc989cf4efcd8b88\",\"PublicIp\":\"13.211.203.69\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-13-211-203-69.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-07d3754577c36eeac\"],\"KeyName\":\"stackql-test\",\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-0343d19bfd304188e\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" + "Identifier": "i-00000000000000001", + "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"launch-wizard\"],\"PrivateDnsName\":\"ip-172-31-6-142.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"172.31.6.142\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-0a06b014d09e72b0d\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"172.31.6.142\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"172.31.6.142\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"172.31.6.142\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-07d3754577c36eeac\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-07c9c942f637281b9\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-0035ee596a0a12a7b\",\"InstanceType\":\"t2.nano\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"Fred\",\"Key\":\"Test\"},{\"Value\":\"test\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-00000000000000001\",\"PublicIp\":\"13.211.203.69\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-13-211-203-69.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-07d3754577c36eeac\"],\"KeyName\":\"stackql-test\",\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-0343d19bfd304188e\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" }, { - "Identifier": "i-0d1be1a3ab2a09a88", - "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"launch-wizard\"],\"PrivateDnsName\":\"ip-172-31-10-204.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"172.31.10.204\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-09bbae22caf877d1e\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"172.31.10.204\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"172.31.10.204\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"172.31.10.204\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-07d3754577c36eeac\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-08cab5f9ca9ea8b9d\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-0ec0514235185af79\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"test1\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-0d1be1a3ab2a09a88\",\"PublicIp\":\"54.66.216.138\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":true,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-54-66-216-138.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-07d3754577c36eeac\"],\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-0343d19bfd304188e\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" + "Identifier": "i-00000000000000002", + "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"launch-wizard\"],\"PrivateDnsName\":\"ip-172-31-10-204.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"172.31.10.204\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-09bbae22caf877d1e\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"172.31.10.204\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"172.31.10.204\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"172.31.10.204\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-07d3754577c36eeac\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-0718009e7e2a0f750\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-08cab5f9ca9ea8b9d\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-0ec0514235185af79\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"test1\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-00000000000000002\",\"PublicIp\":\"54.66.216.138\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":true,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-54-66-216-138.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-07d3754577c36eeac\"],\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-0343d19bfd304188e\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" }, { - "Identifier": "i-0cb58b995b19153a8", - "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"aws-stack-dev-web-sg\"],\"PrivateDnsName\":\"ip-10-2-1-156.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"10.2.1.156\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-0e4cb97c9e4ab3e9a\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"10.2.1.156\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"10.2.1.156\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"10.2.1.156\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-08d705c9b6ecfff45\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-050a905d286cb29b5\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-030a5acd7c996ef60\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"aws-stack\",\"Key\":\"StackName\"},{\"Value\":\"stackql\",\"Key\":\"Provisioner\"},{\"Value\":\"dev\",\"Key\":\"StackEnv\"},{\"Value\":\"aws-stack-dev-instance\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-0cb58b995b19153a8\",\"PublicIp\":\"13.211.134.69\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-13-211-134-69.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-08d705c9b6ecfff45\"],\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-00e086ac8c9504aec\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" + "Identifier": "i-00000000000000003", + "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"aws-stack-dev-web-sg\"],\"PrivateDnsName\":\"ip-10-2-1-156.ap-southeast-2.compute.internal\",\"PrivateIpAddress\":\"10.2.1.156\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-0000000000000008\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"10.2.1.156\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"10.2.1.156\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"10.2.1.156\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-000000000000005\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-05a763b1b39acc6fe\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-00000000000000006\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-030a5acd7c996ef60\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"aws-stack\",\"Key\":\"StackName\"},{\"Value\":\"stackql\",\"Key\":\"Provisioner\"},{\"Value\":\"dev\",\"Key\":\"StackEnv\"},{\"Value\":\"aws-stack-dev-instance\",\"Key\":\"Name\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-00000000000000003\",\"PublicIp\":\"13.211.134.69\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"ap-southeast-2b\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-13-211-134-69.ap-southeast-2.compute.amazonaws.com\",\"SecurityGroupIds\":[\"sg-000000000000005\"],\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-00e086ac8c9504aec\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" } {% elif region == 'us-east-1' %} { - "Identifier": "i-05259c370b23083bf", - "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"aws-stack-dev-web-sg\"],\"PrivateDnsName\":\"ip-10-2-1-199.ec2.internal\",\"PrivateIpAddress\":\"10.2.1.199\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-0e16ac880cb33918f\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-055d48a8b1a6259bf\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"10.2.1.199\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"10.2.1.199\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"10.2.1.199\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-073ac14cc1a4ddfb0\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-0e16ac880cb33918f\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-04ad5c232cc4c4a2e\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-012967cc5a8c9f891\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"aws-stack-dev-instance\",\"Key\":\"Name\"},{\"Value\":\"aws-stack\",\"Key\":\"StackName\"},{\"Value\":\"dev\",\"Key\":\"StackEnv\"},{\"Value\":\"stackql\",\"Key\":\"Provisioner\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-05259c370b23083bf\",\"PublicIp\":\"18.234.253.185\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"us-east-1e\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-18-234-253-185.compute-1.amazonaws.com\",\"SecurityGroupIds\":[\"sg-073ac14cc1a4ddfb0\"],\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-0df79c273d7be8cce\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" + "Identifier": "i-00000000000000004", + "Properties": "{\"Tenancy\":\"default\",\"SecurityGroups\":[\"aws-stack-dev-web-sg\"],\"PrivateDnsName\":\"ip-10-2-1-199.ec2.internal\",\"PrivateIpAddress\":\"10.2.1.199\",\"BlockDeviceMappings\":[],\"SubnetId\":\"subnet-0e16ac880cb33918f\",\"EbsOptimized\":false,\"Volumes\":[{\"VolumeId\":\"vol-055d48a8b1a6259bf\",\"Device\":\"/dev/xvda\"}],\"PrivateIp\":\"10.2.1.199\",\"EnclaveOptions\":{\"Enabled\":false},\"NetworkInterfaces\":[{\"AssociateCarrierIpAddress\":false,\"PrivateIpAddress\":\"10.2.1.199\",\"PrivateIpAddresses\":[{\"PrivateIpAddress\":\"10.2.1.199\",\"Primary\":true}],\"SecondaryPrivateIpAddressCount\":0,\"DeviceIndex\":\"0\",\"Ipv6AddressCount\":0,\"GroupSet\":[\"sg-073ac14cc1a4ddfb0\"],\"Ipv6Addresses\":[],\"SubnetId\":\"subnet-0e16ac880cb33918f\",\"AssociatePublicIpAddress\":true,\"NetworkInterfaceId\":\"eni-04ad5c232cc4c4a2e\",\"DeleteOnTermination\":true}],\"ImageId\":\"ami-012967cc5a8c9f891\",\"InstanceType\":\"t2.micro\",\"Monitoring\":false,\"Tags\":[{\"Value\":\"aws-stack-dev-instance\",\"Key\":\"Name\"},{\"Value\":\"aws-stack\",\"Key\":\"StackName\"},{\"Value\":\"dev\",\"Key\":\"StackEnv\"},{\"Value\":\"stackql\",\"Key\":\"Provisioner\"}],\"HibernationOptions\":{\"Configured\":false},\"LicenseSpecifications\":[],\"InstanceId\":\"i-00000000000000004\",\"PublicIp\":\"18.234.253.185\",\"CpuOptions\":{\"ThreadsPerCore\":1,\"CoreCount\":1},\"AvailabilityZone\":\"us-east-1e\",\"PrivateDnsNameOptions\":{\"EnableResourceNameDnsARecord\":false,\"HostnameType\":\"ip-name\",\"EnableResourceNameDnsAAAARecord\":false},\"PublicDnsName\":\"ec2-18-234-253-185.compute-1.amazonaws.com\",\"SecurityGroupIds\":[\"sg-073ac14cc1a4ddfb0\"],\"SourceDestCheck\":true,\"PlacementGroupName\":\"\",\"VpcId\":\"vpc-0df79c273d7be8cce\",\"State\":{\"Code\":\"16\",\"Name\":\"running\"}}" } {% endif %} ], diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index c1470b6b..6e195fa7 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -7363,13 +7363,13 @@ View Tuple Replacement Working As Exemplified by AWS EC2 Instances List and Deta ... |----------------|---------------------|---------|--------------------------| ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}region${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}instance_id${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}tenancy${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}security_groups${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}| ... |----------------|---------------------|---------|--------------------------| - ... |${SPACE}ap-southeast-2${SPACE}|${SPACE}i-0cb58b995b19153a8${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| + ... |${SPACE}ap-southeast-2${SPACE}|${SPACE}i-00000000000000003${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| ... |----------------|---------------------|---------|--------------------------| - ... |${SPACE}ap-southeast-2${SPACE}|${SPACE}i-0cb58b995b19153a8${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| + ... |${SPACE}ap-southeast-2${SPACE}|${SPACE}i-00000000000000003${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| ... |----------------|---------------------|---------|--------------------------| - ... |${SPACE}ap-southeast-2${SPACE}|${SPACE}i-0cb58b995b19153a8${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| + ... |${SPACE}ap-southeast-2${SPACE}|${SPACE}i-00000000000000003${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| ... |----------------|---------------------|---------|--------------------------| - ... |${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}i-0cb58b995b19153a8${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| + ... |${SPACE}us-east-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}i-00000000000000003${SPACE}|${SPACE}default${SPACE}|${SPACE}\["aws-stack-dev-web-sg"]${SPACE}| ... |----------------|---------------------|---------|--------------------------| Should Stackql Exec Inline Equal Both Streams ... ${STACKQL_EXE}