diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f50c3971..ea1d3827 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,6 +11,8 @@ on: - "**/*.md" - "docs/**" - "licenses/**" + - "test/python/markdown_testing/**" + - ".github/workflows/scenario.yaml" pull_request: branches: - main @@ -20,6 +22,8 @@ on: - "docs/**" - "examples/**" - "licenses/**" + - "test/python/markdown_testing/**" + - ".github/workflows/scenario.yaml" env: GOTESTCMD: "go test -timeout 1200s --tags \"sqlite_stackql\" -v ./..." diff --git a/.github/workflows/scenario.yaml b/.github/workflows/scenario.yaml index 2cc289e4..8b11e827 100644 --- a/.github/workflows/scenario.yaml +++ b/.github/workflows/scenario.yaml @@ -7,6 +7,10 @@ on: env: TAG_NAME: ${{ github.ref_name }} + RUNTYPE_READ_ONLY: readonly + RUNTYPE_READ_WRITE: readwrite + RUNTYPE_DEPLOY: deploy + RUNTYPE_ALL: all jobs: scenario-testing: @@ -15,8 +19,28 @@ jobs: steps: - name: Ref Parse run: | + _defaultRunType="${{ env.RUNTYPE_READ_ONLY }}" + runID=$(echo -n '${{ github.ref_name }}' | cut -d '-' -f 2)" + runType=$(echo -n '${{ github.ref_name }}' | cut -d '-' -f 3)" + repositoryShorthand=$(echo -n '${{ github.ref_name }}' | rev | cut -d '-' -f 1 | rev) + artifactRepositoryFullName='stackql/stackql' + if [ "$repositoryShorthand" = "devel" ]; then + artifactRepositoryFullName='stackql/stackql-devel' + fi + if [ "$runType" = "" ]; then + runType="$_defaultRunType" + elif [ "$runType" != "${{ env.RUNTYPE_READ_ONLY }}" ] \ + && [ "$runType" != "${{ env.RUNTYPE_READ_WRITE }}" ] \ + && [ "$runType" != "${{ env.RUNTYPE_DEPLOY }}" \ + && [ "$runType" != "${{ env.RUNTYPE_ALL }}" \ + ]; then + echo "Invalid run type: $runType" + exit 1 + fi { echo "runID=$(echo -n '${{ github.ref_name }}' | cut -d '-' -f 2)" + echo "artifactRepositoryFullName=$artifactRepositoryFullName" + echo "runType=$runType" } >> "${GITHUB_ENV}" - name: Check out code into the Go module directory @@ -28,7 +52,7 @@ jobs: name: stackql_linux_amd64 path: build github-token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }} - repository: stackql/stackql + repository: ${{ env.artifactRepositoryFullName }} run-id: ${{ env.runID }} - name: Stackql permissions @@ -46,26 +70,45 @@ jobs: - name: Persist secrets run: | echo "$GCP_RO_SECRET" >> cicd/keys/testing/google-ro-credentials.json + echo "$GCP_RW_SECRET" >> cicd/keys/testing/google-rw-credentials.json shell: bash env: GCP_RO_SECRET: ${{ secrets.CI_SCENARIO_GCP_RO_SECRET }} + GCP_RW_SECRET: ${{ secrets.CI_SCENARIO_GCP_RW_SECRET }} - name: Install Python dependencies run: | pip3 install -r cicd/requirements.txt - - name: Run Walkthrough Scenarios + - name: Run Read Only Walkthrough Scenarios + if: ${{ env.runType == env.RUNTYPE_READ_ONLY || env.runType == env.RUNTYPE_ALL }} env: AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RO_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RO_AWS_SECRET_ACCESS_KEY }} run: | - python3 test/python/markdown_testing/markdown_testing.py 2>&1 | tee cicd/log/markdown-testing-results.log + python3 test/python/markdown_testing/markdown_testing.py --test-root=docs/walkthroughs/readonly 2>&1 | tee cicd/log/markdown-readonly-testing-results.log + - name: Run Read Write Walkthrough Scenarios + if: ${{ env.runType == env.RUNTYPE_READ_WRITE || env.runType == env.RUNTYPE_ALL }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RW_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RW_AWS_SECRET_ACCESS_KEY }} + run: | + python3 test/python/markdown_testing/markdown_testing.py --test-root=docs/walkthroughs/readwrite 2>&1 | tee cicd/log/markdown-readwrite-testing-results.log + + - name: Run Deploy Walkthrough Scenarios + if: ${{ env.runType == env.RUNTYPE_DEPLOY || env.runType == env.RUNTYPE_ALL }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RW_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RW_AWS_SECRET_ACCESS_KEY }} + run: | + python3 test/python/markdown_testing/markdown_testing.py --test-root=docs/walkthroughs/deploy 2>&1 | tee cicd/log/markdown-deploy-testing-results.log + - name: Upload Test Results uses: actions/upload-artifact@v4.3.1 with: name: scenario_test_results - path: cicd/log/markdown-testing-results.log + path: cicd/log/markdown-*.log - name: Cleanup if: always() diff --git a/docs/walkthroughs/deploy/README.md b/docs/walkthroughs/deploy/README.md new file mode 100644 index 00000000..0aab2e41 --- /dev/null +++ b/docs/walkthroughs/deploy/README.md @@ -0,0 +1,4 @@ + +## Infra lifecycle managment with `stackql` + + diff --git a/docs/walkthroughs/list-aws-instances.md b/docs/walkthroughs/readonly/list-aws-instances.md similarity index 100% rename from docs/walkthroughs/list-aws-instances.md rename to docs/walkthroughs/readonly/list-aws-instances.md diff --git a/docs/walkthroughs/list-google-accelerator-types.md b/docs/walkthroughs/readonly/list-google-accelerator-types.md similarity index 100% rename from docs/walkthroughs/list-google-accelerator-types.md rename to docs/walkthroughs/readonly/list-google-accelerator-types.md diff --git a/docs/walkthroughs/list-google-vms.md b/docs/walkthroughs/readonly/list-google-vms.md similarity index 100% rename from docs/walkthroughs/list-google-vms.md rename to docs/walkthroughs/readonly/list-google-vms.md diff --git a/docs/walkthroughs/readwrite/README.md b/docs/walkthroughs/readwrite/README.md new file mode 100644 index 00000000..af5e892f --- /dev/null +++ b/docs/walkthroughs/readwrite/README.md @@ -0,0 +1,4 @@ + +## Lifecycle managment with `stackql-deploy` + + diff --git a/test/python/markdown_testing/markdown_testing.py b/test/python/markdown_testing/markdown_testing.py index 9daaf0f9..3d18d0ce 100644 --- a/test/python/markdown_testing/markdown_testing.py +++ b/test/python/markdown_testing/markdown_testing.py @@ -10,7 +10,8 @@ from tabulate import tabulate -_REPOSITORY_ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..')) +import argparse + """ Intentions: @@ -19,6 +20,19 @@ - Support sequential markdown code block execution, leveraging [info strings](https://spec.commonmark.org/0.30/#info-string). """ + +def parse_args() -> argparse.Namespace: + """ + Parse the arguments. + """ + parser = argparse.ArgumentParser(description='Create a token.') + parser.add_argument('--test-root', type=str, help='The test root.', default=os.path.join(_REPOSITORY_ROOT_PATH, 'docs', 'walkthroughs')) + return parser.parse_args() + + +_REPOSITORY_ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..')) + + def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) @@ -363,10 +377,6 @@ def run_all(self, walkthrough_inodes: List[str], recursive=True, skip_readme=Tru e2e: SimpleRunner = SimpleRunner(workload) result = e2e.run() results.append(result) - if recursive: - for dir in dirs: - dir_path = os.path.join(root, dir) - results += self.run_all([dir_path], recursive) continue is_file = os.path.isfile(inode_path) if is_file: @@ -382,7 +392,7 @@ def run_all(self, walkthrough_inodes: List[str], recursive=True, skip_readme=Tru raise FileNotFoundError(f'Path not tractable: {inode_path}') return results -def collate_results(results: List[WalkthroughResult]) -> bool: +def _collate_results(results: List[WalkthroughResult]) -> bool: failed: int = 0 for result in results: if result.rc != 0 or not result.passes_stdout_check or not result.passes_stderr_check: @@ -391,14 +401,32 @@ def collate_results(results: List[WalkthroughResult]) -> bool: print(tabulate([[result.name, result.rc, result.passes_stdout_check, result.passes_stderr_check] for result in results], headers=['Test Name', 'Return Code', 'Passes Stdout Checks', 'Passes Stderr Checks'])) return failed == 0 -def main(): +def run_tests(root_dir: str) -> List[WalkthroughResult]: + """ + Run all tests. + A decent entry point for a test harness. + + :param root_dir: The root directory. + + :return: The results. + """ runner: AllWalkthroughsRunner = AllWalkthroughsRunner() - results: List[WalkthroughResult] = runner.run_all([os.path.join(_REPOSITORY_ROOT_PATH, 'docs', 'walkthroughs')]) - if collate_results(results): + results: List[WalkthroughResult] = runner.run_all([root_dir]) + return results + + +def _process_tests(root_dir: str) -> List[WalkthroughResult]: + results: List[WalkthroughResult] = run_tests(root_dir) + if _collate_results(results): print('All tests passed.') sys.exit(0) print('Some tests failed.') sys.exit(1) +def _main() -> None: + args :argparse.Namespace = parse_args() + _process_tests(args.test_root) + + if __name__ == '__main__': - main() + _main()