diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index cf49dfc..11a0660 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -10,6 +10,18 @@ common: project-number: "911195782929" steps: + - label: ":shell: Shellcheck" + command: "make shellcheck" + agents: + provider: "gcp" + image: family/core-ubuntu-2204 + + - label: ":sparkles: Lint" + command: "make lint" + agents: + provider: "gcp" + image: family/core-ubuntu-2204 + - label: ":gcp: Google Cloud Auth OIDC Test" agents: provider: gcp diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..bdd7c1d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,297 @@ +# Contributing to oblt-google-auth Buildkite Plugin + +Thank you for your interest in contributing to the oblt-google-auth Buildkite plugin! This document provides guidelines and information for contributors. + +## Table of Contents + +- [Getting Started](#getting-started) +- [Development Setup](#development-setup) +- [Plugin Architecture](#plugin-architecture) +- [Testing](#testing) +- [Code Style and Standards](#code-style-and-standards) +- [Making Changes](#making-changes) +- [Submitting Pull Requests](#submitting-pull-requests) +- [Release Process](#release-process) +- [Getting Help](#getting-help) + +## Getting Started + +This plugin enables Buildkite pipelines to authenticate with Google Cloud Platform (GCP) using [Workload Identity Federation](https://cloud.google.com/iam/docs/workload-identity-federation). It provides two authentication modes: +- **OIDC Token Mode**: Uses Buildkite's OIDC token with external account credentials (default) +- **Service Account Mode**: Creates temporary service accounts for enhanced security + +### Prerequisites + +Before contributing, ensure you have: +- Docker and Docker Compose installed +- Basic understanding of Buildkite plugins +- Familiarity with Google Cloud IAM and Workload Identity Federation +- Knowledge of Bash scripting + +## Development Setup + +1. **Fork and Clone** + ```bash + git clone https://github.com/your-username/oblt-google-auth-buildkite-plugin.git + cd oblt-google-auth-buildkite-plugin + ``` + +2. **Install Development Dependencies** + The project uses Docker for all development tasks, so you only need Docker installed locally. + +3. **Verify Setup** + ```bash + make lint shellcheck + ``` + +## Plugin Architecture + +### File Structure + +``` +├── hooks/ +│ ├── environment # Main authentication logic +│ └── pre-exit # Cleanup and service account management +├── libs/ +│ ├── gcloud # Google Cloud operations +│ └── slack # Slack notification functions +├── docker-compose.yml # Development and testing containers +├── plugin.yml # Plugin metadata and configuration schema +└── Makefile # Development commands +``` + +### Hook Lifecycle + +1. **Environment Hook** (`hooks/environment`) + - Executes before the build command + - Sets up authentication credentials + - Configures environment variables for GCP access + +2. **Pre-exit Hook** (`hooks/pre-exit`) + - Executes after the build command (success or failure) + - Cleans up temporary files and credentials + - Handles service account deletion and notifications + +### Authentication Modes + +#### OIDC Token Mode (Default) +- Uses `buildkite-agent oidc request-token` to get OIDC token +- Creates external account credentials file +- Suitable for most use cases + +#### Service Account Mode +- Creates temporary service accounts with specific permissions +- Provides enhanced security and audit capabilities +- Automatically cleaned up after job completion +- Sends Slack notifications on cleanup failures + +## Testing + +### Test Framework + +The project uses [BATS (Bash Automated Testing System)](https://github.com/bats-core/bats-core) with additional libraries: +- `bats-assert` for assertions +- `bats-mock` for command stubbing +- `bats-support` for helper functions + +### Running Tests + +```bash +# Run all tests +make tests + +# Run specific test files +docker compose run --rm tests bats /plugin/tests/environment.bats +docker compose run --rm tests bats /plugin/tests/pre-exit.bats + +# Run individual tests +docker compose run --rm tests bats /plugin/tests/environment.bats -f "test name" +``` + +### Test Structure + +Tests are organized into two main files: +- `tests/environment.bats` - Tests for the environment hook +- `tests/pre-exit.bats` - Tests for the pre-exit hook + +Each test file includes: +- Setup and teardown functions for test isolation +- Mock library functions for external dependencies +- Stub commands for system calls (mktemp, buildkite-agent, curl, etc.) + +### Writing Tests + +When adding new functionality: + +1. **Add test cases** that cover both success and failure scenarios +2. **Use proper mocking** for external dependencies: + ```bash + # Example: Mock buildkite-agent command + stub buildkite-agent "oidc request-token --audience \* --lifetime \* : echo 'mock-token'" + ``` +3. **Test environment isolation** - ensure tests don't affect each other +4. **Mock external services** - don't make real API calls in tests + +### Test Debugging + +To debug failing tests: + +```bash +# Enable debug output for specific commands +export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty +export CURL_STUB_DEBUG=/dev/tty + +# Run tests with verbose output +docker compose run --rm tests bats --verbose-run /plugin/tests/environment.bats +``` + +## Code Style and Standards + +### Bash Style Guidelines + +1. **Use strict error handling** + ```bash + set -euo pipefail # Exit on error, undefined vars, pipe failures + ``` + +2. **Quote variables** + ```bash + echo "Value: $VARIABLE" + [[ -n "${OPTIONAL_VAR:-}" ]] + ``` + +3. **Use meaningful function names** + ```bash + create_service_account() { ... } + cleanup_temporary_files() { ... } + ``` + +4. **Add comments for complex logic** + ```bash + # Generate workload identity provider ID from repository hash + REPO_HASH=$(echo -n "$BUILDKITE_REPO" | sha256sum | cut -c1-27) + ``` + +### Linting and Code Quality + +The project uses automated code quality checks: + +```bash +# Run all quality checks +make lint shellcheck + +# Individual checks +make lint # Plugin-specific linting +make shellcheck # Bash script analysis +``` + +**ShellCheck Rules**: The project follows ShellCheck recommendations with specific exceptions documented in the code using `# shellcheck disable=SCxxxx` comments. + +## Making Changes + +### Branching Strategy + +1. **Create feature branches** from `main`: + ```bash + git checkout -b feature/your-feature-name + ``` + +2. **Use descriptive commit messages**: + ``` + feat: add support for custom GCP regions + + - Add region configuration option + - Update authentication logic for regional resources + - Add tests for region-specific authentication + ``` + +3. **Keep changes focused** - one feature or fix per pull request + +### Configuration Changes + +When modifying plugin configuration: + +1. **Update `plugin.yml`** with new properties +2. **Update `README.md`** with documentation +3. **Add validation** in the environment hook +4. **Add test coverage** for new configuration options + +### Adding New Features + +1. **Design consideration**: Ensure features align with the plugin's purpose +2. **Backward compatibility**: Don't break existing usage +3. **Error handling**: Provide clear error messages +4. **Documentation**: Update README and add inline comments +5. **Testing**: Add comprehensive test coverage + +## Submitting Pull Requests + +### Pre-submission Checklist + +- [ ] All tests pass (`make tests`) +- [ ] Code passes linting (`make lint shellcheck`) +- [ ] Documentation is updated (README.md, inline comments) +- [ ] New features have test coverage +- [ ] Commit messages are descriptive +- [ ] No sensitive information is included + +### Pull Request Process + +1. **Create pull request** with descriptive title and description +2. **Link related issues** if applicable +3. **Request review** from maintainers +4. **Address feedback** promptly +5. **Ensure CI passes** before requesting final review + +## Release Process + +### Versioning + +The project follows [Semantic Versioning](https://semver.org/): +- **MAJOR**: Incompatible API changes +- **MINOR**: New functionality (backward compatible) +- **PATCH**: Bug fixes (backward compatible) + +### Release Steps + +1. **Update version** in relevant files +2. **Create release notes** documenting changes +3. **Tag the release**: `git tag v1.x.x` +4. **Publish release** through GitHub releases +5. **Update documentation** if needed + +## Getting Help + +### Communication Channels + +- **GitHub Issues**: For bugs, feature requests, and questions +- **Pull Request Reviews**: For code-specific discussions +- **Elastic Internal**: Slack channels for team members + +### Reporting Issues + +When reporting bugs or issues: + +1. **Search existing issues** first +2. **Provide reproduction steps** +3. **Include relevant logs** and error messages +4. **Specify environment details** (Buildkite, GCP setup) + +### Contributing Guidelines + +- Be respectful and constructive +- Follow the code of conduct +- Provide clear documentation for changes +- Test thoroughly before submitting +- Respond promptly to review feedback + +## Additional Resources + +- [Buildkite Plugin Documentation](https://buildkite.com/docs/plugins) +- [Google Cloud Workload Identity Federation](https://cloud.google.com/iam/docs/workload-identity-federation) +- [BATS Testing Framework](https://github.com/bats-core/bats-core) +- [ShellCheck Documentation](https://github.com/koalaman/shellcheck) + +--- + +Thank you for contributing to the oblt-google-auth Buildkite plugin! 🚀 \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c253ee2 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +.PHONY: all lint shellcheck clean + +all: lint shellcheck + +lint: + -docker compose run lint + +shellcheck: + -docker compose run shellcheck + +clean: + -docker compose \ + rm --remove-orphans --force --stop diff --git a/README.md b/README.md index 6d5b30a..81eed38 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,29 @@ This is an opinionated plugin to authenticate to any Google Cloud project from B The Workload Identity Provider uses a hash for the GitHub repository with the format `owner/repo`, the hash has a length of 28 characters. +## How It Works + +This plugin supports two authentication modes: + +### OIDC Token Mode (Default) +- Uses Buildkite's built-in OIDC token via `buildkite-agent oidc request-token` +- Creates a Google Cloud external account credentials file +- Suitable for most use cases with minimal setup + +### Service Account Mode +- Creates temporary service accounts with specific permissions +- Provides enhanced security and audit capabilities +- Automatically cleaned up after job completion +- Sends Slack notifications if cleanup fails + +## Environment Variables Set + +The plugin sets the following environment variables for your build steps: + +- `GOOGLE_APPLICATION_CREDENTIALS` - Path to the credentials file +- `CLOUDSDK_CORE_PROJECT` - GCP project ID for gcloud CLI +- `GOOGLE_CLOUD_PROJECT` - GCP project ID for client libraries + ## Properties | Name | Description | Required | Default | diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d1acc54 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +services: + lint: + image: buildkite/plugin-linter + command: ['--id', 'elastic/oblt-google-auth'] + volumes: + - ".:/plugin" + + shellcheck: + image: koalaman/shellcheck:latest + command: -x hooks/environment hooks/pre-exit + working_dir: /plugin + volumes: + - ".:/plugin" diff --git a/hooks/environment b/hooks/environment index 6fe57e5..59cbba5 100755 --- a/hooks/environment +++ b/hooks/environment @@ -31,6 +31,7 @@ if [[ "$OSTYPE" =~ ^(win|msys|cygwin) ]] ; then # Convert paths to Windows format TOKEN_FILE="$(cygpath -w "$TOKEN_FILE")" # Escape backslashes for JSON by doubling them + # shellcheck disable=SC2001 TOKEN_FILE="$(echo "$TOKEN_FILE" | sed 's/\\/\\\\/g')" fi @@ -60,7 +61,7 @@ if [ "$USE_SERVICE_ACCOUNT" = true ]; then echo "~~~ :gcloud: Generating temporary Service Account credentials" # Get the name of the repository - REPO_BASENAME=$(basename ${BUILDKITE_REPO}) + REPO_BASENAME=$(basename "${BUILDKITE_REPO}") REPO_NAME=${REPO_BASENAME%.*} # Set some variables @@ -68,7 +69,9 @@ if [ "$USE_SERVICE_ACCOUNT" = true ]; then SA_ACCOUNT="$SA_NAME@$PROJECT_ID.iam.gserviceaccount.com" # Redact the token file contents - echo "$(cat $TOKEN_FILE)" | buildkite-agent redactor add + # see official docs https://buildkite.com/docs/agent/v3/cli-redactor + # shellcheck disable=SC2005 + echo "$(cat "$TOKEN_FILE")" | buildkite-agent redactor add # shellcheck disable=SC1091 . "$CWD/../libs/gcloud" diff --git a/hooks/pre-exit b/hooks/pre-exit index 0de4686..b20bd59 100755 --- a/hooks/pre-exit +++ b/hooks/pre-exit @@ -7,10 +7,10 @@ SLACK_CHANNEL="${BUILDKITE_PLUGIN_OBLT_GOOGLE_AUTH_SLACK_CHANNEL:-"#observablt-b if [[ -n ${GOOGLE_APPLICATION_SERVICE_ACCOUNT:-} ]]; then echo "~~~ :gcloud: Cleaning up Service Account ${GOOGLE_APPLICATION_SERVICE_ACCOUNT}" - TOKEN_FILE=$(cat $GOOGLE_APPLICATION_CREDENTIALS | jq -r '.credentials_source.file') + TOKEN_FILE=$(cat "$GOOGLE_APPLICATION_CREDENTIALS" | jq -r '.credentials_source.file') curl -X DELETE \ - -H "Authorization: Bearer $(cat $TOKEN_FILE)" \ + -H "Authorization: Bearer $(cat "$TOKEN_FILE")" \ "https://iam.googleapis.com/v1/projects/$GOOGLE_CLOUD_PROJECT/serviceAccounts/$GOOGLE_APPLICATION_SERVICE_ACCOUNT" || error=true if [[ -n ${error:-} ]]; then @@ -29,7 +29,7 @@ EOM buildkite-agent annotate "$MESSAGE" --style "warning" --context "gcloud-auth-service-account-cleanup-failure" - send_message $SLACK_CHANNEL "$MESSAGE" + send_message "$SLACK_CHANNEL" "$MESSAGE" else echo "~~~ :gcloud: Service Account ${GOOGLE_APPLICATION_SERVICE_ACCOUNT} cleaned up" fi diff --git a/libs/gcloud b/libs/gcloud index 0f2543f..80cfd5b 100644 --- a/libs/gcloud +++ b/libs/gcloud @@ -7,7 +7,7 @@ set -euo pipefail LABEL_ORG="obs" LABEL_DIVISION="engineering" LABEL_TEAM="eng-productivity" -LABEL_PROJECT=${BUILDKITE_REPO} +LABEL_PROJECT="${BUILDKITE_REPO}" LABEL_EPHEMERAL="true" # Create the service account @@ -36,7 +36,7 @@ function create_service_account() { JSON curl -X POST \ - -H "Authorization: Bearer $(cat $TOKEN_FILE)" \ + -H "Authorization: Bearer $(cat "$TOKEN_FILE")" \ -H "Content-Type: application/json; charset=utf-8" \ -d @$REQUEST_FILE \ "https://iam.googleapis.com/v1/projects/$PROJECT_ID/serviceAccounts" @@ -69,7 +69,7 @@ function set_iam_policy() { JSON curl -X POST \ - -H "Authorization: Bearer $(cat $TOKEN_FILE)" \ + -H "Authorization: Bearer $(cat "$TOKEN_FILE")" \ -H "Content-Type: application/json; charset=utf-8" \ -d @$REQUEST_FILE \ "https://cloudresourcemanager.googleapis.com/v1/projects/$PROJECT_ID:setIamPolicy" @@ -88,7 +88,7 @@ function create_service_account_key() { local -r PROJECT_ID="$3" local -r SA_KEY_FILE="$4" curl -X POST \ - -H "Authorization: Bearer $(cat $TOKEN_FILE)" \ + -H "Authorization: Bearer $(cat "$TOKEN_FILE")" \ -H "Content-Type: application/json; charset=utf-8" \ "https://iam.googleapis.com/v1/projects/$PROJECT_ID/serviceAccounts/$SA_ACCOUNT/keys" > "$SA_KEY_FILE" } \ No newline at end of file