From d0909ee931914e67d8e14b836d927ec83a89d29c Mon Sep 17 00:00:00 2001 From: Chinedu Uzoka Date: Thu, 28 May 2020 12:23:07 +0100 Subject: [PATCH 1/8] remove: SAAS-281 - remove third party pipeline --- ci/files/datadog_addon.yml | 33 ----- ci/files/manifest.yml | 133 -------------------- ci/files/use_old_stemcell.yml | 14 --- ci/pipeline.yml | 194 +++++++++++------------------- ci/tasks/create_final_release.sh | 13 -- ci/tasks/create_final_release.yml | 18 --- ci/tasks/create_release.sh | 11 -- ci/tasks/create_release.yml | 17 --- ci/tasks/system_test.sh | 32 ----- ci/tasks/system_test.yml | 15 --- 10 files changed, 71 insertions(+), 409 deletions(-) delete mode 100644 ci/files/datadog_addon.yml delete mode 100644 ci/files/manifest.yml delete mode 100644 ci/files/use_old_stemcell.yml delete mode 100755 ci/tasks/create_final_release.sh delete mode 100644 ci/tasks/create_final_release.yml delete mode 100755 ci/tasks/create_release.sh delete mode 100644 ci/tasks/create_release.yml delete mode 100755 ci/tasks/system_test.sh delete mode 100644 ci/tasks/system_test.yml diff --git a/ci/files/datadog_addon.yml b/ci/files/datadog_addon.yml deleted file mode 100644 index 865d9f54..00000000 --- a/ci/files/datadog_addon.yml +++ /dev/null @@ -1,33 +0,0 @@ -- type: replace - path: /releases?/- - value: - name: datadog-agent - version: latest - -- type: replace - path: /addons?/- - value: - name: dd-agent - jobs: - - name: dd-agent - release: datadog-agent - properties: - dd: - use_dogstatsd: yes - api_key: ((datadog_api_key)) - friendly_hostname: false - use_uuid_hostname: true - tags: - - "product:enterprise_archive" - - "customer:cs" - - "environment:deployment_testing" - - "cloud:vsphere" - - "country:USA" - - "region:nam" - - "version:latest" - -- type: replace - path: /tags? - value: - owner: datadog - email: support@datadoghq.com diff --git a/ci/files/manifest.yml b/ci/files/manifest.yml deleted file mode 100644 index 79aa6c57..00000000 --- a/ci/files/manifest.yml +++ /dev/null @@ -1,133 +0,0 @@ -instance_groups: -- azs: - - z1 - - z2 - - z3 - env: - persistent_disk_fs: xfs - instances: 3 - env: - persistent_disk_fs: xfs - jobs: - - name: bpm - release: bpm - - name: cfgsvr - release: mongodb-services - properties: - port: 23758 - replication: - replica_set_name: mongo-config-shared - server_certificate: - certificate: ((mongo_ca.certificate)) - private_key: ((mongo_ca.private_key)) - name: mdbconf-shared - networks: - - name: default - release: mongodb-services - stemcell: ubuntu - vm_type: default -- azs: - - z1 - - z2 - - z3 - instances: 1 - jobs: - - name: bpm - release: bpm - - name: mongos - release: mongodb-services - properties: - server_certificate: - certificate: ((mongo_ca.certificate)) - private_key: ((mongo_ca.private_key)) - users: - - name: root - password: ((root_password)) - roles: - - role: root - db: admin - - name: smarsh - password: myP@ssword - roles: - - role: userAdminAnyDatabase - db: admin - name: mongos-shared - networks: - - name: default - stemcell: ubuntu - vm_type: default -- azs: - - z1 - - z2 - - z3 - env: - persistent_disk_fs: xfs - instances: 3 - jobs: - - name: bpm - release: bpm - - name: shardsvr - release: mongodb-services - properties: - server_certificate: - certificate: ((mongo_ca.certificate)) - private_key: ((mongo_ca.private_key)) - replication: - replica_set_name: mongo-data-shared - users: - - name: root - password: ((root_password)) - roles: - - role: root - db: admin - name: mdb-shared - networks: - - name: default - release: mongodb-services - stemcell: ubuntu - vm_type: default - -- name: acceptance_tests - lifecycle: errand - azs: [z1] - instances: 1 - vm_type: default - stemcell: ubuntu - networks: - - name: default - jobs: - - name: mongodb-acceptance-tests - release: mongodb-services - properties: - mongo_test_suite: [ readwrite, replicaset ] - user: - username: root - password: ((root_password)) -name: mongodb -releases: -- name: mongodb-services - version: latest -- name: "bpm" - version: "1.0.4" - url: "https://bosh.io/d/github.com/cloudfoundry-incubator/bpm-release?v=1.0.4" - sha1: "41df19697d6a69d2552bc2c132928157fa91abe0" -stemcells: -- alias: ubuntu - os: ubuntu-xenial - version: latest -update: - canaries: 1 - canary_watch_time: 30000-240000 - max_in_flight: 1 - serial: false - update_watch_time: 30000-240000 -variables: -- name: root_password - type: password -- name: mongo_ca - type: certificate - options: - is_ca: true - common_name: MongoDB CA -features: - use_dns_addresses: false diff --git a/ci/files/use_old_stemcell.yml b/ci/files/use_old_stemcell.yml deleted file mode 100644 index c1464129..00000000 --- a/ci/files/use_old_stemcell.yml +++ /dev/null @@ -1,14 +0,0 @@ -- type: replace - path: /stemcells/- - value: - alias: old_xenial - os: ubuntu-xenial - version: '621.50' - -- type: replace - path: /instance_groups/name=mdb-shared/stemcell - value: old_xenial - -- type: replace - path: /instance_groups/name=mdbconf-shared/stemcell - value: old_xenial diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 163098b9..0eb97cc4 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -1,134 +1,82 @@ +--- anchors: - blobstore_credentials: &blobstore_credentials - AWS_ACCESS_KEY_ID: ((smarsh_bosh_releases_bucket_access_key_id)) - AWS_SECRET_ACCESS_KEY: ((smarsh_bosh_releases_bucket_secret_access_key)) smarsh_releases_bucket: &smarsh_releases_bucket bucket: smarsh-bosh-releases - access_key_id: ((smarsh_bosh_releases_bucket_access_key_id)) - secret_access_key: ((smarsh_bosh_releases_bucket_secret_access_key)) + access_key_id: ((aws-access-key-id)) + secret_access_key: ((aws-secret-access-key)) + blobstore_credentials: &blobstore_credentials + AWS_ACCESS_KEY_ID: ((aws-access-key-id)) + AWS_SECRET_ACCESS_KEY: ((aws-secret-access-key)) + bosh-creds: &bosh-creds + jumpbox_key: ((jumpbox-key)) + ca_cert: ((ca-cert)) + BOSH_ENVIRONMENT: ((bosh-environment)) resources: -# Docker images -- name: pcf-ops - type: docker-image - source: - repository: engineerbetter/pcf-ops - tag: latest - -# Git -- name: mongodb_bosh_release_git - type: git - source: - ignore_paths: - - releases - - .final_builds - - README.md - branch: master - uri: https://github.com/Smarsh/mongodb-boshrelease.git + - name: boshcli2 + type: registry-image + source: + repository: afamuzoka/boshcli2 + username: ((docker.username)) + password: ((docker.password)) + tag: latest -- name: mongodb_bosh_release_version - type: semver - source: - driver: git - branch: version - file: version - uri: git@github.com:Smarsh/mongodb-boshrelease.git - private_key: ((mongodb_bosh_release_deploy_key)) + - name: version + type: semver + source: + driver: s3 + << : *smarsh_releases_bucket + region: eu-west-2 + key: sredataservices/((bosh-release-name))/version + initial_version: ((initial_version)) -# S3 -- name: mongodb_bosh_candidate_release_s3 - type: s3 - source: - << : *smarsh_releases_bucket - regexp: mongodb/candidates/mongodb-(.*).tgz + - name: git-repo + type: git + icon: github-circle + source: + uri: ((url)) + branch: SAAS-281 + + - name: s3-final-release + type: s3 + source: + << : *smarsh_releases_bucket + regexp: sredataservices/release/((bosh-release-name))-(.*).tgz -- name: mongodb_bosh_release_s3 - type: s3 - source: - << : *smarsh_releases_bucket - regexp: mongodb/mongodb-(.*).tgz - -# Stemcells -- name: old_xenial_stemcell - type: bosh-io-stemcell - source: - name: bosh-warden-boshlite-ubuntu-xenial-go_agent - version_family: '621.50' - -- name: xenial_stemcell - type: bosh-io-stemcell - source: - name: bosh-warden-boshlite-ubuntu-xenial-go_agent jobs: -- name: create_candidate_release - serial: true - plan: - - get: mongodb_bosh_release_version - params: - bump: patch - pre: rc - - get: mongodb_bosh_release_git - trigger: true - - task: create_release - file: mongodb_bosh_release_git/ci/tasks/create_release.yml - params: - <<: *blobstore_credentials - - put: mongodb_bosh_candidate_release_s3 - params: - file: release_tarball/mongodb-*.tgz - - put: mongodb_bosh_release_version - params: - file: mongodb_bosh_release_version/version - -- name: system_test_candidate_release - serial: true - serial_groups: - - mongodb_release - plan: - - get: mongodb_bosh_candidate_release_s3 - passed: - - create_candidate_release - - get: mongodb_bosh_release_git - trigger: true - passed: - - create_candidate_release - - get: mongodb_bosh_release_version - passed: - - create_candidate_release - - get: xenial_stemcell - - get: old_xenial_stemcell - - task: system_tests - file: mongodb_bosh_release_git/ci/tasks/system_test.yml - privileged: true - timeout: 90m -- name: create_final_release - serial: true - serial_groups: - - mongodb_release - plan: - - get: pcf-ops - - get: mongodb_bosh_candidate_release_s3 - trigger: true - passed: - - system_test_candidate_release - - get: mongodb_bosh_release_git - passed: - - system_test_candidate_release - - get: mongodb_bosh_release_version - passed: - - system_test_candidate_release - params: - bump: final - - task: create_final_release - file: mongodb_bosh_release_git/ci/tasks/create_final_release.yml - params: - <<: *blobstore_credentials - privileged: true - - put: mongodb_bosh_release_s3 - params: - file: release_tarball/mongodb-*.tgz - - put: mongodb_bosh_release_version - params: - file: mongodb_bosh_release_version/version + - name: cut-release + public: true + serial: true + plan: + - in_parallel: + - get: git-repo + trigger: true + version: latest + - get: boshcli2 + - get: version + trigger: false + params: + bump: patch + - task: add-blobs + file: git-repo/ci/tasks/add-blobs.yml + image: boshcli2 + params: + << : *blobstore_credentials + << : *bosh-creds + BLOBSTORE: smarsh-bosh-release-blobs + - task: create-final-release + file: git-repo/ci/tasks/create-final-release.yml + image: boshcli2 + params: + << : *blobstore_credentials + << : *bosh-creds + BLOBSTORE: smarsh-bosh-release-blobs + - put: s3-final-release + params: + file: release-tarball/((bosh-release-name))-*.tgz + - put: version + params: {file: version/number} + + diff --git a/ci/tasks/create_final_release.sh b/ci/tasks/create_final_release.sh deleted file mode 100755 index 633c0544..00000000 --- a/ci/tasks/create_final_release.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -eux - -version="$(cat mongodb_bosh_release_version/version)" -cd mongodb_bosh_release_git/ -RELEASE_YML=$PWD/releases/mongodb-services/mongodb-services-${version}.yml - -echo "creating final release" -bosh finalize-release ../mongodb_bosh_candidate_release_s3/mongodb-*.tgz \ - --version "$version" \ - -bosh create-release "${RELEASE_YML}" \ - --tarball "../release_tarball/mongodb-$version.tgz" \ No newline at end of file diff --git a/ci/tasks/create_final_release.yml b/ci/tasks/create_final_release.yml deleted file mode 100644 index 7177e90a..00000000 --- a/ci/tasks/create_final_release.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: - repository: engineerbetter/pcf-ops - tag: latest - -inputs: -- name: mongodb_bosh_candidate_release_s3 -- name: mongodb_bosh_release_git -- name: mongodb_bosh_release_version - -outputs: -- name: release_tarball - -run: - path: mongodb_bosh_release_git/ci/tasks/create_final_release.sh diff --git a/ci/tasks/create_release.sh b/ci/tasks/create_release.sh deleted file mode 100755 index e665b610..00000000 --- a/ci/tasks/create_release.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -eux - -version="$(cat mongodb_bosh_release_version/version)" -cd mongodb_bosh_release_git - -# Create candidate release -bosh create-release --force \ - --version "$version" \ - --tarball "../release_tarball/mongodb-$version.tgz" diff --git a/ci/tasks/create_release.yml b/ci/tasks/create_release.yml deleted file mode 100644 index 11c95e1a..00000000 --- a/ci/tasks/create_release.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -platform: linux -image_resource: - type: docker-image - source: - repository: engineerbetter/pcf-ops - tag: latest - -inputs: -- name: mongodb_bosh_release_git -- name: mongodb_bosh_release_version - -outputs: -- name: release_tarball - -run: - path: mongodb_bosh_release_git/ci/tasks/create_release.sh diff --git a/ci/tasks/system_test.sh b/ci/tasks/system_test.sh deleted file mode 100755 index cd08de82..00000000 --- a/ci/tasks/system_test.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -set -eux - -export BOSH_NON_INTERACTIVE=true -export BOSH_DEPLOYMENT=mongodb - -start-bosh \ - -o /usr/local/bosh-deployment/uaa.yml \ - -o /usr/local/bosh-deployment/credhub.yml -source /tmp/local-bosh/director/env - -# Upload releases and stemcell -bosh upload-stemcell xenial_stemcell/*.tgz -bosh upload-release mongodb_bosh_candidate_release_s3/*.tgz - -# Deploy and run acceptance test errand -bosh deploy mongodb_bosh_release_git/ci/files/manifest.yml -bosh run-errand acceptance_tests --keep-alive - -# Recreate all vms in the deployment and run acceptance test errand -bosh recreate -bosh run-errand acceptance_tests --keep-alive - -# Restart processes on instances and run acceptance test errand -bosh restart -bosh run-errand acceptance_tests --keep-alive - -bosh upload-stemcell old_xenial_stemcell/*.tgz -bosh deploy mongodb_bosh_release_git/ci/files/manifest.yml \ - -o mongodb_bosh_release_git/ci/files/use_old_stemcell.yml -bosh run-errand acceptance_tests --keep-alive diff --git a/ci/tasks/system_test.yml b/ci/tasks/system_test.yml deleted file mode 100644 index 1b5621ea..00000000 --- a/ci/tasks/system_test.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -platform: linux -image_resource: - type: registry-image - source: - repository: bosh/main-bosh-docker - -inputs: -- name: mongodb_bosh_release_git -- name: mongodb_bosh_candidate_release_s3 -- name: old_xenial_stemcell -- name: xenial_stemcell - -run: - path: mongodb_bosh_release_git/ci/tasks/system_test.sh From 3168e2ad42c8d64a447710a4c7dea92b178500e6 Mon Sep 17 00:00:00 2001 From: Chinedu Uzoka Date: Thu, 28 May 2020 12:26:53 +0100 Subject: [PATCH 2/8] change: new internal pipeline --- ci/VERSIONS | 1 + ci/dl-packages-src | 5 + ci/tasks/add-blobs.sh | 173 ++++++++++++++++++++++++++++++ ci/tasks/add-blobs.yml | 11 ++ ci/tasks/create-final-release.sh | 89 +++++++++++++++ ci/tasks/create-final-release.yml | 17 +++ ci/tasks/create-release.sh | 95 ++++++++++++++++ ci/tasks/create-release.yml | 8 ++ config/blobs.yml | 7 +- 9 files changed, 402 insertions(+), 4 deletions(-) create mode 100644 ci/VERSIONS create mode 100644 ci/dl-packages-src create mode 100755 ci/tasks/add-blobs.sh create mode 100644 ci/tasks/add-blobs.yml create mode 100755 ci/tasks/create-final-release.sh create mode 100644 ci/tasks/create-final-release.yml create mode 100755 ci/tasks/create-release.sh create mode 100644 ci/tasks/create-release.yml diff --git a/ci/VERSIONS b/ci/VERSIONS new file mode 100644 index 00000000..79c524b3 --- /dev/null +++ b/ci/VERSIONS @@ -0,0 +1 @@ +MONGODB_VERSION=3.6.18 diff --git a/ci/dl-packages-src b/ci/dl-packages-src new file mode 100644 index 00000000..7b51ccb2 --- /dev/null +++ b/ci/dl-packages-src @@ -0,0 +1,5 @@ +# the file creates an associate array that is iterated over to do two things: +# 1. downloads the vendor packages using the value part of the associative array +# 2. creates a new config/blobs.yml file using the key part of the associative array +declare -A downloads +downloads[mongodb/mongodb-linux-x86_64-${MONGODB_VERSION}.tar.gz]="https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-${MONGODB_VERSION}.tgz" diff --git a/ci/tasks/add-blobs.sh b/ci/tasks/add-blobs.sh new file mode 100755 index 00000000..4c64764e --- /dev/null +++ b/ci/tasks/add-blobs.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $(echo $TERM | grep -v xterm) ]]; then + export TERM=xterm +fi + +SHELL=/bin/bash +ROOT_DIR=$(pwd) +OUTPUT_DIR=add-blobs +SOURCE_DL_DIR=.downloads +BOSH_RELEASE_VERSION_FILE=../version/number +SOURCE_VERSION_FILE="$(pwd)/ci/VERSIONS" +RELEASE_NAME=$(bosh int config/final.yml --path /final_name) +PRERELEASE_REPO=../git-prerelease-repo +RUN_PIPELINE=0 # if script is running locally then 0 if in consourse pipeline then 1 + +BOLD=$(tput bold) +RED=$(tput setaf 1) +GREEN=$(tput setaf 2) +YELLOW=$(tput setaf 3) +MAGENTA=$(tput setaf 5) +RESET=$(tput sgr0) + + +logerror() { + echo + echo 1>&2 "###" + echo 1>&2 "###" + printf "### ${BOLD}${RED}${1}${RESET}\n" 1>&2 + echo 1>&2 "###" + echo 1>&2 +} + +loginfo() { + echo + echo "###" + echo "###" + printf "### ${BOLD}${GREEN}${1}${RESET}\n" + echo "###" + echo +} + +download() { + local file=${1} + local url=${2} + + loginfo "Downloading ${url} ..." + curl --fail -L -o $file $url +} + +addBlob() { + local path=${1} + local blobPath=${2} + + loginfo "Track blob ${blobPath} for inclusion in release" + bosh add-blob $path ${blobPath} +} + +main() { + + if [[ -f ${SOURCE_VERSION_FILE} ]] ; then + source ${SOURCE_VERSION_FILE} + if [[ -f ci/dl-packages-src ]] ; then + source ci/dl-packages-src + else + logerror "Missing local versioning file in ci/dl-packages-src" + exit 2 + fi + else + logerror "Missing source download file in ci/VERSIONS" + exit 2 + fi + + + if [[ -f ${BOSH_RELEASE_VERSION_FILE} ]] ; then + BOSH_RELEASE_VERSION=$(cat ${BOSH_RELEASE_VERSION_FILE}) + RUN_PIPELINE=1 + else + BOSH_RELEASE_VERSION='SNAPSHOT' + fi + + + [[ ! -d ${SOURCE_DL_DIR} ]] && mkdir ${SOURCE_DL_DIR} + + if [[ ${RUN_PIPELINE} -eq 1 ]] ; then + tarBallPath=${SOURCE_DL_DIR}/${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz + else + + tarBallPath=/tmp/bosh-release-$$/${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz + fi + + + for key in "${!downloads[@]}" + do + local file=${SOURCE_DL_DIR}/$(basename ${key}) + local blobPath=${key} + + download ${file} ${downloads[${key}]} + + done + + # remove blobs + > config/blobs.yml + + for key in "${!downloads[@]}" + do + local file=${SOURCE_DL_DIR}/$(basename ${key}) + local blobPath=${key} + + addBlob ${file} ${blobPath} + + done + + + if [[ ${RUN_PIPELINE} -eq 1 ]] ; then + + loginfo "Create release version ${BOSH_RELEASE_VERSION}" + + # fix - removing .final_builds folder is not necessary when running locally however when running in a pipeline + # which uses bosh version 6.2.1 bosh create-release --force fails + # that requires this hidden directory to be renamed/removed + [[ -f ${BOSH_RELEASE_VERSION_FILE} ]] && rm -fr .final_builds + + bosh create-release --force --name ${RELEASE_NAME} --version=${BOSH_RELEASE_VERSION} --timestamp-version --tarball=${tarBallPath} + + BRANCH=$(git name-rev --name-only $(git rev-list HEAD --date-order --max-count 1)) + + + cat << EOF > config/final.yml +--- +blobstore: + provider: s3 + options: + bucket_name: ${BLOBSTORE} +final_name: ${RELEASE_NAME} +EOF + +## Create private.yml for BOSH to use our AWS keys + cat << EOF > config/private.yml +--- +blobstore: + provider: s3 + options: + credentials_source: env_or_profile +EOF + + loginfo "Upload blobs ${BOSH_RELEASE_VERSION}" + + bosh blobs + bosh -n upload-blobs + + if [[ -n "$(git status --porcelain)" ]]; then + + git config --global user.email "CI@localhost" + git config --global user.name "CI Bot " + + git checkout ${BRANCH} + git status + git add config .final_builds || true + git update-index --assume-unchanged config/final.yml + git status + git commit -m "Adding blobs to blobs store ${BLOBSTORE} via concourse" + + git clone -b ${BRANCH} . ${PRERELEASE_REPO} + fi + fi + +} + +bosh reset-release +main diff --git a/ci/tasks/add-blobs.yml b/ci/tasks/add-blobs.yml new file mode 100644 index 00000000..b5687ce1 --- /dev/null +++ b/ci/tasks/add-blobs.yml @@ -0,0 +1,11 @@ +--- +platform: linux +inputs: +- name: git-repo +- name: version +outputs: +- name: add-blobs +- name: git-prerelease-repo +run: + path: ci/tasks/add-blobs.sh + dir: git-repo \ No newline at end of file diff --git a/ci/tasks/create-final-release.sh b/ci/tasks/create-final-release.sh new file mode 100755 index 00000000..cf32f357 --- /dev/null +++ b/ci/tasks/create-final-release.sh @@ -0,0 +1,89 @@ +#!/bin/bash +set -euo pipefail + +SHELL=/bin/bash +ROOT_DIR=$(pwd) + +if [[ $(echo $TERM | grep -v xterm) ]]; then + export TERM=xterm +fi + +GITHUB_REPO="git-repo" +BOSH_RELEASE_VERSION=$(cat ${ROOT_DIR}/version/version) +BOSH_RELEASE_VERSION_FILE=../version/number +RELEASE_NAME=$(bosh int ${GITHUB_REPO}/config/final.yml --path /final_name) +PRERELEASE_REPO=./git-prerelease-repo +RUN_PIPELINE=0 # if script is running locally then 0 if in consourse pipeline then 1 + +BOLD=$(tput bold) +RED=$(tput setaf 1) +GREEN=$(tput setaf 2) +YELLOW=$(tput setaf 3) +MAGENTA=$(tput setaf 5) +RESET=$(tput sgr0) + +loginfo() { + echo + echo "###" + echo "###" + printf "### ${BOLD}${GREEN}${1}${RESET}\n" + echo "###" + echo +} + +loginfo "Configuring files, keys, certs and directories" + +set +x + +echo "$jumpbox_key" | jq -r .private_key > jumpbox.key +echo "$ca_cert" | jq -r .certificate > ca_cert.crt + +set -x + +loginfo "Configuring BOSH environment" + +bosh alias-env $BOSH_ENVIRONMENT -e $BOSH_ENVIRONMENT --ca-cert ${ROOT_DIR}/ca_cert.crt +export BOSH_ALL_PROXY=ssh+socks5://jumpbox@${BOSH_ENVIRONMENT}:22?private-key=${ROOT_DIR}/jumpbox.key + +## change directories into the master branch of the repository that is cloned, not the branched clone +pushd $PRERELEASE_REPO + +git config --global user.email "ci@localhost" +git config --global user.name "CI Bot" + +loginfo "Cutting a final release" + +## Download all of the blobs and packages from the boshrelease bucket that is read only + + + cat << EOF > config/final.yml +--- +blobstore: + provider: s3 + options: + bucket_name: ${BLOBSTORE} +name: ${RELEASE_NAME} +EOF + +## Create private.yml for BOSH to use our AWS keys + cat << EOF > config/private.yml +--- +blobstore: + provider: s3 + options: + credentials_source: env_or_profile +EOF + +git update-index --assume-unchanged config/final.yml + +[[ -d .final_builds ]] && rm -fr .final_builds + +git status + +bosh create-release --final --version=${BOSH_RELEASE_VERSION} --tarball "../release-tarball/${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz" + +git status +git add config .final_builds releases || true +git commit -am "Final release stage change, ${BOSH_RELEASE_VERSION} via concourse" + +popd \ No newline at end of file diff --git a/ci/tasks/create-final-release.yml b/ci/tasks/create-final-release.yml new file mode 100644 index 00000000..833a838c --- /dev/null +++ b/ci/tasks/create-final-release.yml @@ -0,0 +1,17 @@ +--- +platform: linux +inputs: +- name: git-repo +- name: git-prerelease-repo +- name: add-blobs +- name: version +outputs: +- name: release-tarball +- name: git-release-repo + +run: + path: git-repo/ci/tasks/create-final-release.sh + +params: + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: diff --git a/ci/tasks/create-release.sh b/ci/tasks/create-release.sh new file mode 100755 index 00000000..8b813929 --- /dev/null +++ b/ci/tasks/create-release.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $(echo $TERM | grep -v xterm) ]]; then + export TERM=xterm +fi + +SHELL=/bin/bash +ROOT_DIR=$(pwd) +USE_PIPELINE=0 + +ROOT_PATH=${PWD} +DEV_RELEASE_DIR=${ROOT_PATH}/s3-dev-release/kafka-dev-release.tgz +BOSH_RELEASE_VERSION=$(cat ../version/version) +INPUT_DIR=git-bosh-release +OUTPUT_DIR=create-release +PROMOTED_REPO=${INPUT_DIR}-pr + +BOLD=$(tput bold) +RED=$(tput setaf 1) +GREEN=$(tput setaf 2) +YELLOW=$(tput setaf 3) +MAGENTA=$(tput setaf 5) +RESET=$(tput sgr0) + + +[[ ${USE_PIPELINE} -ne 0 ]] && pushd ./${INPUT_DIR} + +VERSION_FILE="${ROOT_DIR}/VERSIONS" + +KAFKA_VERSION=${KAFKA_VERSION:?required} + + +if [[ ! -d ../${DEV_RELEASE_DIR} ]] ; then + tarBallFile=./kafka-dev-release.tgz +else + tarBallFile=../${DEV_RELEASE_DIR}/kafka-dev-release.tgz +fi + + +printf "\n${BOLD}${GREEN}Create final release${RESET}\n" + +git config --global user.email "ci@localhost" +git config --global user.name "CI Bot" + +popd + +pushd ./${INPUT_DIR} +tag_name="v${VERSION}" + +tag_annotation="Final release ${VERSION} tagged via concourse" + +git tag -a "${tag_name}" -m "${tag_annotation}" +popd + +git clone ./${INPUT_DIR} $PROMOTED_REPO + +pushd $PROMOTED_REPO +git status + +git checkout master +git status + +cat > config/final.yml < config/private.yml < Date: Thu, 28 May 2020 13:10:20 +0100 Subject: [PATCH 3/8] refactor: parameterize branch name --- ci/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 0eb97cc4..edb98899 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -34,8 +34,8 @@ resources: type: git icon: github-circle source: - uri: ((url)) - branch: SAAS-281 + uri: ((git.url)) + branch: ((git.branch)) - name: s3-final-release type: s3 From 9a2030fc98c923b9fb091e4af0937412ef0cb2d6 Mon Sep 17 00:00:00 2001 From: Chinedu Uzoka Date: Thu, 28 May 2020 14:45:19 +0100 Subject: [PATCH 4/8] refactor: remove redundant code --- ci/tasks/add-blobs.sh | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/ci/tasks/add-blobs.sh b/ci/tasks/add-blobs.sh index 4c64764e..e6ec6caa 100755 --- a/ci/tasks/add-blobs.sh +++ b/ci/tasks/add-blobs.sh @@ -74,6 +74,8 @@ main() { fi + # Are we running inside a concourse pipeline + # i wonder if theres something else i can use if [[ -f ${BOSH_RELEASE_VERSION_FILE} ]] ; then BOSH_RELEASE_VERSION=$(cat ${BOSH_RELEASE_VERSION_FILE}) RUN_PIPELINE=1 @@ -81,16 +83,7 @@ main() { BOSH_RELEASE_VERSION='SNAPSHOT' fi - [[ ! -d ${SOURCE_DL_DIR} ]] && mkdir ${SOURCE_DL_DIR} - - if [[ ${RUN_PIPELINE} -eq 1 ]] ; then - tarBallPath=${SOURCE_DL_DIR}/${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz - else - - tarBallPath=/tmp/bosh-release-$$/${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz - fi - for key in "${!downloads[@]}" do @@ -122,11 +115,10 @@ main() { # which uses bosh version 6.2.1 bosh create-release --force fails # that requires this hidden directory to be renamed/removed [[ -f ${BOSH_RELEASE_VERSION_FILE} ]] && rm -fr .final_builds - + tarBallPath=${OUTPUT_DIR}/${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz bosh create-release --force --name ${RELEASE_NAME} --version=${BOSH_RELEASE_VERSION} --timestamp-version --tarball=${tarBallPath} BRANCH=$(git name-rev --name-only $(git rev-list HEAD --date-order --max-count 1)) - cat << EOF > config/final.yml --- From a88d47eeb1f0ce6312548fcf4efc5f1afbadd738 Mon Sep 17 00:00:00 2001 From: Chinedu Uzoka Date: Thu, 28 May 2020 17:19:19 +0100 Subject: [PATCH 5/8] feature: create final bosh release --- ci/tasks/create-release.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/tasks/create-release.sh b/ci/tasks/create-release.sh index 8b813929..2adb843f 100755 --- a/ci/tasks/create-release.sh +++ b/ci/tasks/create-release.sh @@ -80,13 +80,13 @@ options: credentials_source: env_or_profile EOF -echo bosh finalize-release --version $VERSION $DEV_RELEASE_PATH -echo bosh create-release --final --name kafka --version=${VERSION} --timestamp-version --tarball ${tarBallFile} +bosh finalize-release --version $VERSION $DEV_RELEASE_PATH +bosh create-release --final --name kafka --version=${VERSION} --timestamp-version --tarball ${tarBallFile} -git add -A git status +git add -A -git commit -m "Adding final release $VERSION via concourse" +[[ -n "$(git status --porcelain > /dev/null 2>&1)" ]] && git commit -m "Adding final release $VERSION via concourse" popd From 19033f6dc0c5e8e5dd7e88ef27a2e5efaefd66a0 Mon Sep 17 00:00:00 2001 From: Chinedu Uzoka Date: Fri, 29 May 2020 09:29:39 +0100 Subject: [PATCH 6/8] feature: add golang vendor dependency support --- ci/pipeline.yml | 10 ++++++++++ ci/tasks/add-blobs.sh | 12 +++++++++--- ci/tasks/add-blobs.yml | 1 + 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index edb98899..8c86e38f 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -37,6 +37,13 @@ resources: uri: ((git.url)) branch: ((git.branch)) + - name: git-golang + type: git + icon: github-circle + source: + uri: https://github.com/bosh-packages/golang-release.git + branch: v0.9.0 + - name: s3-final-release type: s3 source: @@ -54,6 +61,9 @@ jobs: - get: git-repo trigger: true version: latest + - get: git-golang + trigger: false + input_mapping: {golang-release: git-golang} - get: boshcli2 - get: version trigger: false diff --git a/ci/tasks/add-blobs.sh b/ci/tasks/add-blobs.sh index e6ec6caa..fd4a4f5f 100755 --- a/ci/tasks/add-blobs.sh +++ b/ci/tasks/add-blobs.sh @@ -106,6 +106,7 @@ main() { done + rm -fr ${SOURCE_DL_DIR} if [[ ${RUN_PIPELINE} -eq 1 ]] ; then @@ -138,6 +139,10 @@ blobstore: credentials_source: env_or_profile EOF + loginfo "Download GoLang vendor package" + + bosh vendor-package golang-1.9-linux ../golang-release + loginfo "Upload blobs ${BOSH_RELEASE_VERSION}" bosh blobs @@ -150,11 +155,12 @@ EOF git checkout ${BRANCH} git status - git add config .final_builds || true git update-index --assume-unchanged config/final.yml + git add -A git status - git commit -m "Adding blobs to blobs store ${BLOBSTORE} via concourse" - + if [[ -n "$(git status --porcelain)" ]]; then + git commit -m "Adding blobs to blobs store ${BLOBSTORE} via concourse" + fi git clone -b ${BRANCH} . ${PRERELEASE_REPO} fi fi diff --git a/ci/tasks/add-blobs.yml b/ci/tasks/add-blobs.yml index b5687ce1..0d6c26f5 100644 --- a/ci/tasks/add-blobs.yml +++ b/ci/tasks/add-blobs.yml @@ -3,6 +3,7 @@ platform: linux inputs: - name: git-repo - name: version +- name: golang-release outputs: - name: add-blobs - name: git-prerelease-repo From 7063cc3fa543df6fef33a7895e868142e22ada94 Mon Sep 17 00:00:00 2001 From: Chinedu Uzoka Date: Fri, 29 May 2020 09:42:30 +0100 Subject: [PATCH 7/8] fix: goland vendor support run before dev release tarball creation --- ci/dl-packages-src | 2 +- ci/pipeline.yml | 19 +++++++++----- ci/tasks/add-blobs.sh | 43 ++++++++++++++++++------------- ci/tasks/add-blobs.yml | 2 +- ci/tasks/create-final-release.sh | 35 ++++++++++++++----------- ci/tasks/create-final-release.yml | 2 +- 6 files changed, 61 insertions(+), 42 deletions(-) diff --git a/ci/dl-packages-src b/ci/dl-packages-src index 7b51ccb2..edaf19f8 100644 --- a/ci/dl-packages-src +++ b/ci/dl-packages-src @@ -2,4 +2,4 @@ # 1. downloads the vendor packages using the value part of the associative array # 2. creates a new config/blobs.yml file using the key part of the associative array declare -A downloads -downloads[mongodb/mongodb-linux-x86_64-${MONGODB_VERSION}.tar.gz]="https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-${MONGODB_VERSION}.tgz" +downloads[mongodb/mongodb-linux-x86_64-${MONGODB_VERSION}.tar.gz]="https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-${MONGODB_VERSION}.tgz" \ No newline at end of file diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 8c86e38f..7e212663 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -36,13 +36,15 @@ resources: source: uri: ((git.url)) branch: ((git.branch)) - + private_key: ((git-private-key.private_key)) + - name: git-golang type: git icon: github-circle source: uri: https://github.com/bosh-packages/golang-release.git - branch: v0.9.0 + fetch_tags: true + branch: v0.9.0 - name: s3-final-release type: s3 @@ -63,7 +65,6 @@ jobs: version: latest - get: git-golang trigger: false - input_mapping: {golang-release: git-golang} - get: boshcli2 - get: version trigger: false @@ -76,6 +77,7 @@ jobs: << : *blobstore_credentials << : *bosh-creds BLOBSTORE: smarsh-bosh-release-blobs + input_mapping: {golang-release: git-golang} - task: create-final-release file: git-repo/ci/tasks/create-final-release.yml image: boshcli2 @@ -86,7 +88,12 @@ jobs: - put: s3-final-release params: file: release-tarball/((bosh-release-name))-*.tgz - - put: version - params: {file: version/number} - + # - put: git-repo + # params: + # rebase: true + # repository: finalized-release + # tag: version/tag-name + # annotate: version/annotate-msg + # - put: version + # params: {file: version/number} diff --git a/ci/tasks/add-blobs.sh b/ci/tasks/add-blobs.sh index fd4a4f5f..022cde3e 100755 --- a/ci/tasks/add-blobs.sh +++ b/ci/tasks/add-blobs.sh @@ -8,11 +8,12 @@ fi SHELL=/bin/bash ROOT_DIR=$(pwd) -OUTPUT_DIR=add-blobs +OUTPUT_DIR=../add-blob-release SOURCE_DL_DIR=.downloads BOSH_RELEASE_VERSION_FILE=../version/number SOURCE_VERSION_FILE="$(pwd)/ci/VERSIONS" RELEASE_NAME=$(bosh int config/final.yml --path /final_name) +BOSH_RELEASE_FILE=${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz PRERELEASE_REPO=../git-prerelease-repo RUN_PIPELINE=0 # if script is running locally then 0 if in consourse pipeline then 1 @@ -108,19 +109,32 @@ main() { rm -fr ${SOURCE_DL_DIR} + if [[ ${RUN_PIPELINE} -eq 1 ]] ; then - + loginfo "Create release version ${BOSH_RELEASE_VERSION}" # fix - removing .final_builds folder is not necessary when running locally however when running in a pipeline # which uses bosh version 6.2.1 bosh create-release --force fails # that requires this hidden directory to be renamed/removed - [[ -f ${BOSH_RELEASE_VERSION_FILE} ]] && rm -fr .final_builds - tarBallPath=${OUTPUT_DIR}/${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz - bosh create-release --force --name ${RELEASE_NAME} --version=${BOSH_RELEASE_VERSION} --timestamp-version --tarball=${tarBallPath} + rm -fr .final_builds BRANCH=$(git name-rev --name-only $(git rev-list HEAD --date-order --max-count 1)) + git status + + git checkout ${BRANCH} + + if [[ -d ../golang-release ]]; then + loginfo "Download GoLang vendor package" + bosh vendor-package golang-1.9-linux ../golang-release + fi + + tarBallPath=${OUTPUT_DIR}/${BOSH_RELEASE_FILE} + + loginfo "Create release version ${BOSH_RELEASE_VERSION}" + bosh create-release --force --name ${RELEASE_NAME} --version=${BOSH_RELEASE_VERSION} --timestamp-version --tarball=${tarBallPath} + cat << EOF > config/final.yml --- blobstore: @@ -139,29 +153,22 @@ blobstore: credentials_source: env_or_profile EOF - loginfo "Download GoLang vendor package" - - bosh vendor-package golang-1.9-linux ../golang-release - - loginfo "Upload blobs ${BOSH_RELEASE_VERSION}" - - bosh blobs - bosh -n upload-blobs + loginfo "Upload blobs ${BOSH_RELEASE_VERSION}" + bosh blobs + bosh -n upload-blobs if [[ -n "$(git status --porcelain)" ]]; then git config --global user.email "CI@localhost" git config --global user.name "CI Bot " - git checkout ${BRANCH} git status git update-index --assume-unchanged config/final.yml git add -A git status - if [[ -n "$(git status --porcelain)" ]]; then - git commit -m "Adding blobs to blobs store ${BLOBSTORE} via concourse" - fi - git clone -b ${BRANCH} . ${PRERELEASE_REPO} + git commit -m "Adding blob, ${BOSH_RELEASE_FILE} to ${BLOBSTORE} via concourse" + [[ -d ${PRERELEASE_REPO} ]] && mkdir -p ${PRERELEASE_REPO} + cp -r . ${PRERELEASE_REPO} fi fi diff --git a/ci/tasks/add-blobs.yml b/ci/tasks/add-blobs.yml index 0d6c26f5..570d82f9 100644 --- a/ci/tasks/add-blobs.yml +++ b/ci/tasks/add-blobs.yml @@ -5,7 +5,7 @@ inputs: - name: version - name: golang-release outputs: -- name: add-blobs +- name: add-blob-release - name: git-prerelease-repo run: path: ci/tasks/add-blobs.sh diff --git a/ci/tasks/create-final-release.sh b/ci/tasks/create-final-release.sh index cf32f357..1b7563d4 100755 --- a/ci/tasks/create-final-release.sh +++ b/ci/tasks/create-final-release.sh @@ -12,6 +12,7 @@ GITHUB_REPO="git-repo" BOSH_RELEASE_VERSION=$(cat ${ROOT_DIR}/version/version) BOSH_RELEASE_VERSION_FILE=../version/number RELEASE_NAME=$(bosh int ${GITHUB_REPO}/config/final.yml --path /final_name) +BOSH_RELEASE_FILE=${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz PRERELEASE_REPO=./git-prerelease-repo RUN_PIPELINE=0 # if script is running locally then 0 if in consourse pipeline then 1 @@ -48,15 +49,15 @@ export BOSH_ALL_PROXY=ssh+socks5://jumpbox@${BOSH_ENVIRONMENT}:22?private-key=${ ## change directories into the master branch of the repository that is cloned, not the branched clone pushd $PRERELEASE_REPO -git config --global user.email "ci@localhost" -git config --global user.name "CI Bot" + git config --global user.email "ci@localhost" + git config --global user.name "CI Bot" -loginfo "Cutting a final release" + loginfo "Cutting a final release" -## Download all of the blobs and packages from the boshrelease bucket that is read only + ## Download all of the blobs and packages from the boshrelease bucket that is read only - - cat << EOF > config/final.yml + + cat << EOF > config/final.yml --- blobstore: provider: s3 @@ -65,8 +66,8 @@ blobstore: name: ${RELEASE_NAME} EOF -## Create private.yml for BOSH to use our AWS keys - cat << EOF > config/private.yml + ## Create private.yml for BOSH to use our AWS keys + cat << EOF > config/private.yml --- blobstore: provider: s3 @@ -74,16 +75,20 @@ blobstore: credentials_source: env_or_profile EOF -git update-index --assume-unchanged config/final.yml + git update-index --assume-unchanged config/final.yml -[[ -d .final_builds ]] && rm -fr .final_builds + # [[ -d .final_builds ]] && rm -fr .final_builds -git status + git status -bosh create-release --final --version=${BOSH_RELEASE_VERSION} --tarball "../release-tarball/${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz" + loginfo "Create final release" + bosh finalize-release --name=${RELEASE_NAME} --version=${BOSH_RELEASE_VERSION} ../add-blob-release/${BOSH_RELEASE_FILE} -git status -git add config .final_builds releases || true -git commit -am "Final release stage change, ${BOSH_RELEASE_VERSION} via concourse" + git status + git add config .final_builds releases || true + [[ -n "$(git status --porcelain)" ]] && git commit -am "Final release stage change, ${BOSH_RELEASE_VERSION} via concourse" + + loginfo "Create release final release tarball" + bosh create-release --tarball=../release-tarball/${BOSH_RELEASE_FILE} --final popd \ No newline at end of file diff --git a/ci/tasks/create-final-release.yml b/ci/tasks/create-final-release.yml index 833a838c..6ba0ec0b 100644 --- a/ci/tasks/create-final-release.yml +++ b/ci/tasks/create-final-release.yml @@ -3,7 +3,7 @@ platform: linux inputs: - name: git-repo - name: git-prerelease-repo -- name: add-blobs +- name: add-blob-release - name: version outputs: - name: release-tarball From a23ea5b4891d37813747246bdfb5ec97b6147bee Mon Sep 17 00:00:00 2001 From: Chinedu Uzoka Date: Tue, 2 Jun 2020 13:53:22 +0100 Subject: [PATCH 8/8] fix: put the bosh releae var in the wrong place --- ci/tasks/add-blobs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/tasks/add-blobs.sh b/ci/tasks/add-blobs.sh index 022cde3e..bdc8d8aa 100755 --- a/ci/tasks/add-blobs.sh +++ b/ci/tasks/add-blobs.sh @@ -13,7 +13,6 @@ SOURCE_DL_DIR=.downloads BOSH_RELEASE_VERSION_FILE=../version/number SOURCE_VERSION_FILE="$(pwd)/ci/VERSIONS" RELEASE_NAME=$(bosh int config/final.yml --path /final_name) -BOSH_RELEASE_FILE=${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz PRERELEASE_REPO=../git-prerelease-repo RUN_PIPELINE=0 # if script is running locally then 0 if in consourse pipeline then 1 @@ -84,6 +83,8 @@ main() { BOSH_RELEASE_VERSION='SNAPSHOT' fi + BOSH_RELEASE_FILE=${RELEASE_NAME}-${BOSH_RELEASE_VERSION}.tgz + [[ ! -d ${SOURCE_DL_DIR} ]] && mkdir ${SOURCE_DL_DIR} for key in "${!downloads[@]}"