[9.1] (backport #18375) Use logstash-versions.yml for observabilitySRE stack versions (#18401)

* Use logstash-versions.yml for observabilitySRE stack versions (#18375)

* Use logstash-versions.yml for observabilitySRE stack versions

Previously the version of logstash checked in to the logstash repo was used to
determine stack version for smoke and acceptance testing of the observabilitySRE
artifact. Given the ES/Filebeat containers are subject to a unified release the
delay between a logstash version bump and those artifacts being available was
causing failures when the smoke/acceptance tests were run. In plugin testing the
way we deal with this is to ensure the logstash-versions.yml file is updated
when stack components are ready. This commit introduces the use of that pattern
to the observabilitySRE artifact testing. The important step will be to set the
STACK_VERSION to correspond to the desired stack version. For example the 9.2
logstash branch should look up `9.current`.

* track stream in versions.yml

* Finalize name/location for versions.yml mapping

* better comment

(cherry picked from commit d9fda37)

# Conflicts:
#	ci/observabilitySREacceptance_tests.sh
#	ci/observabilitySREsmoke_tests.sh

* Update versions.yml

---------

Co-authored-by: Cas Donoghue <cas.donoghue@gmail.com>

Author: mergify[bot]

.ci/updatecli/bump-logstash-version.yml

@@ -56,6 +56,16 @@ targets:
       key: $.logstash-core
       value: '{{ requiredEnv "LOGSTASH_VERSION" }}'
 
+  update_release_track:
+      name: Update logstash-release-track in versions.yml
+      kind: yaml
+      disablesourceinput: true
+      scmid: default
+      spec:
+        file: versions.yml
+        key: $.logstash-release-track
+        value: '{{ requiredEnv "LOGSTASH_RELEASE_TRACK" }}'
+
   update_gemfile_lock_dependency:
     name: Update logstash-core dependency in lockfile
     kind: file

.github/workflows/bump-logstash.yml

@@ -11,7 +11,10 @@ on:
         description: 'Logstash branch (example: 9.1)'
         required: true
         type: string
-
+      logstash_release_track:
+        description: 'Release track (example: 9.current, 9.previous, main)'
+        required: true
+        type: string
 jobs:
   bump:
     permissions:
@@ -29,3 +32,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           LOGSTASH_BRANCH: "${{ github.event.inputs.logstash_branch }}"
           LOGSTASH_VERSION: "${{ github.event.inputs.logstash_version }}"
+          LOGSTASH_RELEASE_TRACK: "${{ github.event.inputs.logstash_release_track }}"

ci/observabilitySREacceptance_tests.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+# Look up corresponding LOGSTASH_RELEASE_TRACK from versions.yml
+LOGSTASH_RELEASE_TRACK=$(ruby -ryaml -e "puts YAML.load_file('versions.yml')['logstash-release-track']")
+# Use logstash stream to find the corresponding stack verstion from logstash-versions.yml
+STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_RELEASE_TRACK}']")
+export OBSERVABILITY_SRE_IMAGE_VERSION="${OBSERVABILITY_SRE_IMAGE_VERSION:-$STACK_VERSION}"
+export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$STACK_VERSION}"
+export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$STACK_VERSION}"
+
+./gradlew observabilitySREacceptanceTests --stacktrace

ci/observabilitySREsmoke_tests.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)"
+# Look up corresponding LOGSTASH_RELEASE_TRACK from versions.yml
+LOGSTASH_RELEASE_TRACK=$(ruby -ryaml -e "puts YAML.load_file('versions.yml')['logstash-release-track']")
+# Use logstash stream to find the corresponding stack verstion from logstash-versions.yml
+STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_RELEASE_TRACK}']")
+export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$STACK_VERSION}"
+export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$STACK_VERSION}"
+
+./gradlew --stacktrace artifactDockerObservabilitySRE -PfedrampHighMode=true
+
+docker run docker.elastic.co/logstash/logstash-observability-sre:${QUALIFIED_VERSION} \
+  logstash -e 'input { generator { count => 3 } } output { stdout { codec => rubydebug } }'
+
+docker tag docker.elastic.co/logstash/logstash-observability-sre:${QUALIFIED_VERSION} pr-built-observability-sre-image
+
+./gradlew observabilitySREsmokeTests --stacktrace

versions.yml

@@ -3,6 +3,9 @@
 logstash: 9.1.7
 logstash-core: 9.1.7
 logstash-core-plugin-api: 2.1.16
+# Logstash release track corresponding to keys in https://github.com/logstash-plugins/.ci/blob/1.x/logstash-versions.yml
+# For example 9.current, 9.previous, etc
+logstash-release-track: 9.previous
 
 bundled_jdk:
   # for AdoptOpenJDK/OpenJDK jdk-14.0.1+7.1, the revision is 14.0.1 while the build is 7.1