diff --git a/.circleci/compile.sh b/.circleci/compile.sh deleted file mode 100644 index 32c672a..0000000 --- a/.circleci/compile.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -export SHELL=/bin/bash - -set -e - -echo "Building Docker logging plugin binary.." -make diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index ec4ecc4..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,33 +0,0 @@ -version: 2 -jobs: - build: - resource_class: large - machine: - image: ubuntu-1604:201903-01 - working_directory: ~/.go_workspace/src/repo - steps: - - checkout - - run: - name: Builder - command: | - bash .circleci/compile.sh - - - run: - name: Run unit tests - command: | - bash .circleci/unit_tests.sh - - - run: - name: Run functional tests partial log - command: | - bash .circleci/functional_tests_partial.sh - - - run: - name: Run functional tests config params - command: | - bash .circleci/functional_tests_config.sh - - - run: - name: Run functional tests malformed data - command: | - bash .circleci/functional_tests_malformed.sh diff --git a/.circleci/functional_tests_config.sh b/.circleci/functional_tests_config.sh deleted file mode 100755 index e05039d..0000000 --- a/.circleci/functional_tests_config.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -export SHELL=/bin/bash - -set -e - -echo "Running functional tests..." - -#sudo su - -# Start the plugin -sudo /home/circleci/.go_workspace/src/repo/splunk-logging-plugin/rootfs/bin/splunk-logging-plugin & -rm -rf /opt/circleci/.pyenv -echo "Creating virtual env to run functional tests..." -cd test -curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash -sudo apt-get update -sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev git -export PATH="~/.pyenv/bin:$PATH" -pyenv install 3.7.0 -pyenv global 3.7.0 - - -sudo pip install --upgrade pip -sudo -H pip install virtualenv -virtualenv --python=python3.5 venv -source venv/bin/activate -pip install -r requirements.txt -deactivate - -#Run functional tests from within virtualenv -tests=( "test_splunk_index_1" "test_splunk_index_2" "test_splunk_source_1" "test_splunk_source_2" "test_splunk_source_type_1" "test_splunk_source_type_2" "test_splunk_ca" "test_splunk_format_json" "test_splunk_format_inline" "test_splunk_format_raw" "test_splunk_verify_connection" "test_splunk_gzip_1" "test_splunk_gzip_2" "test_splunk_gzip_3" "test_splunk_gzip_4" "test_splunk_gzip_5" "test_splunk_tag" ) -for i in "${tests[@]}" -do - sudo venv/bin/python -m pytest --cache-clear \ - --splunkd-url https://$SPLUNK_HEC_HOST:8089 \ - --splunk-user admin \ - --splunk-password notchangeme \ - --splunk-hec-url https://$SPLUNK_HEC_HOST:8088 \ - --splunk-hec-token $SPLUNK_HEC_TOKEN \ - --docker-plugin-path /home/circleci/.go_workspace/src/repo/splunk-logging-plugin/rootfs/bin/splunk-logging-plugin \ - --fifo-path /home/circleci/.go_workspace/src/repo/pipe \ - -p no:warnings config_params/test_cofig_params.py::${tests[i]} -done diff --git a/.circleci/functional_tests_malformed.sh b/.circleci/functional_tests_malformed.sh deleted file mode 100755 index 5b30ac3..0000000 --- a/.circleci/functional_tests_malformed.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -export SHELL=/bin/bash - -set -e - -echo "Running functional tests..." - -#sudo su - -# Start the plugin -sudo /home/circleci/.go_workspace/src/repo/splunk-logging-plugin/rootfs/bin/splunk-logging-plugin & -rm -rf /opt/circleci/.pyenv -echo "Creating virtual env to run functional tests..." -cd test -curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash -sudo apt-get update -sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev git -export PATH="~/.pyenv/bin:$PATH" -pyenv install 3.7.0 -pyenv global 3.7.0 - - -sudo pip install --upgrade pip -sudo -H pip install virtualenv -virtualenv --python=python3.5 venv -source venv/bin/activate -pip install -r requirements.txt -deactivate - -#Run functional tests from within virtualenv -tests=( "test_malformed_empty_string_1" "test_malformed_empty_string_2" "test_malformed_empty_string_3" "test_malformed_empty_string_4" "test_malformed_empty_string_5" ) -for i in "${tests[@]}" -do - sudo venv/bin/python -m pytest --cache-clear \ - --splunkd-url https://$SPLUNK_HEC_HOST:8089 \ - --splunk-user admin \ - --splunk-password notchangeme \ - --splunk-hec-url https://$SPLUNK_HEC_HOST:8088 \ - --splunk-hec-token $SPLUNK_HEC_TOKEN \ - --docker-plugin-path /home/circleci/.go_workspace/src/repo/splunk-logging-plugin/rootfs/bin/splunk-logging-plugin \ - --fifo-path /home/circleci/.go_workspace/src/repo/pipe \ - -p no:warnings malformed_data/test_malformed_events.py::${tests[i]} -done diff --git a/.circleci/functional_tests_partial.sh b/.circleci/functional_tests_partial.sh deleted file mode 100755 index 45b4f66..0000000 --- a/.circleci/functional_tests_partial.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -export SHELL=/bin/bash - -set -e - -echo "Running functional tests..." - -#sudo su - -# Start the plugin -sudo /home/circleci/.go_workspace/src/repo/splunk-logging-plugin/rootfs/bin/splunk-logging-plugin & -rm -rf /opt/circleci/.pyenv -echo "Creating virtual env to run functional tests..." -cd test -curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash -sudo apt-get update -sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev git -export PATH="~/.pyenv/bin:$PATH" -pyenv install 3.7.0 -pyenv global 3.7.0 - - -sudo pip install --upgrade pip -sudo -H pip install virtualenv -virtualenv --python=python3.5 venv -source venv/bin/activate -pip install -r requirements.txt -deactivate - -#Run functional tests from within virtualenv -tests=( "test_partial_log_1" "test_partial_log_2" "test_partial_log_flush_timeout_1" "test_partial_log_flush_timeout_2" "test_partial_log_flush_size_limit" ) -for i in "${tests[@]}" -do - sudo venv/bin/python -m pytest --cache-clear \ - --splunkd-url https://$SPLUNK_HEC_HOST:8089 \ - --splunk-user admin \ - --splunk-password notchangeme \ - --splunk-hec-url https://$SPLUNK_HEC_HOST:8088 \ - --splunk-hec-token $SPLUNK_HEC_TOKEN \ - --docker-plugin-path /home/circleci/.go_workspace/src/repo/splunk-logging-plugin/rootfs/bin/splunk-logging-plugin \ - --fifo-path /home/circleci/.go_workspace/src/repo/pipe \ - -p no:warnings partial_log/test_partial_log.py::${tests[i]} -done diff --git a/.circleci/unit_tests.sh b/.circleci/unit_tests.sh deleted file mode 100644 index 6fa005c..0000000 --- a/.circleci/unit_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -export SHELL=/bin/bash - -set -e - -mkdir /home/circleci/.go_workspace/bin -curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh -dep ensure - -echo "Running Golang unit tests..." - -go test diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..e3bdf98 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,42 @@ +name: ci test +on: [push, workflow_dispatch] +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '>=1.19.0' + + - name: Install Python 3.8 + uses: actions/setup-python@v2.2.2 + with: + python-version: 3.8 + + - name: Update apt repositories + run: sudo apt update + + - name: Upgrade python packaging tools + run: python -m pip install --upgrade pip setuptools wheel + + - name: Compile + run: make + + - name: Run unit tests + run: make test + + - name: Setup dependencies + run: cd test && make setup-venv && make setup-splunk + + - name: Run functional tests partial log + run: cd test && make test-partial + + - name: Run functional tests config params + run: cd test && make test-config + + - name: Run functional tests malformed data + run: cd test && make test-malformed diff --git a/Dockerfile b/Dockerfile index 95a1f61..a27b808 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,9 @@ -FROM golang:1.9.2 +FROM golang:1.19.0 WORKDIR /go/src/github.com/splunk/splunk-logging-plugin/ COPY . /go/src/github.com/splunk/splunk-logging-plugin/ -# install dep -RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - -RUN cd /go/src/github.com/splunk/splunk-logging-plugin && dep ensure - RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /bin/splunk-logging-plugin . FROM alpine:3.7 diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index db02bc6..0000000 --- a/Gopkg.lock +++ /dev/null @@ -1,162 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - branch = "master" - name = "github.com/Azure/go-ansiterm" - packages = [ - ".", - "winterm" - ] - revision = "d6e3b3328b783f23731bc4d058875b0371ff8109" - -[[projects]] - name = "github.com/Microsoft/go-winio" - packages = ["."] - revision = "7da180ee92d8bd8bb8c37fc560e673e6557c392f" - version = "v0.4.7" - -[[projects]] - branch = "master" - name = "github.com/Nvveen/Gotty" - packages = ["."] - revision = "cd527374f1e5bff4938207604a14f2e38a9cf512" - -[[projects]] - name = "github.com/Sirupsen/logrus" - packages = ["."] - revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc" - version = "v1.0.5" - -[[projects]] - name = "github.com/coreos/go-systemd" - packages = ["activation"] - revision = "40e2722dffead74698ca12a750f64ef313ddce05" - version = "v16" - -[[projects]] - name = "github.com/docker/docker" - packages = [ - "api/types", - "api/types/backend", - "api/types/blkiodev", - "api/types/container", - "api/types/filters", - "api/types/mount", - "api/types/network", - "api/types/plugins/logdriver", - "api/types/registry", - "api/types/strslice", - "api/types/swarm", - "api/types/versions", - "daemon/logger", - "daemon/logger/jsonfilelog", - "daemon/logger/loggerutils", - "pkg/filenotify", - "pkg/ioutils", - "pkg/jsonlog", - "pkg/jsonmessage", - "pkg/longpath", - "pkg/plugingetter", - "pkg/plugins", - "pkg/plugins/transport", - "pkg/progress", - "pkg/pubsub", - "pkg/random", - "pkg/streamformatter", - "pkg/stringid", - "pkg/tailfile", - "pkg/templates", - "pkg/term", - "pkg/term/windows", - "pkg/urlutil" - ] - revision = "90d35abf7b3535c1c319c872900fbd76374e521c" - version = "v17.03.0-ce" - -[[projects]] - name = "github.com/docker/go-connections" - packages = [ - "nat", - "sockets", - "tlsconfig" - ] - revision = "3ede32e2033de7505e6500d6c868c2b9ed9f169d" - version = "v0.3.0" - -[[projects]] - branch = "master" - name = "github.com/docker/go-plugins-helpers" - packages = ["sdk"] - revision = "61cb8e2334204460162c8bd2417cd43cb71da66f" - -[[projects]] - name = "github.com/docker/go-units" - packages = ["."] - revision = "47565b4f722fb6ceae66b95f853feed578a4a51c" - version = "v0.3.3" - -[[projects]] - name = "github.com/fsnotify/fsnotify" - packages = ["."] - revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" - version = "v1.4.7" - -[[projects]] - name = "github.com/gogo/protobuf" - packages = [ - "io", - "proto" - ] - revision = "1adfc126b41513cc696b209667c8656ea7aac67c" - version = "v1.0.0" - -[[projects]] - name = "github.com/pkg/errors" - packages = ["."] - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" - -[[projects]] - branch = "master" - name = "github.com/tonistiigi/fifo" - packages = ["."] - revision = "3d5202aec260678c48179c56f40e6f38a095738c" - -[[projects]] - branch = "master" - name = "golang.org/x/crypto" - packages = ["ssh/terminal"] - revision = "d6449816ce06963d9d136eee5a56fca5b0616e7e" - -[[projects]] - branch = "master" - name = "golang.org/x/net" - packages = [ - "context", - "internal/socks", - "proxy" - ] - revision = "61147c48b25b599e5b561d2e9c4f3e1ef489ca41" - -[[projects]] - branch = "master" - name = "golang.org/x/sys" - packages = [ - "unix", - "windows" - ] - revision = "f6f352972f061230a99fbf49d1eb8073ebdb36cb" - -[[projects]] - branch = "master" - name = "golang.org/x/time" - packages = ["rate"] - revision = "fbb02b2291d28baffd63558aa44b4b56f178d650" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "02df9713899c4a90650c4e69742fe73c5efdd7f1ec434d41792d4c27a5fd82de" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index c274c37..0000000 --- a/Gopkg.toml +++ /dev/null @@ -1,54 +0,0 @@ -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" -# -# [prune] -# non-go = false -# go-tests = true -# unused-packages = true - - -[[constraint]] - name = "github.com/Sirupsen/logrus" - version = "1.0.5" - -[[constraint]] - name = "github.com/docker/docker" - version = "17.3.0-ce" - -[[constraint]] - branch = "master" - name = "github.com/docker/go-plugins-helpers" - -[[constraint]] - name = "github.com/gogo/protobuf" - version = "1.0.0" - -[[constraint]] - name = "github.com/pkg/errors" - version = "0.8.0" - -[[constraint]] - branch = "master" - name = "github.com/tonistiigi/fifo" - -[prune] - go-tests = true - unused-packages = true diff --git a/Makefile b/Makefile index 3e1913d..2e1d015 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ PLUGIN_NAME=splunk/docker-logging-plugin PLUGIN_TAG=latest PLUGIN_DIR=./splunk-logging-plugin +.PHONY: test + all: clean docker rootfs create package: clean docker rootfs zip @@ -39,3 +41,7 @@ enable: push: clean docker rootfs create enable @echo "### push plugin ${PLUGIN_NAME}:${PLUGIN_TAG}" docker plugin push ${PLUGIN_NAME}:${PLUGIN_TAG} + +test: + @echo "### running unit tests" + go test \ No newline at end of file diff --git a/README.md b/README.md index 43af889..3a7961b 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ splunk-index | Event index. (Note that HEC token must be configured to accept th splunk-capath | Path to root certificate. (Must be specified if splunk-insecureskipverify is false) | splunk-caname | Name to use for validating server certificate; by default the hostname of the splunk-url is used. | splunk-insecureskipverify| "false" means that the service certificates are validated and "true" means that server certificates are not validated. | false -splunk-format | Message format. Values can be inline, json, or raw. For more infomation about formats see the Messageformats option. | inline +splunk-format | Message format. Values can be inline, json, hec or raw. For more infomation about formats see the Messageformats option. | inline splunk-verify-connection| Upon plug-in startup, verify that Splunk Connect for Docker can connect to Splunk HEC endpoint. False indicates that Splunk Connect for Docker will start up and continue to try to connect to HEC and will push logs to buffer until connection has been establised. Logs will roll off buffer once buffer is full. True indicates that Splunk Connect for Docker will not start up if connection to HEC cannot be established. | false splunk-gzip | Enable/disable gzip compression to send events to Splunk Enterprise or Splunk Cloud instance. | false splunk-gzip-level | Set compression level for gzip. Valid values are -1 (default), 0 (no compression), 1 (best speed) … 9 (best compression). | -1 @@ -169,10 +169,11 @@ SPLUNK_TELEMETRY | Determines if telemetry is enabled. | true ### Message formats -There are three logging plug-in messaging formats set under the optional variable splunk-format: +There are four logging plug-in messaging formats set under the optional variable splunk-format: * inline (this is the default format) * json +* hec * raw The default format is inline, where each log message is embedded as a string and is assigned to "line" field. For example: @@ -227,6 +228,14 @@ To format messages as json objects, set --log-opt splunk-format=json. The plug-i } } ``` +If your log messages are already in the format expected by the Splunk HEC endpoint, set --log-opt splunk-format=hec. The plug-in will try to parse every line as a JSON object to match the expected structure for the Splunk HEC endpoint. Labels or environment variables specified in the log options will be added as additional fields for Splunk to index. If a tag is specified e.g. --log-opt tag="{{.ID}}" this will be added as an additional field named "container_tag" for Splunk to index. If it cannot parse the message, it is sent inline. For example: +``` +//Example #1 +{"event": "my message", "time": "1660656993.605501", "index": "myindex", "source": "mysource", "sourcetype": "mysourcetype", "host": "myhost" } + +//Example #2 +{ "event": { "foo": "bar" }, "time": "1660656993.605501", "index": "myindex", "source": "mysource", "sourcetype": "mysourcetype", "host": "myhost" } +``` If --log-opt splunk-format=raw, each message together with attributes (environment variables and labels) and tags are combined in a raw string. Attributes and tags are prefixed to the message. For example: ``` MyImage/MyContainer env1=val1 label1=label1 my message diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f768aef --- /dev/null +++ b/go.mod @@ -0,0 +1,30 @@ +module github.com/splunk/docker-logging-plugin + +go 1.19 + +require ( + github.com/Sirupsen/logrus v1.0.5 + github.com/docker/docker v1.4.2-0.20170502054910-90d35abf7b35 + github.com/docker/go-plugins-helpers v0.0.0-20180116160015-61cb8e233420 + github.com/gogo/protobuf v1.0.0 + github.com/pkg/errors v0.8.0 + github.com/tonistiigi/fifo v0.0.0-20180307165137-3d5202aec260 +) + +require ( + github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect + github.com/Microsoft/go-winio v0.4.7 // indirect + github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect + github.com/coreos/go-systemd v0.0.0-20180202092358-40e2722dffea // indirect + github.com/docker/go-connections v0.3.0 // indirect + github.com/docker/go-units v0.3.3 // indirect + github.com/fsnotify/fsnotify v1.4.7 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + github.com/stretchr/testify v1.8.0 // indirect + golang.org/x/crypto v0.0.0-20180411161317-d6449816ce06 // indirect + golang.org/x/net v0.0.0-20180406214816-61147c48b25b // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect + golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect + gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect + gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..393f64f --- /dev/null +++ b/go.sum @@ -0,0 +1,55 @@ +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Microsoft/go-winio v0.4.7 h1:vOvDiY/F1avSWlCWiKJjdYKz2jVjTK3pWPHndeG4OAY= +github.com/Microsoft/go-winio v0.4.7/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/Sirupsen/logrus v1.0.5 h1:447dy9LxSj+Iaa2uN3yoFHOzU9yJcJYiQPtNz8OXtv0= +github.com/Sirupsen/logrus v1.0.5/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U= +github.com/coreos/go-systemd v0.0.0-20180202092358-40e2722dffea h1:IHPWgevPcOUjTvj3n7Qgm+nie6xs/xV8dmO5MddNTpc= +github.com/coreos/go-systemd v0.0.0-20180202092358-40e2722dffea/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/docker v1.4.2-0.20170502054910-90d35abf7b35 h1:/C46Ovt6t+BTjgMe2c6K1sOJYyxGR2TY/uOP6zzO09M= +github.com/docker/docker v1.4.2-0.20170502054910-90d35abf7b35/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF+n1M6o= +github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-plugins-helpers v0.0.0-20180116160015-61cb8e233420 h1:WlllZ18+D+G9eDazaI6pmfcBBtPGFYGNkSQAiUZjHiY= +github.com/docker/go-plugins-helpers v0.0.0-20180116160015-61cb8e233420/go.mod h1:LFyLie6XcDbyKGeVK6bHe+9aJTYCxWLBg5IrJZOaXKA= +github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/gogo/protobuf v1.0.0 h1:2jyBKDKU/8v3v2xVR2PtiWQviFUyiaGk2rpfyFT8rTM= +github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/tonistiigi/fifo v0.0.0-20180307165137-3d5202aec260 h1:SFARkeCX7m3XQDD36gm5RLkwjBie8fezFQghCZWXils= +github.com/tonistiigi/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:Q5IRRDY+cjIaiOjTAnXN5LKQV5MPqVx5ofQn85Jy5Yw= +golang.org/x/crypto v0.0.0-20180411161317-d6449816ce06 h1:EOqG0JqGlLr+punVB69jvWCv/ErZKGlC7PMdyHfv+Bc= +golang.org/x/crypto v0.0.0-20180411161317-d6449816ce06/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20180406214816-61147c48b25b h1:7rskAFQwNXGW6AD8E/6y0LDHW5mT9rsLD7ViLVFfh5w= +golang.org/x/net v0.0.0-20180406214816-61147c48b25b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/splunk_logger.go b/splunk_logger.go index c0ec833..3bd162b 100644 --- a/splunk_logger.go +++ b/splunk_logger.go @@ -121,6 +121,10 @@ type splunkLoggerJSON struct { *splunkLoggerInline } +type splunkLoggerHEC struct { + *splunkLoggerInline +} + type splunkLoggerRaw struct { *splunkLogger @@ -128,13 +132,14 @@ type splunkLoggerRaw struct { } type splunkMessage struct { - Event interface{} `json:"event"` - Time string `json:"time"` - Host string `json:"host"` - Source string `json:"source,omitempty"` - SourceType string `json:"sourcetype,omitempty"` - Index string `json:"index,omitempty"` - Entity string `json:"entity,omitempty"` + Event interface{} `json:"event"` + Time string `json:"time"` + Host string `json:"host"` + Source string `json:"source,omitempty"` + SourceType string `json:"sourcetype,omitempty"` + Index string `json:"index,omitempty"` + Fields map[string]string `json:"fields,omitempty"` + Entity string `json:"entity,omitempty"` } type splunkMessageEvent struct { @@ -148,6 +153,7 @@ const ( splunkFormatRaw = "raw" splunkFormatJSON = "json" splunkFormatInline = "inline" + splunkFormatHEC = "hec" ) /* @@ -302,8 +308,9 @@ func New(info logger.Info) (logger.Logger, error) { case splunkFormatInline: case splunkFormatJSON: case splunkFormatRaw: + case splunkFormatHEC: default: - return nil, fmt.Errorf("unknown format specified %s, supported formats are inline, json and raw", splunkFormat) + return nil, fmt.Errorf("unknown format specified %s, supported formats are inline, json, hec and raw", splunkFormat) } splunkFormat = splunkFormatParsed } else { @@ -327,6 +334,13 @@ func New(info logger.Info) (logger.Logger, error) { } loggerWrapper = &splunkLoggerJSON{&splunkLoggerInline{logger, nullEvent}} + case splunkFormatHEC: + nullEvent := &splunkMessageEvent{ + Tag: tag, + Attrs: attrs, + } + + loggerWrapper = &splunkLoggerHEC{&splunkLoggerInline{logger, nullEvent}} case splunkFormatRaw: var prefix bytes.Buffer if tag != "" { @@ -418,7 +432,7 @@ func parseURL(info logger.Info) (*url.URL, error) { } /* - parseURL() makes sure that the URL is the format of: scheme://dns_name_or_ip:port +parseURL() makes sure that the URL is the format of: scheme://dns_name_or_ip:port */ func composeHealthCheckURL(splunkURL *url.URL) string { return splunkURL.Scheme + "://" + splunkURL.Host + "/services/collector/health" @@ -495,6 +509,32 @@ func (l *splunkLoggerJSON) Log(msg *logger.Message) error { return l.queueMessageAsync(message) } +func (l *splunkLoggerHEC) Log(msg *logger.Message) error { + message := l.createSplunkMessage(msg) + event := *l.nullEvent + + if err := json.Unmarshal(msg.Line, &message); err == nil { + fields := make(map[string]string) + for k, v := range message.Fields { + fields[k] = v + } + for k, v := range event.Attrs { + fields[k] = v + } + if event.Tag != "" { + fields["container_tag"] = event.Tag + } + message.Fields = fields + } else { + event.Line = string(msg.Line) + event.Source = msg.Source + message.Event = &event + } + + logger.PutMessage(msg) + return l.queueMessageAsync(message) +} + func (l *splunkLoggerRaw) Log(msg *logger.Message) error { message := l.createSplunkMessage(msg) diff --git a/splunk_test.go b/splunk_test.go index 1c24b45..e228fd9 100644 --- a/splunk_test.go +++ b/splunk_test.go @@ -83,7 +83,7 @@ func TestNewMissedUrl(t *testing.T) { } } -//splunk-url needs to be in the format of scheme://dns_name_or_ip<:port> +// splunk-url needs to be in the format of scheme://dns_name_or_ip<:port> func TestUrlFormat(t *testing.T) { info := logger.Info{ Config: map[string]string{ @@ -554,6 +554,127 @@ func TestJsonFormat(t *testing.T) { } } +// Verify HEC format +func TestHECFormat(t *testing.T) { + hec := NewHTTPEventCollectorMock(t) + + go hec.Serve() + + info := logger.Info{ + Config: map[string]string{ + splunkURLKey: hec.URL(), + splunkTokenKey: hec.token, + splunkFormatKey: splunkFormatHEC, + splunkGzipCompressionKey: "true", + splunkGzipCompressionLevelKey: "1", + tagKey: "{{.ImageName}}/{{.Name}}", + labelsKey: "a", + envRegexKey: "^foo", + }, + ContainerID: "containeriid", + ContainerName: "/container_name", + ContainerImageID: "contaimageid", + ContainerImageName: "container_image_name", + ContainerLabels: map[string]string{ + "a": "b", + }, + ContainerEnv: []string{"foo_finder=bar"}, + } + + hostname, err := info.Hostname() + if err != nil { + t.Fatal(err) + } + + loggerDriver, err := New(info) + if err != nil { + t.Fatal(err) + } + + if hec.connectionVerified { + t.Fatal("By default connection should not be verified") + } + + splunkLoggerDriver, ok := loggerDriver.(*splunkLoggerHEC) + if !ok { + t.Fatal("Unexpected Splunk Logging Driver type") + } + + if splunkLoggerDriver.hec.url != hec.URL()+"/services/collector/event/1.0" || + splunkLoggerDriver.hec.auth != "Splunk "+hec.token || + splunkLoggerDriver.nullMessage.Host != hostname || + splunkLoggerDriver.nullMessage.Source != "" || + splunkLoggerDriver.nullMessage.SourceType != "splunk_connect_docker" || + splunkLoggerDriver.nullMessage.Index != "" || + splunkLoggerDriver.hec.gzipCompression != true || + splunkLoggerDriver.hec.gzipCompressionLevel != gzip.BestSpeed || + splunkLoggerDriver.hec.postMessagesFrequency != defaultPostMessagesFrequency || + splunkLoggerDriver.hec.postMessagesBatchSize != defaultPostMessagesBatchSize || + splunkLoggerDriver.hec.bufferMaximum != defaultBufferMaximum || + cap(splunkLoggerDriver.stream) != defaultStreamChannelSize { + t.Fatal("Values do not match configuration.") + } + + message1Time := time.Now() + if err := loggerDriver.Log(&logger.Message{Line: []byte("{\"event\":\"test message\", \"time\": \"1659644859\", \"host\": \"container_host\", \"source\": \"container_source\", \"index\": \"custom_index\", \"sourcetype\": \"custom_sourcetype\"}"), Source: "stdout", Timestamp: message1Time}); err != nil { + t.Fatal(err) + } + message2Time := time.Now() + if err := loggerDriver.Log(&logger.Message{Line: []byte("nothec"), Source: "stdout", Timestamp: message2Time}); err != nil { + t.Fatal(err) + } + + err = loggerDriver.Close() + if err != nil { + t.Fatal(err) + } + + if len(hec.messages) != 2 { + t.Fatal("Expected two messages") + } + + message1 := hec.messages[0] + if message1.Event != "test message" || + message1.Time != "1659644859" || + message1.Host != "container_host" || + message1.Source != "container_source" || + message1.SourceType != "custom_sourcetype" || + message1.Fields["a"] != "b" || + message1.Fields["container_tag"] != "container_image_name/container_name" || + message1.Fields["foo_finder"] != "bar" || + message1.Index != "custom_index" { + t.Fatalf("Unexpected values of message 1 %v", message1) + } + + message2 := hec.messages[1] + if message2.Time != fmt.Sprintf("%f", float64(message2Time.UnixNano())/float64(time.Second)) || + message2.Host != hostname || + message2.Source != "" || + message2.SourceType != "splunk_connect_docker" || + message2.Index != "" { + t.Fatalf("Unexpected values of message 2 %v", message2) + } + + // If message cannot be parsed as JSON - it should be sent as a line + if event, err := message2.EventAsMap(); err != nil { + t.Fatal(err) + } else { + if event["line"] != "nothec" || + event["source"] != "stdout" || + event["tag"] != "container_image_name/container_name" || + event["attrs"].(map[string]interface{})["a"] != "b" || + event["attrs"].(map[string]interface{})["foo_finder"] != "bar" || + len(event) != 4 { + t.Fatalf("Unexpected event in message 2 %v", event) + } + } + + err = hec.Close() + if err != nil { + t.Fatal(err) + } +} + // Verify raw format func TestRawFormat(t *testing.T) { hec := NewHTTPEventCollectorMock(t) diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..eba74f4 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +venv/ \ No newline at end of file diff --git a/test/LogEntry_pb2.py b/test/LogEntry_pb2.py index e5a9a50..47395bf 100644 --- a/test/LogEntry_pb2.py +++ b/test/LogEntry_pb2.py @@ -1,13 +1,11 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: LogEntry.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -15,76 +13,13 @@ -DESCRIPTOR = _descriptor.FileDescriptor( - name='LogEntry.proto', - package='', - syntax='proto2', - serialized_pb=_b('\n\x0eLogEntry.proto\"L\n\x08LogEntry\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x11\n\ttime_nano\x18\x02 \x01(\x03\x12\x0c\n\x04line\x18\x03 \x02(\x0c\x12\x0f\n\x07partial\x18\x04 \x01(\x08') -) - - - - -_LOGENTRY = _descriptor.Descriptor( - name='LogEntry', - full_name='LogEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='source', full_name='LogEntry.source', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='time_nano', full_name='LogEntry.time_nano', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='line', full_name='LogEntry.line', index=2, - number=3, type=12, cpp_type=9, label=2, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='partial', full_name='LogEntry.partial', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=18, - serialized_end=94, -) - -DESCRIPTOR.message_types_by_name['LogEntry'] = _LOGENTRY -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -LogEntry = _reflection.GeneratedProtocolMessageType('LogEntry', (_message.Message,), dict( - DESCRIPTOR = _LOGENTRY, - __module__ = 'LogEntry_pb2' - # @@protoc_insertion_point(class_scope:LogEntry) - )) -_sym_db.RegisterMessage(LogEntry) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0eLogEntry.proto\"L\n\x08LogEntry\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x11\n\ttime_nano\x18\x02 \x01(\x03\x12\x0c\n\x04line\x18\x03 \x02(\x0c\x12\x0f\n\x07partial\x18\x04 \x01(\x08') +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'LogEntry_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _LOGENTRY._serialized_start=18 + _LOGENTRY._serialized_end=94 # @@protoc_insertion_point(module_scope) diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 0000000..1e36f54 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,46 @@ +PARTIAL_TESTS = test_partial_log_1 test_partial_log_2 test_partial_log_flush_timeout_1 test_partial_log_flush_timeout_2 test_partial_log_flush_size_limit +CONFIG_TESTS = test_splunk_index_1 test_splunk_index_2 test_splunk_source_1 test_splunk_source_2 test_splunk_source_type_1 test_splunk_source_type_2 test_splunk_format_json test_splunk_format_inline test_splunk_format_raw test_splunk_verify_connection test_splunk_gzip_1 test_splunk_gzip_2 test_splunk_gzip_3 test_splunk_gzip_4 test_splunk_gzip_5 test_splunk_tag +MALFORMED_TESTS = test_malformed_empty_string_1 test_malformed_empty_string_2 test_malformed_empty_string_3 test_malformed_empty_string_4 test_malformed_empty_string_5 + +.PHONY: setup-venv setup-splunk test-partial test-config test-malformed + +setup-venv: + if [ ! -d venv/ ] ; then python3.8 -m venv venv ; fi + . venv/bin/activate && pip install --upgrade pip && pip install -r requirements.txt + +setup-splunk: + docker compose up -d + ./wait-for-splunk.sh + +test-partial: + $(foreach var,$(PARTIAL_TESTS),sudo venv/bin/python -m pytest --cache-clear \ + --splunkd-url https://localhost:8089 \ + --splunk-user admin \ + --splunk-password notchangeme \ + --splunk-hec-url https://localhost:8088 \ + --splunk-hec-token abcd1234 \ + --docker-plugin-path ../splunk-logging-plugin/rootfs/bin/splunk-logging-plugin \ + --fifo-path /tmp/pipe \ + -p no:warnings partial_log/test_partial_log.py::$(var) || exit;) + +test-config: + $(foreach var,$(CONFIG_TESTS),sudo venv/bin/python -m pytest --cache-clear \ + --splunkd-url https://localhost:8089 \ + --splunk-user admin \ + --splunk-password notchangeme \ + --splunk-hec-url https://localhost:8088 \ + --splunk-hec-token abcd1234 \ + --docker-plugin-path ../splunk-logging-plugin/rootfs/bin/splunk-logging-plugin \ + --fifo-path /tmp/pipe \ + -p no:warnings config_params/test_cofig_params.py::$(var) || exit;) + +test-malformed: + $(foreach var,$(MALFORMED_TESTS),sudo venv/bin/python -m pytest --cache-clear \ + --splunkd-url https://localhost:8089 \ + --splunk-user admin \ + --splunk-password notchangeme \ + --splunk-hec-url https://localhost:8088 \ + --splunk-hec-token abcd1234 \ + --docker-plugin-path ../splunk-logging-plugin/rootfs/bin/splunk-logging-plugin \ + --fifo-path /tmp/pipe \ + -p no:warnings malformed_data/test_malformed_events.py::$(var) || exit;) diff --git a/test/README.md b/test/README.md index d7b1dff..7301c88 100644 --- a/test/README.md +++ b/test/README.md @@ -5,12 +5,12 @@ * Python version must be > 3.x # Testing Instructions -0. (Optional) Use a virtual environment for the test - `virtualenv --python=python3.6 venv` - `source venv/bin/activate` -1. Install the dependencies - `pip install -r requirements.txt` -2. Start the test with the required options configured +0. Make sure all commands below are run from within the test directory +1. Install the dependencies in a virtual environment + `make setup-venv` +2. Run Splunk within a Docker container if required + `make setup-splunk` +4. Start the test with the required options configured `python -m pytest ` **Options are:** diff --git a/test/docker-compose.yml b/test/docker-compose.yml new file mode 100644 index 0000000..d34b189 --- /dev/null +++ b/test/docker-compose.yml @@ -0,0 +1,15 @@ +version: "3.6" + +services: + so1: + image: splunk/splunk:latest + container_name: so1 + environment: + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_HEC_TOKEN=abcd1234 + - SPLUNK_PASSWORD=notchangeme + ports: + - 8000:8000 + - 8088:8088 + - 8089:8089 + diff --git a/test/wait-for-splunk.sh b/test/wait-for-splunk.sh new file mode 100755 index 0000000..a6503f7 --- /dev/null +++ b/test/wait-for-splunk.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +COUNT=0 +output=$(curl -s -u admin:notchangeme -k https://localhost:8089/services/server/info/server-info?output_mode=json | jq -r ".entry[0].content.kvStoreStatus") + +while [ "$output" != "ready" ] && [ $COUNT -lt 12 ] +do + echo "Waiting for Splunk to be ready (iteration $COUNT)..." + sleep 10 + output=$(curl -s -u admin:notchangeme -k https://localhost:8089/services/server/info/server-info?output_mode=json | jq -r ".entry[0].content.kvStoreStatus") + ((COUNT=COUNT+1)) +done + +if [ "$output" = "ready" ] ; then + echo "Splunk now ready." +else + echo "Failed waiting for Splunk" + exit 1 +fi