From 548b59a68de8568a5e48807092c841cc570f7371 Mon Sep 17 00:00:00 2001 From: Anshul Sirur Date: Mon, 28 Jun 2021 23:23:41 +0200 Subject: [PATCH 1/3] refactor: improve logging on startup - Prints some of the parameters out when skuttle starts. - Add a message when adding providers. - Add a message for when we exit cleanly. --- cmd/skuttle/main.go | 10 ++++++++++ {test => integration}/kind.yaml | 0 {test => integration}/resources.yaml | 0 3 files changed, 10 insertions(+) rename {test => integration}/kind.yaml (100%) rename {test => integration}/resources.yaml (100%) diff --git a/cmd/skuttle/main.go b/cmd/skuttle/main.go index fec7e31..67973a7 100644 --- a/cmd/skuttle/main.go +++ b/cmd/skuttle/main.go @@ -86,6 +86,10 @@ func main() { // Create Kubernetes client log.Info("init") + if argDryRun { + log.Info("*** dry run mode - nodes will not be deleted! ***") + } + if argKubeconfig != "" { log.Info("using config from: %s\n", argKubeconfig) } else { @@ -136,10 +140,13 @@ func main() { log.Fatalf("error creating provider %v: %v", prefix, err) } + log.Info("added provider for prefix: %s", prefix) providerStore.Add(prefix, p) } // Create node informer + log.Info("using node selector: %s", argNodeSelector) + log.Info("informer refresh duration: %v", argRefreshDuration) tweakListOptions := informers.WithTweakListOptions(func(opts *metav1.ListOptions) { opts.LabelSelector = argNodeSelector }) @@ -147,6 +154,7 @@ func main() { nodeInformer := informerFactory.Core().V1().Nodes().Informer() // Create controller + log.Info("not ready duration: %v", argNotReadyDuration) cfg := &controller.Config{ DryRun: argDryRun, NotReadyDuration: argNotReadyDuration, @@ -169,6 +177,8 @@ func main() { if err = ctx.Err(); err != nil { runtime.HandleError(err) } + + log.Info("clean exit") } func StringEnv(key string, defaultVal string) string { diff --git a/test/kind.yaml b/integration/kind.yaml similarity index 100% rename from test/kind.yaml rename to integration/kind.yaml diff --git a/test/resources.yaml b/integration/resources.yaml similarity index 100% rename from test/resources.yaml rename to integration/resources.yaml From 5a42ef261d033ebc66bd2c16e7df411359b7d3c8 Mon Sep 17 00:00:00 2001 From: Anshul Sirur Date: Mon, 28 Jun 2021 23:30:19 +0200 Subject: [PATCH 2/3] refactor: update logging in controller Changing log level for adding and removing nodes from the node informer to `info` because it's useful to know which nodes have been picked up on startup and that deletions have made it to the API. --- internal/controller/controller.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index a84e67b..e7d75ec 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -54,10 +54,10 @@ func NewController( return controller } -// When a new node gets created +// When a new node gets added to API func (c *Controller) Add(obj interface{}) { n := coerce(obj) - log.Debug("add node %s", n.Name()) + log.Info("node added: %s", n.Name()) if err := c.Handle(n); err != nil { log.Error(err.Error()) } @@ -66,7 +66,7 @@ func (c *Controller) Add(obj interface{}) { // When a node gets updated func (c *Controller) Update(_ interface{}, obj interface{}) { n := coerce(obj) - log.Debug("update node %s", n.Name()) + log.Debug("node updated: %s", n.Name()) if err := c.Handle(n); err != nil { log.Error(err.Error()) } @@ -75,7 +75,7 @@ func (c *Controller) Update(_ interface{}, obj interface{}) { // When a node gets deleted func (c *Controller) Delete(obj interface{}) { n := coerce(obj) - log.Debug("remove node %s", n.Name()) + log.Info("node deleted: %s", n.Name()) } // Handle a node From ebb5b0d4f67237699d5c8fce2959093bace269d4 Mon Sep 17 00:00:00 2001 From: Anshul Sirur Date: Mon, 28 Jun 2021 23:38:19 +0200 Subject: [PATCH 3/3] wip: mvp integration test with kind --- .github/workflows/ci.yaml | 8 +++-- Makefile | 12 ++++++-- integration/kind.yaml | 18 ++++++++++++ integration/test.sh | 61 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 5 deletions(-) create mode 100755 integration/test.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 59900a0..8a05a0c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,8 +16,12 @@ jobs: - uses: actions/setup-go@v2 with: go-version: '^1.16' - - name: run-tests - run: go test -v ./... + - name: unit-tests + run: | + make test + - name: integration-tests + run: | + make integration-test image: runs-on: ubuntu-latest if: github.event_name == 'push' diff --git a/Makefile b/Makefile index 66bbc21..1369db9 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ DOCKER_TAG ?= $(shell git rev-parse HEAD) .PHONY: default -default: build +default: test build .PHONY: docker-build docker-build: @@ -16,13 +16,19 @@ clean: rm -f skuttle .PHONY: build -build: test +build: CGO_ENABLED=0 go build ./cmd/skuttle .PHONY: test test: - go test ./... + go install github.com/onsi/ginkgo/ginkgo@v1.16.4 + ginkgo -r -skipPackage integration .PHONY: fmt fmt: go fmt ./... + +.PHONY: integration-test +integration-test: build + go install sigs.k8s.io/kind@v0.11.1 + integration/test.sh diff --git a/integration/kind.yaml b/integration/kind.yaml index f4bf78c..d986a7a 100644 --- a/integration/kind.yaml +++ b/integration/kind.yaml @@ -3,5 +3,23 @@ apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + provider-id: "file://node1" - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + provider-id: "file://node2" - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + provider-id: "file://node3" diff --git a/integration/test.sh b/integration/test.sh new file mode 100755 index 0000000..79e905a --- /dev/null +++ b/integration/test.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail + +tmpd="$(mktemp -d -p /tmp skuttle.XXXXX)" +echo "config dir: $tmpd" + +export KUBECONFIG="$tmpd/config" +export NODE_LIST="$tmpd/nodes" + +# set up provider nodes +cat <"$NODE_LIST" +node1 +node3 +EOF + +# create kind cluster +kind delete cluster --name skuttle +kind create cluster \ + --config integration/kind.yaml \ + --name skuttle \ + --wait 2m + +# start skuttle +./skuttle \ + --log-level debug \ + --node-selector "!node-role.kubernetes.io/control-plane" \ + --not-ready-duration 20s \ + --providers file \ + >"$tmpd/log" 2>&1 & + +skuttle_pid="$!" + +# follow logs +tail -f "$tmpd/log" & + +# make a worker not ready +docker exec skuttle-worker2 systemctl stop kubelet + +# wait for worker to be removed by skuttle +exitcode=0 +waits=0 +while kubectl get node skuttle-worker2 >/dev/null 2>/dev/null; do + sleep 1 + waits="$(( waits+1 ))" + + if test "$waits" -eq 30; then + echo "FAIL: timed out waiting for node deletion" + exitcode=1 + fi +done + +echo "PASS: node deleted by skuttle" + +# cleanup +kill "$skuttle_pid" +kind delete cluster --name skuttle +rm -r "$tmpd" + +# exit +exit $exitcode