integration tests

.github/workflows/ci.yaml

@@ -16,8 +16,12 @@ jobs:
       - uses: actions/setup-go@v2
         with:
           go-version: '^1.16'
-      - name: run-tests
-        run: go test -v ./...
+      - name: unit-tests
+        run: |
+          make test
+      - name: integration-tests
+        run: |
+          make integration-test
   image:
     runs-on: ubuntu-latest
     if: github.event_name == 'push'

Makefile

@@ -1,7 +1,7 @@
 DOCKER_TAG ?= $(shell git rev-parse HEAD)
 
 .PHONY: default
-default: build
+default: test build
 
 .PHONY: docker-build
 docker-build:
@@ -16,13 +16,19 @@ clean:
 	rm -f skuttle
 
 .PHONY: build
-build: test
+build:
 	CGO_ENABLED=0 go build ./cmd/skuttle
 
 .PHONY: test
 test:
-	go test ./...
+	go install github.com/onsi/ginkgo/ginkgo@v1.16.4
+	ginkgo -r -skipPackage integration
 
 .PHONY: fmt
 fmt:
 	go fmt ./...
+
+.PHONY: integration-test
+integration-test: build
+	go install sigs.k8s.io/kind@v0.11.1
+	integration/test.sh

cmd/skuttle/main.go

@@ -86,6 +86,10 @@ func main() {
 	// Create Kubernetes client
 	log.Info("init")
 
+	if argDryRun {
+		log.Info("*** dry run mode - nodes will not be deleted! ***")
+	}
+
 	if argKubeconfig != "" {
 		log.Info("using config from: %s\n", argKubeconfig)
 	} else {
@@ -136,17 +140,21 @@ func main() {
 			log.Fatalf("error creating provider %v: %v", prefix, err)
 		}
 
+		log.Info("added provider for prefix: %s", prefix)
 		providerStore.Add(prefix, p)
 	}
 
 	// Create node informer
+	log.Info("using node selector: %s", argNodeSelector)
+	log.Info("informer refresh duration: %v", argRefreshDuration)
 	tweakListOptions := informers.WithTweakListOptions(func(opts *metav1.ListOptions) {
 		opts.LabelSelector = argNodeSelector
 	})
 	informerFactory := informers.NewSharedInformerFactoryWithOptions(clientset, argRefreshDuration, tweakListOptions)
 	nodeInformer := informerFactory.Core().V1().Nodes().Informer()
 
 	// Create controller
+	log.Info("not ready duration: %v", argNotReadyDuration)
 	cfg := &controller.Config{
 		DryRun:           argDryRun,
 		NotReadyDuration: argNotReadyDuration,
@@ -169,6 +177,8 @@ func main() {
 	if err = ctx.Err(); err != nil {
 		runtime.HandleError(err)
 	}
+
+	log.Info("clean exit")
 }
 
 func StringEnv(key string, defaultVal string) string {

integration/kind.yaml

@@ -0,0 +1,25 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+  - role: worker
+    kubeadmConfigPatches:
+      - |
+        kind: JoinConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            provider-id: "file://node1"
+  - role: worker
+    kubeadmConfigPatches:
+      - |
+        kind: JoinConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            provider-id: "file://node2"
+  - role: worker
+    kubeadmConfigPatches:
+      - |
+        kind: JoinConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            provider-id: "file://node3"

integration/test.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail
+
+tmpd="$(mktemp -d -p /tmp skuttle.XXXXX)"
+echo "config dir: $tmpd"
+
+export KUBECONFIG="$tmpd/config"
+export NODE_LIST="$tmpd/nodes"
+
+# set up provider nodes
+cat <<EOF >"$NODE_LIST"
+node1
+node3
+EOF
+
+# create kind cluster
+kind delete cluster --name skuttle
+kind create cluster \
+  --config integration/kind.yaml \
+  --name skuttle \
+  --wait 2m
+
+# start skuttle
+./skuttle \
+  --log-level debug \
+  --node-selector "!node-role.kubernetes.io/control-plane" \
+  --not-ready-duration 20s \
+  --providers file \
+  >"$tmpd/log" 2>&1 &
+
+skuttle_pid="$!"
+
+# follow logs
+tail -f "$tmpd/log" &
+
+# make a worker not ready
+docker exec skuttle-worker2 systemctl stop kubelet
+
+# wait for worker to be removed by skuttle
+exitcode=0
+waits=0
+while kubectl get node skuttle-worker2 >/dev/null 2>/dev/null; do
+  sleep 1
+  waits="$(( waits+1 ))"
+
+  if test "$waits" -eq 30; then
+    echo "FAIL: timed out waiting for node deletion"
+    exitcode=1
+  fi
+done
+
+echo "PASS: node deleted by skuttle"
+
+# cleanup
+kill "$skuttle_pid"
+kind delete cluster --name skuttle
+rm -r "$tmpd"
+
+# exit
+exit $exitcode

internal/controller/controller.go

@@ -54,10 +54,10 @@ func NewController(
 	return controller
 }
 
-// When a new node gets created
+// When a new node gets added to API
 func (c *Controller) Add(obj interface{}) {
 	n := coerce(obj)
-	log.Debug("add node %s", n.Name())
+	log.Info("node added: %s", n.Name())
 	if err := c.Handle(n); err != nil {
 		log.Error(err.Error())
 	}
@@ -66,7 +66,7 @@ func (c *Controller) Add(obj interface{}) {
 // When a node gets updated
 func (c *Controller) Update(_ interface{}, obj interface{}) {
 	n := coerce(obj)
-	log.Debug("update node %s", n.Name())
+	log.Debug("node updated: %s", n.Name())
 	if err := c.Handle(n); err != nil {
 		log.Error(err.Error())
 	}
@@ -75,7 +75,7 @@ func (c *Controller) Update(_ interface{}, obj interface{}) {
 // When a node gets deleted
 func (c *Controller) Delete(obj interface{}) {
 	n := coerce(obj)
-	log.Debug("remove node %s", n.Name())
+	log.Info("node deleted: %s", n.Name())
 }
 
 // Handle a node