Sync container and pod metrics

cmd/icinga-kubernetes/main.go

@@ -8,26 +8,30 @@ import (
 	"github.com/icinga/icinga-go-library/logging"
 	"github.com/icinga/icinga-kubernetes/internal"
 	"github.com/icinga/icinga-kubernetes/pkg/contracts"
+	"github.com/icinga/icinga-kubernetes/pkg/metrics"
 	"github.com/icinga/icinga-kubernetes/pkg/schema"
 	"github.com/icinga/icinga-kubernetes/pkg/sync"
 	"github.com/okzk/sdnotify"
 	"github.com/pkg/errors"
+	promapi "github.com/prometheus/client_golang/api"
+	promv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"golang.org/x/sync/errgroup"
 	kinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	kclientcmd "k8s.io/client-go/tools/clientcmd"
+	"strconv"
 )
 
 func main() {
 	kconfig, err := kclientcmd.NewNonInteractiveDeferredLoadingClientConfig(
 		kclientcmd.NewDefaultClientConfigLoadingRules(), &kclientcmd.ConfigOverrides{}).ClientConfig()
 	if err != nil {
-		logging.Fatal(errors.Wrap(err, "can't configure Kubernetes client"))
+		logging.Fatal(errors.Wrap(err, "can't configure Kubernetes promClient"))
 	}
 
 	k, err := kubernetes.NewForConfig(kconfig)
 	if err != nil {
-		logging.Fatal(errors.Wrap(err, "can't create Kubernetes client"))
+		logging.Fatal(errors.Wrap(err, "can't create Kubernetes promClient"))
 	}
 
 	flags, err := config.ParseFlags[internal.Flags]()
@@ -40,6 +44,13 @@ func main() {
 		logging.Fatal(errors.Wrap(err, "can't create configuration"))
 	}
 
+	promClient, err := promapi.NewClient(promapi.Config{Address: cfg.Prometheus.Host + ":" + strconv.Itoa(cfg.Prometheus.Port)})
+	if err != nil {
+		logging.Fatal(errors.Wrap(err, "error creating promClient"))
+	}
+
+	promApiClient := promv1.NewAPI(promClient)
+
 	logs, err := logging.NewLoggingFromConfig("Icinga Kubernetes", &cfg.Logging)
 	if err != nil {
 		logging.Fatal(errors.Wrap(err, "can't configure logging"))
@@ -71,19 +82,19 @@ func main() {
 
 	g, ctx := errgroup.WithContext(ctx)
 
-	forwardUpsertPodsChannel := make(chan database.Entity)
-	defer close(forwardUpsertPodsChannel)
-
-	forwardDeletePodsChannel := make(chan any)
-	defer close(forwardDeletePodsChannel)
+	// node forward channels
+	forwardDeleteNodesToMetricChannel := make(chan contracts.KDelete)
 
 	g.Go(func() error {
 		return sync.NewSync(
 			db,
 			schema.NewNode,
 			informers.Core().V1().Nodes().Informer(),
 			logs.GetChildLogger("Nodes"),
-		).Run(ctx)
+		).Run(
+			ctx,
+			sync.WithForwardDeleteToMetric(forwardDeleteNodesToMetricChannel),
+		)
 	})
 
 	g.Go(func() error {
@@ -95,9 +106,12 @@ func main() {
 		).Run(ctx)
 	})
 
+	// pod forward channels
 	forwardUpsertPodsToLogChannel := make(chan contracts.KUpsert)
 	forwardDeletePodsToLogChannel := make(chan contracts.KDelete)
 
+	//forwardDeletePodsToMetricChannel := make(chan contracts.KDelete)
+
 	g.Go(func() error {
 
 		defer close(forwardUpsertPodsToLogChannel)
@@ -115,6 +129,7 @@ func main() {
 		)
 	})
 
+	// sync logs
 	logSync := sync.NewLogSync(k, db, logs.GetChildLogger("ContainerLogs"))
 
 	g.Go(func() error {
@@ -125,6 +140,11 @@ func main() {
 		return logSync.Run(ctx)
 	})
 
+	// sync prometheus metrics
+	promMetricSync := metrics.NewPromMetricSync(promApiClient, db)
+
+	g.Go(func() error { return promMetricSync.Run(ctx) })
+
 	if err := g.Wait(); err != nil {
 		logging.Fatal(errors.Wrap(err, "can't sync"))
 	}

config.example.yml

@@ -39,3 +39,12 @@ logging:
   # Valid units are "ms", "s", "m", "h".
   # Defaults to "20s".
 #  interval: 20s
+
+# Configuration for Prometheus metrics API.
+prometheus:
+
+  # Prometheus host
+#  host: http://localhost
+
+    # Prometheus port
+#  port: 9090

go.mod

@@ -1,32 +1,39 @@
 module github.com/icinga/icinga-kubernetes
 
-go 1.18
+go 1.22.0
+
+toolchain go1.22.2
 
 require (
+	github.com/dustin/go-humanize v1.0.1
 	github.com/icinga/icinga-go-library v0.0.0-20231121080432-c03a40718ed9
 	github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.19.1
+	github.com/prometheus/common v0.48.0
 	go.uber.org/zap v1.26.0
 	golang.org/x/sync v0.5.0
-	k8s.io/apimachinery v0.28.2
-	k8s.io/client-go v0.28.2
+	k8s.io/api v0.30.0
+	k8s.io/apimachinery v0.30.0
+	k8s.io/client-go v0.30.0
+	k8s.io/metrics v0.30.0
 )
 
 require (
 	github.com/creasty/defaults v1.7.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
-	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
 	github.com/go-sql-driver/mysql v1.7.1 // indirect
 	github.com/goccy/go-yaml v1.11.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
@@ -41,27 +48,27 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/ssgreg/journald v1.0.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
-	golang.org/x/net v0.16.0 // indirect
-	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/oauth2 v0.16.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/api v0.28.2 // indirect
-	k8s.io/klog/v2 v2.100.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
-	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
+	k8s.io/klog/v2 v2.120.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )