Doc 0 3 1

.github/workflows/ci.yaml

@@ -29,5 +29,5 @@ jobs:
           verb: call
           args: -s --name my-test build-images --source-folder ../ new-interlink --plugin-endpoint tcp://localhost:4000 --manifests ./manifests test stdout
           cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
-          version: "0.11.9"
+          version: "0.13.0"
           dagger-flags: -d

.goreleaser.yaml

@@ -18,7 +18,7 @@ builds:
       - arm64
       - amd64
     main: ./cmd/virtual-kubelet
-  - id: "interlink"
+  - id: "interlink-api"
     binary: interlink
     hooks:
       pre: bash -c "KUBELET_VERSION={{.Version}} ./cmd/virtual-kubelet/set-version.sh" 
@@ -32,8 +32,8 @@ builds:
       - amd64
       - ppc64le
     main: ./cmd/interlink
-  - id: "interlink-install"
-    binary: interlink-install
+  - id: "installer"
+    binary: interlink-installer
     env:
       - CGO_ENABLED=0
     goos:
@@ -44,6 +44,18 @@ builds:
       - amd64
       - ppc64le
     main: ./cmd/installer
+  - id: "ssh-tunnel"
+    binary: ssh-tunnel
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - arm64
+      - amd64
+      - ppc64le
+    main: ./cmd/ssh-tunnel
 archives:
   - name_template: >-
       {{ .Binary }}_

CONTRIBUTING.md

@@ -31,6 +31,7 @@ you submit the pull request). We have a 3 steps process for contributions.
    understanding and appreciating your pull request, please use the template to
    explain _why_ you are making this contribution, rather than just _what_ the
    contribution entails.
+2. **Run E2E tests with success**. You can follow the steps described [here](https://intertwin-eu.github.io/interLink/docs/Developers)
 
 ### Code Review Process
 
@@ -44,7 +45,7 @@ follows:
 
 1. A maintainer will review your code and merge it if no changes are necessary.
    Your change will be merged into the repository's `main` branch.
-1. If a maintainer has feedback or questions on your changes then they will set
+2. If a maintainer has feedback or questions on your changes then they will set
    `request changes` in the review and provide an explanation.
 
 ## Using git

Makefile

@@ -1,4 +1,4 @@
-all: interlink vk installer
+all: interlink vk installer ssh-tunnel
 
 interlink:
 	CGO_ENABLED=0 OOS=linux go build -o bin/interlink cmd/interlink/main.go
@@ -9,16 +9,16 @@ vk:
 installer:
 	CGO_ENABLED=0 OOS=linux go build -o bin/installer cmd/installer/main.go
 
+ssh-tunnel:
+	CGO_ENABLED=0 OOS=linux go build -o bin/ssh-tunnel cmd/ssh-tunnel/main.go
+
 clean:
 	rm -rf ./bin
 
-dagger_registry_delete:
-	docker rm -fv registry || true
-
 test:
-	dagger_registry_delete
-	docker run -d --rm --name registry -p 5432:5000  registry
-	cd ci
-	dagger go run go main.go k8s.go
-	cd -
+	dagger call -m ./ci \
+    --name my-tests \
+    build-images \
+    new-interlink \
+    test stdout
 

ci/dagger.json

@@ -4,9 +4,8 @@
   "dependencies": [
     {
       "name": "k3s",
-      "source": "github.com/marcosnils/daggerverse/k3s@e0bd6b9f5519c49db4b6eb0689927214720976f9"
+      "source": "github.com/marcosnils/daggerverse/k3s@ce8fe35d309bdb29f9983f7d90ea518e724534fe"
     }
   ],
-  "source": ".",
-  "engineVersion": "v0.11.9"
+  "engineVersion": "v0.13.0"
 }

ci/go.mod

@@ -3,9 +3,9 @@ module dagger/interlink
 go 1.22.2
 
 require (
-	github.com/99designs/gqlgen v0.17.44
+	github.com/99designs/gqlgen v0.17.49
 	github.com/Khan/genqlient v0.7.0
-	github.com/vektah/gqlparser/v2 v2.5.11
+	github.com/vektah/gqlparser/v2 v2.5.16
 	go.opentelemetry.io/otel v1.27.0
 	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88
 	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0
@@ -28,13 +28,21 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
-	github.com/sosodev/duration v1.2.0 // indirect
+	github.com/sosodev/duration v1.3.1 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect
 	go.opentelemetry.io/otel/metric v1.27.0 // indirect
-	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
 )
+
+replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88
+
+replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.3.0
+
+replace go.opentelemetry.io/otel/log => go.opentelemetry.io/otel/log v0.3.0
+
+replace go.opentelemetry.io/otel/sdk/log => go.opentelemetry.io/otel/sdk/log v0.3.0

ci/go.sum

@@ -1,5 +1,5 @@
-github.com/99designs/gqlgen v0.17.44 h1:OS2wLk/67Y+vXM75XHbwRnNYJcbuJd4OBL76RX3NQQA=
-github.com/99designs/gqlgen v0.17.44/go.mod h1:UTCu3xpK2mLI5qcMNw+HKDiEL77it/1XtAjisC4sLwM=
+github.com/99designs/gqlgen v0.17.49 h1:b3hNGexHd33fBSAd4NDT/c3NCcQzcAVkknhN9ym36YQ=
+github.com/99designs/gqlgen v0.17.49/go.mod h1:tC8YFVZMed81x7UJ7ORUwXF4Kn6SXuucFqQBhN8+BU0=
 github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w=
 github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
@@ -27,14 +27,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8=
 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
-github.com/sosodev/duration v1.2.0 h1:pqK/FLSjsAADWY74SyWDCjOcd5l7H8GSnnOGEB9A1Us=
-github.com/sosodev/duration v1.2.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg=
+github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4=
+github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/vektah/gqlparser/v2 v2.5.11 h1:JJxLtXIoN7+3x6MBdtIP59TP1RANnY7pXOaDnADQSf8=
-github.com/vektah/gqlparser/v2 v2.5.11/go.mod h1:1rCcfwB2ekJofmluGWXMSEnPMZgbxzwj6FaZ/4OT8Cc=
+github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0vy5p8=
+github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww=
 go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg=
 go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ=
 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88 h1:oM0GTNKGlc5qHctWeIGTVyda4iFFalOzMZ3Ehj5rwB4=
@@ -63,14 +63,14 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
-golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
 golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ=
 google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI=

ci/helm_test/interlink-config.yaml

@@ -0,0 +1,16 @@
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+#   name: "interlink-config"
+#   namespace: interlink 
+# data:
+#   InterLinkConfig.yaml: |
+    #InterlinkAddress: "unix:///var/run/interlink.socket"
+InterlinkAddress: "http://0.0.0.0"
+InterlinkPort: "3000"
+SidecarURL: "http://plugin"
+SidecarPort: "4000"
+VerboseLogging: true
+ErrorsOnlyLogging: false
+ExportPodData: true
+DataRootFolder: "~/.interlink"

ci/helm_test/kustomization.yaml

@@ -0,0 +1,13 @@
+resources:
+- virtual-kubelet-config.yaml
+- virtual-kubelet.yaml
+#- interlink-config.yaml
+#- interlink.yaml
+#- plugin-k8s-config.yaml
+#- plugin.yaml
+patches:
+- path: virtual-kubelet-merge.yaml
+  target:
+    kind: Deployment
+    labelSelector: nodeName=virtual-kubelet
+

ci/helm_test/plugin-config.yaml

@@ -0,0 +1,21 @@
+InterlinkURL: "http://interlink"
+InterlinkPort: "3000"
+SidecarURL: "http://0.0.0.0"
+SidecarPort: "4000"
+VerboseLogging: true
+ErrorsOnlyLogging: false
+ExportPodData: true
+# NEEDED PATH FOR GITHUB ACTIONS
+#DataRootFolder: "/home/runner/work/interLink/interLink/.interlink/"
+# on your host use something like:
+DataRootFolder: "/home/ubuntu/.interlink/"
+SbatchPath: "/usr/bin/sbatch"
+ScancelPath: "/usr/bin/scancel"
+SqueuePath: "/usr/bin/squeue"
+CommandPrefix: ""
+SingularityPrefix: ""
+Namespace: "vk"
+Tsocks: false
+TsocksPath: "$WORK/tsocks-1.8beta5+ds1/libtsocks.so"
+TsocksLoginNode: "login01"
+BashPath: /bin/bash

ci/helm_test/plugin-k8s-config.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "plugin-config"
+  namespace: interlink 
+data:
+    InterLinkConfig.yaml: |
+      InterlinkURL: "http://localhost"
+      InterlinkPort: "3000"
+      SidecarURL: "http://0.0.0.0"
+      SidecarPort: "4000"
+      VerboseLogging: true
+      ErrorsOnlyLogging: false
+      ExportPodData: true
+      DataRootFolder: "/home/runner/work/interLink/interLink/.interlink/"
+      SbatchPath: "/usr/bin/sbatch"
+      ScancelPath: "/usr/bin/scancel"
+      SqueuePath: "/usr/bin/squeue"
+      CommandPrefix: ""
+      SingularityPrefix: ""
+      Namespace: "vk"
+      Tsocks: false
+      TsocksPath: "$WORK/tsocks-1.8beta5+ds1/libtsocks.so"
+      TsocksLoginNode: "login01"
+      BashPath: /bin/bash

ci/helm_test/plugin.yaml

@@ -0,0 +1,54 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: plugin 
+  namespace: interlink
+spec:
+  selector:
+    app: plugin 
+  ports:
+    - protocol: TCP
+      port: 4000
+      targetPort: 4000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: plugin 
+  namespace: interlink
+  labels:
+    app: plugin
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: plugin 
+  template:
+    metadata:
+      labels:
+        app: plugin
+    spec:
+      containers:
+      - name: plugin 
+        image: "dciangot/docker-plugin:v1"
+        #image: "ghcr.io/intertwin-eu/interlink-sidecar-slurm/interlink-sidecar-slurm:0.2.3"
+        imagePullPolicy: Always
+        command:
+          - bash
+          - -c
+        args:
+          - dockerd --mtu 1450 & /sidecar/docker-sidecar
+        securityContext:
+          privileged: true
+        env:
+        - name: INTERLINKCONFIGPATH
+          value: "/etc/interlink/InterLinkConfig.yaml"
+        volumeMounts:
+        - name: config
+          mountPath: /etc/interlink/InterLinkConfig.yaml
+          subPath: InterLinkConfig.yaml
+      volumes:
+      - name: config
+        configMap:
+          # Provide the name of the ConfigMap you want to mount.
+          name: plugin-config

helm/interlink/templates/service-account.yaml → ci/helm_test/service-account.yaml

@@ -1,14 +1,14 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: {{ .Values.nodeName }}
-  namespace: {{ .Release.Namespace }}
+  name: virtual-kubelet 
+  namespace: interlink
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: "{{ .Values.nodeName }}-role"
-  namespace: {{ .Release.Namespace }}
+  name: virtual-kubelet 
+  namespace: interlink
 rules:
 - apiGroups:
   - "coordination.k8s.io"
@@ -75,24 +75,14 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: "{{ .Values.nodeName }}-rolebinding"
-  namespace: {{ .Release.Namespace }}
+  name: virtual-kubelet 
+  namespace: interlink
 subjects:
 - kind: ServiceAccount
-  name: {{ .Values.nodeName }}
-  namespace: {{ .Release.Namespace }}
+  name: virtual-kubelet
+  namespace: interlink
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: "{{ .Values.nodeName }}-role"
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: "{{ .Values.nodeName }}-secret"
-  namespace: "{{ .Release.Namespace }}"
-  annotations:
-    kubernetes.io/service-account.name: {{ .Values.nodeName }}
-  labels:
-    kubernetes.io/service-account.name: {{ .Values.nodeName }}
-type: kubernetes.io/service-account-token
+  name: virtual-kubelet
+

ci/helm_test/virtual-kubelet-config.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "virtual-kubelet-config"
+  namespace: interlink 
+data:
+  InterLinkConfig.yaml: |
+    #InterlinkURL: unix:///var/run/interlink.socket
+    InterlinkURL: "http://interlink"
+    InterlinkPort: "3000" 
+    ExportPodData: true
+    VerboseLogging: true
+    ErrorsOnlyLogging: false
+    ServiceAccount: "virtual-kubelet"
+    Namespace: interlink 
+    VKTokenFile: ""
+    CPU: "100"
+    Memory: "128Gi"
+    Pods: "100"