From 3fb1eece816e815ff325096ab32e8eaf338b9709 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Tue, 19 Apr 2022 16:57:59 -0400 Subject: [PATCH 1/2] Add rclone push example without credentials --- inst/helm-chart/templates/_helpers.tpl | 21 +++++ .../templates/extrafilemappings.yaml | 23 +++++ .../templates/manager-deployment.yaml | 85 ++++++++++++++++++- inst/helm-chart/values.yaml | 34 ++++++++ 4 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 inst/helm-chart/templates/extrafilemappings.yaml diff --git a/inst/helm-chart/templates/_helpers.tpl b/inst/helm-chart/templates/_helpers.tpl index 7f82a81..78ebe1a 100644 --- a/inst/helm-chart/templates/_helpers.tpl +++ b/inst/helm-chart/templates/_helpers.tpl @@ -17,3 +17,24 @@ csi: {{- end -}} {{- end -}} {{- end -}} + +{{/* +Extract the filename portion from a file path +*/}} +{{- define "biockubeinstall.getFilenameFromPath" -}} +{{- printf "%s" (. | splitList "/" | last) -}} +{{- end -}} + +{{/* +Make string DNS-compliant by turning to lowercase then removing all noncompliant characters +*/}} +{{- define "biockubeinstall.makeDnsCompliant" -}} +{{- (printf "%s" (regexReplaceAll "[^a-z0-9-]" (. | lower) "")) | trunc 63 | trimSuffix "-" }} +{{- end -}} + +{{/* +Get unique name for extra files +*/}} +{{- define "biockubeinstall.getExtraFilesUniqueName" -}} +{{- (printf "%s" (include "biockubeinstall.makeDnsCompliant" (printf "extra-%s-%s" (include "biockubeinstall.getFilenameFromPath" .) (. | sha256sum)))) }} +{{- end -}} diff --git a/inst/helm-chart/templates/extrafilemappings.yaml b/inst/helm-chart/templates/extrafilemappings.yaml new file mode 100644 index 0000000..d0dc452 --- /dev/null +++ b/inst/helm-chart/templates/extrafilemappings.yaml @@ -0,0 +1,23 @@ +{{- range $key, $entry := .Values.extraFileMappings -}} +{{- if $entry }} +apiVersion: v1 +metadata: + # Extract the filename portion only + name: {{ printf "%s-%s" "extrafiles" (include "biockubeinstall.getExtraFilesUniqueName" $key) }} +{{- if $entry.useSecret }} +kind: Secret +type: Opaque +stringData: +{{- else }} +kind: ConfigMap +data: +{{- end }} + {{- include "biockubeinstall.getFilenameFromPath" $key | nindent 2 }}: | + {{- if $entry.tpl }} + {{- tpl (tpl $entry.content $) $ | nindent 4 }} + {{- else }} + {{- $entry.content | nindent 4 }} + {{- end }} +--- +{{- end }} +{{- end }} \ No newline at end of file diff --git a/inst/helm-chart/templates/manager-deployment.yaml b/inst/helm-chart/templates/manager-deployment.yaml index b8e2722..b4b64c5 100644 --- a/inst/helm-chart/templates/manager-deployment.yaml +++ b/inst/helm-chart/templates/manager-deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: app: rstudio spec: - containers: + initContainers: - name: manager image: {{ .Values.manager.image.repository }}:{{ .Values.manager.image.tag }} imagePullPolicy: {{ .Values.pullPolicy }} @@ -14,6 +14,13 @@ spec: volumeMounts: - name: nfs-data mountPath: {{ .Values.persistence.mountPath }} + {{- range $key, $entry := .Values.extraFileMappings -}} + {{- if $entry.applyToManager }} + - name: {{ include "biockubeinstall.getExtraFilesUniqueName" $key }} + mountPath: {{ $key }} + subPath: {{ include "biockubeinstall.getFilenameFromPath" $key }} + {{- end }} + {{- end }} env: - name: PASSWORD value: {{ .Values.rstudioPassword | quote }} @@ -26,8 +33,82 @@ spec: {{- tpl .Values.manager.extraCommand $ }}; {{- end -}} {{ tpl .Values.manager.defaultCommand . }} + containers: + - name: osn-rclone-copy + image: rclone/rclone:latest + command: + - /bin/sh + - -c + args: + - rclone copyto /host/binary_{{ .Values.biocVersion | replace "." "_" }} osn:bir190004-bucket01/bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker -v && rclone copyto /host/library_{{ .Values.biocVersion | replace "." "_" }} osn:bir190004-bucket01/bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/contrib -v && rclone copyto /host/logs_{{ .Values.biocVersion | replace "." "_" }} osn:bir190004-bucket01/bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/package_logs -v + volumeMounts: + - name: nfs-data + mountPath: {{ .Values.persistence.mountPath }} + {{- range $key, $entry := .Values.extraFileMappings -}} + {{- if $entry.applyToRclone }} + - name: {{ include "biockubeinstall.getExtraFilesUniqueName" $key }} + mountPath: {{ $key }} + subPath: {{ include "biockubeinstall.getFilenameFromPath" $key }} + {{- end }} + {{- end }} + resources: + requests: + memory: 100Mi + cpu: 100m + - name: azure-rclone-copy + image: rclone/rclone:latest + command: + - /bin/sh + - -c + args: + - rclone copyto /host/binary_{{ .Values.biocVersion | replace "." "_" }} azurebuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker -v && rclone copyto /host/library_{{ .Values.biocVersion | replace "." "_" }} azurebuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/contrib -v && rclone copyto /host/logs_{{ .Values.biocVersion | replace "." "_" }} azurebuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/package_logs -v + volumeMounts: + - name: nfs-data + mountPath: {{ .Values.persistence.mountPath }} + {{- range $key, $entry := .Values.extraFileMappings -}} + {{- if $entry.applyToRclone }} + - name: {{ include "biockubeinstall.getExtraFilesUniqueName" $key }} + mountPath: {{ $key }} + subPath: {{ include "biockubeinstall.getFilenameFromPath" $key }} + {{- end }} + {{- end }} + resources: + requests: + memory: 100Mi + cpu: 100m + - name: gcp-rclone-copy + image: rclone/rclone:latest + command: + - /bin/sh + - -c + args: + - rclone copyto /host/binary_{{ .Values.biocVersion | replace "." "_" }} gcpbuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker -v && rclone copyto /host/library_{{ .Values.biocVersion | replace "." "_" }} gcpbuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/contrib -v && rclone copyto /host/logs_{{ .Values.biocVersion | replace "." "_" }} gcpbuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/package_logs -v + volumeMounts: + - name: nfs-data + mountPath: {{ .Values.persistence.mountPath }} + {{- range $key, $entry := .Values.extraFileMappings -}} + {{- if $entry.applyToRclone }} + - name: {{ include "biockubeinstall.getExtraFilesUniqueName" $key }} + mountPath: {{ $key }} + subPath: {{ include "biockubeinstall.getFilenameFromPath" $key }} + {{- end }} + {{- end }} + resources: + requests: + memory: 100Mi + cpu: 100m restartPolicy: {{ .Values.restartPolicy | quote}} volumes: - name: nfs-data persistentVolumeClaim: - claimName: {{.Release.Namespace}}-nfs \ No newline at end of file + claimName: {{.Release.Namespace}}-nfs + {{- range $key, $entry := .Values.extraFileMappings -}} + - name: {{ include "biockubeinstall.getExtraFilesUniqueName" $key }} + {{- if $entry.useSecret }} + secret: + secretName: {{ printf "%s-%s" "extrafiles" (include "biockubeinstall.getExtraFilesUniqueName" $key) }} + {{- else }} + configMap: + name: {{ printf "%s-%s" "extrafiles" (include "biockubeinstall.getExtraFilesUniqueName" $key) }} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/inst/helm-chart/values.yaml b/inst/helm-chart/values.yaml index 5d89838..5fbea92 100644 --- a/inst/helm-chart/values.yaml +++ b/inst/helm-chart/values.yaml @@ -90,3 +90,37 @@ redis: redisImage: redis redisImageTag: '5.0' containerPort: 6379 + +extraFileMappings: + /root/.rclone.conf: + useSecret: true + applyToRclone: true + applyToManager: false + tpl: false + content: | + [azurebuckets] + sas_url = https://bioconductorhubs.blob.core.windows.net/bucketname?sp=rl&sas-secret + type = azureblob + + [osn] + type = s3 + provider = AWS + access_key_id = accesskey + secret_access_key = secretkey + endpoint = https://mghp.osn.xsede.org + acl = private + no_check_bucket = true + + [awsopendata] + type = s3 + provider = AWS + access_key_id = accesskey + secret_access_key = secretkey + acl = private + + [gcpbuckets] + type = google cloud storage + service_account_file = /path/to/service_account + project_number = 12345678 + object_acl = private + bucket_acl = private From dff210514a88c92bec9bb4af80821566ab3ee3f4 Mon Sep 17 00:00:00 2001 From: Alexandru Mahmoud Date: Wed, 4 May 2022 22:02:51 -0400 Subject: [PATCH 2/2] Fix indentations --- .../templates/manager-deployment.yaml | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/inst/helm-chart/templates/manager-deployment.yaml b/inst/helm-chart/templates/manager-deployment.yaml index b4b64c5..5728934 100644 --- a/inst/helm-chart/templates/manager-deployment.yaml +++ b/inst/helm-chart/templates/manager-deployment.yaml @@ -34,13 +34,13 @@ spec: {{- end -}} {{ tpl .Values.manager.defaultCommand . }} containers: - - name: osn-rclone-copy - image: rclone/rclone:latest - command: - - /bin/sh - - -c - args: - - rclone copyto /host/binary_{{ .Values.biocVersion | replace "." "_" }} osn:bir190004-bucket01/bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker -v && rclone copyto /host/library_{{ .Values.biocVersion | replace "." "_" }} osn:bir190004-bucket01/bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/contrib -v && rclone copyto /host/logs_{{ .Values.biocVersion | replace "." "_" }} osn:bir190004-bucket01/bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/package_logs -v + - name: osn-rclone-copy + image: rclone/rclone:latest + command: + - /bin/sh + - -c + args: + - rclone copyto /host/binary_{{ .Values.biocVersion | replace "." "_" }} osn:bir190004-bucket01/bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker -v && rclone copyto /host/library_{{ .Values.biocVersion | replace "." "_" }} osn:bir190004-bucket01/bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/contrib -v && rclone copyto /host/logs_{{ .Values.biocVersion | replace "." "_" }} osn:bir190004-bucket01/bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/package_logs -v volumeMounts: - name: nfs-data mountPath: {{ .Values.persistence.mountPath }} @@ -51,17 +51,17 @@ spec: subPath: {{ include "biockubeinstall.getFilenameFromPath" $key }} {{- end }} {{- end }} - resources: - requests: - memory: 100Mi - cpu: 100m - - name: azure-rclone-copy - image: rclone/rclone:latest - command: - - /bin/sh - - -c - args: - - rclone copyto /host/binary_{{ .Values.biocVersion | replace "." "_" }} azurebuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker -v && rclone copyto /host/library_{{ .Values.biocVersion | replace "." "_" }} azurebuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/contrib -v && rclone copyto /host/logs_{{ .Values.biocVersion | replace "." "_" }} azurebuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/package_logs -v + resources: + requests: + memory: 100Mi + cpu: 100m + - name: azure-rclone-copy + image: rclone/rclone:latest + command: + - /bin/sh + - -c + args: + - rclone copyto /host/binary_{{ .Values.biocVersion | replace "." "_" }} azurebuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker -v && rclone copyto /host/library_{{ .Values.biocVersion | replace "." "_" }} azurebuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/contrib -v && rclone copyto /host/logs_{{ .Values.biocVersion | replace "." "_" }} azurebuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/package_logs -v volumeMounts: - name: nfs-data mountPath: {{ .Values.persistence.mountPath }} @@ -72,17 +72,17 @@ spec: subPath: {{ include "biockubeinstall.getFilenameFromPath" $key }} {{- end }} {{- end }} - resources: - requests: - memory: 100Mi - cpu: 100m - - name: gcp-rclone-copy - image: rclone/rclone:latest - command: - - /bin/sh - - -c - args: - - rclone copyto /host/binary_{{ .Values.biocVersion | replace "." "_" }} gcpbuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker -v && rclone copyto /host/library_{{ .Values.biocVersion | replace "." "_" }} gcpbuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/contrib -v && rclone copyto /host/logs_{{ .Values.biocVersion | replace "." "_" }} gcpbuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/package_logs -v + resources: + requests: + memory: 100Mi + cpu: 100m + - name: gcp-rclone-copy + image: rclone/rclone:latest + command: + - /bin/sh + - -c + args: + - rclone copyto /host/binary_{{ .Values.biocVersion | replace "." "_" }} gcpbuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker -v && rclone copyto /host/library_{{ .Values.biocVersion | replace "." "_" }} gcpbuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/contrib -v && rclone copyto /host/logs_{{ .Values.biocVersion | replace "." "_" }} gcpbuckets:bioconductor-packages/{{ .Values.biocVersion }}/container-binaries/bioconductor_docker/src/package_logs -v volumeMounts: - name: nfs-data mountPath: {{ .Values.persistence.mountPath }} @@ -93,16 +93,16 @@ spec: subPath: {{ include "biockubeinstall.getFilenameFromPath" $key }} {{- end }} {{- end }} - resources: - requests: - memory: 100Mi - cpu: 100m + resources: + requests: + memory: 100Mi + cpu: 100m restartPolicy: {{ .Values.restartPolicy | quote}} volumes: - name: nfs-data persistentVolumeClaim: claimName: {{.Release.Namespace}}-nfs - {{- range $key, $entry := .Values.extraFileMappings -}} + {{- range $key, $entry := .Values.extraFileMappings }} - name: {{ include "biockubeinstall.getExtraFilesUniqueName" $key }} {{- if $entry.useSecret }} secret: