diff --git a/data-archive/helm/data-archive/.gitignore b/data-archive/helm/data-archive/.gitignore deleted file mode 100644 index ee3892e8..00000000 --- a/data-archive/helm/data-archive/.gitignore +++ /dev/null @@ -1 +0,0 @@ -charts/ diff --git a/data-archive/helm/data-archive/.helmignore b/data-archive/helm/data-archive/.helmignore deleted file mode 100644 index 35e62f99..00000000 --- a/data-archive/helm/data-archive/.helmignore +++ /dev/null @@ -1,24 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ -.chart.lock diff --git a/data-archive/helm/data-archive/Chart.yaml b/data-archive/helm/data-archive/Chart.yaml deleted file mode 100644 index 2a706d0e..00000000 --- a/data-archive/helm/data-archive/Chart.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v2 -name: data-archive -description: A Helm chart for Kubernetes to initialize mosip_archive Postgres DB and execute py script to archive data from source db to archive db -type: application -version: 0.0.1-develop -dependencies: - - name: common - repository: https://charts.bitnami.com/bitnami - tags: - - bitnami-common - version: 1.x.x -home: https://mosip.io -keywords: - - postgres - - db - - database -maintainers: - - email: info@mosip.io - name: MOSIP diff --git a/data-archive/helm/data-archive/templates/_helpers.tpl b/data-archive/helm/data-archive/templates/_helpers.tpl deleted file mode 100644 index 6b92a3a0..00000000 --- a/data-archive/helm/data-archive/templates/_helpers.tpl +++ /dev/null @@ -1,63 +0,0 @@ -{{/* -Return the proper image name -*/}} -{{- define "data-archive.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }} -{{- end -}} - -{{/* -Return the proper image name (for the init container volume-permissions image) -*/}} -{{- define "data-archive.volumePermissions.image" -}} -{{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global ) -}} -{{- end -}} - -{{/* -Return the proper Docker Image Registry Secret Names -*/}} -{{- define "data-archive.imagePullSecrets" -}} -{{- include "common.images.pullSecrets" (dict "images" (list .Values.image .Values.volumePermissions.image) "global" .Values.global) -}} -{{- end -}} - -{{/* -Create the name of the service account to use -*/}} -{{- define "data-archive.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (printf "%s-foo" (include "common.names.fullname" .)) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} -{{- end -}} - -{{/* -Compile all warnings into a single message. -*/}} -{{- define "data-archive.validateValues" -}} -{{- $messages := list -}} -{{- $messages := append $messages (include "data-archive.validateValues.foo" .) -}} -{{- $messages := append $messages (include "data-archive.validateValues.bar" .) -}} -{{- $messages := without $messages "" -}} -{{- $message := join "\n" $messages -}} - -{{- if $message -}} -{{- printf "\nVALUES VALIDATION:\n%s" $message -}} -{{- end -}} -{{- end -}} - -{{/* -Return podAnnotations -*/}} -{{- define "data-archive.podAnnotations" -}} -{{- if .Values.podAnnotations }} -{{ include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) }} -{{- end }} -{{- if and .Values.metrics.enabled .Values.metrics.podAnnotations }} -{{ include "common.tplvalues.render" (dict "value" .Values.metrics.podAnnotations "context" $) }} -{{- end }} -{{- end -}} - -{{/* Create the name for restart cronjob */}} -{{- define "data-archive.cronjob" -}} -{{ default (printf "cronjob-%s" (include "common.names.fullname" .)) .Values.serviceAccount.name }} -{{- end -}} \ No newline at end of file diff --git a/data-archive/helm/data-archive/templates/all-db-tables-info-configmap.yaml b/data-archive/helm/data-archive/templates/all-db-tables-info-configmap.yaml deleted file mode 100644 index e34b2cc6..00000000 --- a/data-archive/helm/data-archive/templates/all-db-tables-info-configmap.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }} - labels: - {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: your-component-name - {{- if .Values.commonLabels }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- end }} - {{- if .Values.commonAnnotations }} - annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} -data: - audit_archive_table_info: {{ .Values.databases.all_db_tables_info.audit | toJson | quote }} - credential_archive_table_info: {{ .Values.databases.all_db_tables_info.credential | toJson | quote }} - esignet_archive_table_info: {{ .Values.databases.all_db_tables_info.esignet | toJson | quote }} - ida_archive_table_info: {{ .Values.databases.all_db_tables_info.ida | toJson | quote }} - idrepo_archive_table_info: {{ .Values.databases.all_db_tables_info.idrepo | toJson | quote }} - kernel_archive_table_info: {{ .Values.databases.all_db_tables_info.kernel | toJson | quote }} - master_archive_table_info: {{ .Values.databases.all_db_tables_info.master | toJson | quote }} - pms_archive_table_info: {{ .Values.databases.all_db_tables_info.pms | toJson | quote }} - prereg_archive_table_info: {{ .Values.databases.all_db_tables_info.prereg | toJson | quote }} - regprc_archive_table_info: {{ .Values.databases.all_db_tables_info.regprc | toJson | quote }} - resident_archive_table_info: {{ .Values.databases.all_db_tables_info.resident | toJson | quote }} \ No newline at end of file diff --git a/data-archive/helm/data-archive/templates/archive-db-secret.yaml b/data-archive/helm/data-archive/templates/archive-db-secret.yaml deleted file mode 100644 index ee4c424e..00000000 --- a/data-archive/helm/data-archive/templates/archive-db-secret.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: archive-db-secrets - namespace: {{ .Release.Namespace }} - labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: postgres - {{- if .Values.commonLabels }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- end }} - {{- if .Values.commonAnnotations }} - annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} -type: Opaque -data: - SU_USER_PWD: {{ .Values.databases.archive_db.su_user_pwd | b64enc | quote }} - DBUSER_PWD: {{ .Values.databases.archive_db.db_pwd | b64enc | quote }} - ARCHIVE_DB_PASS: {{ .Values.databases.archive_db.archive_db_password | b64enc | quote }} diff --git a/data-archive/helm/data-archive/templates/archive-env-configmap.yaml b/data-archive/helm/data-archive/templates/archive-env-configmap.yaml deleted file mode 100644 index af125dd0..00000000 --- a/data-archive/helm/data-archive/templates/archive-env-configmap.yaml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: db-archive-init-env-config - namespace: {{ .Release.Namespace }} - labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: postgres - {{- if .Values.commonLabels }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- end }} - {{- if .Values.commonAnnotations }} - annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} -data: - MOSIP_DB_NAME: {{ .Values.databases.archive_db.db_name | quote }} - SU_USER: {{ .Values.databases.archive_db.su_user | quote }} - DB_SERVERIP: {{ .Values.databases.archive_db.host | quote }} - DB_PORT: {{ .Values.databases.archive_db.port | quote }} - DML_FLAG: {{ .Values.databases.archive_db.dml | quote }} - BATCH_SIZE: {{ .Values.databases.archive_db.batch_size | quote }} - AUDIT_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_audit_host | quote }} - AUDIT_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_audit_port | quote }} - AUDIT_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_audit_uname | quote }} - AUDIT_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_audit_dbname | quote }} - AUDIT_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_audit_schemaname | quote }} - CREDENTIAL_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_credential_host | quote }} - CREDENTIAL_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_credential_port | quote }} - CREDENTIAL_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_credential_uname | quote }} - CREDENTIAL_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_credential_dbname | quote }} - CREDENTIAL_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_credential_schemaname | quote }} - ESIGNET_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_esignet_host | quote }} - ESIGNET_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_esignet_port | quote }} - ESIGNET_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_esignet_uname | quote }} - ESIGNET_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_esignet_dbname | quote }} - ESIGNET_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_esignet_schemaname | quote }} - IDA_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_ida_host | quote }} - IDA_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_ida_port | quote }} - IDA_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_ida_uname | quote }} - IDA_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_ida_dbname | quote }} - IDA_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_ida_schemaname | quote }} - IDREPO_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_idrepo_host | quote }} - IDREPO_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_idrepo_port | quote }} - IDREPO_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_idrepo_uname | quote }} - IDREPO_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_idrepo_dbname | quote }} - IDREPO_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_idrepo_schemaname | quote }} - KERNEL_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_kernel_host | quote }} - KERNEL_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_kernel_port | quote }} - KERNEL_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_kernel_uname | quote }} - KERNEL_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_kernel_dbname | quote }} - KERNEL_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_kernel_schemaname | quote }} - MASTER_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_master_host | quote }} - MASTER_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_master_port | quote }} - MASTER_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_master_uname | quote }} - MASTER_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_master_dbname | quote }} - MASTER_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_master_schemaname | quote }} - PMS_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_pms_host | quote }} - PMS_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_pms_port | quote }} - PMS_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_pms_uname | quote }} - PMS_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_pms_dbname | quote }} - PMS_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_pms_schemaname | quote }} - PREREG_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_prereg_host | quote }} - PREREG_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_prereg_port | quote }} - PREREG_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_prereg_uname | quote }} - PREREG_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_prereg_dbname | quote }} - PREREG_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_prereg_schemaname | quote }} - REGPRC_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_regprc_host | quote }} - REGPRC_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_regprc_port | quote }} - REGPRC_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_regprc_uname | quote }} - REGPRC_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_regprc_dbname | quote }} - REGPRC_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_regprc_schemaname | quote }} - RESIDENT_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_resident_host | quote }} - RESIDENT_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_resident_port | quote }} - RESIDENT_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_resident_uname | quote }} - RESIDENT_SOURCE_DB_NAME: {{ .Values.databases.source_db.source_resident_dbname | quote }} - RESIDENT_SOURCE_SCHEMA_NAME: {{ .Values.databases.source_db.source_resident_schemaname | quote }} - ARCHIVE_DB_HOST: {{ .Values.databases.archive_db.archivehost | quote }} - ARCHIVE_DB_PORT: {{ .Values.databases.archive_db.archiveport | quote }} - ARCHIVE_DB_UNAME: {{ .Values.databases.archive_db.archiveuname | quote }} - ARCHIVE_DB_NAME: {{ .Values.databases.archive_db.archive_dbname | quote }} - ARCHIVE_SCHEMA_NAME: {{ .Values.databases.archive_db.archive_schemaname | quote }} \ No newline at end of file diff --git a/data-archive/helm/data-archive/templates/cronjob.yaml b/data-archive/helm/data-archive/templates/cronjob.yaml deleted file mode 100644 index db092a0f..00000000 --- a/data-archive/helm/data-archive/templates/cronjob.yaml +++ /dev/null @@ -1,55 +0,0 @@ -{{- range $dbnames := $.Values.databases.db_names }} -{{- if $dbnames.enabled }} ---- -apiVersion: {{ include "common.capabilities.cronjob.apiVersion" $ }} -kind: CronJob -metadata: - name: {{ template "data-archive.cronjob" $ }}-{{ $dbnames.name }} - namespace: {{ $.Release.Namespace }} -spec: - concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 1 # remove jobs which are successfully executed - failedJobsHistoryLimit: 1 # except 1 recent failed job, remove jobs which are not successfully executed - schedule: {{ $.Values.crontime }} - jobTemplate: - spec: - backoffLimit: 2 # this has very low chance of failing, as all this does - # is prompt kubernetes to schedule new replica set for - # the deployment - # activeDeadlineSeconds: 600 # timeout, makes most sense with - # "waiting for rollout" variant specified below - template: - spec: - restartPolicy: Never - serviceAccountName: {{ template "data-archive.serviceAccountName" $ }} - containers: - - name: {{ template "data-archive.serviceAccountName" $ }}-{{ $dbnames.name }} - image: {{ template "data-archive.image" $ }} - imagePullPolicy: {{ $.Values.image.pullPolicy }} - volumeMounts: - - name: config-volume-{{ $dbnames.name }} # Unique volume mount path for each job - mountPath: {{ $dbnames.container_volume_path | quote }} # Dynamic volume path from values.yaml - env: - - name: CONTAINER_VOLUME_PATH - value: {{ $dbnames.container_volume_path | quote }} # Set dynamic CONTAINER_VOLUME_PATH - - name: container_user - value: {{ $.Values.containerSecurityContext.runAsUser }} - - name: DB_NAMES - value: {{ $dbnames.name | upper }} - envFrom: - - configMapRef: - name: db-archive-init-env-config - - secretRef: - name: source-db-secrets - - secretRef: - name: archive-db-secrets - {{- if $.Values.resources }} - resources: {{- toYaml $.Values.resources | nindent 14 }} - {{- end }} - volumes: - - name: config-volume-{{ $dbnames.name }} # Unique config-volume per job - configMap: - name: {{ $.Release.Name }} - -{{- end }} -{{- end }} \ No newline at end of file diff --git a/data-archive/helm/data-archive/templates/serviceaccount.yaml b/data-archive/helm/data-archive/templates/serviceaccount.yaml deleted file mode 100644 index cedd6dc6..00000000 --- a/data-archive/helm/data-archive/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: {{- include "common.labels.standard" . | nindent 4 }} - {{- if .Values.commonLabels }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- end }} - name: {{ template "data-archive.serviceAccountName" . }} - {{- if .Values.commonAnnotations }} - annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} - namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/data-archive/helm/data-archive/templates/source-db-secret.yaml b/data-archive/helm/data-archive/templates/source-db-secret.yaml deleted file mode 100644 index 2ed02932..00000000 --- a/data-archive/helm/data-archive/templates/source-db-secret.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: source-db-secrets - namespace: {{ .Release.Namespace }} - labels: {{- include "common.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: postgres - {{- if .Values.commonLabels }} - {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }} - {{- end }} - {{- if .Values.commonAnnotations }} - annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} - {{- end }} -type: Opaque -data: - AUDIT_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_audit_db_pass | b64enc | quote }} - CREDENTIAL_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_credential_db_pass | b64enc | quote }} - ESIGNET_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_esignet_db_pass | b64enc | quote }} - IDA_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_ida_db_pass | b64enc | quote }} - IDREPO_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_idrepo_db_pass | b64enc | quote }} - KERNEL_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_kernel_db_pass | b64enc | quote }} - MASTER_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_master_db_pass | b64enc | quote }} - PMS_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_pms_db_pass | b64enc | quote }} - PREREG_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_prereg_db_pass | b64enc | quote }} - REGPRC_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_regprc_db_pass | b64enc | quote }} - RESIDENT_SOURCE_DB_PASS: {{ .Values.databases.source_db.source_resident_db_pass | b64enc | quote }} \ No newline at end of file diff --git a/data-archive/helm/data-archive/values.yaml b/data-archive/helm/data-archive/values.yaml deleted file mode 100644 index 66f999e8..00000000 --- a/data-archive/helm/data-archive/values.yaml +++ /dev/null @@ -1,610 +0,0 @@ -## Global Docker image parameters -## Please, note that this will override the image parameters, including dependencies, configured to use the global value -## Current available global Docker image parameters: imageRegistry and imagePullSecrets -## -# global: -# imageRegistry: myRegistryName -# imagePullSecrets: -# - myRegistryKeySecretName -# storageClass: myStorageClass - -## Add labels to all the deployed resources -## -commonLabels: - app.kubernetes.io/component: mosip - -## Add annotations to all the deployed resources -## -commonAnnotations: {} - -## Kubernetes Cluster Domain -## -clusterDomain: cluster.local - -## Extra objects to deploy (value evaluated as a template) -## -extraDeploy: [] - -## Number of nodes -## -replicaCount: 1 - -image: - registry: docker.io - repository: mosipdev/data-archive - tag: develop - - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: Always - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ - ## - # pullSecrets: - # - myRegistryKeySecretName - -## -# existingConfigmap: - -## Command and args for running the container (set to default if not set). Use array form -## -command: [] -args: [] - -## Deployment pod host aliases -## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ -## -hostAliases: [] - -## ref: http://kubernetes.io/docs/user-guide/compute-resources/ -## -resources: - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - limits: - cpu: 1000m - memory: 1000Mi - requests: - cpu: 300m - memory: 700Mi - -# additionalResources: - ## Specify any JAVA_OPTS string here. These typically will be specified in conjunction with above resources - ## Example: java_opts: "-Xms500M -Xmx500M" - # javaOpts: "-Xms2600M -Xmx2600M" - -## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container -## Clamav container already runs as 'mosip' user, so we may not need to enable this -containerSecurityContext: - enabled: false - runAsUser: mosip - runAsNonRoot: true - -## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod -## -podSecurityContext: - enabled: false - fsGroup: 1001 - -## Pod affinity preset -## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity -## Allowed values: soft, hard -## -podAffinityPreset: "" - -## Pod anti-affinity preset -## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity -## Allowed values: soft, hard -## -podAntiAffinityPreset: soft - -## Node affinity preset -## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity -## Allowed values: soft, hard -## -nodeAffinityPreset: - ## Node affinity type - ## Allowed values: soft, hard - ## - type: "" - ## Node label key to match - ## E.g. - ## key: "kubernetes.io/e2e-az-name" - ## - key: "" - ## Node label values to match - ## E.g. - ## values: - ## - e2e-az1 - ## - e2e-az2 - ## - values: [] - -## Affinity for pod assignment. Evaluated as a template. -## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity -## -affinity: {} - -## Node labels for pod assignment. Evaluated as a template. -## ref: https://kubernetes.io/docs/user-guide/node-selection/ -## -## Some common set of users are created across all postgres servers with same password. TODO: change this later. -## These user names are currently hardcoded in the sql scripts, so can't change from here. - -nodeSelector: {} - -## Tolerations for pod assignment. Evaluated as a template. -## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ -## -tolerations: [] - -## Pod extra labels -## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ -## -podLabels: {} - -## Annotations for server pods. -## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ -## -podAnnotations: {} - -## pods' priority. -## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ -## -# priorityClassName: "" - -## lifecycleHooks for the container to automate configuration before or after startup. -## -lifecycleHooks: {} - -## Custom Liveness probes for -## -customLivenessProbe: {} - -## Custom Rediness probes -## -customReadinessProbe: {} - -## Update strategy - only really applicable for deployments with RWO PVs attached -## If replicas = 1, an update can get "stuck", as the previous pod remains attached to the -## PV, and the "incoming" pod can never start. Changing the strategy to "Recreate" will -## terminate the single previous pod, so that the new, incoming pod can attach to the PV -## -updateStrategy: - type: RollingUpdate - -## Additional environment variables to set -## Example: -## extraEnvVars: -## - name: FOO -## value: "bar" -## -extraEnvVars: [] - -## ConfigMap with extra environment variables -## -# extraEnvVarsCM: - # - s3 - -## Secret with extra environment variables -## -# extraEnvVarsSecret: - # - s3 - -## Extra volumes to add to the deployment -## -extraVolumes: [] - -## Extra volume mounts to add to the container -## -extraVolumeMounts: [] - -## Add init containers to the pods. -## Example: -## initContainers: -## - name: your-image-name -## image: your-image -## imagePullPolicy: Always -## ports: -## - name: portname -## containerPort: 1234 -## -## initContainers: {} - -## Add sidecars to the pods. -## Example: -## sidecars: -## - name: your-image-name -## image: your-image -## imagePullPolicy: Always -## ports: -## - name: portname -## containerPort: 1234 -## -sidecars: {} - -persistence: - enabled: false - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. (gp2 on AWS, standard on - ## GKE, AWS & OpenStack). - ## - # storageClass: "-" - ## - ## If you want to reuse an existing claim, you can pass the name of the PVC using - ## the existingClaim variable - # existingClaim: your-claim - ## ReadWriteMany not supported by AWS gp2 - storageClass: - accessModes: - - ReadWriteOnce - size: 10M - existingClaim: - # Dir where config and keys are written inside container - mountDir: - -## Init containers parameters: -## volumePermissions: Change the owner and group of the persistent volume mountpoint to runAsUser:fsGroup values from the securityContext section. -## -volumePermissions: - enabled: false - image: - registry: docker.io - repository: bitnami/bitnami-shell - tag: "10" - pullPolicy: Always - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ - ## - pullSecrets: [] - ## - myRegistryKeySecretName - ## Init containers' resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## - resources: - ## We usually recommend not to specify default resources and to leave this as a conscious - ## choice for the user. This also increases chances charts run on environments with little - ## resources, such as Minikube. If you do want to specify resources, uncomment the following - ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. - ## - # limits: {} - ## cpu: 100m - ## memory: 128Mi - ## - # requests: {} - ## cpu: 100m - ## memory: 128Mi - ## - -## Specifies whether RBAC resources should be created -## -rbac: - create: true - -## Specifies whether a ServiceAccount should be created -## -serviceAccount: - create: true - ## The name of the ServiceAccount to use. - ## If not set and create is true, a name is generated using the fullname template - ## - name: - - -crontime: "0 4 * * *" ## run cronjob every day at 4 AM in GMT which is 9.30 AM in IST (time hr: 0-23 ) - -databases: - archive_db: - db_name: "mosip_archive" - host: "api-internal.sandbox.xyz.net" - port: 5432 - su_user: "postgres" - su_user_pwd: "" - db_pwd: "" - dml: 0 - archivehost: "api-internal.sandbox.xyz.net" - archiveport: 5432 - archiveuname: "archiveuser" - archive_dbname: "mosip_archive" - archive_schemaname: "archive" - archive_db_password: "" - batch_size: 1000 - - db_names: - - name: audit - enabled: true - container_volume_path: /audit-table-info ## volume mount path for table info inside cronjob container - - name: ida - enabled: true - container_volume_path: /ida-table-info ## volume mount path for table info inside cronjob container - - name: credential - enabled: true - container_volume_path: /credential-table-info ## volume mount path for table info inside cronjob container - - name: kernel - enabled: true - container_volume_path: /kernek-table-info ## volume mount path for table info inside cronjob container - - name: master - enabled: true - container_volume_path: /master-table-info ## volume mount path for table info inside cronjob container - - name: pms - enabled: true - container_volume_path: /pms-table-info ## volume mount path for table info inside cronjob container - - name: regprc - enabled: true - container_volume_path: /regprc-table-info ## volume mount path for table info inside cronjob container - - name: resident - enabled: true - container_volume_path: /resident-table-info ## volume mount path for table info inside cronjob container - - source_db: - source_audit_host: "api-internal.sandbox.xyz.net" - source_audit_port: 5432 - source_audit_uname: "audituser" - source_audit_dbname: "mosip_audit" - source_audit_schemaname: "audit" - source_audit_db_pass: "" - source_credential_host: "api-internal.sandbox.xyz.net" - source_credential_port: 5432 - source_credential_uname: "credentialuser" - source_credential_dbname: "mosip_credential" - source_credential_schemaname: "credential" - source_credential_db_pass: "" - source_esignet_host: "api-internal.sandbox.xyz.net" - source_esignet_port: 5432 - source_esignet_uname: "esignetuser" - source_esignet_dbname: "mosip_esignet" - source_esignet_schemaname: "esignet" - source_esignet_db_pass: "" - source_ida_host: "api-internal.sandbox.xyz.net" - source_ida_port: 5432 - source_ida_uname: "idauser" - source_ida_dbname: "mosip_ida" - source_ida_schemaname: "ida" - source_ida_db_pass: "" - source_idrepo_host: "api-internal.sandbox.xyz.net" - source_idrepo_port: 5432 - source_idrepo_uname: "idrepouser" - source_idrepo_dbname: "mosip_idrepo" - source_idrepo_schemaname: "idrepo" - source_idrepo_db_pass: "" - source_kernel_host: "api-internal.sandbox.xyz.net" - source_kernel_port: 5432 - source_kernel_uname: "kerneluser" - source_kernel_dbname: "mosip_kernel" - source_kernel_schemaname: "kernel" - source_kernel_db_pass: "" - source_master_host: "api-internal.sandbox.xyz.net" - source_master_port: 5432 - source_master_uname: "masteruser" - source_master_dbname: "mosip_master" - source_master_schemaname: "master" - source_master_db_pass: "" - source_pms_host: "api-internal.sandbox.xyz.net" - source_pms_port: 5432 - source_pms_uname: "pmsuser" - source_pms_dbname: "mosip_pms" - source_pms_schemaname: "pms" - source_pms_db_pass: "" - source_prereg_host: "api-internal.sandbox.xyz.net" - source_prereg_port: 5432 - source_prereg_uname: "prereguser" - source_prereg_dbname: "mosip_prereg" - source_prereg_schemaname: "prereg" - source_prereg_db_pass: "" - source_regprc_host: "api-internal.sandbox.xyz.net" - source_regprc_port: 5432 - source_regprc_uname: "regprcuser" - source_regprc_dbname: "mosip_regprc" - source_regprc_schemaname: "regprc" - source_regprc_db_pass: "" - source_resident_host: "api-internal.sandbox.xyz.net" - source_resident_port: 5432 - source_resident_uname: "residentuser" - source_resident_dbname: "mosip_resident" - source_resident_schemaname: "resident" - source_resident_db_pass: "" - all_db_tables_info: - audit: - tables_info: - - source_table: "app_audit_log" - archive_table: "mosip_audit_app_audit_log" - id_column: "log_id" - date_column: "log_dtimes" - retention_days: 30 - operation_type: "archive_delete" - credential: - tables_info: - - source_table: "credential_transaction" - archive_table: "mosip_credential_credential_transaction" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 30 - operation_type: "archive_delete" - esignet: - tables_info: - - source_table: "consent_history" - archive_table: "mosip_esignet_consent_history" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 30 - operation_type: "none" - ida: - tables_info: - - source_table: "credential_event_store" - archive_table: "mosip_ida_credential_event_store" - id_column: "event_id" - date_column: "cr_dtimes" - retention_days: 30 - operation_type: "archive_delete" - - source_table: "otp_transaction" - archive_table: "mosip_ida_otp_transaction" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 30 - operation_type: 'delete' - idrepo: - tables_info: - - source_table: "anonymous_profile" - archive_table: "mosip_idrepo_anonymous_profile" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 30 - operation_type: "archive_delete" - - source_table: "credential_request_status" - archive_table: "mosip_idrepo_credential_request_status" - id_column: "individual_id" - date_column: "cr_dtimes" - retention_days: 30 - operation_type: "archive_delete" - - source_table: "uin_draft" - archive_table: "mosip_idrepo_uin_draft" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 30 - operation_type: "archive_delete" - kernel: - tables_info: - - source_table: "otp_transaction" - archive_table: "mosip_kernel_otp_transaction" - id_column: "id" - date_column: "generated_dtimes" - retention_days: 7 - operation_type: "delete" - master: - tables_info: - - source_table: "bulkupload_transaction" - archive_table: "mosip_master_bulkupload_transaction" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 91 - operation_type: "archive_delete" - - source_table: "device_master_h" - archive_table: "mosip_master_device_master_h" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 365 - operation_type: "archive_delete" - - source_table: "machine_master_h" - archive_table: "mosip_master_machine_master_h" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 183 - operation_type: "archive_delete" - - source_table: "registration_center_h" - archive_table: "mosip_master_registration_center_h" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 365 - operation_type: "archive_delete" - - source_table: "user_detail_h" - archive_table: "mosip_master_user_detail_h" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 183 - operation_type: "archive_delete" - - source_table: "zone_user_h" - archive_table: "mosip_master_zone_user_h" - id_column: "usr_id" - date_column: "cr_dtimes" - retention_days: 183 - operation_type: "archive_delete" - pms: - tables_info: - - source_table: "auth_policy_h" - archive_table: "mosip_pms_auth_policy_h" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 183 - operation_type: "archive_delete" - - source_table: "secure_biometric_interface_h" - archive_table: "mosip_pms_secure_biometric_interface_h" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 183 - operation_type: "archive_delete" - - source_table: "partner_h" - archive_table: "mosip_pms_partner_h" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 183 - operation_type: "archive_delete" - prereg: - tables_info: - - source_table: "otp_transaction" - archive_table: "mosip_prereg_otp_transaction" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 30 - operation_type: "delete" - regprc: - tables_info: - - source_table: "abis_response_det" - archive_table: "mosip_regprc_abis_response_det" - id_column: "abis_resp_id" - date_column: "cr_dtimes" - retention_days: 183 - operation_type: "archive_delete" - - source_table: "abis_response" - archive_table: "mosip_regprc_abis_response" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 183 - operation_type: "archive_delete" - - source_table: "abis_request" - archive_table: "mosip_regprc_abis_request" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 183 - operation_type: "archive_delete" - - source_table: "reg_demo_dedupe_list" - archive_table: "mosip_regprc_reg_demo_dedupe_list" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 183 - operation_type: "archive_delete" - - source_table: "registration_transaction" - archive_table: "mosip_regprc_registration_transaction" - id_column: "regtrn_id" - date_column: "cr_dtimes" - retention_days: 183 - operation_type: "archive_delete" - resident: - tables_info: - - source_table: "otp_transaction" - archive_table: "mosip_resident_otp_transaction" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 30 - operation_type: "delete" - - source_table: "resident_grievance_ticket" - archive_table: "mosip_resident_grievance_ticket" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 365 - operation_type: "archive_delete" - - source_table: "resident_session" - archive_table: "mosip_resident_session" - id_column: "session_id" - date_column: "login_dtimes" - retention_days: 30 - operation_type: "archive_delete" - - source_table: "resident_transaction" - archive_table: "mosip_resident_transaction" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 365 - operation_type: "archive_delete" - - source_table: "resident_user_actions" - archive_table: "mosip_resident_user_actions" - id_column: "ida_token" - date_column: "last_bell_notif_click_dtimes" - retention_days: 365 - operation_type: "archive_delete" diff --git a/data-archive/deploy/README.md b/deploy/README.md similarity index 100% rename from data-archive/deploy/README.md rename to deploy/README.md diff --git a/deploy/data-archive/README.md b/deploy/data-archive/README.md deleted file mode 100644 index cb97246c..00000000 --- a/deploy/data-archive/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Database Archiving Configuration - -This configuration file is used for setting up database connections and defining archiving parameters. Please follow the guidelines below to fill in the required information. - -## Database Connections - -### Archive Database Connection (archive_db) - -- `db_name`: Name of the archive database. -- `host`: Destination host for the archive database. -- `port`: Port number for the archive database connection. -- `su_user`: Superuser for the archive database. -- `su_user_pwd`: Password for the superuser. -- `db_pwd`: Password for the archive database. -- `archivehost`: Destination host for the archive database. -- `archiveport`: Port number for the archive database connection. -- `archiveuname`: Archive database username. -- `archive_dbname`: Archive database name. -- `archive_schemaname`: Archive schema name. -- `archive_db_password`: Password for the archive database. - -### Source Database Connections (source_db) - -For each source database (audit, credential, esignet, ida, idrepo, kernel, master, pms, prereg, regprc, resident), provide the following information: - -- `source__host`: Source database host. -- `source__port`: Port number for the source database connection. -- `source__uname`: Source database username. -- `source__dbname`: Source database name. -- `source__schemaname`: Source schema name. -- `source__db_pass`: Password for the source database. - -- `provide_db_names_to_archive`: Comma-separated list of database names to archive (e.g., "AUDIT,CREDENTIAL,IDA,.....").(in CAPS) - - -## Container Volume Path -container_volume_path: Path where JSON files containing information about all databases will be stored - -## Archiving Information (all_db_tables_info) - -For each database, specify tables_info with details for archiving. Example: - -```yaml -audit: - tables_info: - - source_table: "app_audit_log" - archive_table: "mosip_audit_app_audit_log" - id_column: "log_id" - date_column: "log_dtimes" - retention_days: 30 - operation_type: "archive_delete" - -source_table: Name of the table in the source database. -archive_table: Name of the table in the archive database. -id_column: Column representing the unique identifier. -date_column: Column representing the date of the record. -retention_days: Number of days to retain the archived data. -operation_type: Type of operation for archiving (e.g., archive_delete, delete, none). -- Delete: Delete records from the source table. -- Archive and Delete: Archive records to an archive table and then delete them from the source table. -- Archive (No Delete): Archive records to an archive table without deleting them from the source table. -- None: Skip archival for the specified table. diff --git a/deploy/data-archive/delete.sh b/deploy/data-archive/delete.sh deleted file mode 100755 index 521fad2e..00000000 --- a/deploy/data-archive/delete.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Uninstalls data-archive -## Usage: ./delete.sh [kubeconfig] - -if [ $# -ge 1 ] ; then - export KUBECONFIG=$1 -fi - -function deleting_data-archive() { - NS=data-archive - while true; do - read -p "Are you sure you want to delete data-archive helm charts?(Y/n) " yn - if [ $yn = "Y" ] - then - helm -n $NS delete data-archive - break - else - break - fi - done - return 0 -} - -# set commands for error handling. -set -e -set -o errexit ## set -e : exit the script if any statement returns a non-true return value -set -o nounset ## set -u : exit the script if you try to use an uninitialised variable -set -o errtrace # trace ERR through 'time command' and other functions -set -o pipefail # trace ERR through pipes -deleting_data-archive # calling function diff --git a/deploy/data-archive/install.sh b/deploy/data-archive/install.sh deleted file mode 100755 index d54e8086..00000000 --- a/deploy/data-archive/install.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# Installs data-archive -## Usage: ./install.sh [kubeconfig] - -if [ $# -ge 1 ]; then - export KUBECONFIG=$1 -fi - -NS=data-archive -CHART_VERSION=0.0.1-develop - -echo Create $NS namespace -kubectl create ns $NS - -function installing_data-archive() { - echo Updating repos - helm repo add mosip https://mosip.github.io/mosip-helm - helm repo update - - read -p "Is values.yaml for data-archive chart set correctly as part of Pre-requisites?(Y/n) " yn; - if [ "$yn" != "Y" ]; then - echo "ERROR: values.yaml not set correctly; EXITING;"; - exit 1; - fi - - read -p "Please enter the time(hr) to run the cronjob every day (time: 0-23) : " time - if [ -z "$time" ]; then - echo "ERROR: Time cannot be empty; EXITING;"; - exit 1; - fi - if ! [ $time -eq $time ] 2>/dev/null; then - echo "ERROR: Time $time is not a number; EXITING;"; - exit 1; - fi - if [ $time -gt 23 ] || [ $time -lt 0 ]; then - echo "ERROR: Time should be in range ( 0-23 ); EXITING;"; - exit 1; - fi - - read -p "Is archival running for sandbox installation? (Y/N): " archival_running - if [ "$archival_running" == "Y" ]; then - echo "Sandbox installation selected. This will use superuser PostgreSQL secrets for creating archivedb." - super_user_password=$(kubectl get secret --namespace postgres postgres-postgresql -o jsonpath={.data.postgres-password} | base64 --decode) - echo "Common secrets will be used as passwords for all the db users." - db_common_password=$(kubectl get secret --namespace postgres db-common-secrets -o jsonpath={.data.db-dbuser-password} | base64 --decode) - set_db_pwd="--set databases.archive_db.su_user_pwd=$super_user_password \ - --set databases.source_db.source_audit_db_pass=$db_common_password \ - --set databases.source_db.source_credential_db_pass=$db_common_password \ - --set databases.source_db.source_esignet_db_pass=$db_common_password \ - --set databases.source_db.source_ida_db_pass=$db_common_password \ - --set databases.source_db.source_idrepo_db_pass=$db_common_password \ - --set databases.source_db.source_kernel_db_pass=$db_common_password \ - --set databases.source_db.source_master_db_pass=$db_common_password \ - --set databases.source_db.source_pms_db_pass=$db_common_password \ - --set databases.source_db.source_prereg_db_pass=$db_common_password \ - --set databases.source_db.source_regprc_db_pass=$db_common_password \ - --set databases.source_db.source_resident_db_pass=$db_common_password \ - --set databases.archive_db.db_pwd=$db_common_password \ - --set databases.archive_db.archive_db_password=$db_common_password" - - elif [ "$archival_running" == "N" ]; then - echo "Other installation selected.This will Use individual secrets for db passwords from values.yaml" - set_db_pwd="" - else - echo "Incorrect input; EXITING;" - exit 1; - fi - - # Install data-archive - helm -n $NS install data-archive mosip/data-archive --set crontime="0 $time * * *" -f values.yaml $set_db_pwd --version $CHART_VERSION - - echo Installed data-archive - return 0 -} - -# set commands for error handling. -set -e -set -o errexit ## set -e : exit the script if any statement returns a non-true return value -set -o nounset ## set -u : exit the script if you try to use an uninitialised variable -set -o errtrace # trace ERR through 'time command' and other functions -set -o pipefail # trace ERR through pipes -installing_data-archive # calling function diff --git a/deploy/data-archive/values.yaml b/deploy/data-archive/values.yaml deleted file mode 100644 index 64e3cd4f..00000000 --- a/deploy/data-archive/values.yaml +++ /dev/null @@ -1,281 +0,0 @@ -databases: - archive_db: - db_name: "mosip_archive" - host: "" - port: 5432 - su_user: "postgres" - su_user_pwd: "" - db_pwd: "" - dml: 0 - archivehost: "" - archiveport: 5432 - archiveuname: "archiveuser" - archive_dbname: "mosip_archive" - archive_schemaname: "archive" - archive_db_password: "" - source_db: - provide_db_names_to_archive: "AUDIT,CREDENTIAL,IDA" - source_audit_host: "" - source_audit_port: 5432 - source_audit_uname: "audituser" - source_audit_dbname: "mosip_audit" - source_audit_schemaname: "audit" - source_audit_db_pass: "" - source_credential_host: "" - source_credential_port: 5432 - source_credential_uname: "credentialuser" - source_credential_dbname: "mosip_credential" - source_credential_schemaname: "credential" - source_credential_db_pass: "" - source_esignet_host: "" - source_esignet_port: 5432 - source_esignet_uname: "esignetuser" - source_esignet_dbname: "mosip_esignet" - source_esignet_schemaname: "esignet" - source_esignet_db_pass: "" - source_ida_host: "" - source_ida_port: 5432 - source_ida_uname: "idauser" - source_ida_dbname: "mosip_ida" - source_ida_schemaname: "ida" - source_ida_db_pass: "" - source_idrepo_host: "" - source_idrepo_port: 5432 - source_idrepo_uname: "idrepouser" - source_idrepo_dbname: "mosip_idrepo" - source_idrepo_schemaname: "idrepo" - source_idrepo_db_pass: "" - source_kernel_host: "" - source_kernel_port: 5432 - source_kernel_uname: "kerneluser" - source_kernel_dbname: "mosip_kernel" - source_kernel_schemaname: "kernel" - source_kernel_db_pass: "" - source_master_host: "" - source_master_port: 5432 - source_master_uname: "masteruser" - source_master_dbname: "mosip_master" - source_master_schemaname: "master" - source_master_db_pass: "" - source_pms_host: "" - source_pms_port: 5432 - source_pms_uname: "pmsuser" - source_pms_dbname: "mosip_pms" - source_pms_schemaname: "pms" - source_pms_db_pass: "" - source_prereg_host: "" - source_prereg_port: 5432 - source_prereg_uname: "prereguser" - source_prereg_dbname: "mosip_prereg" - source_prereg_schemaname: "prereg" - source_prereg_db_pass: "" - source_regprc_host: "" - source_regprc_port: 5432 - source_regprc_uname: "regprcuser" - source_regprc_dbname: "mosip_regprc" - source_regprc_schemaname: "regprc" - source_regprc_db_pass: "" - source_resident_host: "" - source_resident_port: 5432 - source_resident_uname: "residentuser" - source_resident_dbname: "mosip_resident" - source_resident_schemaname: "resident" - source_resident_db_pass: "" - container_volume_path: "/all-db-info-json" - all_db_tables_info: - audit: - tables_info: - - source_table: "app_audit_log" - archive_table: "mosip_audit_app_audit_log" - id_column: "log_id" - date_column: "log_dtimes" - retention_days: 30 - operation_type: "archive_delete" - credential: - tables_info: - - source_table: "credential_transaction" - archive_table: "mosip_credential_credential_transaction" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 30 - operation_type: "archive_delete" - esignet: - tables_info: - - source_table: "consent_history" - archive_table: "mosip_esignet_consent_history" - id_column: "id" - date_column: "cr_dtimes" - retention_days: 30 - operation_type: "none" - ida: - tables_info: - - source_table: "credential_event_store" - archive_table: "mosip_ida_credential_event_store" - id_column: "event_id" - date_column: "cr_dtimes" - retension_days: 30 - operation_type: "archive_delete" - - source_table: "otp_transaction" - archive_table: "mosip_ida_otp_transaction" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 30 - operation_type: 'delete' - idrepo: - tables_info: - - source_table: "anonymous_profile" - archive_table: "mosip_idrepo_anonymous_profile" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 30 - operation_type: "archive_delete" - - source_table: "credential_request_status" - archive_table: "mosip_idrepo_credential_request_status" - id_column: "individual_id" - date_column: "cr_dtimes" - retension_days: 30 - operation_type: "archive_delete" - - source_table: "uin_draft" - archive_table: "mosip_idrepo_uin_draft" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 30 - operation_type: "archive_delete" - kernel: - tables_info: - - source_table: "otp_transaction" - archive_table: "mosip_kernel_otp_transaction" - id_column: "id" - date_column: "generated_dtimes" - retension_days: 7 - operation_type: "delete" - master: - tables_info: - - source_table: "bulkupload_transaction" - archive_table: "mosip_master_bulkupload_transaction" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 91 - operation_type: "archive_delete" - - source_table: "device_master_h" - archive_table: "mosip_master_device_master_h" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 365 - operation_type: "archive_delete" - - source_table: "machine_master_h" - archive_table: "mosip_master_machine_master_h" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 183 - operation_type: "archive_delete" - - source_table: "registration_center_h" - archive_table: "mosip_master_registration_center_h" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 365 - operation_type: "archive_delete" - - source_table: "user_detail_h" - archive_table: "mosip_master_user_detail_h" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 183 - operation_type: "archive_delete" - - source_table: "zone_user_h" - archive_table: "mosip_master_zone_user_h" - id_column: "usr_id" - date_column: "cr_dtimes" - retension_days: 183 - operation_type: "archive_delete" - pms: - tables_info: - - source_table: "auth_policy_h" - archive_table: "mosip_pms_auth_policy_h" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 183 - operation_type: "archive_delete" - - source_table: "secure_biometric_interface_h" - archive_table: "mosip_pms_secure_biometric_interface_h" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 183 - operation_type: "archive_delete" - - source_table: "partner_h" - archive_table: "mosip_pms_partner_h" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 183 - operation_type: "archive_delete" - prereg: - tables_info: - - source_table: "otp_transaction" - archive_table: "mosip_prereg_otp_transaction" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 30 - operation_type: "delete" - regprc: - tables_info: - - source_table: "abis_response_det" - archive_table: "mosip_regprc_abis_response_det" - id_column: "abis_resp_id" - date_column: "cr_dtimes" - retension_days: 183 - operation_type: "archive_delete" - - source_table: "abis_response" - archive_table: "mosip_regprc_abis_response" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 183 - operation_type: "archive_delete" - - source_table: "abis_request" - archive_table: "mosip_regprc_abis_request" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 183 - operation_type: "archive_delete" - - source_table: "reg_demo_dedupe_list" - archive_table: "mosip_regprc_reg_demo_dedupe_list" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 183 - operation_type: "archive_delete" - - source_table: "registration_transaction" - archive_table: "mosip_regprc_registration_transaction" - id_column: "regtrn_id" - date_column: "cr_dtimes" - retension_days: 183 - operation_type: "archive_delete" - resident: - tables_info: - - source_table: "otp_transaction" - archive_table: "mosip_resident_otp_transaction" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 30 - operation_type: "delete" - - source_table: "resident_grievance_ticket" - archive_table: "mosip_resident_grievance_ticket" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 365 - operation_type: "archive_delete" - - source_table: "resident_session" - archive_table: "mosip_resident_session" - id_column: "session_id" - date_column: "login_dtimes" - retension_days: 30 - operation_type: "archive_delete" - - source_table: "resident_transaction" - archive_table: "mosip_resident_transaction" - id_column: "id" - date_column: "cr_dtimes" - retension_days: 365 - operation_type: "archive_delete" - - source_table: "resident_user_actions" - archive_table: "mosip_resident_user_actions" - id_column: "ida_token" - date_column: "last_bell_notif_click_dtimes" - retension_days: 365 - operation_type: "archive_delete" \ No newline at end of file diff --git a/data-archive/deploy/delete.sh b/deploy/delete.sh similarity index 100% rename from data-archive/deploy/delete.sh rename to deploy/delete.sh diff --git a/data-archive/deploy/install.sh b/deploy/install.sh similarity index 100% rename from data-archive/deploy/install.sh rename to deploy/install.sh diff --git a/data-archive/deploy/values.yaml b/deploy/values.yaml similarity index 97% rename from data-archive/deploy/values.yaml rename to deploy/values.yaml index 3fa43cc7..4305f1fa 100644 --- a/data-archive/deploy/values.yaml +++ b/deploy/values.yaml @@ -40,6 +40,15 @@ databases: - name: resident enabled: true container_volume_path: /resident-db-info-json + - name: idrepo + enabled: true + container_volume_path: /idrepo-db-info-json + - name: prereg + enabled: true + container_volume_path: /prereg-db-info-json + - name: esignet + enabled: true + container_volume_path: /esignet-db-info-json source_db: source_audit_host: "" # Host address for the source audit database @@ -209,7 +218,7 @@ databases: operation_type: "archive_delete" - source_table: "zone_user_h" archive_table: "mosip_master_zone_user_h" - id_column: "usr_id" + id_column: "zone_code" date_column: "cr_dtimes" retention_days: 183 operation_type: "archive_delete" diff --git a/helm/data-archive/templates/archive-env-configmap.yaml b/helm/data-archive/templates/archive-env-configmap.yaml index 7833668a..af125dd0 100644 --- a/helm/data-archive/templates/archive-env-configmap.yaml +++ b/helm/data-archive/templates/archive-env-configmap.yaml @@ -17,7 +17,7 @@ data: DB_SERVERIP: {{ .Values.databases.archive_db.host | quote }} DB_PORT: {{ .Values.databases.archive_db.port | quote }} DML_FLAG: {{ .Values.databases.archive_db.dml | quote }} - DB_NAMES: {{ .Values.databases.source_db.provide_db_names_to_archive | quote }} + BATCH_SIZE: {{ .Values.databases.archive_db.batch_size | quote }} AUDIT_SOURCE_DB_HOST: {{ .Values.databases.source_db.source_audit_host | quote }} AUDIT_SOURCE_DB_PORT: {{ .Values.databases.source_db.source_audit_port | quote }} AUDIT_SOURCE_DB_UNAME: {{ .Values.databases.source_db.source_audit_uname | quote }} @@ -77,5 +77,4 @@ data: ARCHIVE_DB_PORT: {{ .Values.databases.archive_db.archiveport | quote }} ARCHIVE_DB_UNAME: {{ .Values.databases.archive_db.archiveuname | quote }} ARCHIVE_DB_NAME: {{ .Values.databases.archive_db.archive_dbname | quote }} - ARCHIVE_SCHEMA_NAME: {{ .Values.databases.archive_db.archive_schemaname | quote }} - CONTAINER_VOLUME_PATH: {{ .Values.databases.container_volume_path | quote }} \ No newline at end of file + ARCHIVE_SCHEMA_NAME: {{ .Values.databases.archive_db.archive_schemaname | quote }} \ No newline at end of file diff --git a/helm/data-archive/templates/cronjob.yaml b/helm/data-archive/templates/cronjob.yaml index 5a808e78..db092a0f 100644 --- a/helm/data-archive/templates/cronjob.yaml +++ b/helm/data-archive/templates/cronjob.yaml @@ -1,13 +1,15 @@ +{{- range $dbnames := $.Values.databases.db_names }} +{{- if $dbnames.enabled }} +--- apiVersion: {{ include "common.capabilities.cronjob.apiVersion" $ }} kind: CronJob metadata: - name: {{ template "data-archive.cronjob" $ }} + name: {{ template "data-archive.cronjob" $ }}-{{ $dbnames.name }} namespace: {{ $.Release.Namespace }} spec: concurrencyPolicy: Forbid successfulJobsHistoryLimit: 1 # remove jobs which are successfully executed failedJobsHistoryLimit: 1 # except 1 recent failed job, remove jobs which are not successfully executed - #schedule: '*/3 * * * *' # cron spec of time, here, 8 o'clock schedule: {{ $.Values.crontime }} jobTemplate: spec: @@ -19,24 +21,35 @@ spec: template: spec: restartPolicy: Never - containers: - - name: {{ template "data-archive.serviceAccountName" $ }} - image: {{ template "data-archive.image" $ }} - imagePullPolicy: {{ $.Values.image.pullPolicy }} - volumeMounts: - - name: config-volume - mountPath: /all-db-info-json - env: - - name: container_user - value: {{ $.Values.containerSecurityContext.runAsUser }} - envFrom: - - configMapRef: - name: db-archive-init-env-config - - secretRef: - name: source-db-secrets - - secretRef: - name: archive-db-secrets + serviceAccountName: {{ template "data-archive.serviceAccountName" $ }} + containers: + - name: {{ template "data-archive.serviceAccountName" $ }}-{{ $dbnames.name }} + image: {{ template "data-archive.image" $ }} + imagePullPolicy: {{ $.Values.image.pullPolicy }} + volumeMounts: + - name: config-volume-{{ $dbnames.name }} # Unique volume mount path for each job + mountPath: {{ $dbnames.container_volume_path | quote }} # Dynamic volume path from values.yaml + env: + - name: CONTAINER_VOLUME_PATH + value: {{ $dbnames.container_volume_path | quote }} # Set dynamic CONTAINER_VOLUME_PATH + - name: container_user + value: {{ $.Values.containerSecurityContext.runAsUser }} + - name: DB_NAMES + value: {{ $dbnames.name | upper }} + envFrom: + - configMapRef: + name: db-archive-init-env-config + - secretRef: + name: source-db-secrets + - secretRef: + name: archive-db-secrets + {{- if $.Values.resources }} + resources: {{- toYaml $.Values.resources | nindent 14 }} + {{- end }} volumes: - - name: config-volume - configMap: - name: {{ .Release.Name }} \ No newline at end of file + - name: config-volume-{{ $dbnames.name }} # Unique config-volume per job + configMap: + name: {{ $.Release.Name }} + +{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/data-archive/values.yaml b/helm/data-archive/values.yaml index c99093c7..73e1d571 100644 --- a/helm/data-archive/values.yaml +++ b/helm/data-archive/values.yaml @@ -31,7 +31,7 @@ replicaCount: 1 image: registry: docker.io - repository: mosipqa/data-archive + repository: mosipdev/data-archive tag: develop ## Specify a imagePullPolicy @@ -68,10 +68,10 @@ resources: # lines, adjust them as necessary, and remove the curly braces after 'resources:'. limits: cpu: 1000m - memory: 3500Mi + memory: 1000Mi requests: - cpu: 1000m - memory: 3500Mi + cpu: 300m + memory: 700Mi # additionalResources: ## Specify any JAVA_OPTS string here. These typically will be specified in conjunction with above resources @@ -298,11 +298,6 @@ serviceAccount: ## name: -## Admin swagger should have only internal access. Hence linked to internal gateway -istio: - enabled: false - gateways: - - istio-system/internal crontime: "0 4 * * *" ## run cronjob every day at 4 AM in GMT which is 9.30 AM in IST (time hr: 0-23 ) @@ -321,8 +316,43 @@ databases: archive_dbname: "mosip_archive" archive_schemaname: "archive" archive_db_password: "" + batch_size: 1000 + + db_names: + - name: audit + enabled: true + container_volume_path: /audit-table-info ## volume mount path for table info inside cronjob container + - name: ida + enabled: true + container_volume_path: /ida-table-info ## volume mount path for table info inside cronjob container + - name: credential + enabled: true + container_volume_path: /credential-table-info ## volume mount path for table info inside cronjob container + - name: kernel + enabled: true + container_volume_path: /kernel-table-info ## volume mount path for table info inside cronjob container + - name: master + enabled: true + container_volume_path: /master-table-info ## volume mount path for table info inside cronjob container + - name: pms + enabled: true + container_volume_path: /pms-table-info ## volume mount path for table info inside cronjob container + - name: regprc + enabled: true + container_volume_path: /regprc-table-info ## volume mount path for table info inside cronjob container + - name: resident + enabled: true + container_volume_path: /resident-table-info ## volume mount path for table info inside cronjob container + - name: idrepo + enabled: true + container_volume_path: /idrepo-db-info-json + - name: prereg + enabled: true + container_volume_path: /prereg-db-info-json + - name: esignet + enabled: true + container_volume_path: /esignet-db-info-json source_db: - provide_db_names_to_archive: "AUDIT,IDA" source_audit_host: "api-internal.sandbox.xyz.net" source_audit_port: 5432 source_audit_uname: "audituser" @@ -389,7 +419,6 @@ databases: source_resident_dbname: "mosip_resident" source_resident_schemaname: "resident" source_resident_db_pass: "" - container_volume_path: "/all-db-info-json" ## volume mount path for table info inside cronjob container all_db_tables_info: audit: tables_info: @@ -405,7 +434,7 @@ databases: archive_table: "mosip_credential_credential_transaction" id_column: "id" date_column: "cr_dtimes" - retension_days: 30 + retention_days: 30 operation_type: "archive_delete" esignet: tables_info: @@ -421,13 +450,13 @@ databases: archive_table: "mosip_ida_credential_event_store" id_column: "event_id" date_column: "cr_dtimes" - retension_days: 30 + retention_days: 30 operation_type: "archive_delete" - source_table: "otp_transaction" archive_table: "mosip_ida_otp_transaction" id_column: "id" date_column: "cr_dtimes" - retension_days: 30 + retention_days: 30 operation_type: 'delete' idrepo: tables_info: @@ -435,19 +464,19 @@ databases: archive_table: "mosip_idrepo_anonymous_profile" id_column: "id" date_column: "cr_dtimes" - retension_days: 30 + retention_days: 30 operation_type: "archive_delete" - source_table: "credential_request_status" archive_table: "mosip_idrepo_credential_request_status" id_column: "individual_id" date_column: "cr_dtimes" - retension_days: 30 + retention_days: 30 operation_type: "archive_delete" - source_table: "uin_draft" archive_table: "mosip_idrepo_uin_draft" id_column: "id" date_column: "cr_dtimes" - retension_days: 30 + retention_days: 30 operation_type: "archive_delete" kernel: tables_info: @@ -455,7 +484,7 @@ databases: archive_table: "mosip_kernel_otp_transaction" id_column: "id" date_column: "generated_dtimes" - retension_days: 7 + retention_days: 7 operation_type: "delete" master: tables_info: @@ -463,37 +492,37 @@ databases: archive_table: "mosip_master_bulkupload_transaction" id_column: "id" date_column: "cr_dtimes" - retension_days: 91 + retention_days: 91 operation_type: "archive_delete" - source_table: "device_master_h" archive_table: "mosip_master_device_master_h" id_column: "id" date_column: "cr_dtimes" - retension_days: 365 + retention_days: 365 operation_type: "archive_delete" - source_table: "machine_master_h" archive_table: "mosip_master_machine_master_h" id_column: "id" date_column: "cr_dtimes" - retension_days: 183 + retention_days: 183 operation_type: "archive_delete" - source_table: "registration_center_h" archive_table: "mosip_master_registration_center_h" id_column: "id" date_column: "cr_dtimes" - retension_days: 365 + retention_days: 365 operation_type: "archive_delete" - source_table: "user_detail_h" archive_table: "mosip_master_user_detail_h" id_column: "id" date_column: "cr_dtimes" - retension_days: 183 + retention_days: 183 operation_type: "archive_delete" - source_table: "zone_user_h" archive_table: "mosip_master_zone_user_h" - id_column: "usr_id" + id_column: "zone_code" date_column: "cr_dtimes" - retension_days: 183 + retention_days: 183 operation_type: "archive_delete" pms: tables_info: @@ -501,19 +530,19 @@ databases: archive_table: "mosip_pms_auth_policy_h" id_column: "id" date_column: "cr_dtimes" - retension_days: 183 + retention_days: 183 operation_type: "archive_delete" - source_table: "secure_biometric_interface_h" archive_table: "mosip_pms_secure_biometric_interface_h" id_column: "id" date_column: "cr_dtimes" - retension_days: 183 + retention_days: 183 operation_type: "archive_delete" - source_table: "partner_h" archive_table: "mosip_pms_partner_h" id_column: "id" date_column: "cr_dtimes" - retension_days: 183 + retention_days: 183 operation_type: "archive_delete" prereg: tables_info: @@ -521,7 +550,7 @@ databases: archive_table: "mosip_prereg_otp_transaction" id_column: "id" date_column: "cr_dtimes" - retension_days: 30 + retention_days: 30 operation_type: "delete" regprc: tables_info: @@ -529,31 +558,31 @@ databases: archive_table: "mosip_regprc_abis_response_det" id_column: "abis_resp_id" date_column: "cr_dtimes" - retension_days: 183 + retention_days: 183 operation_type: "archive_delete" - source_table: "abis_response" archive_table: "mosip_regprc_abis_response" id_column: "id" date_column: "cr_dtimes" - retension_days: 183 + retention_days: 183 operation_type: "archive_delete" - source_table: "abis_request" archive_table: "mosip_regprc_abis_request" id_column: "id" date_column: "cr_dtimes" - retension_days: 183 + retention_days: 183 operation_type: "archive_delete" - source_table: "reg_demo_dedupe_list" archive_table: "mosip_regprc_reg_demo_dedupe_list" id_column: "id" date_column: "cr_dtimes" - retension_days: 183 + retention_days: 183 operation_type: "archive_delete" - source_table: "registration_transaction" archive_table: "mosip_regprc_registration_transaction" id_column: "regtrn_id" date_column: "cr_dtimes" - retension_days: 183 + retention_days: 183 operation_type: "archive_delete" resident: tables_info: @@ -561,29 +590,29 @@ databases: archive_table: "mosip_resident_otp_transaction" id_column: "id" date_column: "cr_dtimes" - retension_days: 30 + retention_days: 30 operation_type: "delete" - source_table: "resident_grievance_ticket" archive_table: "mosip_resident_grievance_ticket" id_column: "id" date_column: "cr_dtimes" - retension_days: 365 + retention_days: 365 operation_type: "archive_delete" - source_table: "resident_session" archive_table: "mosip_resident_session" id_column: "session_id" date_column: "login_dtimes" - retension_days: 30 + retention_days: 30 operation_type: "archive_delete" - source_table: "resident_transaction" archive_table: "mosip_resident_transaction" id_column: "id" date_column: "cr_dtimes" - retension_days: 365 + retention_days: 365 operation_type: "archive_delete" - source_table: "resident_user_actions" archive_table: "mosip_resident_user_actions" id_column: "ida_token" date_column: "last_bell_notif_click_dtimes" - retension_days: 365 + retention_days: 365 operation_type: "archive_delete"