From ac8113b6302b60dc71709754efeeea8988385abc Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:29:02 +0800 Subject: [PATCH 01/49] Create config.yml --- .circleci/config.yml | 167 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..02c12973 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,167 @@ +version: 2.1 +orbs: + moneysmart-orb: moneysmartco/circleci-orb@1.0.5 + aws-cli: circleci/aws-cli@3.1 + helm: circleci/helm@1.0 +defaults: &defaults + working_directory: ~/app + docker: + # the Docker image with Cypress dependencies + - image: cypress/base:16.14.0 + environment: + ## this enables colors in the output + TERM: xterm +aliases: + - &prod_aws_creds + aws-cli/setup: + aws-access-key-id: AWS_ACCESS_KEY + aws-secret-access-key: AWS_SECRET_ACCESS_KEY + aws-region: AWS_REGION + - &stg_aws_creds + aws-cli/setup: + aws-access-key-id: STAGING_AWS_ACCESS_KEY + aws-secret-access-key: STAGING_AWS_SECRET_ACCESS_KEY + aws-region: AWS_REGION +default-image: &default-image + machine: + image: ubuntu-2004:current + docker_layer_caching: true +jobs: + deploy_to_sg_staging_eks: + docker: + - image: 'cimg/python:3.10' + steps: + - checkout + - *stg_aws_creds + - attach_workspace: + at: /tmp/workspace + - moneysmart-orb/pull_helm_charts: + cluster-name: product-listing-staging + chart-name: kafka-rest + branch-name: production + - run: + name: Copy ingress templaets + command: | + cp -R charts/identity-provider-fe/ingress/* ms-eks-objects/helm-charts/templates + - helm/upgrade-helm-chart: + chart: charts/cp-kafka-rest + release-name: kafka-rest-sg + namespace: kafka-rest + values-to-override: appLabels.region=sg,image.tag=staging-${CIRCLE_SHA1} + values: charts/cp-kakfa-rest/values.yaml + timeout: 600s + helm-version: v3.9.0 + + deploy_to_hk_staging_eks: + docker: + - image: 'cimg/python:3.10' + steps: + - checkout + - *stg_aws_creds + - attach_workspace: + at: /tmp/workspace + - moneysmart-orb/pull_helm_charts: + cluster-name: product-listing-staging + chart-name: kafka-rest + branch-name: production + - run: + name: Copy ingress templaets + command: | + cp -R charts/identity-provider-fe/ingress/* charts/cp-kafka-rest/templates + - helm/upgrade-helm-chart: + chart: charts/cp-kafka-rest + release-name: kafka-rest-hk + namespace: kafka-rest + values-to-override: appLabels.region=hk,image.tag=staging-${CIRCLE_SHA1} + values: charts/cp-kakfa-rest/values.yaml + timeout: 600s + helm-version: v3.9.0 +#--------------------------------------------- +# campaign_service SG and HK Production Jobs +#-------------------------------------------- + deploy_to_sg_prod_eks: + docker: + - image: 'cimg/python:3.10' + steps: + - checkout + - *prod_aws_creds + - attach_workspace: + at: /tmp/workspace + - moneysmart-orb/pull_helm_charts: + cluster-name: product-listing-prod + chart-name: kafka-rest + branch-name: production + - helm/upgrade-helm-chart: + chart: charts/cp-kafka-rest + release-name: kafka-rest-sg + namespace: kafka-rest + values-to-override: appLabels.region=sg + values: charts/cp-kakfa-rest/values.yaml + timeout: 600s + helm-version: v3.9.0 + + deploy_to_hk_prod_eks: + docker: + - image: 'cimg/python:3.10' + steps: + - checkout + - *prod_aws_creds + - attach_workspace: + at: /tmp/workspace + - moneysmart-orb/pull_helm_charts: + cluster-name: product-listing-prod + chart-name: kafka-rest + branch-name: production + - helm/upgrade-helm-chart: + chart: charts/cp-kafka-rest + release-name: kafka-rest-hk + namespace: kafka-rest + values-to-override: appLabels.region=hk + values: charts/cp-kakfa-rest/values.yaml + timeout: 600s + helm-version: v3.9.0 +workflows: + version: 2.1 +#--------------------------------------------- +# kafka-rest SG and HK Staging Pipeline +#--------------------------------------------- + staging-build-deploy: + jobs: + - deploy_to_sg_staging_eks: + context: common-eks-vars + requires: + - publish_image_to_ecr_staging + filters: + branches: + only: + - update-cicd + + - deploy_to_hk_staging_eks: + context: common-eks-vars + requires: + - publish_image_to_ecr_staging + filters: + branches: + only: + - update-cicd + +#--------------------------------------------- +# campaign_service SG and HK Production Pipeline +#--------------------------------------------- + prod-build-deploy: + jobs: + - deploy_to_sg_prod_eks: + context: common-eks-vars + filters: + tags: + only: /^v.*/ + branches: + ignore: /.*/ + + - deploy_to_hk_prod_eks: + context: common-eks-vars + filters: + tags: + only: /^v.*/ + branches: + ignore: /.*/ From 9141fc92de8947bf8f6711da099ef83b0232fba4 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:34:54 +0800 Subject: [PATCH 02/49] Update config.yml --- .circleci/config.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 02c12973..04595648 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,8 +129,6 @@ workflows: jobs: - deploy_to_sg_staging_eks: context: common-eks-vars - requires: - - publish_image_to_ecr_staging filters: branches: only: @@ -138,8 +136,6 @@ workflows: - deploy_to_hk_staging_eks: context: common-eks-vars - requires: - - publish_image_to_ecr_staging filters: branches: only: From 31ca2a0d5835a8e49d8dd51b6531f83ba4f20e63 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:39:03 +0800 Subject: [PATCH 03/49] Create ingress.yml --- charts/cp-kafka-rest/templates/ingress.yml | 31 ++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 charts/cp-kafka-rest/templates/ingress.yml diff --git a/charts/cp-kafka-rest/templates/ingress.yml b/charts/cp-kafka-rest/templates/ingress.yml new file mode 100644 index 00000000..6e89f31f --- /dev/null +++ b/charts/cp-kafka-rest/templates/ingress.yml @@ -0,0 +1,31 @@ +{{- $fullName := template "cp-kafka-rest.name" . -}} +{{- $ingressPath := .Values.envs.kakfa-rest.paths -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + alb.ingress.kubernetes.io/certificate-arn: {{ .Values.envs.kakfa-rest.certificate_arn }} + alb.ingress.kubernetes.io/group.name: eks-lb-{{ .Values.appLabels.region }} + alb.ingress.kubernetes.io/group.order: '10' + alb.ingress.kubernetes.io/healthcheck-path: /_ping + kubernetes.io/ingress.class: alb + nginx.ingress.kubernetes.io/use-regex: "true" + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/name: {{ .Release.Name }} + region: {{ .Values.appLabels.region }} + name: {{ .Release.Name }} +spec: + rules: + - host: {{ .Values.envs.kakfa-rest.ingress_host }} + http: + paths: + {{- range $ingressPath }} + - path: {{ . }} + backend: + service: + name: {{ $fullName }} + port: + number: 80 + pathType: ImplementationSpecific + {{- end }} From a59a17f68e762d0b6108e40d45b9dd72141125d9 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:39:51 +0800 Subject: [PATCH 04/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index d73ed033..887d363a 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -1,6 +1,13 @@ # Default values for cp-kafka-rest. # This is a YAML-formatted file. # Declare variables to be passed into your templates. +envs: + kakfa-rest: + ingress_host: kafka-client.mssgdev.com + name: kafka-client + certificate_arn: arn:aws:acm:ap-southeast-1:856075066843:certificate/3d8d374c-c52f-4e59-8deb-e5b44c615db3 + paths: + - /* replicaCount: 1 From 9b4c552e9a5f8c950c7df18bb3e9c24754c5343c Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:41:29 +0800 Subject: [PATCH 05/49] Update config.yml --- .circleci/config.yml | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 04595648..d93c475a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,19 +35,11 @@ jobs: - *stg_aws_creds - attach_workspace: at: /tmp/workspace - - moneysmart-orb/pull_helm_charts: - cluster-name: product-listing-staging - chart-name: kafka-rest - branch-name: production - - run: - name: Copy ingress templaets - command: | - cp -R charts/identity-provider-fe/ingress/* ms-eks-objects/helm-charts/templates - helm/upgrade-helm-chart: chart: charts/cp-kafka-rest release-name: kafka-rest-sg namespace: kafka-rest - values-to-override: appLabels.region=sg,image.tag=staging-${CIRCLE_SHA1} + values-to-override: appLabels.region=sg values: charts/cp-kakfa-rest/values.yaml timeout: 600s helm-version: v3.9.0 @@ -60,19 +52,11 @@ jobs: - *stg_aws_creds - attach_workspace: at: /tmp/workspace - - moneysmart-orb/pull_helm_charts: - cluster-name: product-listing-staging - chart-name: kafka-rest - branch-name: production - - run: - name: Copy ingress templaets - command: | - cp -R charts/identity-provider-fe/ingress/* charts/cp-kafka-rest/templates - helm/upgrade-helm-chart: chart: charts/cp-kafka-rest release-name: kafka-rest-hk namespace: kafka-rest - values-to-override: appLabels.region=hk,image.tag=staging-${CIRCLE_SHA1} + values-to-override: appLabels.region=hk values: charts/cp-kakfa-rest/values.yaml timeout: 600s helm-version: v3.9.0 From c856dc99a7da4f3de3351ede3fa0d3911b9c32b4 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 15:52:12 +0800 Subject: [PATCH 06/49] Update config.yml --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d93c475a..47595020 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,6 +35,10 @@ jobs: - *stg_aws_creds - attach_workspace: at: /tmp/workspace + - moneysmart-orb/pull_helm_charts: + cluster-name: product-listing-staging + chart-name: identity-provider-fe + branch-name: production - helm/upgrade-helm-chart: chart: charts/cp-kafka-rest release-name: kafka-rest-sg @@ -52,6 +56,10 @@ jobs: - *stg_aws_creds - attach_workspace: at: /tmp/workspace + - moneysmart-orb/pull_helm_charts: + cluster-name: product-listing-staging + chart-name: identity-provider-fe + branch-name: production - helm/upgrade-helm-chart: chart: charts/cp-kafka-rest release-name: kafka-rest-hk From a60e5818b93a96e9f70a933b8e40cd92a314fbbb Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:00:47 +0800 Subject: [PATCH 07/49] Update config.yml --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 47595020..b1c39faa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,7 +65,7 @@ jobs: release-name: kafka-rest-hk namespace: kafka-rest values-to-override: appLabels.region=hk - values: charts/cp-kakfa-rest/values.yaml + values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 #--------------------------------------------- @@ -88,7 +88,7 @@ jobs: release-name: kafka-rest-sg namespace: kafka-rest values-to-override: appLabels.region=sg - values: charts/cp-kakfa-rest/values.yaml + values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 @@ -109,7 +109,7 @@ jobs: release-name: kafka-rest-hk namespace: kafka-rest values-to-override: appLabels.region=hk - values: charts/cp-kakfa-rest/values.yaml + values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 workflows: From d2dda9fb199117dd176f5a2ff06b02a7d83554d3 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:02:41 +0800 Subject: [PATCH 08/49] Update config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1c39faa..fba4c045 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,7 +44,7 @@ jobs: release-name: kafka-rest-sg namespace: kafka-rest values-to-override: appLabels.region=sg - values: charts/cp-kakfa-rest/values.yaml + values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 From 912694ef4d87fd061704051ba4ab37ba393faea1 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:03:46 +0800 Subject: [PATCH 09/49] Update ingress.yml --- charts/cp-kafka-rest/templates/ingress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/templates/ingress.yml b/charts/cp-kafka-rest/templates/ingress.yml index 6e89f31f..13954d83 100644 --- a/charts/cp-kafka-rest/templates/ingress.yml +++ b/charts/cp-kafka-rest/templates/ingress.yml @@ -1,4 +1,4 @@ -{{- $fullName := template "cp-kafka-rest.name" . -}} +{{- $fullName := include ".Values.name.fullname" . -}} {{- $ingressPath := .Values.envs.kakfa-rest.paths -}} apiVersion: networking.k8s.io/v1 kind: Ingress From 3857982e7c1a41f966b8861685ca6987c9ddae6a Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:04:39 +0800 Subject: [PATCH 10/49] Update ingress.yml --- charts/cp-kafka-rest/templates/ingress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/templates/ingress.yml b/charts/cp-kafka-rest/templates/ingress.yml index 13954d83..425c35af 100644 --- a/charts/cp-kafka-rest/templates/ingress.yml +++ b/charts/cp-kafka-rest/templates/ingress.yml @@ -1,5 +1,5 @@ {{- $fullName := include ".Values.name.fullname" . -}} -{{- $ingressPath := .Values.envs.kakfa-rest.paths -}} +{{- $ingressPath := .Values.envs.kafka-rest.paths -}} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: From df444e51850fb4570b27f2df3a024d43d6e38625 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:04:49 +0800 Subject: [PATCH 11/49] Update ingress.yml --- charts/cp-kafka-rest/templates/ingress.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/cp-kafka-rest/templates/ingress.yml b/charts/cp-kafka-rest/templates/ingress.yml index 425c35af..02705d20 100644 --- a/charts/cp-kafka-rest/templates/ingress.yml +++ b/charts/cp-kafka-rest/templates/ingress.yml @@ -4,7 +4,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: - alb.ingress.kubernetes.io/certificate-arn: {{ .Values.envs.kakfa-rest.certificate_arn }} + alb.ingress.kubernetes.io/certificate-arn: {{ .Values.envs.kafka-rest.certificate_arn }} alb.ingress.kubernetes.io/group.name: eks-lb-{{ .Values.appLabels.region }} alb.ingress.kubernetes.io/group.order: '10' alb.ingress.kubernetes.io/healthcheck-path: /_ping @@ -17,7 +17,7 @@ metadata: name: {{ .Release.Name }} spec: rules: - - host: {{ .Values.envs.kakfa-rest.ingress_host }} + - host: {{ .Values.envs.kafka-rest.ingress_host }} http: paths: {{- range $ingressPath }} From cb53196a6e469b08f3e71ecaf23e6297a98aa697 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:05:55 +0800 Subject: [PATCH 12/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index 887d363a..292fb741 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -2,7 +2,7 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. envs: - kakfa-rest: + kafka-rest: ingress_host: kafka-client.mssgdev.com name: kafka-client certificate_arn: arn:aws:acm:ap-southeast-1:856075066843:certificate/3d8d374c-c52f-4e59-8deb-e5b44c615db3 From 7f000e78b408114332b10ee60b4a06243d59942e Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:08:11 +0800 Subject: [PATCH 13/49] Update ingress.yml --- charts/cp-kafka-rest/templates/ingress.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/charts/cp-kafka-rest/templates/ingress.yml b/charts/cp-kafka-rest/templates/ingress.yml index 02705d20..c2149a6f 100644 --- a/charts/cp-kafka-rest/templates/ingress.yml +++ b/charts/cp-kafka-rest/templates/ingress.yml @@ -1,12 +1,11 @@ {{- $fullName := include ".Values.name.fullname" . -}} -{{- $ingressPath := .Values.envs.kafka-rest.paths -}} +{{- $ingressPath := .Values.envs.kafka_rest.paths -}} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: - alb.ingress.kubernetes.io/certificate-arn: {{ .Values.envs.kafka-rest.certificate_arn }} + alb.ingress.kubernetes.io/certificate-arn: {{ .Values.envs.kafka_rest.certificate_arn }} alb.ingress.kubernetes.io/group.name: eks-lb-{{ .Values.appLabels.region }} - alb.ingress.kubernetes.io/group.order: '10' alb.ingress.kubernetes.io/healthcheck-path: /_ping kubernetes.io/ingress.class: alb nginx.ingress.kubernetes.io/use-regex: "true" @@ -17,7 +16,7 @@ metadata: name: {{ .Release.Name }} spec: rules: - - host: {{ .Values.envs.kafka-rest.ingress_host }} + - host: {{ .Values.envs.kafka_rest.ingress_host }} http: paths: {{- range $ingressPath }} From a0d79c558a9cfeb08dcdb1e5ffc44d7d6bb557cd Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:08:18 +0800 Subject: [PATCH 14/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index 292fb741..f0aa6b09 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -2,7 +2,7 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. envs: - kafka-rest: + kafka_rest: ingress_host: kafka-client.mssgdev.com name: kafka-client certificate_arn: arn:aws:acm:ap-southeast-1:856075066843:certificate/3d8d374c-c52f-4e59-8deb-e5b44c615db3 From 9477360f3e4fc9645056578084ad8e3b82ef8e69 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:10:03 +0800 Subject: [PATCH 15/49] Update ingress.yml --- charts/cp-kafka-rest/templates/ingress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/templates/ingress.yml b/charts/cp-kafka-rest/templates/ingress.yml index c2149a6f..c28ad6c8 100644 --- a/charts/cp-kafka-rest/templates/ingress.yml +++ b/charts/cp-kafka-rest/templates/ingress.yml @@ -1,4 +1,4 @@ -{{- $fullName := include ".Values.name.fullname" . -}} +{{- $fullName := include ".cp-kafka-rest.fullname" . -}} {{- $ingressPath := .Values.envs.kafka_rest.paths -}} apiVersion: networking.k8s.io/v1 kind: Ingress From 3b207961385fc3315b06ab8b84e95e9bb6f86908 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:10:14 +0800 Subject: [PATCH 16/49] Update ingress.yml --- charts/cp-kafka-rest/templates/ingress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/templates/ingress.yml b/charts/cp-kafka-rest/templates/ingress.yml index c28ad6c8..29a4108f 100644 --- a/charts/cp-kafka-rest/templates/ingress.yml +++ b/charts/cp-kafka-rest/templates/ingress.yml @@ -1,4 +1,4 @@ -{{- $fullName := include ".cp-kafka-rest.fullname" . -}} +{{- $fullName := include "cp-kafka-rest.fullname" . -}} {{- $ingressPath := .Values.envs.kafka_rest.paths -}} apiVersion: networking.k8s.io/v1 kind: Ingress From ccbf666428b4c122dd834da68ed01bed4a185bbd Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:11:10 +0800 Subject: [PATCH 17/49] Update config.yml --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fba4c045..b2a86c5a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,7 +42,7 @@ jobs: - helm/upgrade-helm-chart: chart: charts/cp-kafka-rest release-name: kafka-rest-sg - namespace: kafka-rest + namespace: kafka-client values-to-override: appLabels.region=sg values: charts/cp-kafka-rest/values.yaml timeout: 600s @@ -63,7 +63,7 @@ jobs: - helm/upgrade-helm-chart: chart: charts/cp-kafka-rest release-name: kafka-rest-hk - namespace: kafka-rest + namespace: kafka-client values-to-override: appLabels.region=hk values: charts/cp-kafka-rest/values.yaml timeout: 600s @@ -86,7 +86,7 @@ jobs: - helm/upgrade-helm-chart: chart: charts/cp-kafka-rest release-name: kafka-rest-sg - namespace: kafka-rest + namespace: kafka-client values-to-override: appLabels.region=sg values: charts/cp-kafka-rest/values.yaml timeout: 600s @@ -107,7 +107,7 @@ jobs: - helm/upgrade-helm-chart: chart: charts/cp-kafka-rest release-name: kafka-rest-hk - namespace: kafka-rest + namespace: kafka-client values-to-override: appLabels.region=hk values: charts/cp-kafka-rest/values.yaml timeout: 600s From fb80eccaa55cf4776386f8a5db25c32cef729be7 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:24:04 +0800 Subject: [PATCH 18/49] Update deployment.yaml --- charts/cp-kafka-rest/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/templates/deployment.yaml b/charts/cp-kafka-rest/templates/deployment.yaml index 39bca499..82bd1adf 100644 --- a/charts/cp-kafka-rest/templates/deployment.yaml +++ b/charts/cp-kafka-rest/templates/deployment.yaml @@ -75,7 +75,7 @@ spec: fieldRef: fieldPath: status.podIP - name: KAFKA_REST_BOOTSTRAP_SERVERS - value: {{ template "cp-kafka-rest.kafka.bootstrapServers" . }} + value: {{ .Values.KAFKA_REST_BOOTSTRAP_SERVERS }} - name: KAFKA_REST_SCHEMA_REGISTRY_URL value: {{ template "cp-kafka-rest.cp-schema-registry.service-name" . }} - name: KAFKAREST_HEAP_OPTS From e4c7d1ccb06552929e4ced0150415a21a328be9a Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:27:25 +0800 Subject: [PATCH 19/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index f0aa6b09..d5c624e7 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -11,6 +11,7 @@ envs: replicaCount: 1 +KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://b-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096,PLAINTEXT://b-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096 ## Image Info ## ref: https://hub.docker.com/r/confluentinc/cp-kafka/ image: confluentinc/cp-kafka-rest From 782af51b5546eea7cfed9119712edad5b24573f1 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:32:27 +0800 Subject: [PATCH 20/49] Update ingress.yml --- charts/cp-kafka-rest/templates/ingress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/templates/ingress.yml b/charts/cp-kafka-rest/templates/ingress.yml index 29a4108f..126e7e50 100644 --- a/charts/cp-kafka-rest/templates/ingress.yml +++ b/charts/cp-kafka-rest/templates/ingress.yml @@ -25,6 +25,6 @@ spec: service: name: {{ $fullName }} port: - number: 80 + number: 8082 pathType: ImplementationSpecific {{- end }} From a96c22c0e94eaebae6290735d421e8a283307a2f Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:34:27 +0800 Subject: [PATCH 21/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index d5c624e7..59f194b7 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -11,7 +11,7 @@ envs: replicaCount: 1 -KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://b-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096,PLAINTEXT://b-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096 +KAFKA_REST_BOOTSTRAP_SERVERS: kafka://b-1.sg-staging.365fra.c2.kafka.ap-southeast-1.amazonaws.com:9094,kafka://b-2.sg-staging.365fra.c2.kafka.ap-southeast-1.amazonaws.com:9094 ## Image Info ## ref: https://hub.docker.com/r/confluentinc/cp-kafka/ image: confluentinc/cp-kafka-rest From 39bdc7d188b66934033192366c2e7c4f06cdf71c Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:39:44 +0800 Subject: [PATCH 22/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index 59f194b7..d5c624e7 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -11,7 +11,7 @@ envs: replicaCount: 1 -KAFKA_REST_BOOTSTRAP_SERVERS: kafka://b-1.sg-staging.365fra.c2.kafka.ap-southeast-1.amazonaws.com:9094,kafka://b-2.sg-staging.365fra.c2.kafka.ap-southeast-1.amazonaws.com:9094 +KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://b-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096,PLAINTEXT://b-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096 ## Image Info ## ref: https://hub.docker.com/r/confluentinc/cp-kafka/ image: confluentinc/cp-kafka-rest From 83b59e3eb7b50b01a52f612e331b2421cddecacf Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:46:36 +0800 Subject: [PATCH 23/49] Update config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b2a86c5a..fcc3a7cf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ jobs: chart: charts/cp-kafka-rest release-name: kafka-rest-sg namespace: kafka-client - values-to-override: appLabels.region=sg + values-to-override: appLabels.region=sg,cp-zookeeper.url=z-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-3.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181 values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 @@ -64,7 +64,7 @@ jobs: chart: charts/cp-kafka-rest release-name: kafka-rest-hk namespace: kafka-client - values-to-override: appLabels.region=hk + values-to-override: appLabels.region=hk,cp-zookeeper.url=z-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-3.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181 values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 From aee84bea18d5c49460c98a90981f7585da4acdb1 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:47:43 +0800 Subject: [PATCH 24/49] Update config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fcc3a7cf..08dc3414 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ jobs: chart: charts/cp-kafka-rest release-name: kafka-rest-sg namespace: kafka-client - values-to-override: appLabels.region=sg,cp-zookeeper.url=z-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-3.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181 + values-to-override: appLabels.region=sg,cp-zookeeper.url="z-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-3.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181" values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 @@ -64,7 +64,7 @@ jobs: chart: charts/cp-kafka-rest release-name: kafka-rest-hk namespace: kafka-client - values-to-override: appLabels.region=hk,cp-zookeeper.url=z-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-3.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181 + values-to-override: appLabels.region=hk,cp-zookeeper.url="z-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-3.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181" values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 From 2fa40f308977100af7604442b071c71985229cb8 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:54:55 +0800 Subject: [PATCH 25/49] Update deployment.yaml --- charts/cp-kafka-rest/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/templates/deployment.yaml b/charts/cp-kafka-rest/templates/deployment.yaml index 82bd1adf..39bca499 100644 --- a/charts/cp-kafka-rest/templates/deployment.yaml +++ b/charts/cp-kafka-rest/templates/deployment.yaml @@ -75,7 +75,7 @@ spec: fieldRef: fieldPath: status.podIP - name: KAFKA_REST_BOOTSTRAP_SERVERS - value: {{ .Values.KAFKA_REST_BOOTSTRAP_SERVERS }} + value: {{ template "cp-kafka-rest.kafka.bootstrapServers" . }} - name: KAFKA_REST_SCHEMA_REGISTRY_URL value: {{ template "cp-kafka-rest.cp-schema-registry.service-name" . }} - name: KAFKAREST_HEAP_OPTS From 77e68422a78fd3dc2da5fab3dd5c3638487eeb3c Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:55:09 +0800 Subject: [PATCH 26/49] Update config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08dc3414..b2a86c5a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ jobs: chart: charts/cp-kafka-rest release-name: kafka-rest-sg namespace: kafka-client - values-to-override: appLabels.region=sg,cp-zookeeper.url="z-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-3.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181" + values-to-override: appLabels.region=sg values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 @@ -64,7 +64,7 @@ jobs: chart: charts/cp-kafka-rest release-name: kafka-rest-hk namespace: kafka-client - values-to-override: appLabels.region=hk,cp-zookeeper.url="z-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181,z-3.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181" + values-to-override: appLabels.region=hk values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 From d0a3d9cdfaba61313dfa5600c96be2ae29a171a5 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:57:29 +0800 Subject: [PATCH 27/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index d5c624e7..6fac1f2e 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -100,7 +100,7 @@ external: ## If the Kafka Chart is disabled a URL of zookeeper are required to connect ## e.g. ungaged-sparrow-cp-zookeeper:2181 cp-zookeeper: - url: "" + url: "z-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:2181" ## If the Kafka Chart is disabled a URL and port are required to connect ## e.g. gnoble-panther-cp-schema-registry:8081 @@ -108,4 +108,4 @@ cp-schema-registry: url: "" cp-kafka: - bootstrapServers: "" + bootstrapServers: "b-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096" From e1db290eaf1fdb74e3a168501ba5728ef04c9828 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Thu, 5 Oct 2023 09:55:40 +0800 Subject: [PATCH 28/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index 6fac1f2e..62f915cd 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -64,8 +64,8 @@ configurationOverrides: # "consumer.request.timeout.ms": 5000 ## Additional env variables -customEnv: {} - # ZOOKEEPER_SASL_ENABLED: "false" +customEnv: + KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/kafka_jaas.conf" ## Monitoring ## Kafka REST JMX Settings From 6737342a348d8ef34df9e50cf0d9810ef902d50e Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Thu, 5 Oct 2023 09:56:26 +0800 Subject: [PATCH 29/49] Update deployment.yaml --- charts/cp-kafka-rest/templates/deployment.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/cp-kafka-rest/templates/deployment.yaml b/charts/cp-kafka-rest/templates/deployment.yaml index 39bca499..dc08e5c8 100644 --- a/charts/cp-kafka-rest/templates/deployment.yaml +++ b/charts/cp-kafka-rest/templates/deployment.yaml @@ -56,6 +56,8 @@ spec: - name: jmx-config mountPath: /etc/jmx-kafka-rest {{- end }} + - name: jaas-config + mountPath: /etc/kafka/secrets - name: {{ template "cp-kafka-rest.name" . }}-server image: "{{ .Values.image }}:{{ .Values.imageTag }}" imagePullPolicy: "{{ .Values.imagePullPolicy }}" @@ -102,6 +104,9 @@ spec: configMap: name: {{ template "cp-kafka-rest.fullname" . }}-jmx-configmap {{- end }} + - name: jaas-config + secret: + secretName: my-jaas-config {{- if .Values.nodeSelector }} nodeSelector: {{ toYaml .Values.nodeSelector | indent 8 }} From 7c47a0998ee8327ec4e77ca4eb3dda439d2336d2 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Thu, 5 Oct 2023 10:06:01 +0800 Subject: [PATCH 30/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index 62f915cd..d4156c34 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -108,4 +108,4 @@ cp-schema-registry: url: "" cp-kafka: - bootstrapServers: "b-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096" + bootstrapServers: "b-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096,b-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096" From 7cd1874b91469ce77f06c686bad5676870158446 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:19:18 +0800 Subject: [PATCH 31/49] Update deployment.yaml --- charts/cp-kafka-rest/templates/deployment.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/charts/cp-kafka-rest/templates/deployment.yaml b/charts/cp-kafka-rest/templates/deployment.yaml index dc08e5c8..420c56b2 100644 --- a/charts/cp-kafka-rest/templates/deployment.yaml +++ b/charts/cp-kafka-rest/templates/deployment.yaml @@ -60,6 +60,8 @@ spec: mountPath: /etc/kafka/secrets - name: {{ template "cp-kafka-rest.name" . }}-server image: "{{ .Values.image }}:{{ .Values.imageTag }}" + command: ["/bin/bash", "-c"] + args: ["kafka-rest-start /etc/kafka-rest-cloud/kafka-rest.properties"] imagePullPolicy: "{{ .Values.imagePullPolicy }}" ports: - name: rest-proxy @@ -69,6 +71,10 @@ spec: - containerPort: {{ .Values.jmx.port }} name: jmx {{- end }} + volumeMounts: + - name: kafka-rest-config + mountPath: /etc/kafka-rest-cloud/ + readOnly: true resources: {{ toYaml .Values.resources | indent 12 }} env: @@ -104,9 +110,9 @@ spec: configMap: name: {{ template "cp-kafka-rest.fullname" . }}-jmx-configmap {{- end }} - - name: jaas-config + - name: kafka-rest-config secret: - secretName: my-jaas-config + secretName: kafka-rest-config {{- if .Values.nodeSelector }} nodeSelector: {{ toYaml .Values.nodeSelector | indent 8 }} From 7b0ca3bf8921a151ada8146ed25a722d85a8418c Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:20:58 +0800 Subject: [PATCH 32/49] Update deployment.yaml --- charts/cp-kafka-rest/templates/deployment.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/cp-kafka-rest/templates/deployment.yaml b/charts/cp-kafka-rest/templates/deployment.yaml index 420c56b2..c2ef628d 100644 --- a/charts/cp-kafka-rest/templates/deployment.yaml +++ b/charts/cp-kafka-rest/templates/deployment.yaml @@ -56,8 +56,6 @@ spec: - name: jmx-config mountPath: /etc/jmx-kafka-rest {{- end }} - - name: jaas-config - mountPath: /etc/kafka/secrets - name: {{ template "cp-kafka-rest.name" . }}-server image: "{{ .Values.image }}:{{ .Values.imageTag }}" command: ["/bin/bash", "-c"] From 0e6acfd49ddac4528081a64bfd84f8618d732903 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:31:29 +0800 Subject: [PATCH 33/49] Update service.yaml --- charts/cp-kafka-rest/templates/service.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/cp-kafka-rest/templates/service.yaml b/charts/cp-kafka-rest/templates/service.yaml index 4d9765f9..4cdd6f3d 100644 --- a/charts/cp-kafka-rest/templates/service.yaml +++ b/charts/cp-kafka-rest/templates/service.yaml @@ -11,6 +11,7 @@ spec: ports: - name: rest-proxy port: {{ .Values.servicePort }} + type: {{ .Values.serviceType }} {{- if .Values.prometheus.jmx.enabled }} - name: metrics port: {{ .Values.prometheus.jmx.port }} From 8a0c2ab79fd5ab4b7a16abbd90688a04c7908128 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:31:48 +0800 Subject: [PATCH 34/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index d4156c34..5dc34c67 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -27,6 +27,7 @@ imagePullPolicy: IfNotPresent imagePullSecrets: servicePort: 8082 +serviceType: NodePort ## Kafka rest JVM Heap Option heapOptions: "-Xms512M -Xmx512M" From 99dbf979a7f9a544b8d7b0338607dcae99e5f371 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:32:03 +0800 Subject: [PATCH 35/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index 5dc34c67..c38b389a 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. envs: kafka_rest: - ingress_host: kafka-client.mssgdev.com + ingress_host: event-proxy.mssgdev.com name: kafka-client certificate_arn: arn:aws:acm:ap-southeast-1:856075066843:certificate/3d8d374c-c52f-4e59-8deb-e5b44c615db3 paths: From 637753838b2431073dd7928280e4be87295dedc3 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:33:25 +0800 Subject: [PATCH 36/49] Update service.yaml --- charts/cp-kafka-rest/templates/service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/templates/service.yaml b/charts/cp-kafka-rest/templates/service.yaml index 4cdd6f3d..971d5c41 100644 --- a/charts/cp-kafka-rest/templates/service.yaml +++ b/charts/cp-kafka-rest/templates/service.yaml @@ -11,7 +11,6 @@ spec: ports: - name: rest-proxy port: {{ .Values.servicePort }} - type: {{ .Values.serviceType }} {{- if .Values.prometheus.jmx.enabled }} - name: metrics port: {{ .Values.prometheus.jmx.port }} @@ -19,3 +18,4 @@ spec: selector: app: {{ template "cp-kafka-rest.name" . }} release: {{ .Release.Name }} + type: {{ .Values.serviceType }} From 416953af381434e2486d22500d25c4a14201393a Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:45:57 +0800 Subject: [PATCH 37/49] Update values.yaml --- charts/cp-kafka-rest/values.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values.yaml index c38b389a..51bfc66b 100644 --- a/charts/cp-kafka-rest/values.yaml +++ b/charts/cp-kafka-rest/values.yaml @@ -11,7 +11,6 @@ envs: replicaCount: 1 -KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://b-2.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096,PLAINTEXT://b-1.mskproductlisting.cya0qq.c3.kafka.ap-southeast-1.amazonaws.com:9096 ## Image Info ## ref: https://hub.docker.com/r/confluentinc/cp-kafka/ image: confluentinc/cp-kafka-rest @@ -65,8 +64,8 @@ configurationOverrides: # "consumer.request.timeout.ms": 5000 ## Additional env variables -customEnv: - KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/kafka_jaas.conf" +customEnv: {} + # ZOOKEEPER_SASL_ENABLED: "false" ## Monitoring ## Kafka REST JMX Settings From 4702907dceed4ccbef45510b1cfb3222931d6f3c Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:49:27 +0800 Subject: [PATCH 38/49] Update config.yml --- .circleci/config.yml | 60 +------------------------------------------- 1 file changed, 1 insertion(+), 59 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b2a86c5a..8dcaba3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,29 +47,8 @@ jobs: values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 - - deploy_to_hk_staging_eks: - docker: - - image: 'cimg/python:3.10' - steps: - - checkout - - *stg_aws_creds - - attach_workspace: - at: /tmp/workspace - - moneysmart-orb/pull_helm_charts: - cluster-name: product-listing-staging - chart-name: identity-provider-fe - branch-name: production - - helm/upgrade-helm-chart: - chart: charts/cp-kafka-rest - release-name: kafka-rest-hk - namespace: kafka-client - values-to-override: appLabels.region=hk - values: charts/cp-kafka-rest/values.yaml - timeout: 600s - helm-version: v3.9.0 #--------------------------------------------- -# campaign_service SG and HK Production Jobs +# kakfa-rest SG and #-------------------------------------------- deploy_to_sg_prod_eks: docker: @@ -91,27 +70,6 @@ jobs: values: charts/cp-kafka-rest/values.yaml timeout: 600s helm-version: v3.9.0 - - deploy_to_hk_prod_eks: - docker: - - image: 'cimg/python:3.10' - steps: - - checkout - - *prod_aws_creds - - attach_workspace: - at: /tmp/workspace - - moneysmart-orb/pull_helm_charts: - cluster-name: product-listing-prod - chart-name: kafka-rest - branch-name: production - - helm/upgrade-helm-chart: - chart: charts/cp-kafka-rest - release-name: kafka-rest-hk - namespace: kafka-client - values-to-override: appLabels.region=hk - values: charts/cp-kafka-rest/values.yaml - timeout: 600s - helm-version: v3.9.0 workflows: version: 2.1 #--------------------------------------------- @@ -125,14 +83,6 @@ workflows: branches: only: - update-cicd - - - deploy_to_hk_staging_eks: - context: common-eks-vars - filters: - branches: - only: - - update-cicd - #--------------------------------------------- # campaign_service SG and HK Production Pipeline #--------------------------------------------- @@ -145,11 +95,3 @@ workflows: only: /^v.*/ branches: ignore: /.*/ - - - deploy_to_hk_prod_eks: - context: common-eks-vars - filters: - tags: - only: /^v.*/ - branches: - ignore: /.*/ From 4710d835cab40a3d56c327e18c63301c61a646d9 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Mon, 9 Oct 2023 18:55:54 +0800 Subject: [PATCH 39/49] Update ingress.yml --- charts/cp-kafka-rest/templates/ingress.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/cp-kafka-rest/templates/ingress.yml b/charts/cp-kafka-rest/templates/ingress.yml index 126e7e50..932b7efc 100644 --- a/charts/cp-kafka-rest/templates/ingress.yml +++ b/charts/cp-kafka-rest/templates/ingress.yml @@ -16,8 +16,7 @@ metadata: name: {{ .Release.Name }} spec: rules: - - host: {{ .Values.envs.kafka_rest.ingress_host }} - http: + - http: paths: {{- range $ingressPath }} - path: {{ . }} From a4ea25c63beb568999032f444ca90914f60c1657 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Tue, 10 Oct 2023 11:09:07 +0800 Subject: [PATCH 40/49] Update ingress.yml --- charts/cp-kafka-rest/templates/ingress.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/cp-kafka-rest/templates/ingress.yml b/charts/cp-kafka-rest/templates/ingress.yml index 932b7efc..1cfadc6d 100644 --- a/charts/cp-kafka-rest/templates/ingress.yml +++ b/charts/cp-kafka-rest/templates/ingress.yml @@ -7,6 +7,7 @@ metadata: alb.ingress.kubernetes.io/certificate-arn: {{ .Values.envs.kafka_rest.certificate_arn }} alb.ingress.kubernetes.io/group.name: eks-lb-{{ .Values.appLabels.region }} alb.ingress.kubernetes.io/healthcheck-path: /_ping + alb.ingress.kubernetes.io/group.order: '150' kubernetes.io/ingress.class: alb nginx.ingress.kubernetes.io/use-regex: "true" labels: From de932c39376f2602bc7fc5c942f503155da1271f Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:39:31 +0800 Subject: [PATCH 41/49] Update config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8dcaba3d..848d9871 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,7 +77,7 @@ workflows: #--------------------------------------------- staging-build-deploy: jobs: - - deploy_to_sg_staging_eks: + - deploy_to_sg_prod_eks: context: common-eks-vars filters: branches: From aab5c2a679699a0a4fe974883057dfe85f6986b4 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:46:53 +0800 Subject: [PATCH 42/49] Update config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 848d9871..62a402e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,7 +37,7 @@ jobs: at: /tmp/workspace - moneysmart-orb/pull_helm_charts: cluster-name: product-listing-staging - chart-name: identity-provider-fe + chart-name: kafka-rest branch-name: production - helm/upgrade-helm-chart: chart: charts/cp-kafka-rest From c6bc9bc6d858cb4d03febd4e81bce038ce27dd70 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:49:51 +0800 Subject: [PATCH 43/49] Update config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 62a402e7..6f02b5e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,7 +67,7 @@ jobs: release-name: kafka-rest-sg namespace: kafka-client values-to-override: appLabels.region=sg - values: charts/cp-kafka-rest/values.yaml + values: charts/cp-kafka-rest/values-prod.yaml timeout: 600s helm-version: v3.9.0 workflows: @@ -82,7 +82,7 @@ workflows: filters: branches: only: - - update-cicd + - update-cicd2 #--------------------------------------------- # campaign_service SG and HK Production Pipeline #--------------------------------------------- From 36b7947c5c6650f3cd71dcc265c44b742aa1944c Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:50:44 +0800 Subject: [PATCH 44/49] Rename values.yaml to values-dev.yaml --- charts/cp-kafka-rest/{values.yaml => values-dev.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename charts/cp-kafka-rest/{values.yaml => values-dev.yaml} (100%) diff --git a/charts/cp-kafka-rest/values.yaml b/charts/cp-kafka-rest/values-dev.yaml similarity index 100% rename from charts/cp-kafka-rest/values.yaml rename to charts/cp-kafka-rest/values-dev.yaml From 24b68c92c6fa518a7fa8af31b9d45b21743e05a1 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:55:34 +0800 Subject: [PATCH 45/49] Create values-prod.yml --- charts/cp-kafka-rest/values-prod.yml | 111 +++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 charts/cp-kafka-rest/values-prod.yml diff --git a/charts/cp-kafka-rest/values-prod.yml b/charts/cp-kafka-rest/values-prod.yml new file mode 100644 index 00000000..2300c8ae --- /dev/null +++ b/charts/cp-kafka-rest/values-prod.yml @@ -0,0 +1,111 @@ +# Default values for cp-kafka-rest. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +envs: + kafka_rest: + ingress_host: event-proxy.moneysmart.sg + name: kafka-client + certificate_arn: arn:aws:acm:ap-southeast-1:814507470717:certificate/989bd44a-305f-47ab-acb9-574fd7a4dd9e + paths: + - /* + +replicaCount: 1 + +## Image Info +## ref: https://hub.docker.com/r/confluentinc/cp-kafka/ +image: confluentinc/cp-kafka-rest +imageTag: 6.1.0 + +## Specify a imagePullPolicy +## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images +imagePullPolicy: IfNotPresent + +## Specify an array of imagePullSecrets. +## Secrets must be manually created in the namespace. +## ref: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod +imagePullSecrets: + +servicePort: 8082 +serviceType: NodePort + +## Kafka rest JVM Heap Option +heapOptions: "-Xms512M -Xmx512M" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +## Custom pod annotations +podAnnotations: {} + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ +nodeSelector: {} + +## Taints to tolerate on node assignment: +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +## Pod scheduling constraints +## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +affinity: {} + +## Kafka REST configuration options +## ref: https://docs.confluent.io/current/kafka-rest/docs/config.html +configurationOverrides: + # "consumer.request.timeout.ms": 5000 + +## Additional env variables +customEnv: {} + # ZOOKEEPER_SASL_ENABLED: "false" + +## Monitoring +## Kafka REST JMX Settings +## ref: https://docs.confluent.io/current/kafka-rest/docs/monitoring.html +jmx: + port: 5555 + +## Prometheus Exporter Configuration +## ref: https://prometheus.io/docs/instrumenting/exporters/ +prometheus: + ## JMX Exporter Configuration + ## ref: https://github.com/prometheus/jmx_exporter + jmx: + enabled: true + image: solsson/kafka-prometheus-jmx-exporter@sha256 + imageTag: 6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 + imagePullPolicy: IfNotPresent + port: 5556 + + ## Resources configuration for the JMX exporter container. + ## See the `resources` documentation above for details. + resources: {} + +## External Access +## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer +external: + enabled: false + type: LoadBalancer + externalTrafficPolicy: Cluster + port: 8082 + +## If the Kafka Chart is disabled a URL of zookeeper are required to connect +## e.g. ungaged-sparrow-cp-zookeeper:2181 +cp-zookeeper: + url: "z-1.mskproductlisting.x95avm.c3.kafka.ap-southeast-1.amazonaws.com:2181" + +## If the Kafka Chart is disabled a URL and port are required to connect +## e.g. gnoble-panther-cp-schema-registry:8081 +cp-schema-registry: + url: "" + +cp-kafka: + bootstrapServers: "b-1.mskproductlisting.x95avm.c3.kafka.ap-southeast-1.amazonaws.com:9096,b-2.mskproductlisting.x95avm.c3.kafka.ap-southeast-1.amazonaws.com" From e371c28f1bfa6f1e225ba592835035a91a25d6b0 Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Tue, 30 Jan 2024 12:59:38 +0800 Subject: [PATCH 46/49] Update config.yml --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6f02b5e3..b478f764 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,7 +44,7 @@ jobs: release-name: kafka-rest-sg namespace: kafka-client values-to-override: appLabels.region=sg - values: charts/cp-kafka-rest/values.yaml + values: charts/cp-kafka-rest/values-dev.yaml timeout: 600s helm-version: v3.9.0 #--------------------------------------------- @@ -82,7 +82,7 @@ workflows: filters: branches: only: - - update-cicd2 + - update-cicd #--------------------------------------------- # campaign_service SG and HK Production Pipeline #--------------------------------------------- From 02cfc0b9cf8515fe641502e36c62806ecf93b35d Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Tue, 30 Jan 2024 13:02:18 +0800 Subject: [PATCH 47/49] Rename values-prod.yml to values-prod.yaml --- charts/cp-kafka-rest/{values-prod.yml => values-prod.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename charts/cp-kafka-rest/{values-prod.yml => values-prod.yaml} (100%) diff --git a/charts/cp-kafka-rest/values-prod.yml b/charts/cp-kafka-rest/values-prod.yaml similarity index 100% rename from charts/cp-kafka-rest/values-prod.yml rename to charts/cp-kafka-rest/values-prod.yaml From 75790225feb48301c10f17c8ab6eac06b41d7dec Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:18:34 +0800 Subject: [PATCH 48/49] Update ingress.yml --- charts/cp-kafka-rest/templates/ingress.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/cp-kafka-rest/templates/ingress.yml b/charts/cp-kafka-rest/templates/ingress.yml index 1cfadc6d..9e95f5ab 100644 --- a/charts/cp-kafka-rest/templates/ingress.yml +++ b/charts/cp-kafka-rest/templates/ingress.yml @@ -17,7 +17,8 @@ metadata: name: {{ .Release.Name }} spec: rules: - - http: + - host: event-proxy.{{ .Values.domain }} + http: paths: {{- range $ingressPath }} - path: {{ . }} From c026b7f64e8972df046ffae30fc221a7600e045e Mon Sep 17 00:00:00 2001 From: petercipriano-ms <105843078+petercipriano-ms@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:18:38 +0800 Subject: [PATCH 49/49] Update values-prod.yaml --- charts/cp-kafka-rest/values-prod.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/cp-kafka-rest/values-prod.yaml b/charts/cp-kafka-rest/values-prod.yaml index 2300c8ae..2456f0ec 100644 --- a/charts/cp-kafka-rest/values-prod.yaml +++ b/charts/cp-kafka-rest/values-prod.yaml @@ -11,6 +11,8 @@ envs: replicaCount: 1 +domain: moneysmart.sg + ## Image Info ## ref: https://hub.docker.com/r/confluentinc/cp-kafka/ image: confluentinc/cp-kafka-rest