Skip to content

Commit 280c33c

Browse files
mayuthombreJoshArmi
mayuthombre
authored and
JoshArmi
committed
build custom grafana image
1 parent 1366a3b commit 280c33c

File tree

15 files changed

+433
-307
lines changed

15 files changed

+433
-307
lines changed

‎.github/workflows/gcp_cicd.yaml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,17 +53,26 @@ jobs:
5353
with:
5454
directory: infrastructure/gcp/cdktf.out/stacks/
5555
framework: terraform_plan
56-
# skip_check: CKV_GCP_102
5756

5857
- name: Deploy base infrastructure
5958
run: make gcp-deploy-base INFRA_ARGS=--auto-approve
6059

6160
- name: Build & push docker image
62-
run: make gcp-build-image
61+
run: make gcp-core-image
62+
63+
- name: Build & push Grafana image
64+
run: make gcp-grafana-image
65+
env:
66+
GRAFANA_PASSWORD: ${{ secrets.GRAFANA_PASSWORD }}
6367

6468
- name: Deploy main infrastructure
6569
run: make gcp-deploy-core INFRA_ARGS=--auto-approve
6670

71+
- name: Deploy grafana dashboard
72+
run: make gcp-deploy-grafana INFRA_ARGS=--auto-approve
73+
env:
74+
GRAFANA_PASSWORD: ${{ secrets.GRAFANA_PASSWORD }}
75+
6776
- name: Perform full test and check coverage
6877
run: make gcp-test
6978

‎Makefile‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -125,11 +125,17 @@ gcp-build-dependencies:
125125
@echo "\n\n---GCP-BUILD-DEPENDENCIES---\n"
126126
cd infrastructure/gcp; cdktf provider add grafana/grafana
127127

128-
gcp-build-image:
128+
gcp-core-image:
129129
@echo "\n\n---GCP-BUILD-IMAGE---\n"
130130
gcloud auth configure-docker australia-southeast1-docker.pkg.dev
131131
pipenv requirements | tee requirements.txt
132-
docker buildx build --platform=linux/amd64 --push . -t australia-southeast1-docker.pkg.dev/contino-squad0-fc/flight-controller-event-receiver/event_receiver:${COMMIT}
132+
docker buildx build --platform=linux/amd64 -f infrastructure/gcp/Core.Dockerfile . --push -t australia-southeast1-docker.pkg.dev/contino-squad0-fc/flight-controller-event-receiver/event_receiver:${COMMIT}
133+
134+
gcp-grafana-image:
135+
@echo "\n\n---BUILD-GRAFANA-IMAGE---\n"
136+
cd infrastructure/gcp
137+
gcloud auth configure-docker australia-southeast1-docker.pkg.dev
138+
docker buildx build --platform=linux/amd64 --build-arg GRAFANA_PASSWORD=${GRAFANA_PASSWORD} -f infrastructure/gcp/Grafana.Dockerfile . --push -t australia-southeast1-docker.pkg.dev/contino-squad0-fc/flight-controller-event-receiver/grafana:${COMMIT}
133139

134140
gcp-synth: gcp-build-dependencies
135141
@echo "\n\n---GCP-SYNTH---\n"
@@ -170,7 +176,7 @@ gcp-deploy-grafana:
170176
@echo "\n\n---GCP-DEPLOY-GRAFANA---\n"
171177
cd infrastructure/gcp;cdktf deploy gcp_grafana ${INFRA_ARGS}
172178

173-
gcp-deploy-all: gcp-deploy-base gcp-build-image gcp-deploy-core #gcp-deploy-grafana
179+
gcp-deploy-all: gcp-deploy-base gcp-core-image gcp-grafana-image gcp-deploy-core gcp-deploy-grafana
174180

175181
gcp-destroy-base:
176182
@echo "\n\n---GCP-DESTROY-BASE---\n"
@@ -186,8 +192,4 @@ gcp-destroy-grafana:
186192

187193
gcp-destroy-all:
188194
@echo "\n\n---GCP-DESTROY-ALL---\n"
189-
cd infrastructure/gcp;cdktf destroy gcp_base gcp_core gcp_grafana
190-
191-
local-gcp-plan: gcp-plan-base gcp-plan-core
192-
193-
local-gcp-deploy: gcp-deploy-base gcp-deploy-core
195+
cd infrastructure/gcp;cdktf destroy gcp_base gcp_core gcp_grafana

‎Pipfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,4 @@ dirhash = "*"
3232
python_version = "3.9"
3333

3434
[pipenv]
35-
allow_prereleases = true
35+
allow_prereleases = false

‎Pipfile.lock‎

Lines changed: 274 additions & 246 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Dockerfile‎ renamed to ‎infrastructure/gcp/Core.Dockerfile‎

File renamed without changes.

‎infrastructure/gcp/Grafana.Dockerfile‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
ARG GRAFANA_VERSION="latest"
2+
3+
# Download base grafana
4+
FROM grafana/grafana:${GRAFANA_VERSION}
5+
6+
ARG GRAFANA_PASSWORD
7+
8+
USER grafana
9+
10+
# Set the admin password
11+
ENV GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
12+
13+
# Install BigQuery data source plugin
14+
RUN grafana-cli plugins install simpod-json-datasource && \
15+
grafana-cli plugins install grafana-bigquery-datasource && \
16+
mkdir -p /var/lib/grafana/plugins/grafana-bigquery-datasource
17+
18+
# Set the default port
19+
ENV GF_SERVER_HTTP_PORT=8080
20+
21+
# Expose Grafana's default port
22+
EXPOSE 8080

‎infrastructure/gcp/cdktf.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"projectId": "ab6ba306-5f3e-4a6e-88ce-c23e56898e9d",
55
"sendCrashReports": "false",
66
"terraformProviders": [
7-
"grafana/grafana@~> 1.39"
7+
"grafana/grafana@~> 1.40"
88
],
99
"terraformModules": [],
1010
"codeMakerOutput": "imports",

‎infrastructure/gcp/dashboard.json‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,24 +11,19 @@
1111
"hide": true,
1212
"iconColor": "rgba(0, 211, 255, 1)",
1313
"name": "Annotations & Alerts",
14-
"target": {
15-
"limit": 100,
16-
"matchAny": false,
17-
"tags": [],
18-
"type": "dashboard"
19-
},
2014
"type": "dashboard"
2115
}
2216
]
2317
},
18+
"description": "",
2419
"editable": true,
2520
"fiscalYearStartMonth": 0,
2621
"graphTooltip": 0,
22+
"id": 3,
2723
"links": [],
2824
"liveNow": false,
2925
"panels": [],
3026
"refresh": "",
31-
"revision": 1,
3227
"schemaVersion": 38,
3328
"style": "dark",
3429
"tags": [],
@@ -41,7 +36,8 @@
4136
},
4237
"timepicker": {},
4338
"timezone": "",
44-
"title": "Flight Controller",
45-
"version": 0,
39+
"title": "Flight Controller Dashboard",
40+
"uid": "d5c3a93e-f160-4ed5-b3b8-aefbc1532bf4",
41+
"version": 2,
4642
"weekStart": ""
4743
}

‎infrastructure/gcp/grafana_cloudrun_component.py‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
1-
from cdktf_cdktf_provider_google import (cloud_run_service,
2-
cloud_run_service_iam_member,
3-
service_account)
1+
import subprocess
2+
3+
from cdktf_cdktf_provider_google import (
4+
cloud_run_service,
5+
cloud_run_service_iam_member,
6+
)
47
from constructs import Construct
58

9+
COMMIT = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode("utf-8")
10+
611

712
class GrafanaComponent(Construct):
813
def __init__(
@@ -15,11 +20,14 @@ def __init__(
1520
):
1621
super().__init__(scope, id)
1722

23+
24+
# Grafana service setup
1825
self.service = cloud_run_service.CloudRunService(
1926
self,
2027
"grafana_service",
2128
name=f"{name_prefix}-grafana",
2229
location=location,
30+
autogenerate_revision_name=True,
2331
project=project_id,
2432
metadata={
2533
"annotations": {
@@ -30,7 +38,7 @@ def __init__(
3038
"spec": {
3139
"containers": [
3240
{
33-
"image": "mirror.gcr.io/grafana/grafana:latest",
41+
"image": f"australia-southeast1-docker.pkg.dev/contino-squad0-fc/flight-controller-event-receiver/grafana:{COMMIT}",
3442
"ports": [
3543
{
3644
"container_port": 8080,
@@ -40,12 +48,18 @@ def __init__(
4048
{
4149
"name": "GF_SERVER_HTTP_PORT",
4250
"value": "8080",
43-
}
51+
},
4452
],
4553
}
4654
]
4755
}
4856
},
57+
traffic=[
58+
{
59+
"percent": 100,
60+
"latest_revision": True,
61+
}
62+
],
4963
)
5064

5165
self.allowusers = cloud_run_service_iam_member.CloudRunServiceIamMember(

‎infrastructure/gcp/grafana_dashboard_component.py‎

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,67 @@
11
# Import the necessary modules
22
import json
3+
from typing import Any
34

45
from constructs import Construct
56
from imports.grafana.dashboard import Dashboard
7+
from imports.grafana.data_source import DataSource
68
from imports.grafana.folder import Folder
79

8-
# Load the dashboard configuration from a JSON file
9-
with open("dashboard.json", "r") as f:
10-
data = json.load(f)
11-
12-
# Convert the data to a JSON string
13-
data = json.dumps(data)
1410

11+
# Function for updating the datasource in the static dashboard.json config file.
12+
# This allowing the datasource and config to be updated at the same time. Aswell as removes production config issues where it uses dev/different datasource uid.
13+
def update_uid(obj: dict, uid: str) -> None:
14+
if isinstance(obj, dict):
15+
for key in obj:
16+
if key == "targets":
17+
for target in obj[key]:
18+
if target["datasource"]["type"] == "grafana-timestream-datasource":
19+
if target["datasource"]["uid"] != uid:
20+
target["datasource"]["uid"] = uid
21+
else:
22+
update_uid(obj[key], uid)
23+
elif isinstance(obj, list):
24+
for item in obj:
25+
update_uid(item, uid)
1526

1627
# Create a new component
1728
class GrafanaDashboardComponent(Construct):
1829
def __init__(
1930
self,
2031
scope: Construct,
2132
id: str,
33+
data: Any,
2234
):
2335
super().__init__(scope, id)
2436

25-
self.folder = Folder(self, "folder", title="AWS Flight Controller")
37+
self.folder = Folder(
38+
self,
39+
"folder",
40+
title="GCP Flight Controller"
41+
)
42+
43+
datasource_config = json.dumps({
44+
'authenticationType': 'gce',
45+
'defaultProject': 'contino-squad0-fc',
46+
'processingLocation': 'australia-southeast1',
47+
})
48+
49+
50+
self.datasource = DataSource(
51+
self,
52+
"bigquery-datasource",
53+
name="Flight Controller Bigquery",
54+
type="grafana-bigquery-datasource",
55+
is_default=True,
56+
json_data_encoded=datasource_config
57+
)
58+
59+
# Update datasource UID
60+
update_uid(data, self.datasource.uid)
61+
62+
# Convert the data to a JSON string
63+
data = json.dumps(data)
64+
2665

2766
# Create a new Grafana dashboard
2867
self.dashboard = Dashboard(

0 commit comments

Comments
 (0)