Skip to content

datalake physical component change updated #62

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

datalake physical component change updated #62

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fd80f7c
datalake physical component change updated
nerudadhich Mar 7, 2022
3382474
join on removed and query fixed for alb and apigateway
nerudadhich Mar 7, 2022
862156f
[datalake] fix alb and elb modules -- replace status_histo with gauge
arjunmahishi Mar 8, 2022
b4a2c9c
fixes
Mar 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
232 changes: 67 additions & 165 deletions modules/datalake/aws/alb.hcl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ ingester aws_alb_cloudwatch module {

physical_component {
type = "alb"
name = "$input{LoadBalancer}"
name = "alb"
}

data_for_graph_node {
type = "alb"
name = "$input{LoadBalancer}"
name = "$output{LoadBalancer}"
}

gauge "throughput" {
Expand All @@ -38,11 +38,18 @@ ingester aws_alb_cloudwatch module {
aggregator = "SUM"

source prometheus "throughput" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (throughput)"
query = "sum by (LoadBalancer, tag_service, tag_namespace) (throughput{LoadBalancer != ''})"
}
}

join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
gauge "lcu" {
index = 3
input_unit = "count"
output_unit = "count"
aggregator = "SUM"

source prometheus "lcu" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (lcu{LoadBalancer != ''})"
}
}

Expand All @@ -53,11 +60,7 @@ ingester aws_alb_cloudwatch module {
aggregator = "SUM"

source prometheus "new_connections" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (new_connections)"

join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
query = "sum by (LoadBalancer, tag_service, tag_namespace) (new_connections{LoadBalancer != ''})"
}
}

Expand All @@ -68,11 +71,7 @@ ingester aws_alb_cloudwatch module {
aggregator = "SUM"

source prometheus "rejected_connections" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (rejected_connections)"

join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
query = "sum by (LoadBalancer, tag_service, tag_namespace) (rejected_connections{LoadBalancer != ''})"
}
}

Expand All @@ -83,136 +82,84 @@ ingester aws_alb_cloudwatch module {
aggregator = "SUM"

source prometheus "processed_bytes" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (processed_bytes)"

join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
query = "sum by (LoadBalancer, tag_service, tag_namespace) (processed_bytes{LoadBalancer != ''})"
}
}

gauge "lcu" {
index = 3
input_unit = "count"
output_unit = "count"
aggregator = "SUM"

source prometheus "lcu" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (lcu)"

join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
}
}

status_histo status_5xx {
index = 5
gauge lb_4xx {
index = 7
input_unit = "count"
output_unit = "rpm"
aggregator = "SUM"

source prometheus "status_5xx" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (status_5xx)"

join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
source prometheus "lb_4xx" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (response{code='4xx_lb', LoadBalancer != ''})"
}
}

status_histo status_4xx {
index = 4
gauge lb_5xx {
index = 8
input_unit = "count"
output_unit = "rpm"
aggregator = "SUM"

source prometheus "status_4xx" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (status_4xx)"

join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
source prometheus "lb_5xx" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (response{code='5xx_lb', LoadBalancer != ''})"
}
}

gauge lb_5xx {
index = 8
gauge status_5xx {
index = 9
input_unit = "count"
output_unit = "rpm"
aggregator = "SUM"

source prometheus "lb_5xx" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (lb_5xx)"

join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
source prometheus "status_5xx" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (response{code='5xx_target', LoadBalancer != ''})"
}
}

gauge lb_4xx {
index = 7
gauge status_4xx {
index = 11
input_unit = "count"
output_unit = "rpm"
aggregator = "SUM"

source prometheus "lb_4xx" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (lb_4xx)"

join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
source prometheus "status_4xx" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (response{code='4xx_target', LoadBalancer != ''})"
}
}

latency "latency_histo" {
index = 6
input_unit = "ms"
output_unit = "ms"
aggregator = "PERCENTILE"
error_margin = 0.05

source prometheus "throughput" {
query = "sum by (LoadBalancer, tag_service, tag_namespace) (throughput)"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
}
gauge latency_min {
index = 12
input_unit = "s"
output_unit = "s"
aggregator = "MIN"

source prometheus "p50" {
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p50'})"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
source prometheus "latency_min" {
query = "min by (LoadBalancer, tag_service, tag_namespace) (latency{stat='min', LoadBalancer != ''})"
}
}

source prometheus "p75" {
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p75'})"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
}
gauge latency_max {
index = 13
input_unit = "s"
output_unit = "s"
aggregator = "MAX"

source prometheus "p90" {
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p90'})"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
source prometheus "latency_max" {
query = "max by (LoadBalancer, tag_service, tag_namespace) (latency{stat='max', LoadBalancer != ''})"
}
}

source prometheus "p99" {
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p99'})"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
}
gauge latency_avg {
index = 14
input_unit = "s"
output_unit = "s"
aggregator = "AVG"

source prometheus "p100" {
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p100'})"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
}
source prometheus "latency_avg" {
query = "avg by (LoadBalancer, tag_service, tag_namespace) (latency{stat='avg', LoadBalancer != ''})"
}
}
}
Expand Down Expand Up @@ -243,7 +190,7 @@ ingester aws_alb_tg_cloudwatch module {

physical_component {
type = "alb"
name = "$input{LoadBalancer}"
name = "alb"
}

physical_address {
Expand All @@ -262,12 +209,7 @@ ingester aws_alb_tg_cloudwatch module {
output_unit = "rpm"
aggregator = "SUM"
source prometheus "throughput" {
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (throughput)"

join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
"$output{TargetGroup}" = "$input{TargetGroup}"
}
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (throughput{LoadBalancer != '', TargetGroup != ''})"
}
}

Expand All @@ -278,51 +220,27 @@ ingester aws_alb_tg_cloudwatch module {
aggregator = "PERCENTILE"
error_margin = 0.05
source prometheus "throughput" {
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (throughput)"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
"$output{TargetGroup}" = "$input{TargetGroup}"
}
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (throughput{LoadBalancer != '', TargetGroup != ''})"
}

source prometheus "p50" {
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p50'})"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
"$output{TargetGroup}" = "$input{TargetGroup}"
}
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p50', LoadBalancer != '', TargetGroup != ''})"
}

source prometheus "p75" {
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p75'})"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
"$output{TargetGroup}" = "$input{TargetGroup}"
}
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p75', LoadBalancer != '', TargetGroup != ''})"
}

source prometheus "p90" {
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p90'})"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
"$output{TargetGroup}" = "$input{TargetGroup}"
}
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p90', LoadBalancer != '', TargetGroup != ''})"
}

source prometheus "p99" {
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p99'})"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
"$output{TargetGroup}" = "$input{TargetGroup}"
}
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p99', LoadBalancer != '', TargetGroup != ''})"
}

source prometheus "p100" {
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p100'})"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
"$output{TargetGroup}" = "$input{TargetGroup}"
}
query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p100', LoadBalancer != '', TargetGroup != ''})"
}
}

Expand All @@ -332,11 +250,7 @@ ingester aws_alb_tg_cloudwatch module {
output_unit = "rpm"
aggregator = "SUM"
source prometheus "status_500" {
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_5xx)"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
"$output{TargetGroup}" = "$input{TargetGroup}"
}
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_5xx{LoadBalancer != '', TargetGroup != ''})"
}
}

Expand All @@ -346,11 +260,7 @@ ingester aws_alb_tg_cloudwatch module {
output_unit = "rpm"
aggregator = "SUM"
source prometheus "status_400" {
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_4xx)"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
"$output{TargetGroup}" = "$input{TargetGroup}"
}
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_4xx{LoadBalancer != '', TargetGroup != ''})"
}
}

Expand All @@ -360,11 +270,7 @@ ingester aws_alb_tg_cloudwatch module {
output_unit = "rpm"
aggregator = "SUM"
source prometheus "status_300" {
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_3xx)"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
"$output{TargetGroup}" = "$input{TargetGroup}"
}
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_3xx{LoadBalancer != '', TargetGroup != ''})"
}
}

Expand All @@ -374,11 +280,7 @@ ingester aws_alb_tg_cloudwatch module {
output_unit = "rpm"
aggregator = "SUM"
source prometheus "status_200" {
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_2xx)"
join_on = {
"$output{LoadBalancer}" = "$input{LoadBalancer}"
"$output{TargetGroup}" = "$input{TargetGroup}"
}
query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_2xx{LoadBalancer != '', TargetGroup != ''})"
}
}
}
Loading