From fd80f7cd42a2e0e9f26288587a7261bda84d1a41 Mon Sep 17 00:00:00 2001
From: Neeraj Sharma <neeraj@last9.io>
Date: Mon, 7 Mar 2022 15:54:03 +0530
Subject: [PATCH 1/4] datalake physical component change updated

---
 modules/datalake/aws/alb.hcl           | 4 ++--
 modules/datalake/aws/apigateway.hcl    | 2 +-
 modules/datalake/aws/aurora.hcl        | 4 ++--
 modules/datalake/aws/cloudfront.hcl    | 2 +-
 modules/datalake/aws/dax.hcl           | 2 +-
 modules/datalake/aws/dynamodb.hcl      | 2 +-
 modules/datalake/aws/ec2.hcl           | 2 +-
 modules/datalake/aws/eks.hcl           | 8 ++++----
 modules/datalake/aws/elasticcache.hcl  | 4 ++--
 modules/datalake/aws/elasticsearch.hcl | 4 ++--
 modules/datalake/aws/elb.hcl           | 8 ++++----
 modules/datalake/aws/lambda.hcl        | 2 +-
 modules/datalake/aws/msk.hcl           | 6 +++---
 modules/datalake/aws/nlb.hcl           | 2 +-
 modules/datalake/aws/rds.hcl           | 2 +-
 modules/datalake/aws/sqs.hcl           | 2 +-
 16 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/modules/datalake/aws/alb.hcl b/modules/datalake/aws/alb.hcl
index 456b095..d714ad5 100644
--- a/modules/datalake/aws/alb.hcl
+++ b/modules/datalake/aws/alb.hcl
@@ -23,7 +23,7 @@ ingester aws_alb_cloudwatch module {
 
   physical_component {
     type = "alb"
-    name = "$input{LoadBalancer}"
+    name = "alb"
   }
 
   data_for_graph_node {
@@ -243,7 +243,7 @@ ingester aws_alb_tg_cloudwatch module {
 
   physical_component {
     type = "alb"
-    name = "$input{LoadBalancer}"
+    name = "alb"
   }
 
   physical_address {
diff --git a/modules/datalake/aws/apigateway.hcl b/modules/datalake/aws/apigateway.hcl
index 1cc7afb..e0d633e 100644
--- a/modules/datalake/aws/apigateway.hcl
+++ b/modules/datalake/aws/apigateway.hcl
@@ -24,7 +24,7 @@ ingester aws_apigateway module {
 
   physical_component {
     type = "aws_apigateway"
-    name = "$input{ApiName}-$input{Stage}"
+    name = "aws_apigateway"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/aurora.hcl b/modules/datalake/aws/aurora.hcl
index 66c7f14..f886c55 100644
--- a/modules/datalake/aws/aurora.hcl
+++ b/modules/datalake/aws/aurora.hcl
@@ -17,7 +17,7 @@ ingester aws_aurora_instance_logical module {
 
   physical_component {
     type = "aurora_instance"
-    name = "$input{DBInstanceIdentifier}"
+    name = "aurora_instance"
   }
 
   data_for_graph_node {
@@ -192,7 +192,7 @@ ingester aws_aurora_instance_physical module {
 
   physical_component {
     type = "aurora_instance"
-    name = "$input{DBInstanceIdentifier}"
+    name = "aurora_instance"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/cloudfront.hcl b/modules/datalake/aws/cloudfront.hcl
index 5872d67..622e3f2 100644
--- a/modules/datalake/aws/cloudfront.hcl
+++ b/modules/datalake/aws/cloudfront.hcl
@@ -23,7 +23,7 @@ ingester aws_cloudfront module {
 
   physical_component {
     type = "cloudfront_distribution"
-    name = "$input{DistributionId}"
+    name = "cloudfront_distribution"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/dax.hcl b/modules/datalake/aws/dax.hcl
index 1c8c090..72a92bb 100644
--- a/modules/datalake/aws/dax.hcl
+++ b/modules/datalake/aws/dax.hcl
@@ -22,7 +22,7 @@ ingester aws_dax module {
 
   physical_component {
     type = "dax_cluster"
-    name = "$input{ClusterId}"
+    name = "dax_cluster"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/dynamodb.hcl b/modules/datalake/aws/dynamodb.hcl
index b9f6aa4..c3b6ec6 100644
--- a/modules/datalake/aws/dynamodb.hcl
+++ b/modules/datalake/aws/dynamodb.hcl
@@ -24,7 +24,7 @@ ingester aws_dynamodb_table_operation module {
 
   physical_component {
     type = "dynamodb"
-    name = "$input{TableName}"
+    name = "dynamodb"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/ec2.hcl b/modules/datalake/aws/ec2.hcl
index 0fcec53..2558f6b 100644
--- a/modules/datalake/aws/ec2.hcl
+++ b/modules/datalake/aws/ec2.hcl
@@ -24,7 +24,7 @@ ingester aws_ec2 module {
 
   physical_component {
     type = "ec2_instance"
-    name = "$input{InstanceId}"
+    name = "ec2_instance"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/eks.hcl b/modules/datalake/aws/eks.hcl
index 22c600f..8d4a33d 100644
--- a/modules/datalake/aws/eks.hcl
+++ b/modules/datalake/aws/eks.hcl
@@ -23,7 +23,7 @@ ingester aws_eks_containerinsights_deployment_without_service module {
 
   physical_component {
     type = "eks_cluster"
-    name = "$input{ClusterName}"
+    name = "eks_cluster"
   }
 
   data_for_graph_node {
@@ -181,7 +181,7 @@ ingester aws_eks_containerinsights_node module {
 
   physical_component {
     type = "eks_cluster"
-    name = "$input{ClusterName}"
+    name = "eks_cluster"
   }
 
   data_for_graph_node {
@@ -301,7 +301,7 @@ ingester aws_eks_containerinsights_service module {
 
   physical_component {
     type = "eks_cluster"
-    name = "$input{ClusterName}"
+    name = "eks_cluster"
   }
 
   data_for_graph_node {
@@ -455,7 +455,7 @@ ingester aws_eks_containerinsights_eks_cluster module {
 
   physical_component {
     type = "eks_cluster"
-    name = "$input{ClusterName}"
+    name = "eks_cluster"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/elasticcache.hcl b/modules/datalake/aws/elasticcache.hcl
index bceaad5..6e096f7 100644
--- a/modules/datalake/aws/elasticcache.hcl
+++ b/modules/datalake/aws/elasticcache.hcl
@@ -19,7 +19,7 @@ ingester aws_elasticache_redis module {
 
   physical_component {
     type = "elasticache_cluster"
-    name = "$input{CacheClusterId}"
+    name = "elasticache_cluster"
   }
 
   physical_address {
@@ -219,7 +219,7 @@ ingester aws_elasticache_cluster module {
 
   physical_component {
     type = "elasticache_cluster"
-    name = "$input{CacheClusterId}"
+    name = "elasticache_cluster"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/elasticsearch.hcl b/modules/datalake/aws/elasticsearch.hcl
index 3267173..23b1cd4 100644
--- a/modules/datalake/aws/elasticsearch.hcl
+++ b/modules/datalake/aws/elasticsearch.hcl
@@ -17,7 +17,7 @@ ingester aws_elasticsearch module {
 
   physical_component {
     type = "aws_elasticsearch_cluster"
-    name = "$input{DomainName}"
+    name = "aws_elasticsearch_cluster"
   }
 
   data_for_graph_node {
@@ -236,7 +236,7 @@ ingester aws_elasticsearch_master module {
 
   physical_component {
     type = "aws_elasticsearch_cluster"
-    name = "$input{DomainName}"
+    name = "aws_elasticsearch_cluster"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/elb.hcl b/modules/datalake/aws/elb.hcl
index 5905837..9d118e5 100644
--- a/modules/datalake/aws/elb.hcl
+++ b/modules/datalake/aws/elb.hcl
@@ -19,7 +19,7 @@ ingester aws_elb module {
 
   physical_component {
     type = "elb"
-    name = "$input{LoadBalancerName}"
+    name = "elb"
   }
 
   data_for_graph_node {
@@ -208,7 +208,7 @@ ingester aws_elb module {
 
 //   physical_component {
 //     type = "internalElb"
-//     name = "$input{LoadBalancerName}"
+//     name = "internalElb"
 //   }
 
 //   data_for_graph_node {
@@ -393,7 +393,7 @@ ingester aws_elb module {
 
 //   physical_component {
 //     type = "elb"
-//     name = "$input{LoadBalancerName}"
+//     name = "elb"
 //   }
 
 //   data_for_graph_node {
@@ -537,7 +537,7 @@ ingester aws_elb module {
 
 //   physical_component {
 //     type = "internalElb"
-//     name = "$input{LoadBalancerName}"
+//     name = "internalElb"
 //   }
 
 //   data_for_graph_node {
diff --git a/modules/datalake/aws/lambda.hcl b/modules/datalake/aws/lambda.hcl
index 8f3075b..52f2a0a 100644
--- a/modules/datalake/aws/lambda.hcl
+++ b/modules/datalake/aws/lambda.hcl
@@ -24,7 +24,7 @@ ingester aws_lambda module {
 
   physical_component {
     type = "aws_lambda"
-    name = "$input{FunctionName}"
+    name = "aws_lambda"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/msk.hcl b/modules/datalake/aws/msk.hcl
index 2e5a2fa..c60b110 100644
--- a/modules/datalake/aws/msk.hcl
+++ b/modules/datalake/aws/msk.hcl
@@ -24,7 +24,7 @@ ingester aws_msk_topic_per_broker module {
 
   physical_component {
     type = "aws_msk_cluster"
-    name = "$input{ClusterName}"
+    name = "aws_msk_cluster"
   }
 
   data_for_graph_node {
@@ -265,7 +265,7 @@ ingester aws_msk_cluster module {
 
   physical_component {
     type = "aws_msk_cluster"
-    name = "$input{ClusterName}"
+    name = "aws_msk_cluster"
   }
 
   data_for_graph_node {
@@ -332,7 +332,7 @@ ingester aws_msk_broker module {
 
   physical_component {
     type = "aws_msk_cluster"
-    name = "$input{ClusterName}"
+    name = "aws_msk_cluster"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/nlb.hcl b/modules/datalake/aws/nlb.hcl
index c247aa6..71a5f79 100644
--- a/modules/datalake/aws/nlb.hcl
+++ b/modules/datalake/aws/nlb.hcl
@@ -19,7 +19,7 @@ ingester aws_nlb module {
 
   physical_component {
     type = "nlb"
-    name = "$input{LoadBalancer}"
+    name = "nlb"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/rds.hcl b/modules/datalake/aws/rds.hcl
index fc9e2f6..01674f1 100644
--- a/modules/datalake/aws/rds.hcl
+++ b/modules/datalake/aws/rds.hcl
@@ -19,7 +19,7 @@ ingester aws_rds module {
 
   physical_component {
     type = "rds"
-    name = "$input{DBInstanceIdentifier}"
+    name = "rds"
   }
 
   data_for_graph_node {
diff --git a/modules/datalake/aws/sqs.hcl b/modules/datalake/aws/sqs.hcl
index fb4dd01..73b2ce5 100644
--- a/modules/datalake/aws/sqs.hcl
+++ b/modules/datalake/aws/sqs.hcl
@@ -12,7 +12,7 @@ ingester aws_sqs module {
 
   physical_component {
     type = "sqs"
-    name = "$input{QueueName}"
+    name = "sqs"
   }
 
   data_for_graph_node {

From 338247461cb40d26edc56c444e2900a43e3711a1 Mon Sep 17 00:00:00 2001
From: Neeraj Sharma <neeraj@last9.io>
Date: Mon, 7 Mar 2022 19:57:42 +0530
Subject: [PATCH 2/4] join on removed and query fixed for alb and apigateway

---
 modules/datalake/aws/alb.hcl        | 151 +++++-----------------------
 modules/datalake/aws/apigateway.hcl |  72 +++----------
 2 files changed, 38 insertions(+), 185 deletions(-)

diff --git a/modules/datalake/aws/alb.hcl b/modules/datalake/aws/alb.hcl
index d714ad5..755708e 100644
--- a/modules/datalake/aws/alb.hcl
+++ b/modules/datalake/aws/alb.hcl
@@ -38,11 +38,7 @@ ingester aws_alb_cloudwatch module {
     aggregator  = "SUM"
 
     source prometheus "throughput" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (throughput)"
-
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (throughput{LoadBalancer != ''})"
     }
   }
 
@@ -53,11 +49,7 @@ ingester aws_alb_cloudwatch module {
     aggregator  = "SUM"
 
     source prometheus "new_connections" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (new_connections)"
-
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (new_connections{LoadBalancer != ''})"
     }
   }
 
@@ -68,11 +60,7 @@ ingester aws_alb_cloudwatch module {
     aggregator  = "SUM"
 
     source prometheus "rejected_connections" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (rejected_connections)"
-
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (rejected_connections{LoadBalancer != ''})"
     }
   }
 
@@ -83,11 +71,7 @@ ingester aws_alb_cloudwatch module {
     aggregator  = "SUM"
 
     source prometheus "processed_bytes" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (processed_bytes)"
-
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (processed_bytes{LoadBalancer != ''})"
     }
   }
 
@@ -98,11 +82,7 @@ ingester aws_alb_cloudwatch module {
     aggregator  = "SUM"
 
     source prometheus "lcu" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (lcu)"
-
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (lcu{LoadBalancer != ''})"
     }
   }
 
@@ -113,11 +93,7 @@ ingester aws_alb_cloudwatch module {
     aggregator  = "SUM"
 
     source prometheus "status_5xx" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (status_5xx)"
-
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (status_5xx{LoadBalancer != ''})"
     }
   }
 
@@ -128,11 +104,7 @@ ingester aws_alb_cloudwatch module {
     aggregator  = "SUM"
 
     source prometheus "status_4xx" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (status_4xx)"
-
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (status_4xx{LoadBalancer != ''})"
     }
   }
 
@@ -143,11 +115,7 @@ ingester aws_alb_cloudwatch module {
     aggregator  = "SUM"
 
     source prometheus "lb_5xx" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (lb_5xx)"
-
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (lb_5xx{LoadBalancer != ''})"
     }
   }
 
@@ -158,11 +126,7 @@ ingester aws_alb_cloudwatch module {
     aggregator  = "SUM"
 
     source prometheus "lb_4xx" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (lb_4xx)"
-
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (lb_4xx{LoadBalancer != ''})"
     }
   }
 
@@ -174,45 +138,27 @@ ingester aws_alb_cloudwatch module {
     error_margin = 0.05
 
     source prometheus "throughput" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (throughput)"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (throughput{LoadBalancer != ''})"
     }
 
     source prometheus "p50" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p50'})"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p50', LoadBalancer != ''})"
     }
 
     source prometheus "p75" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p75'})"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p75', LoadBalancer != ''})"
     }
 
     source prometheus "p90" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p90'})"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p90', LoadBalancer != ''})"
     }
 
     source prometheus "p99" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p99'})"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p99', LoadBalancer != ''})"
     }
 
     source prometheus "p100" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p100'})"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-      }
+      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p100', LoadBalancer != ''})"
     }
   }
 }
@@ -262,12 +208,7 @@ ingester aws_alb_tg_cloudwatch module {
     output_unit = "rpm"
     aggregator  = "SUM"
     source prometheus "throughput" {
-      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (throughput)"
-
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-        "$output{TargetGroup}"  = "$input{TargetGroup}"
-      }
+      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (throughput{LoadBalancer != '', TargetGroup != ''})"
     }
   }
 
@@ -278,51 +219,27 @@ ingester aws_alb_tg_cloudwatch module {
     aggregator   = "PERCENTILE"
     error_margin = 0.05
     source prometheus "throughput" {
-      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (throughput)"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-        "$output{TargetGroup}"  = "$input{TargetGroup}"
-      }
+      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (throughput{LoadBalancer != '', TargetGroup != ''})"
     }
 
     source prometheus "p50" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p50'})"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-        "$output{TargetGroup}"  = "$input{TargetGroup}"
-      }
+      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p50', LoadBalancer != '', TargetGroup != ''})"
     }
 
     source prometheus "p75" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p75'})"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-        "$output{TargetGroup}"  = "$input{TargetGroup}"
-      }
+      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p75', LoadBalancer != '', TargetGroup != ''})"
     }
 
     source prometheus "p90" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p90'})"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-        "$output{TargetGroup}"  = "$input{TargetGroup}"
-      }
+      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p90', LoadBalancer != '', TargetGroup != ''})"
     }
 
     source prometheus "p99" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p99'})"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-        "$output{TargetGroup}"  = "$input{TargetGroup}"
-      }
+      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p99', LoadBalancer != '', TargetGroup != ''})"
     }
 
     source prometheus "p100" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p100'})"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-        "$output{TargetGroup}"  = "$input{TargetGroup}"
-      }
+      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p100', LoadBalancer != '', TargetGroup != ''})"
     }
   }
 
@@ -332,11 +249,7 @@ ingester aws_alb_tg_cloudwatch module {
     output_unit = "rpm"
     aggregator  = "SUM"
     source prometheus "status_500" {
-      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_5xx)"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-        "$output{TargetGroup}"  = "$input{TargetGroup}"
-      }
+      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_5xx{LoadBalancer != '', TargetGroup != ''})"
     }
   }
 
@@ -346,11 +259,7 @@ ingester aws_alb_tg_cloudwatch module {
     output_unit = "rpm"
     aggregator  = "SUM"
     source prometheus "status_400" {
-      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_4xx)"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-        "$output{TargetGroup}"  = "$input{TargetGroup}"
-      }
+      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_4xx{LoadBalancer != '', TargetGroup != ''})"
     }
   }
 
@@ -360,11 +269,7 @@ ingester aws_alb_tg_cloudwatch module {
     output_unit = "rpm"
     aggregator  = "SUM"
     source prometheus "status_300" {
-      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_3xx)"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-        "$output{TargetGroup}"  = "$input{TargetGroup}"
-      }
+      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_3xx{LoadBalancer != '', TargetGroup != ''})"
     }
   }
 
@@ -374,11 +279,7 @@ ingester aws_alb_tg_cloudwatch module {
     output_unit = "rpm"
     aggregator  = "SUM"
     source prometheus "status_200" {
-      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_2xx)"
-      join_on = {
-        "$output{LoadBalancer}" = "$input{LoadBalancer}"
-        "$output{TargetGroup}"  = "$input{TargetGroup}"
-      }
+      query = "sum by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (status_2xx{LoadBalancer != '', TargetGroup != ''})"
     }
   }
 }
diff --git a/modules/datalake/aws/apigateway.hcl b/modules/datalake/aws/apigateway.hcl
index e0d633e..f09f9ee 100644
--- a/modules/datalake/aws/apigateway.hcl
+++ b/modules/datalake/aws/apigateway.hcl
@@ -39,11 +39,7 @@ ingester aws_apigateway module {
     aggregator  = "SUM"
 
     source prometheus "throughput" {
-      query = "sum by (ApiName, Stage, tag_namespace, tag_service) (throughput)"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (throughput{ApiName != '', Stage != ''})"
     }
   }
 
@@ -57,59 +53,31 @@ ingester aws_apigateway module {
 
 
     source prometheus "throughput" {
-      query = "avg by (ApiName, Stage, tag_namespace, tag_service) (latency_histo{le='throughput'})"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (throughput{ApiName != '', Stage != ''})"
     }
 
     source prometheus "p50" {
-      query = "avg by (ApiName, Stage, tag_namespace, tag_service) (latency_histo{le='p50'})"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (latency{latency='p50', ApiName != '', Stage != ''})"
     }
 
     source prometheus "latency" {
-      query = "avg by (ApiName, Stage, tag_namespace, tag_service) (latency_histo{le='latency'})"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (latency{ApiName != '', Stage != ''})"
     }
 
     source prometheus "p75" {
-      query = "avg by (ApiName, Stage, tag_namespace, tag_service) (latency_histo{le='p75'})"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (latency{latency='p75', ApiName != '', Stage != ''})"
     }
 
     source prometheus "p90" {
-      query = "avg by (ApiName, Stage, tag_namespace, tag_service) (latency_histo{le='p90'})"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (latency{latency='p90', ApiName != '', Stage != ''})"
     }
 
     source prometheus "p99" {
-      query = "avg by (ApiName, Stage, tag_namespace, tag_service) (latency_histo{le='p99'})"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (latency{latency='p99', ApiName != '', Stage != ''})"
     }
 
     source prometheus "p100" {
-      query = "avg by (ApiName, Stage, tag_namespace, tag_service) (latency_histo{le='p100'})"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (latency{latency='p100', ApiName != '', Stage != ''})"
     }
 
   }
@@ -121,11 +89,7 @@ ingester aws_apigateway module {
     aggregator  = "AVG"
 
     source prometheus "integration_latency" {
-      query = "avg by (ApiName, Stage, tag_namespace, tag_service) (integration_latency)"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (integration_latency{ApiName != '', Stage != ''})"
     }
   }
 
@@ -138,11 +102,7 @@ ingester aws_apigateway module {
     aggregator  = "SUM"
 
     source prometheus "cache_miss" {
-      query = "sum by (ApiName, Stage, tag_namespace, tag_service) (cache_miss)"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (cache_miss{ApiName != '', Stage != ''})"
     }
   }
 
@@ -155,11 +115,7 @@ ingester aws_apigateway module {
     aggregator  = "SUM"
 
     source prometheus "status_4xx" {
-      query = "sum by (ApiName, Stage, tag_namespace, tag_service) (status_4xx)"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (status_4xx{ApiName != '', Stage != ''})"
     }
   }
 
@@ -172,11 +128,7 @@ ingester aws_apigateway module {
     aggregator  = "SUM"
 
     source prometheus "status_5xx" {
-      query = "sum by (ApiName, Stage, tag_namespace, tag_service) (status_5xx)"
-      join_on = {
-        "$output{ApiName}" = "$input{ApiName}"
-        "$output{Stage}"   = "$input{Stage}"
-      }
+      query = "sum by (ApiName, Stage, tag_service, tag_namespace) (status_5xx{ApiName != '', Stage != ''})"
     }
   }
 }

From 862156f424766d0699a093635691aa88cc1c93a6 Mon Sep 17 00:00:00 2001
From: Arjun Mahishi <arjun.mahishi@gmail.com>
Date: Tue, 8 Mar 2022 17:18:57 +0530
Subject: [PATCH 3/4] [datalake] fix alb and elb modules -- replace
 status_histo with gauge

---
 modules/datalake/aws/alb.hcl | 6 +++---
 modules/datalake/aws/elb.hcl | 9 +++++----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/modules/datalake/aws/alb.hcl b/modules/datalake/aws/alb.hcl
index 755708e..6ee5921 100644
--- a/modules/datalake/aws/alb.hcl
+++ b/modules/datalake/aws/alb.hcl
@@ -28,7 +28,7 @@ ingester aws_alb_cloudwatch module {
 
   data_for_graph_node {
     type = "alb"
-    name = "$input{LoadBalancer}"
+    name = "$output{LoadBalancer}"
   }
 
   gauge "throughput" {
@@ -86,7 +86,7 @@ ingester aws_alb_cloudwatch module {
     }
   }
 
-  status_histo status_5xx {
+  gauge status_5xx {
     index       = 5
     input_unit  = "count"
     output_unit = "rpm"
@@ -97,7 +97,7 @@ ingester aws_alb_cloudwatch module {
     }
   }
 
-  status_histo status_4xx {
+  gauge status_4xx {
     index       = 4
     input_unit  = "count"
     output_unit = "rpm"
diff --git a/modules/datalake/aws/elb.hcl b/modules/datalake/aws/elb.hcl
index 9d118e5..69f7bec 100644
--- a/modules/datalake/aws/elb.hcl
+++ b/modules/datalake/aws/elb.hcl
@@ -24,7 +24,7 @@ ingester aws_elb module {
 
   data_for_graph_node {
     type = "elb"
-    name = "$input{LoadBalancerName}"
+    name = "$output{LoadBalancerName}"
   }
 
   using = {
@@ -83,7 +83,7 @@ ingester aws_elb module {
       }
     }
   }
-  status_histo status_5xx {
+  gauge status_5xx {
     index       = 5
     input_unit  = "count"
     output_unit = "rpm"
@@ -96,7 +96,8 @@ ingester aws_elb module {
       }
     }
   }
-  status_histo status_4xx {
+
+  gauge status_4xx {
     index       = 4
     input_unit  = "count"
     output_unit = "rpm"
@@ -143,7 +144,7 @@ ingester aws_elb module {
     aggregator   = "PERCENTILE"
     error_margin = 0.05
 
-    source prometheus "latency_histo" {
+    source prometheus "throughput" {
       query = "avg by (LoadBalancerName, tag_namespace, tag_service) (latency_histo{le='throughput'})"
       join_on = {
         "$output{LoadBalancerName}" = "$input{LoadBalancerName}"

From b4a2c9c785a2658ae92e9a0c3674c0af38048652 Mon Sep 17 00:00:00 2001
From: Mohan Dutt Parashar <kraken@Mohans-MacBook-Pro.local>
Date: Wed, 9 Mar 2022 12:41:02 +0530
Subject: [PATCH 4/4] fixes

---
 modules/datalake/aws/alb.hcl | 99 ++++++++++++++++++------------------
 1 file changed, 50 insertions(+), 49 deletions(-)

diff --git a/modules/datalake/aws/alb.hcl b/modules/datalake/aws/alb.hcl
index 6ee5921..b2b175f 100644
--- a/modules/datalake/aws/alb.hcl
+++ b/modules/datalake/aws/alb.hcl
@@ -42,6 +42,17 @@ ingester aws_alb_cloudwatch module {
     }
   }
 
+  gauge "lcu" {
+    index       = 3
+    input_unit  = "count"
+    output_unit = "count"
+    aggregator  = "SUM"
+
+    source prometheus "lcu" {
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (lcu{LoadBalancer != ''})"
+    }
+  }
+
   gauge "new_connections" {
     index       = 4
     input_unit  = "count"
@@ -75,90 +86,80 @@ ingester aws_alb_cloudwatch module {
     }
   }
 
-  gauge "lcu" {
-    index       = 3
-    input_unit  = "count"
-    output_unit = "count"
-    aggregator  = "SUM"
-
-    source prometheus "lcu" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (lcu{LoadBalancer != ''})"
-    }
-  }
-
-  gauge status_5xx {
-    index       = 5
+  gauge lb_4xx {
+    index       = 7
     input_unit  = "count"
     output_unit = "rpm"
     aggregator  = "SUM"
 
-    source prometheus "status_5xx" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (status_5xx{LoadBalancer != ''})"
+    source prometheus "lb_4xx" {
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (response{code='4xx_lb', LoadBalancer != ''})"
     }
   }
 
-  gauge status_4xx {
-    index       = 4
+  gauge lb_5xx {
+    index       = 8
     input_unit  = "count"
     output_unit = "rpm"
     aggregator  = "SUM"
 
-    source prometheus "status_4xx" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (status_4xx{LoadBalancer != ''})"
+    source prometheus "lb_5xx" {
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (response{code='5xx_lb', LoadBalancer != ''})"
     }
   }
 
-  gauge lb_5xx {
-    index       = 8
+  gauge status_5xx {
+    index       = 9
     input_unit  = "count"
     output_unit = "rpm"
     aggregator  = "SUM"
 
-    source prometheus "lb_5xx" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (lb_5xx{LoadBalancer != ''})"
+    source prometheus "status_5xx" {
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (response{code='5xx_target', LoadBalancer != ''})"
     }
   }
 
-  gauge lb_4xx {
-    index       = 7
+  gauge status_4xx {
+    index       = 11
     input_unit  = "count"
     output_unit = "rpm"
     aggregator  = "SUM"
 
-    source prometheus "lb_4xx" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (lb_4xx{LoadBalancer != ''})"
+    source prometheus "status_4xx" {
+      query = "sum by (LoadBalancer, tag_service, tag_namespace) (response{code='4xx_target', LoadBalancer != ''})"
     }
   }
 
-  latency "latency_histo" {
-    index        = 6
-    input_unit   = "ms"
-    output_unit  = "ms"
-    aggregator   = "PERCENTILE"
-    error_margin = 0.05
+  gauge latency_min {
+    index       = 12
+    input_unit  = "s"
+    output_unit = "s"
+    aggregator  = "MIN"
 
-    source prometheus "throughput" {
-      query = "sum by (LoadBalancer, tag_service, tag_namespace) (throughput{LoadBalancer != ''})"
+    source prometheus "latency_min" {
+      query = "min by (LoadBalancer, tag_service, tag_namespace) (latency{stat='min', LoadBalancer != ''})"
     }
+  }
 
-    source prometheus "p50" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p50', LoadBalancer != ''})"
-    }
+  gauge latency_max {
+    index       = 13
+    input_unit  = "s"
+    output_unit = "s"
+    aggregator  = "MAX"
 
-    source prometheus "p75" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p75', LoadBalancer != ''})"
+    source prometheus "latency_max" {
+      query = "max by (LoadBalancer, tag_service, tag_namespace) (latency{stat='max', LoadBalancer != ''})"
     }
+  }
 
-    source prometheus "p90" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p90', LoadBalancer != ''})"
-    }
+  gauge latency_avg {
+    index       = 14
+    input_unit  = "s"
+    output_unit = "s"
+    aggregator  = "AVG"
 
-    source prometheus "p99" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p99', LoadBalancer != ''})"
-    }
-
-    source prometheus "p100" {
-      query = "avg by (LoadBalancer, TargetGroup, tag_service, tag_namespace) (latency{latency='p100', LoadBalancer != ''})"
+    source prometheus "latency_avg" {
+      query = "avg by (LoadBalancer, tag_service, tag_namespace) (latency{stat='avg', LoadBalancer != ''})"
     }
   }
 }