From 570a8855553391daa8689ba3e9db1529a2129fd3 Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Fri, 25 Jul 2025 11:41:33 +0800 Subject: [PATCH 01/15] add loki bucket for aws BYOC poolmebers --- modules/dns-bucket/README.md | 11 ++++++++--- modules/dns-bucket/bucket.tf | 15 +++++++++++++++ modules/dns-bucket/outputs.tf | 4 ++++ modules/dns-bucket/variables.tf | 16 ++++++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/modules/dns-bucket/README.md b/modules/dns-bucket/README.md index cfe5de2..ad971b6 100644 --- a/modules/dns-bucket/README.md +++ b/modules/dns-bucket/README.md @@ -28,9 +28,9 @@ A basic module used to create Route53 Zone and S3 Buckets. | Name | Version | |------|---------| -| [aws](#provider\_aws) | 5.76.0 | -| [aws.source](#provider\_aws.source) | 5.76.0 | -| [aws.target](#provider\_aws.target) | 5.76.0 | +| [aws](#provider\_aws) | n/a | +| [aws.source](#provider\_aws.source) | n/a | +| [aws.target](#provider\_aws.target) | n/a | ## Modules @@ -42,6 +42,7 @@ No modules. |------|------| | [aws_route53_record.delegate](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route53_record) | resource | | [aws_route53_zone.zone](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route53_zone) | resource | +| [aws_s3_bucket.loki](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | | [aws_s3_bucket.tiered_storage](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | | [aws_s3_bucket.velero](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | | [aws_s3_bucket_server_side_encryption_configuration.velero](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | @@ -52,11 +53,14 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [bucket\_location](#input\_bucket\_location) | The location of the bucket | `string` | n/a | yes | | [custom\_dns\_zone\_id](#input\_custom\_dns\_zone\_id) | if specified, then a streamnative zone will not be created, and this zone will be used instead. Otherwise, we will provision a new zone and delegate access | `string` | `""` | no | | [custom\_dns\_zone\_name](#input\_custom\_dns\_zone\_name) | must be passed if custom\_dns\_zone\_id is passed, this is the zone name to use | `string` | `""` | no | +| [enable\_loki](#input\_enable\_loki) | Enable loki storage bucket creation | `bool` | `false` | no | | [extra\_aws\_tags](#input\_extra\_aws\_tags) | Additional to apply to the resources. Note that this module sets the tags Name, Type, and Vendor by default. They can be overwritten, but it is not recommended. | `map(string)` | `{}` | no | | [parent\_zone\_name](#input\_parent\_zone\_name) | The parent zone in which we create the delegation records | `string` | n/a | yes | | [pm\_name](#input\_pm\_name) | The name of the poolmember, for new clusters, this should be like `pm-` | `string` | n/a | yes | +| [pm\_namespace](#input\_pm\_namespace) | The namespace of the poolmember | `string` | n/a | yes | | [s3\_encryption\_kms\_key\_arn](#input\_s3\_encryption\_kms\_key\_arn) | KMS key ARN to use for S3 encryption. If not set, the default AWS S3 key will be used. | `string` | `""` | no | ## Outputs @@ -65,6 +69,7 @@ No modules. |------|-------------| | [backup\_bucket](#output\_backup\_bucket) | n/a | | [backup\_bucket\_kms\_key\_id](#output\_backup\_bucket\_kms\_key\_id) | n/a | +| [loki\_bucket](#output\_loki\_bucket) | n/a | | [tiered\_storage\_bucket](#output\_tiered\_storage\_bucket) | n/a | | [zone\_id](#output\_zone\_id) | n/a | | [zone\_name](#output\_zone\_name) | n/a | diff --git a/modules/dns-bucket/bucket.tf b/modules/dns-bucket/bucket.tf index 7fdf755..84e527a 100644 --- a/modules/dns-bucket/bucket.tf +++ b/modules/dns-bucket/bucket.tf @@ -36,6 +36,21 @@ resource "aws_s3_bucket" "tiered_storage" { } } +resource "aws_s3_bucket" "loki" { + count = var.enable_loki ? 1 : 0 + provider = aws.source + region = var.bucket_location + bucket = format("loki-%s-%s", var.pm_namespace, var.pm_name) + tags = merge({ "Attributes" = "loki", "Name" = "logs-byoc" }, local.tags) + force_destroy = true + + lifecycle { + ignore_changes = [ + bucket, + ] + } +} + data "aws_kms_key" "s3_default" { key_id = "alias/aws/s3" } diff --git a/modules/dns-bucket/outputs.tf b/modules/dns-bucket/outputs.tf index 970b59c..d48b6c6 100644 --- a/modules/dns-bucket/outputs.tf +++ b/modules/dns-bucket/outputs.tf @@ -30,4 +30,8 @@ output "backup_bucket_kms_key_id" { output "tiered_storage_bucket" { value = aws_s3_bucket.tiered_storage.bucket +} + +output "loki_bucket" { + value = var.enable_loki ? aws_s3_bucket.loki.bucket : "" } \ No newline at end of file diff --git a/modules/dns-bucket/variables.tf b/modules/dns-bucket/variables.tf index 90670ab..c392921 100644 --- a/modules/dns-bucket/variables.tf +++ b/modules/dns-bucket/variables.tf @@ -12,6 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +variable "pm_namespace" { + type = string + description = "The namespace of the poolmember" +} + variable "pm_name" { description = "The name of the poolmember, for new clusters, this should be like `pm-`" type = string @@ -34,6 +39,11 @@ variable "custom_dns_zone_name" { description = "must be passed if custom_dns_zone_id is passed, this is the zone name to use" } +variable "bucket_location" { + type = string + description = "The location of the bucket" +} + variable "s3_encryption_kms_key_arn" { default = "" description = "KMS key ARN to use for S3 encryption. If not set, the default AWS S3 key will be used." @@ -51,3 +61,9 @@ locals { "Vendor" = "StreamNative" }, var.extra_aws_tags) } + +variable "enable_loki" { + type = bool + default = false + description = "Enable loki storage bucket creation" +} \ No newline at end of file From 132c7294baf8df26df1d7f050a7844a96569e53b Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Fri, 25 Jul 2025 12:26:17 +0800 Subject: [PATCH 02/15] fix output loki_bucket --- modules/dns-bucket/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dns-bucket/outputs.tf b/modules/dns-bucket/outputs.tf index d48b6c6..1682f7c 100644 --- a/modules/dns-bucket/outputs.tf +++ b/modules/dns-bucket/outputs.tf @@ -33,5 +33,5 @@ output "tiered_storage_bucket" { } output "loki_bucket" { - value = var.enable_loki ? aws_s3_bucket.loki.bucket : "" + value = var.enable_loki ? aws_s3_bucket.loki[0].bucket : "" } \ No newline at end of file From 3a03eaec0b089f2c3b9e0c12a298c638356b8974 Mon Sep 17 00:00:00 2001 From: ignacioli Date: Mon, 28 Jul 2025 19:44:45 +0800 Subject: [PATCH 03/15] fix bucket_region --- modules/dns-bucket/bucket.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dns-bucket/bucket.tf b/modules/dns-bucket/bucket.tf index 84e527a..abea44c 100644 --- a/modules/dns-bucket/bucket.tf +++ b/modules/dns-bucket/bucket.tf @@ -39,7 +39,7 @@ resource "aws_s3_bucket" "tiered_storage" { resource "aws_s3_bucket" "loki" { count = var.enable_loki ? 1 : 0 provider = aws.source - region = var.bucket_location + bucket_region = var.bucket_location bucket = format("loki-%s-%s", var.pm_namespace, var.pm_name) tags = merge({ "Attributes" = "loki", "Name" = "logs-byoc" }, local.tags) force_destroy = true From 4e1852eb09eb62578af4eb76b75fb932f633adb7 Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Mon, 28 Jul 2025 21:21:19 +0800 Subject: [PATCH 04/15] fix bucket_location --- modules/dns-bucket/bucket.tf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/dns-bucket/bucket.tf b/modules/dns-bucket/bucket.tf index abea44c..657f6cb 100644 --- a/modules/dns-bucket/bucket.tf +++ b/modules/dns-bucket/bucket.tf @@ -39,7 +39,13 @@ resource "aws_s3_bucket" "tiered_storage" { resource "aws_s3_bucket" "loki" { count = var.enable_loki ? 1 : 0 provider = aws.source - bucket_region = var.bucket_location + # Only required if region is NOT us-east-1 + dynamic "create_bucket_configuration" { + for_each = var.bucket_location == "us-east-1" ? [] : [1] + content { + location_constraint = var.bucket_location + } + } bucket = format("loki-%s-%s", var.pm_namespace, var.pm_name) tags = merge({ "Attributes" = "loki", "Name" = "logs-byoc" }, local.tags) force_destroy = true From 425e4b94ba9fb568111aba6144f0f26b1f065294 Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Mon, 28 Jul 2025 21:40:43 +0800 Subject: [PATCH 05/15] add region for aws provider --- modules/dns-bucket/bucket.tf | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/modules/dns-bucket/bucket.tf b/modules/dns-bucket/bucket.tf index 657f6cb..91880f7 100644 --- a/modules/dns-bucket/bucket.tf +++ b/modules/dns-bucket/bucket.tf @@ -24,6 +24,11 @@ resource "aws_s3_bucket" "velero" { } } +provider "aws" { + alias = "source" + region = var.bucket_location +} + resource "aws_s3_bucket" "tiered_storage" { bucket = format("%s-tiered-storage-snc", var.pm_name) tags = merge({ "Attributes" = "tiered-storage" }, local.tags) @@ -39,13 +44,6 @@ resource "aws_s3_bucket" "tiered_storage" { resource "aws_s3_bucket" "loki" { count = var.enable_loki ? 1 : 0 provider = aws.source - # Only required if region is NOT us-east-1 - dynamic "create_bucket_configuration" { - for_each = var.bucket_location == "us-east-1" ? [] : [1] - content { - location_constraint = var.bucket_location - } - } bucket = format("loki-%s-%s", var.pm_namespace, var.pm_name) tags = merge({ "Attributes" = "loki", "Name" = "logs-byoc" }, local.tags) force_destroy = true From 1e84ac8da227111ac6bf776ae99bff6b2953a01c Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Mon, 28 Jul 2025 22:00:41 +0800 Subject: [PATCH 06/15] fix aws provider --- modules/dns-bucket/bucket.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dns-bucket/bucket.tf b/modules/dns-bucket/bucket.tf index 91880f7..a169fe1 100644 --- a/modules/dns-bucket/bucket.tf +++ b/modules/dns-bucket/bucket.tf @@ -24,7 +24,7 @@ resource "aws_s3_bucket" "velero" { } } -provider "aws" { +provider "aws_loki" { alias = "source" region = var.bucket_location } @@ -43,7 +43,7 @@ resource "aws_s3_bucket" "tiered_storage" { resource "aws_s3_bucket" "loki" { count = var.enable_loki ? 1 : 0 - provider = aws.source + provider = aws_loki.source bucket = format("loki-%s-%s", var.pm_namespace, var.pm_name) tags = merge({ "Attributes" = "loki", "Name" = "logs-byoc" }, local.tags) force_destroy = true From 93b20aed60770aae1bb8b6f1d3819c3f9247e5b7 Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Mon, 28 Jul 2025 22:26:04 +0800 Subject: [PATCH 07/15] revert bucket_location --- modules/dns-bucket/bucket.tf | 7 +------ modules/dns-bucket/variables.tf | 4 ---- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/modules/dns-bucket/bucket.tf b/modules/dns-bucket/bucket.tf index a169fe1..0011ea7 100644 --- a/modules/dns-bucket/bucket.tf +++ b/modules/dns-bucket/bucket.tf @@ -24,11 +24,6 @@ resource "aws_s3_bucket" "velero" { } } -provider "aws_loki" { - alias = "source" - region = var.bucket_location -} - resource "aws_s3_bucket" "tiered_storage" { bucket = format("%s-tiered-storage-snc", var.pm_name) tags = merge({ "Attributes" = "tiered-storage" }, local.tags) @@ -43,7 +38,7 @@ resource "aws_s3_bucket" "tiered_storage" { resource "aws_s3_bucket" "loki" { count = var.enable_loki ? 1 : 0 - provider = aws_loki.source + provider = aws.source bucket = format("loki-%s-%s", var.pm_namespace, var.pm_name) tags = merge({ "Attributes" = "loki", "Name" = "logs-byoc" }, local.tags) force_destroy = true diff --git a/modules/dns-bucket/variables.tf b/modules/dns-bucket/variables.tf index c392921..3e92715 100644 --- a/modules/dns-bucket/variables.tf +++ b/modules/dns-bucket/variables.tf @@ -39,10 +39,6 @@ variable "custom_dns_zone_name" { description = "must be passed if custom_dns_zone_id is passed, this is the zone name to use" } -variable "bucket_location" { - type = string - description = "The location of the bucket" -} variable "s3_encryption_kms_key_arn" { default = "" From 26eefd7334b8ceea5388f8f5e93c2ba64fc5df7b Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Tue, 29 Jul 2025 09:23:11 +0800 Subject: [PATCH 08/15] upgrade aws provider to 6.6.0 --- versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.tf b/versions.tf index a845f95..521a0fb 100644 --- a/versions.tf +++ b/versions.tf @@ -18,7 +18,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "5.75.0" + version = "6.6.0" } kubernetes = { source = "hashicorp/kubernetes" From 85312275a4d0c37fb9366111d09ea1129a18da12 Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Tue, 29 Jul 2025 09:24:15 +0800 Subject: [PATCH 09/15] add back var bucket_location --- modules/dns-bucket/bucket.tf | 1 + modules/dns-bucket/variables.tf | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/modules/dns-bucket/bucket.tf b/modules/dns-bucket/bucket.tf index 0011ea7..84e527a 100644 --- a/modules/dns-bucket/bucket.tf +++ b/modules/dns-bucket/bucket.tf @@ -39,6 +39,7 @@ resource "aws_s3_bucket" "tiered_storage" { resource "aws_s3_bucket" "loki" { count = var.enable_loki ? 1 : 0 provider = aws.source + region = var.bucket_location bucket = format("loki-%s-%s", var.pm_namespace, var.pm_name) tags = merge({ "Attributes" = "loki", "Name" = "logs-byoc" }, local.tags) force_destroy = true diff --git a/modules/dns-bucket/variables.tf b/modules/dns-bucket/variables.tf index 3e92715..c392921 100644 --- a/modules/dns-bucket/variables.tf +++ b/modules/dns-bucket/variables.tf @@ -39,6 +39,10 @@ variable "custom_dns_zone_name" { description = "must be passed if custom_dns_zone_id is passed, this is the zone name to use" } +variable "bucket_location" { + type = string + description = "The location of the bucket" +} variable "s3_encryption_kms_key_arn" { default = "" From 8484b3580d607c2814c4c66da8417bab76d5c146 Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Tue, 29 Jul 2025 10:47:49 +0800 Subject: [PATCH 10/15] fix README --- modules/dns-bucket/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/dns-bucket/README.md b/modules/dns-bucket/README.md index ad971b6..931be28 100644 --- a/modules/dns-bucket/README.md +++ b/modules/dns-bucket/README.md @@ -28,9 +28,9 @@ A basic module used to create Route53 Zone and S3 Buckets. | Name | Version | |------|---------| -| [aws](#provider\_aws) | n/a | -| [aws.source](#provider\_aws.source) | n/a | -| [aws.target](#provider\_aws.target) | n/a | +| [aws](#provider\_aws) | 6.6.0 | +| [aws.source](#provider\_aws.source) | 6.6.0 | +| [aws.target](#provider\_aws.target) | 6.6.0 | ## Modules From 2aff2c85660bea8db2e7d4d38db613f65e9b11fa Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Wed, 30 Jul 2025 00:21:43 +0800 Subject: [PATCH 11/15] keep hashicorp/aws version the same with argo ensure-tf-provider.libsonnet --- modules/dns-bucket/README.md | 6 +++--- versions.tf | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/dns-bucket/README.md b/modules/dns-bucket/README.md index 931be28..d37120b 100644 --- a/modules/dns-bucket/README.md +++ b/modules/dns-bucket/README.md @@ -28,9 +28,9 @@ A basic module used to create Route53 Zone and S3 Buckets. | Name | Version | |------|---------| -| [aws](#provider\_aws) | 6.6.0 | -| [aws.source](#provider\_aws.source) | 6.6.0 | -| [aws.target](#provider\_aws.target) | 6.6.0 | +| [aws](#provider\_aws) | 5.75.0 | +| [aws.source](#provider\_aws.source) | 5.75.0 | +| [aws.target](#provider\_aws.target) | 5.75.0 | ## Modules diff --git a/versions.tf b/versions.tf index 521a0fb..a845f95 100644 --- a/versions.tf +++ b/versions.tf @@ -18,7 +18,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "6.6.0" + version = "5.75.0" } kubernetes = { source = "hashicorp/kubernetes" From f2683b277a194c6ab4eb32c65a382f99606e1569 Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Wed, 30 Jul 2025 00:22:45 +0800 Subject: [PATCH 12/15] hashicorp/aws 5.75.0 doesn't support setting bucket region --- modules/dns-bucket/bucket.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/dns-bucket/bucket.tf b/modules/dns-bucket/bucket.tf index 84e527a..0011ea7 100644 --- a/modules/dns-bucket/bucket.tf +++ b/modules/dns-bucket/bucket.tf @@ -39,7 +39,6 @@ resource "aws_s3_bucket" "tiered_storage" { resource "aws_s3_bucket" "loki" { count = var.enable_loki ? 1 : 0 provider = aws.source - region = var.bucket_location bucket = format("loki-%s-%s", var.pm_namespace, var.pm_name) tags = merge({ "Attributes" = "loki", "Name" = "logs-byoc" }, local.tags) force_destroy = true From 760bf0d2156507557083ae21ea53b4e9e07bb19e Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Wed, 30 Jul 2025 00:24:57 +0800 Subject: [PATCH 13/15] remove var bucket_location --- modules/dns-bucket/variables.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/dns-bucket/variables.tf b/modules/dns-bucket/variables.tf index c392921..983065b 100644 --- a/modules/dns-bucket/variables.tf +++ b/modules/dns-bucket/variables.tf @@ -39,11 +39,6 @@ variable "custom_dns_zone_name" { description = "must be passed if custom_dns_zone_id is passed, this is the zone name to use" } -variable "bucket_location" { - type = string - description = "The location of the bucket" -} - variable "s3_encryption_kms_key_arn" { default = "" description = "KMS key ARN to use for S3 encryption. If not set, the default AWS S3 key will be used." From 043deca987eb46b0cc9923316cd5cdfd7f82032f Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Thu, 31 Jul 2025 10:40:00 +0800 Subject: [PATCH 14/15] reove terraform lifecycle Meta-Argument --- modules/dns-bucket/bucket.tf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/modules/dns-bucket/bucket.tf b/modules/dns-bucket/bucket.tf index 0011ea7..6b8fdfd 100644 --- a/modules/dns-bucket/bucket.tf +++ b/modules/dns-bucket/bucket.tf @@ -42,12 +42,6 @@ resource "aws_s3_bucket" "loki" { bucket = format("loki-%s-%s", var.pm_namespace, var.pm_name) tags = merge({ "Attributes" = "loki", "Name" = "logs-byoc" }, local.tags) force_destroy = true - - lifecycle { - ignore_changes = [ - bucket, - ] - } } data "aws_kms_key" "s3_default" { From c3d4cab3ebf72bfbf921d91dacd7fa799c3286c7 Mon Sep 17 00:00:00 2001 From: Ignacio Li Date: Thu, 31 Jul 2025 12:20:28 +0800 Subject: [PATCH 15/15] remove terraform lifecycle and add provider attribute for backup and tiered_storage --- modules/dns-bucket/bucket.tf | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/modules/dns-bucket/bucket.tf b/modules/dns-bucket/bucket.tf index 6b8fdfd..4c69bd8 100644 --- a/modules/dns-bucket/bucket.tf +++ b/modules/dns-bucket/bucket.tf @@ -13,27 +13,17 @@ # limitations under the License. resource "aws_s3_bucket" "velero" { + provider = aws.target bucket = format("%s-cluster-backup-snc", var.pm_name) tags = merge({ "Attributes" = "backup", "Name" = "velero-backups" }, local.tags) force_destroy = true - - lifecycle { - ignore_changes = [ - bucket, - ] - } } resource "aws_s3_bucket" "tiered_storage" { + provider = aws.target bucket = format("%s-tiered-storage-snc", var.pm_name) tags = merge({ "Attributes" = "tiered-storage" }, local.tags) force_destroy = true - - lifecycle { - ignore_changes = [ - bucket, - ] - } } resource "aws_s3_bucket" "loki" {