From 1286779c06461341f6cfa0d1257c8db1a30d7e34 Mon Sep 17 00:00:00 2001
From: june-hua <junehua@verily.com>
Date: Wed, 18 Mar 2026 13:43:27 -0400
Subject: [PATCH 1/2] Fluent-bit config updates

Specify "DB" to store file offsets so fluent-bit doesn't send duplicated
logs to output
---
 startupscript/butane/aws/fluent-bit.conf | 3 +++
 startupscript/butane/gcp/fluent-bit.conf | 5 ++++-
 startupscript/butane/run-fluent-bit.sh   | 3 +++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/startupscript/butane/aws/fluent-bit.conf b/startupscript/butane/aws/fluent-bit.conf
index bb3fa6ab..53103267 100644
--- a/startupscript/butane/aws/fluent-bit.conf
+++ b/startupscript/butane/aws/fluent-bit.conf
@@ -26,8 +26,11 @@
     Parser docker
     Path /var/lib/docker/containers/*/*.log
     Path_key filepath
+    DB /var/lib/fluent-bit/flb_docker.db
+    Read_from_Head true
     Skip_Long_Lines On
     Mem_Buf_Limit 10MB
+    Refresh_Interval 5
 
 [OUTPUT]
     Name cloudwatch_logs
diff --git a/startupscript/butane/gcp/fluent-bit.conf b/startupscript/butane/gcp/fluent-bit.conf
index 80c2673c..07693f27 100644
--- a/startupscript/butane/gcp/fluent-bit.conf
+++ b/startupscript/butane/gcp/fluent-bit.conf
@@ -26,9 +26,12 @@
     Parser docker
     Path /var/lib/docker/containers/*/*.log
     Path_key filepath
+    DB /var/lib/fluent-bit/flb_docker.db
+    Read_from_Head true
     Skip_Long_Lines On
     Mem_Buf_Limit 10MB
-    
+    Refresh_Interval 5
+
 [OUTPUT]
     name stackdriver
     match vm-*
diff --git a/startupscript/butane/run-fluent-bit.sh b/startupscript/butane/run-fluent-bit.sh
index cf8850d4..3a98a2cd 100755
--- a/startupscript/butane/run-fluent-bit.sh
+++ b/startupscript/butane/run-fluent-bit.sh
@@ -43,6 +43,9 @@ readonly CLOUD="$1"
 # Configuration with defaults
 readonly FLUENT_BIT_IMAGE="${FLUENT_BIT_IMAGE:-cr.fluentbit.io/fluent/fluent-bit:2.0-debug}"
 
+# Db directory to store file offsets
+mkdir -p /var/lib/fluent-bit
+
 # Build Docker run command arguments
 DOCKER_ARGS=(
     --rm

From b4e6b1db733dc459a47340e1d4b7f8eea5f822cf Mon Sep 17 00:00:00 2001
From: june-hua <junehua@verily.com>
Date: Wed, 18 Mar 2026 14:10:38 -0400
Subject: [PATCH 2/2] mount fluent-bit db folder with rw permissions

---
 startupscript/butane/run-fluent-bit.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/startupscript/butane/run-fluent-bit.sh b/startupscript/butane/run-fluent-bit.sh
index 3a98a2cd..b50e8329 100755
--- a/startupscript/butane/run-fluent-bit.sh
+++ b/startupscript/butane/run-fluent-bit.sh
@@ -54,6 +54,7 @@ DOCKER_ARGS=(
     -v /etc/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf:ro
     -v /var/log/journal:/var/log/journal:ro
     -v /var/lib/docker/containers:/var/lib/docker/containers:ro
+    -v /var/lib/fluent-bit:/var/lib/fluent-bit
 )
 
 # Add cloud-specific configuration