feat: Add Lua/OpenResty realtime security capture

.github/workflows/ci.yml

@@ -80,3 +80,45 @@ jobs:
       - name: Cleanup E2E environment
         if: always()
         run: make test-e2e-cleanup
+
+  security-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Run Security E2E tests
+        run: make test-e2e-security
+
+      - name: Cleanup on failure
+        if: failure()
+        run: make test-e2e-security-cleanup
+
+  lua-tests:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21'
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Run Lua/OpenResty E2E tests
+        run: make test-e2e-lua
+
+      - name: Cleanup on failure
+        if: failure()
+        run: make test-e2e-lua-cleanup

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help build run test test-unit test-e2e test-e2e-docker test-e2e-setup test-e2e-run test-e2e-short test-e2e-cleanup test-coverage clean deps lint lint-install fmt vet
+.PHONY: help build run test test-unit test-e2e test-e2e-docker test-e2e-setup test-e2e-run test-e2e-short test-e2e-cleanup test-e2e-security test-e2e-security-setup test-e2e-security-cleanup test-e2e-lua test-e2e-lua-setup test-e2e-lua-cleanup test-coverage clean deps lint lint-install fmt vet
 
 BINARY_NAME=flatrun-agent
 VERSION?=$(shell cat VERSION 2>/dev/null || echo "dev")
@@ -8,6 +8,8 @@ LDFLAGS=-ldflags "-X github.com/flatrun/agent/pkg/version.Version=$(VERSION) -X
 
 E2E_API_URL?=http://localhost:18090/api
 E2E_DEPLOYMENTS_PATH?=/tmp/flatrun-e2e-deployments
+E2E_SECURITY_PATH?=/tmp/flatrun-e2e-security
+E2E_LUA_PATH?=/tmp/flatrun-e2e-lua
 
 help:
 	@echo "FlatRun Agent - Build commands"
@@ -23,6 +25,8 @@ help:
 	@echo "make test-e2e-run       - Run E2E tests against Docker environment"
 	@echo "make test-e2e-short     - Run E2E tests (skip long-running)"
 	@echo "make test-e2e-cleanup   - Clean up Docker test environment"
+	@echo "make test-e2e-security  - Run security E2E tests (static configs)"
+	@echo "make test-e2e-lua       - Run Lua/OpenResty E2E tests (realtime capture)"
 	@echo "make test-coverage      - Run tests with coverage report"
 	@echo "make lint               - Run golangci-lint"
 	@echo "make lint-install       - Install golangci-lint"
@@ -80,13 +84,61 @@ test-e2e-short:
 	FLATRUN_API_URL=$(E2E_API_URL) FLATRUN_DEPLOYMENTS_PATH=$(E2E_DEPLOYMENTS_PATH) go test -v -short -timeout 5m ./test/e2e/...
 
 test-e2e-cleanup:
-	@echo "Cleaning up E2E test environment..."
+	@echo "Cleaning up all E2E test environments..."
 	cd test/e2e && docker compose -f docker-compose.test.yml down -v --remove-orphans 2>/dev/null || true
+	cd test/e2e && docker compose -f docker-compose.security.yml down -v --remove-orphans 2>/dev/null || true
+	cd test/e2e && docker compose -f docker-compose.lua.yml down -v --remove-orphans 2>/dev/null || true
 	@docker network rm proxy 2>/dev/null || true
 	@docker network rm database 2>/dev/null || true
 	@rm -rf $(E2E_DEPLOYMENTS_PATH)/* 2>/dev/null || true
+	@rm -rf $(E2E_SECURITY_PATH)/* 2>/dev/null || true
+	@rm -rf $(E2E_LUA_PATH)/* 2>/dev/null || true
 	@echo "Cleanup complete"
 
+test-e2e-security-setup:
+	@echo "Setting up Security E2E test environment..."
+	@mkdir -p $(E2E_SECURITY_PATH)/nginx/conf.d
+	@chmod -R 777 $(E2E_SECURITY_PATH) 2>/dev/null || true
+	@docker network create proxy 2>/dev/null || true
+	cd test/e2e && docker compose -f docker-compose.security.yml up -d --build
+	@echo "Waiting for services to be healthy..."
+	@timeout 120 bash -c 'until docker exec flatrun-e2e-security-agent wget -q -O /dev/null http://127.0.0.1:8090/api/health 2>/dev/null; do sleep 2; done' || (echo "Security agent failed to start" && exit 1)
+	@chmod -R 777 $(E2E_SECURITY_PATH) 2>/dev/null || true
+	@echo "Security E2E environment ready"
+
+test-e2e-security-cleanup:
+	@echo "Cleaning up Security E2E test environment..."
+	cd test/e2e && docker compose -f docker-compose.security.yml down -v --remove-orphans 2>/dev/null || true
+	@rm -rf $(E2E_SECURITY_PATH)/* 2>/dev/null || true
+	@echo "Security cleanup complete"
+
+test-e2e-security: test-e2e-security-setup
+	@echo "Running Security E2E tests..."
+	FLATRUN_SECURITY_TEST=true go test -v -timeout 5m ./test/e2e/... -run TestSecurityConfigFilesCreated
+	@$(MAKE) test-e2e-security-cleanup
+
+test-e2e-lua-setup:
+	@echo "Setting up Lua/OpenResty E2E test environment..."
+	@mkdir -p $(E2E_LUA_PATH)/nginx/conf.d
+	@chmod -R 777 $(E2E_LUA_PATH) 2>/dev/null || true
+	@docker network create proxy 2>/dev/null || true
+	cd test/e2e && docker compose -f docker-compose.lua.yml up -d --build
+	@echo "Waiting for services to be healthy..."
+	@timeout 120 bash -c 'until docker exec flatrun-e2e-lua-agent wget -q -O /dev/null http://127.0.0.1:8090/api/health 2>/dev/null; do sleep 2; done' || (echo "Lua agent failed to start" && exit 1)
+	@chmod -R 777 $(E2E_LUA_PATH) 2>/dev/null || true
+	@echo "Lua/OpenResty E2E environment ready"
+
+test-e2e-lua-cleanup:
+	@echo "Cleaning up Lua/OpenResty E2E test environment..."
+	cd test/e2e && docker compose -f docker-compose.lua.yml down -v --remove-orphans 2>/dev/null || true
+	@rm -rf $(E2E_LUA_PATH)/* 2>/dev/null || true
+	@echo "Lua cleanup complete"
+
+test-e2e-lua: test-e2e-lua-setup
+	@echo "Running Lua/OpenResty E2E tests..."
+	FLATRUN_LUA_TEST=true go test -v -timeout 5m ./test/e2e/... -run TestLuaRealtimeCapture
+	@$(MAKE) test-e2e-lua-cleanup
+
 test-coverage:
 	@echo "Running tests with coverage..."
 	go test -coverprofile=coverage.out ./...

config.example.yml

@@ -58,3 +58,12 @@ infrastructure:
         host: ""
         port: 6379
         password: ""
+security:
+    enabled: true
+    realtime_capture: false
+    scan_interval: 30s
+    retention_days: 30
+    rate_threshold: 100
+    auto_block_enabled: false
+    auto_block_threshold: 50
+    auto_block_duration: 24h0m0s

internal/api/security_handlers.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/flatrun/agent/internal/security"
+	"github.com/flatrun/agent/pkg/config"
 	"github.com/flatrun/agent/pkg/models"
 	"github.com/gin-gonic/gin"
 )
@@ -420,3 +421,41 @@ func (s *Server) getDeploymentSecurityEvents(c *gin.Context) {
 		"deployment": name,
 	})
 }
+
+// getRealtimeCaptureStatus returns the current realtime capture status
+func (s *Server) getRealtimeCaptureStatus(c *gin.Context) {
+	c.JSON(http.StatusOK, gin.H{
+		"enabled":          s.config.Security.Enabled,
+		"realtime_capture": s.config.Security.RealtimeCapture,
+	})
+}
+
+// setRealtimeCaptureStatus enables or disables realtime capture
+func (s *Server) setRealtimeCaptureStatus(c *gin.Context) {
+	var req struct {
+		Enabled bool `json:"enabled"`
+	}
+	if err := c.ShouldBindJSON(&req); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	if err := s.infraManager.SetNginxRealtimeCapture(req.Enabled); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	s.config.Security.RealtimeCapture = req.Enabled
+
+	if s.configPath != "" {
+		if err := config.Save(s.config, s.configPath); err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "Config updated but failed to save: " + err.Error()})
+			return
+		}
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"realtime_capture": req.Enabled,
+		"message":          "Realtime capture " + map[bool]string{true: "enabled", false: "disabled"}[req.Enabled],
+	})
+}

internal/api/server.go

@@ -88,6 +88,14 @@ func New(cfg *config.Config, configPath string) *Server {
 		securityManager, err = security.NewManager(cfg.DeploymentsPath)
 		if err != nil {
 			log.Printf("Warning: Failed to initialize security manager: %v", err)
+		} else {
+			nginxConfigPath := cfg.Nginx.ConfigPath
+			if nginxConfigPath == "" {
+				nginxConfigPath = filepath.Join(cfg.DeploymentsPath, "nginx", "conf.d")
+			}
+			if err := securityManager.InitNginxConfigs(nginxConfigPath); err != nil {
+				log.Printf("Warning: Failed to initialize security nginx configs: %v", err)
+			}
 		}
 	}
 
@@ -255,6 +263,8 @@ func (s *Server) setupRoutes() {
 			protected.POST("/security/protected-routes", s.addProtectedRoute)
 			protected.PUT("/security/protected-routes/:id", s.updateProtectedRoute)
 			protected.DELETE("/security/protected-routes/:id", s.deleteProtectedRoute)
+			protected.GET("/security/realtime-capture", s.getRealtimeCaptureStatus)
+			protected.PUT("/security/realtime-capture", s.setRealtimeCaptureStatus)
 			protected.GET("/deployments/:name/security", s.getDeploymentSecurity)
 			protected.PUT("/deployments/:name/security", s.updateDeploymentSecurity)
 			protected.GET("/deployments/:name/security/events", s.getDeploymentSecurityEvents)
@@ -1175,6 +1185,16 @@ func (s *Server) getSettings(c *gin.Context) {
 					"port":      s.config.Infrastructure.Redis.Port,
 				},
 			},
+			"security": gin.H{
+				"enabled":              s.config.Security.Enabled,
+				"realtime_capture":     s.config.Security.RealtimeCapture,
+				"scan_interval":        s.config.Security.ScanInterval.String(),
+				"retention_days":       s.config.Security.RetentionDays,
+				"rate_threshold":       s.config.Security.RateThreshold,
+				"auto_block_enabled":   s.config.Security.AutoBlockEnabled,
+				"auto_block_threshold": s.config.Security.AutoBlockThreshold,
+				"auto_block_duration":  s.config.Security.AutoBlockDuration.String(),
+			},
 		},
 	})
 }
@@ -1224,6 +1244,16 @@ func (s *Server) updateSettings(c *gin.Context) {
 				Password  string `json:"password"`
 			} `json:"redis,omitempty"`
 		} `json:"infrastructure,omitempty"`
+		Security *struct {
+			Enabled            bool   `json:"enabled"`
+			RealtimeCapture    bool   `json:"realtime_capture"`
+			ScanInterval       string `json:"scan_interval"`
+			RetentionDays      int    `json:"retention_days"`
+			RateThreshold      int    `json:"rate_threshold"`
+			AutoBlockEnabled   bool   `json:"auto_block_enabled"`
+			AutoBlockThreshold int    `json:"auto_block_threshold"`
+			AutoBlockDuration  string `json:"auto_block_duration"`
+		} `json:"security,omitempty"`
 	}
 
 	if err := c.ShouldBindJSON(&req); err != nil {
@@ -1312,6 +1342,38 @@ func (s *Server) updateSettings(c *gin.Context) {
 		}
 	}
 
+	if req.Security != nil {
+		prevEnabled := s.config.Security.Enabled
+		prevRealtimeCapture := s.config.Security.RealtimeCapture
+		s.config.Security.Enabled = req.Security.Enabled
+		s.config.Security.RealtimeCapture = req.Security.RealtimeCapture
+		s.config.Security.AutoBlockEnabled = req.Security.AutoBlockEnabled
+		if req.Security.RetentionDays > 0 {
+			s.config.Security.RetentionDays = req.Security.RetentionDays
+		}
+		if req.Security.RateThreshold > 0 {
+			s.config.Security.RateThreshold = req.Security.RateThreshold
+		}
+		if req.Security.AutoBlockThreshold > 0 {
+			s.config.Security.AutoBlockThreshold = req.Security.AutoBlockThreshold
+		}
+		if req.Security.ScanInterval != "" {
+			if d, err := time.ParseDuration(req.Security.ScanInterval); err == nil {
+				s.config.Security.ScanInterval = d
+			}
+		}
+		if req.Security.AutoBlockDuration != "" {
+			if d, err := time.ParseDuration(req.Security.AutoBlockDuration); err == nil {
+				s.config.Security.AutoBlockDuration = d
+			}
+		}
+		if prevRealtimeCapture != s.config.Security.RealtimeCapture || prevEnabled != s.config.Security.Enabled {
+			if err := s.infraManager.SetNginxRealtimeCapture(s.config.Security.RealtimeCapture && s.config.Security.Enabled); err != nil {
+				log.Printf("Warning: failed to update nginx realtime capture: %v", err)
+			}
+		}
+	}
+
 	s.infraManager.UpdateConfig(s.config)
 	s.proxyOrchestrator.UpdateConfig(s.config)
 
@@ -1364,6 +1426,16 @@ func (s *Server) updateSettings(c *gin.Context) {
 					"port":      s.config.Infrastructure.Redis.Port,
 				},
 			},
+			"security": gin.H{
+				"enabled":              s.config.Security.Enabled,
+				"realtime_capture":     s.config.Security.RealtimeCapture,
+				"scan_interval":        s.config.Security.ScanInterval.String(),
+				"retention_days":       s.config.Security.RetentionDays,
+				"rate_threshold":       s.config.Security.RateThreshold,
+				"auto_block_enabled":   s.config.Security.AutoBlockEnabled,
+				"auto_block_threshold": s.config.Security.AutoBlockThreshold,
+				"auto_block_duration":  s.config.Security.AutoBlockDuration.String(),
+			},
 		},
 	})
 }