feat(health): introduce back HealthService with Livez RPC for health status checks; deprecate legacy gRPC endpoint in favor of HTTP

Author: randygrok

pkg/rpc/server/server.go

@@ -287,10 +287,88 @@ func (p *P2PServer) GetNetInfo(
 	}), nil
 }
 
-// NewServiceHandler creates a new HTTP handler for Store, P2P and Config services
+// HealthServer implements the HealthService defined in the proto file
+// DEPRECATED: This is a legacy compatibility shim for external frameworks.
+// New code should use GET /health/live HTTP endpoint instead.
+type HealthServer struct {
+	store  store.Store
+	config config.Config
+	logger zerolog.Logger
+}
+
+// NewHealthServer creates a new HealthServer instance
+func NewHealthServer(store store.Store, config config.Config, logger zerolog.Logger) *HealthServer {
+	return &HealthServer{
+		store:  store,
+		config: config,
+		logger: logger,
+	}
+}
+
+// Livez implements the HealthService.Livez RPC
+// DEPRECATED: Use GET /health/live HTTP endpoint instead. This endpoint exists only
+// for backward compatibility with external testing frameworks.
+func (h *HealthServer) Livez(
+	ctx context.Context,
+	req *connect.Request[emptypb.Empty],
+) (*connect.Response[pb.GetHealthResponse], error) {
+	status := pb.HealthStatus_PASS
+
+	// For aggregator nodes, check if block production is healthy
+	if h.config.Node.Aggregator {
+		state, err := h.store.GetState(ctx)
+		if err != nil {
+			h.logger.Error().Err(err).Msg("Failed to get state for health check")
+			return connect.NewResponse(&pb.GetHealthResponse{
+				Status: pb.HealthStatus_FAIL,
+			}), nil
+		}
+
+		// If we have blocks, check if the last block time is recent
+		if state.LastBlockHeight > 0 {
+			timeSinceLastBlock := time.Since(state.LastBlockTime)
+
+			// Calculate the threshold based on block time
+			blockTime := h.config.Node.BlockTime.Duration
+
+			// For lazy mode, use the lazy block interval instead
+			if h.config.Node.LazyMode {
+				blockTime = h.config.Node.LazyBlockInterval.Duration
+			}
+
+			warnThreshold := blockTime * 3 // healthCheckWarnMultiplier
+			failThreshold := blockTime * 5 // healthCheckFailMultiplier
+
+			if timeSinceLastBlock > failThreshold {
+				h.logger.Error().
+					Dur("time_since_last_block", timeSinceLastBlock).
+					Dur("fail_threshold", failThreshold).
+					Uint64("last_block_height", state.LastBlockHeight).
+					Time("last_block_time", state.LastBlockTime).
+					Msg("Health check: node has stopped producing blocks (FAIL)")
+				status = pb.HealthStatus_FAIL
+			} else if timeSinceLastBlock > warnThreshold {
+				h.logger.Warn().
+					Dur("time_since_last_block", timeSinceLastBlock).
+					Dur("warn_threshold", warnThreshold).
+					Uint64("last_block_height", state.LastBlockHeight).
+					Time("last_block_time", state.LastBlockTime).
+					Msg("Health check: block production is slow (WARN)")
+				status = pb.HealthStatus_WARN
+			}
+		}
+	}
+
+	return connect.NewResponse(&pb.GetHealthResponse{
+		Status: status,
+	}), nil
+}
+
+// NewServiceHandler creates a new HTTP handler for Store, P2P, Health and Config services
 func NewServiceHandler(store store.Store, peerManager p2p.P2PRPC, proposerAddress []byte, logger zerolog.Logger, config config.Config, bestKnown BestKnownHeightProvider) (http.Handler, error) {
 	storeServer := NewStoreServer(store, logger)
 	p2pServer := NewP2PServer(peerManager)
+	healthServer := NewHealthServer(store, config, logger) // Legacy gRPC endpoint
 	configServer := NewConfigServer(config, proposerAddress, logger)
 
 	mux := http.NewServeMux()
@@ -299,6 +377,7 @@ func NewServiceHandler(store store.Store, peerManager p2p.P2PRPC, proposerAddres
 	reflector := grpcreflect.NewStaticReflector(
 		rpc.StoreServiceName,
 		rpc.P2PServiceName,
+		rpc.HealthServiceName, // Legacy gRPC endpoint
 		rpc.ConfigServiceName,
 	)
 	mux.Handle(grpcreflect.NewHandlerV1(reflector, compress1KB))
@@ -312,10 +391,14 @@ func NewServiceHandler(store store.Store, peerManager p2p.P2PRPC, proposerAddres
 	p2pPath, p2pHandler := rpc.NewP2PServiceHandler(p2pServer)
 	mux.Handle(p2pPath, p2pHandler)
 
+	// Register HealthService (legacy gRPC endpoint for backward compatibility)
+	healthPath, healthHandler := rpc.NewHealthServiceHandler(healthServer)
+	mux.Handle(healthPath, healthHandler)
+
 	configPath, configHandler := rpc.NewConfigServiceHandler(configServer)
 	mux.Handle(configPath, configHandler)
 
-	// Register custom HTTP endpoints
+	// Register custom HTTP endpoints (including the preferred /health/live endpoint)
 	RegisterCustomHTTPEndpoints(mux, store, peerManager, config, bestKnown, logger)
 
 	// Use h2c to support HTTP/2 without TLS

proto/evnode/v1/health.proto

@@ -0,0 +1,32 @@
+syntax = "proto3";
+package evnode.v1;
+
+import "google/protobuf/empty.proto";
+
+option go_package = "github.com/evstack/ev-node/types/pb/evnode/v1";
+
+// HealthService defines the RPC service for the health package
+// DEPRECATED: Use HTTP endpoint GET /health/live instead
+service HealthService {
+  // Livez returns the health status of the node
+  // DEPRECATED: Use HTTP endpoint GET /health/live instead
+  rpc Livez(google.protobuf.Empty) returns (GetHealthResponse) {}
+}
+
+// HealthStatus defines the health status of the node
+enum HealthStatus {
+  // Unknown health status
+  UNKNOWN = 0;
+  // Healthy status (Healthy)
+  PASS = 1;
+  // Degraded but still serving
+  WARN = 2;
+  // Hard fail
+  FAIL = 3;
+}
+
+// GetHealthResponse defines the response for retrieving health status
+message GetHealthResponse {
+  // Health status
+  HealthStatus status = 1;
+}

test/docker-e2e/upgrade_test.go

@@ -3,7 +3,6 @@
 package docker_e2e
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"math/big"