Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion config/core/configmaps/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ metadata:
app.kubernetes.io/component: controller
app.kubernetes.io/version: devel
annotations:
knative.dev/example-checksum: "b99000ec"
knative.dev/example-checksum: "555b4826"
data:
# This is the Go import path for the binary that is containerized
# and substituted here.
Expand Down Expand Up @@ -92,6 +92,25 @@ data:
# If omitted, or empty, no rootCA is added to the golang rootCAs
queue-sidecar-rootca: ""

# Sets the minimum TLS version for the queue proxy sidecar's TLS server.
# Accepted values: "1.2", "1.3". Default is "1.3" if not specified.
queue-sidecar-tls-min-version: ""

# Sets the maximum TLS version for the queue proxy sidecar's TLS server.
# Accepted values: "1.2", "1.3". If omitted, the Go default is used.
queue-sidecar-tls-max-version: ""

# Sets the cipher suites for the queue proxy sidecar's TLS server.
# Comma-separated list of cipher suite names (e.g. "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256").
# If omitted, the Go default cipher suites are used.
# Note: cipher suites are not configurable in TLS 1.3.
queue-sidecar-tls-cipher-suites: ""

# Sets the elliptic curve preferences for the queue proxy sidecar's TLS server.
# Comma-separated list of curve names (e.g. "X25519,CurveP256").
# If omitted, the Go default curves are used.
queue-sidecar-tls-curve-preferences: ""

# If set, it automatically configures pod anti-affinity requirements for all Knative services.
# It employs the `preferredDuringSchedulingIgnoredDuringExecution` weighted pod affinity term,
# aligning with the Knative revision label. It yields the configuration below in all workloads' deployments:
Expand Down
23 changes: 23 additions & 0 deletions pkg/deployment/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ const (
queueSidecarTokenAudiencesKey = "queue-sidecar-token-audiences"
queueSidecarRooCAKey = "queue-sidecar-rootca"

// queueSidecar TLS configuration keys.
queueSidecarTLSMinVersionKey = "queue-sidecar-tls-min-version"
queueSidecarTLSMaxVersionKey = "queue-sidecar-tls-max-version"
queueSidecarTLSCipherSuitesKey = "queue-sidecar-tls-cipher-suites"
queueSidecarTLSCurvePreferencesKey = "queue-sidecar-tls-curve-preferences"

defaultAffinityTypeKey = "default-affinity-type"
defaultAffinityTypeValue = PreferSpreadRevisionOverNodes

Expand Down Expand Up @@ -202,6 +208,11 @@ func NewConfigFromMap(configMap map[string]string) (*Config, error) {
cm.AsStringSet(queueSidecarTokenAudiencesKey, &nc.QueueSidecarTokenAudiences),
cm.AsString(queueSidecarRooCAKey, &nc.QueueSidecarRootCA),

cm.AsString(queueSidecarTLSMinVersionKey, &nc.QueueSidecarTLSMinVersion),
cm.AsString(queueSidecarTLSMaxVersionKey, &nc.QueueSidecarTLSMaxVersion),
cm.AsString(queueSidecarTLSCipherSuitesKey, &nc.QueueSidecarTLSCipherSuites),
cm.AsString(queueSidecarTLSCurvePreferencesKey, &nc.QueueSidecarTLSCurvePreferences),

cm.AsString(RuntimeClassNameKey, &runtimeClassNames),

cm.AsBool(podIsAlwaysSchedulableKey, &nc.PodIsAlwaysSchedulable),
Expand Down Expand Up @@ -308,6 +319,18 @@ type Config struct {
// QueueSidecarRootCA is a root certificate to be trusted by the queue proxy sidecar qpoptions.
QueueSidecarRootCA string

// QueueSidecarTLSMinVersion is the minimum TLS version for the queue proxy sidecar (e.g. "1.2", "1.3").
QueueSidecarTLSMinVersion string

// QueueSidecarTLSMaxVersion is the maximum TLS version for the queue proxy sidecar (e.g. "1.2", "1.3").
QueueSidecarTLSMaxVersion string

// QueueSidecarTLSCipherSuites is a comma-separated list of cipher suites for the queue proxy sidecar.
QueueSidecarTLSCipherSuites string

// QueueSidecarTLSCurvePreferences is a comma-separated list of elliptic curves for the queue proxy sidecar.
QueueSidecarTLSCurvePreferences string

// DefaultAffinityType is a string that controls what affinity rules will be automatically
// applied to the PodSpec of all Knative services.
DefaultAffinityType AffinityType
Expand Down
22 changes: 22 additions & 0 deletions pkg/deployment/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,28 @@ kata:
podIsAlwaysSchedulableKey: "true",
QueueSidecarImageKey: defaultSidecarImage,
},
}, {
name: "controller configuration with queue sidecar TLS settings",
wantConfig: &Config{
RegistriesSkippingTagResolving: sets.New("kind.local", "ko.local", "dev.local"),
DigestResolutionTimeout: digestResolutionTimeoutDefault,
QueueSidecarImage: defaultSidecarImage,
QueueSidecarCPURequest: &QueueSidecarCPURequestDefault,
QueueSidecarTokenAudiences: sets.New(""),
ProgressDeadline: ProgressDeadlineDefault,
DefaultAffinityType: defaultAffinityTypeValue,
QueueSidecarTLSMinVersion: "1.2",
QueueSidecarTLSMaxVersion: "1.3",
QueueSidecarTLSCipherSuites: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
QueueSidecarTLSCurvePreferences: "X25519,CurveP256",
},
data: map[string]string{
QueueSidecarImageKey: defaultSidecarImage,
queueSidecarTLSMinVersionKey: "1.2",
queueSidecarTLSMaxVersionKey: "1.3",
queueSidecarTLSCipherSuitesKey: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
queueSidecarTLSCurvePreferencesKey: "X25519,CurveP256",
},
}}

for _, tt := range configTests {
Expand Down
59 changes: 30 additions & 29 deletions pkg/queue/sharedmain/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package sharedmain

import (
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
Expand All @@ -31,6 +30,7 @@ import (

netproxy "knative.dev/networking/pkg/http/proxy"
pkghandler "knative.dev/pkg/network/handlers"
knativetls "knative.dev/pkg/tls"
"knative.dev/serving/pkg/activator"

"github.com/kelseyhightower/envconfig"
Expand Down Expand Up @@ -265,23 +265,10 @@ func Main(opts ...Option) error {
httpServers["profile"] = pprof.Server
}

tlsServers := make(map[string]*http.Server)
var tlsServer *http.Server
var certWatcher *certificate.CertWatcher
var err error

if tlsEnabled {
tlsServers["main"] = mainServer(":"+env.QueueServingTLSPort, mainHandler)
// Keep admin server on HTTP even with TLS enabled since it's only accessed locally by kubelet

certWatcher, err = certificate.NewCertWatcher(certPath, keyPath, 1*time.Minute, logger)
if err != nil {
logger.Fatal("failed to create certWatcher", zap.Error(err))
}
defer certWatcher.Stop()
}

logger.Info("Starting queue-proxy")

errCh := make(chan error)
for name, server := range httpServers {
go func(name string, s *http.Server) {
Expand All @@ -292,20 +279,34 @@ func Main(opts ...Option) error {
}
}(name, server)
}
for name, server := range tlsServers {
go func(name string, s *http.Server) {
logger.Info("Starting tls server ", name, s.Addr)
s.TLSConfig = &tls.Config{
GetCertificate: certWatcher.GetCertificate,
MinVersion: tls.VersionTLS13,
}

if tlsEnabled {
tlsServer = mainServer(":"+env.QueueServingTLSPort, mainHandler)
// Keep admin server on HTTP even with TLS enabled since it's only accessed locally by kubelet

certWatcher, err = certificate.NewCertWatcher(certPath, keyPath, 1*time.Minute, logger)
if err != nil {
logger.Fatal("failed to create certWatcher", zap.Error(err))
}
defer certWatcher.Stop()

tlsCfg, err := knativetls.DefaultConfigFromEnv("QUEUE_PROXY_")
if err != nil {
logger.Fatalw("Failed to read TLS configuration from environment", zap.Error(err))
}
go func() {
logger.Info("Starting tls server main ", tlsServer.Addr)
tlsServer.TLSConfig = tlsCfg
tlsServer.TLSConfig.GetCertificate = certWatcher.GetCertificate
// Don't forward ErrServerClosed as that indicates we're already shutting down.
if err := s.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
errCh <- fmt.Errorf("%s server failed to serve: %w", name, err)
if err := tlsServer.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) {
errCh <- fmt.Errorf("main tls server failed to serve: %w", err)
}
}(name, server)
}()
}

logger.Info("Starting queue-proxy")

// Blocks until we actually receive a TERM signal or one of the servers
// exits unexpectedly. We fold both signals together because we only want
// to act on the first of those to reach here.
Expand All @@ -326,10 +327,10 @@ func Main(opts ...Option) error {
logger.Errorw("Failed to shutdown server", zap.String("server", name), zap.Error(err))
}
}
for name, srv := range tlsServers {
logger.Info("Shutting down server: ", name)
if err := srv.Shutdown(ctx); err != nil {
logger.Errorw("Failed to shutdown server", zap.String("server", name), zap.Error(err))
if tlsServer != nil {
logger.Info("Shutting down server: main tls")
if err := tlsServer.Shutdown(ctx); err != nil {
logger.Errorw("Failed to shutdown server", zap.String("server", "main tls"), zap.Error(err))
}
}

Expand Down
12 changes: 12 additions & 0 deletions pkg/reconciler/revision/resources/deploy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,18 @@ var (
}, {
Name: "ROOT_CA",
Value: "",
}, {
Name: "QUEUE_PROXY_TLS_MIN_VERSION",
Value: "",
}, {
Name: "QUEUE_PROXY_TLS_MAX_VERSION",
Value: "",
}, {
Name: "QUEUE_PROXY_TLS_CIPHER_SUITES",
Value: "",
}, {
Name: "QUEUE_PROXY_TLS_CURVE_PREFERENCES",
Value: "",
}, {
Name: "ENABLE_MULTI_CONTAINER_PROBES",
Value: "false",
Expand Down
12 changes: 12 additions & 0 deletions pkg/reconciler/revision/resources/queue.go
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,18 @@ func makeQueueContainer(rev *v1.Revision, cfg *config.Config) (*corev1.Container
}, {
Name: "ROOT_CA",
Value: cfg.Deployment.QueueSidecarRootCA,
}, {
Name: "QUEUE_PROXY_TLS_MIN_VERSION",
Value: cfg.Deployment.QueueSidecarTLSMinVersion,
}, {
Name: "QUEUE_PROXY_TLS_MAX_VERSION",
Value: cfg.Deployment.QueueSidecarTLSMaxVersion,
}, {
Name: "QUEUE_PROXY_TLS_CIPHER_SUITES",
Value: cfg.Deployment.QueueSidecarTLSCipherSuites,
}, {
Name: "QUEUE_PROXY_TLS_CURVE_PREFERENCES",
Value: cfg.Deployment.QueueSidecarTLSCurvePreferences,
}, {
Name: "ENABLE_MULTI_CONTAINER_PROBES",
Value: strconv.FormatBool(multiContainerProbingEnabled),
Expand Down
4 changes: 4 additions & 0 deletions pkg/reconciler/revision/resources/queue_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1122,6 +1122,10 @@ var defaultEnv = map[string]string{
"SYSTEM_NAMESPACE": system.Namespace(),
"USER_PORT": strconv.Itoa(v1.DefaultUserPort),
"ROOT_CA": "",
"QUEUE_PROXY_TLS_MIN_VERSION": "",
"QUEUE_PROXY_TLS_MAX_VERSION": "",
"QUEUE_PROXY_TLS_CIPHER_SUITES": "",
"QUEUE_PROXY_TLS_CURVE_PREFERENCES": "",
"ENABLE_MULTI_CONTAINER_PROBES": "false",
"OBSERVABILITY_CONFIG": `{"tracing":{},"metrics":{},"runtime":{},"requestMetrics":{}}`,
}
Expand Down
Loading