Add trace context to the downward API

build/lib/release.sh

@@ -355,8 +355,13 @@ function kube::release::create_docker_images_for_server() {
         rm -rf "${docker_build_path}"
         mkdir -p "${docker_build_path}"
         ln "${binary_dir}/${binary_name}" "${docker_build_path}/${binary_name}"
-        printf " FROM ${base_image} \n ADD ${binary_name} /usr/local/bin/${binary_name}\n" > "${docker_file_path}"
-
+        cat <<EOF > "${docker_file_path}"
+FROM alpine:latest as certs
+RUN apk --update add ca-certificates
+FROM ${base_image}
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY ${binary_name} /usr/local/bin/${binary_name}
+EOF
         "${DOCKER[@]}" build --pull -q -t "${docker_image_tag}" "${docker_build_path}" >/dev/null
         "${DOCKER[@]}" tag "${docker_image_tag}" "${deprecated_image_tag}" >/dev/null
         "${DOCKER[@]}" save "${docker_image_tag}" "${deprecated_image_tag}" > "${binary_dir}/${binary_name}.tar"

downward-trace-experiment/downward-application/Dockerfile

@@ -0,0 +1,8 @@
+FROM ubuntu
+RUN apt-get update
+RUN apt-get install -y ca-certificates
+ADD ./main /usr/bin/main
+ADD ./creds.json /usr/bin/creds.json
+EXPOSE 6060
+ENV GOOGLE_APPLICATION_CREDENTIALS /usr/bin/creds.json
+ENTRYPOINT ["/usr/bin/main", "-logtostderr"]

downward-trace-experiment/downward-application/main.go

@@ -0,0 +1,69 @@
+//mple trace_quickstart creates traces incoming and outgoing requests.
+package main
+
+import (
+	"context"
+	"encoding/base64"
+	"log"
+	"os"
+	"time"
+
+	"contrib.go.opencensus.io/exporter/stackdriver"
+	"go.opencensus.io/trace"
+	"go.opencensus.io/trace/propagation"
+)
+
+func main() {
+
+	log.Println("Execution begin...")
+	traceContext := os.Getenv("KUBERNETES_TRACE_CONTEXT")
+	log.Println("Downward API passed trace context: ", traceContext)
+	log.Println("Another test")
+
+	// Create an register a OpenCensus
+	// Stackdriver Trace exporter.
+	exporter, err := stackdriver.NewExporter(stackdriver.Options{
+		ProjectID: "samnaser-gke-dev-217421",
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+	trace.RegisterExporter(exporter)
+
+	log.Println("Stackdriver exporter created.")
+
+	decodedContextBytes, err := base64.StdEncoding.DecodeString(traceContext)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Println("Decoded context.")
+
+	spanContext, ok := propagation.FromBinary(decodedContextBytes)
+	if !ok {
+		log.Fatalf("could not convert raw bytes to trace")
+	}
+
+	log.Println("Trace ID: ", spanContext.TraceID)
+
+	_, span := trace.StartSpan(context.Background(), "ApplicationLevelTrace")
+
+	link := trace.Link{
+		TraceID: spanContext.TraceID,
+		SpanID:  spanContext.SpanID,
+		Type:    trace.LinkTypeChild,
+	}
+
+	log.Println("Linking to span with TraceID -> SpanID: ", link.TraceID, link.SpanID)
+
+	span.AddLink(link)
+
+	time.Sleep(2 * time.Second)
+	span.End()
+
+	log.Println("Span ended.")
+	time.Sleep(time.Minute * 2)
+
+}

downward-trace-experiment/downwardTraceTestCustomConfig/tracedPod.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: traced-pod
+spec:
+  containers:
+    - name: traced-pod
+      image: gcr.io/samnaser-gke-dev-217421/traced-pod:v2
+      env:
+        - name: KUBERNETES_TRACE_CONTEXT
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.traceContext
+        - name: MY_POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+  restartPolicy: Never

pkg/apis/core/pods/helpers.go

@@ -46,6 +46,7 @@ func ConvertDownwardAPIFieldLabel(version, label, value string) (string, string,
 		"metadata.name",
 		"metadata.namespace",
 		"metadata.uid",
+		"metadata.traceContext",
 		"spec.nodeName",
 		"spec.restartPolicy",
 		"spec.serviceAccountName",

pkg/apis/core/v1/conversion.go

@@ -60,6 +60,7 @@ func addConversionFuncs(scheme *runtime.Scheme) error {
 			switch label {
 			case "metadata.name",
 				"metadata.namespace",
+				"metadata.traceContext",
 				"spec.nodeName",
 				"spec.restartPolicy",
 				"spec.schedulerName",

pkg/apis/core/validation/validation.go

@@ -2008,6 +2008,7 @@ var validEnvDownwardAPIFieldPathExpressions = sets.NewString(
 	"metadata.name",
 	"metadata.namespace",
 	"metadata.uid",
+	"metadata.traceContext",
 	"spec.nodeName",
 	"spec.serviceAccountName",
 	"status.hostIP",

pkg/fieldpath/fieldpath.go

@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/golang/glog"
+
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation"
@@ -67,6 +69,9 @@ func ExtractFieldPathAsString(obj interface{}, fieldPath string) (string, error)
 	}
 
 	switch fieldPath {
+	case "metadata.traceContext":
+		glog.Errorf("Here we are!")
+		return accessor.GetTraceContext(), nil
 	case "metadata.annotations":
 		return FormatMap(accessor.GetAnnotations()), nil
 	case "metadata.labels":

pkg/kubelet/kubelet.go

@@ -35,6 +35,7 @@ import (
 
 	cadvisorapi "github.com/google/cadvisor/info/v1"
 	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
+	"go.opencensus.io/trace"
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -75,6 +76,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/eviction"
 	"k8s.io/kubernetes/pkg/kubelet/images"
 	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig"
+	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
 	"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
 	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
 	"k8s.io/kubernetes/pkg/kubelet/logs"
@@ -112,6 +114,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/mount"
 	nodeutil "k8s.io/kubernetes/pkg/util/node"
 	"k8s.io/kubernetes/pkg/util/oom"
+	"k8s.io/kubernetes/pkg/util/trace"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/csi"
 	utilexec "k8s.io/utils/exec"
@@ -2052,9 +2055,27 @@ func (kl *Kubelet) HandlePodAdditions(pods []*v1.Pod) {
 				continue
 			}
 		}
+
+		// Create an register a OpenCensus
+		// Stackdriver Trace exporter.
+		exporter, err := traceutil.DefaultExporter()
+		if err != nil {
+			log.Errorf("could not register default exporter in kubelet")
+		}
+
+		trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+		trace.RegisterExporter(exporter)
+
+		_, remoteSpan, err := traceutil.SpanFromPodEncodedContext(pod, "Kubelet: handle pod addition")
+		if err != nil {
+			trace.ApplyConfig(trace.Config{DefaultSampler: trace.NeverSample()})
+		}
+
 		mirrorPod, _ := kl.podManager.GetMirrorPodByPod(pod)
 		kl.dispatchWork(pod, kubetypes.SyncPodCreate, mirrorPod, start)
 		kl.probeManager.AddPod(pod)
+
+		remoteSpan.End()
 	}
 }
 

pkg/kubelet/kuberuntime/kuberuntime_container.go

@@ -101,7 +101,7 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb
 	trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
 	trace.RegisterExporter(exporter)
 
-	ctx, remoteSpan, err := traceutil.SpanFromPodEncodedContext(pod, "Kuberuntime: initiate start container")
+	ctx, remoteSpan, err := traceutil.SpanFromPodEncodedContext(pod, "Kuberuntime: container start process")
 	if err != nil {
 		trace.ApplyConfig(trace.Config{DefaultSampler: trace.NeverSample()})
 	}
@@ -280,6 +280,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(ctx context.Context,
 		envVarAttr = append(envVarAttr, trace.StringAttribute(e.Name, e.Value))
 	}
 
+	envVarAttr = append(envVarAttr, trace.Int64Attribute("Environment variable count", int64(len(envVarAttr))))
 	containerConfigSpan.Annotate(envVarAttr, "Environment variables added to container")
 	config.Envs = envs
 

pkg/kubelet/status/status_manager.go

@@ -22,22 +22,23 @@ import (
 	"sync"
 	"time"
 
-	clientset "k8s.io/client-go/kubernetes"
-
 	"github.com/golang/glog"
+	"go.opencensus.io/trace"
 	"k8s.io/api/core/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/diff"
 	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
 	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
 	statusutil "k8s.io/kubernetes/pkg/util/pod"
+	"k8s.io/kubernetes/pkg/util/trace"
 )
 
 // A wrapper around v1.PodStatus that includes a version to enforce that stale pod statuses are
@@ -503,6 +504,23 @@ func (m *manager) syncPod(uid types.UID, status versionedPodStatus) {
 	}
 	pod = newPod
 
+	//If transitioned from Pending to Running, then trace it
+	if newPod.Status.Phase == "Running" && oldStatus.Phase == "Pending" {
+		// Create an register a OpenCensus
+		// Stackdriver Trace exporter.
+		exporter, _ := traceutil.DefaultExporter()
+
+		trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+		trace.RegisterExporter(exporter)
+
+		_, podRunningSpan, err := traceutil.SpanFromPodEncodedContext(newPod, "Status manager: pod transitioned from pending to running")
+		if err != nil {
+			trace.ApplyConfig(trace.Config{DefaultSampler: trace.NeverSample()})
+		}
+
+		podRunningSpan.End()
+	}
+
 	glog.V(3).Infof("Status for pod %q updated successfully: (%d, %+v)", format.Pod(pod), status.version, status.status)
 	m.apiStatusVersions[kubetypes.MirrorPodUID(pod.UID)] = status.version
 

pkg/util/trace/traceutil.go

@@ -16,11 +16,6 @@ import (
 // SpanContextFromPodEncodedContext takes a pod to extract a SpanContext from and returns the decoded SpanContext
 func SpanContextFromPodEncodedContext(pod *v1.Pod) (spanContext trace.SpanContext, err error) {
 
-	// If there is no context encoded in the pod, error out
-	if pod.TraceContext == "" {
-		return trace.SpanContext{}, errors.New("could not extract trace context from given pod object")
-	}
-
 	decodedContextBytes, err := base64.StdEncoding.DecodeString(pod.TraceContext)
 	if err != nil {
 		return trace.SpanContext{}, err
@@ -53,11 +48,6 @@ func SpanFromPodEncodedContext(pod *v1.Pod, name string) (ctx context.Context, r
 // Base64 encodes the wire format for the SpanContext, and puts it in the pod's TraceContext field
 func EncodeSpanContextIntoPod(pod *core.Pod, spanContext trace.SpanContext) error {
 
-	if string(pod.Name) == "" {
-		pod.TraceContext = ""
-		return errors.New("will not encode span into pod without name")
-	}
-
 	rawContextBytes := propagation.Binary(spanContext)
 	encodedContext := base64.StdEncoding.EncodeToString(rawContextBytes)
 	pod.TraceContext = encodedContext
@@ -70,9 +60,7 @@ func EncodeSpanContextIntoPod(pod *core.Pod, spanContext trace.SpanContext) erro
 func DefaultExporter() (exporter trace.Exporter, err error) {
 	// Create an register a OpenCensus
 	// Stackdriver Trace exporter.
-	exporter, err = stackdriver.NewExporter(stackdriver.Options{
-		ProjectID: "samnaser-gke-dev-217421",
-	})
+	exporter, err = stackdriver.NewExporter(stackdriver.Options{})
 
 	return exporter, err
 }

staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/meta.go

@@ -31,6 +31,7 @@ type ObjectMetaAccessor interface {
 // not support that field (Name, UID, Namespace on lists) will be a no-op and return
 // a default value.
 type Object interface {
+	GetTraceContext() string
 	GetNamespace() string
 	SetNamespace(namespace string)
 	GetName() string
@@ -128,6 +129,7 @@ func (obj *ObjectMeta) GetObjectMeta() Object { return obj }
 
 // Namespace implements metav1.Object for any object with an ObjectMeta typed field. Allows
 // fast, direct access to metadata fields for API objects.
+func (meta *ObjectMeta) GetTraceContext() string             { return meta.TraceContext }
 func (meta *ObjectMeta) GetNamespace() string                { return meta.Namespace }
 func (meta *ObjectMeta) SetNamespace(namespace string)       { meta.Namespace = namespace }
 func (meta *ObjectMeta) GetName() string                     { return meta.Name }

staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructured.go

@@ -196,6 +196,10 @@ func (u *Unstructured) SetOwnerReferences(references []metav1.OwnerReference) {
 	u.setNestedField(newReferences, "metadata", "ownerReferences")
 }
 
+func (u *Unstructured) GetTraceContext() string {
+	return getNestedString(u.Object, "metadata", "traceContext")
+}
+
 func (u *Unstructured) GetAPIVersion() string {
 	return getNestedString(u.Object, "apiVersion")
 }