From bbf2da28bf2b500d3c7ac2ecec7124ecd27a55c1 Mon Sep 17 00:00:00 2001 From: Sam Naser Date: Thu, 18 Oct 2018 10:17:30 -0700 Subject: [PATCH 1/5] Add additional annotation for environment variable count, more informative span names --- pkg/kubelet/kuberuntime/kuberuntime_container.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index a2d04e8cbf5fb..5928f8ed82673 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -101,7 +101,7 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) trace.RegisterExporter(exporter) - ctx, remoteSpan, err := traceutil.SpanFromPodEncodedContext(pod, "Kuberuntime: initiate start container") + ctx, remoteSpan, err := traceutil.SpanFromPodEncodedContext(pod, "Kuberuntime: container start process") if err != nil { trace.ApplyConfig(trace.Config{DefaultSampler: trace.NeverSample()}) } @@ -280,6 +280,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(ctx context.Context, envVarAttr = append(envVarAttr, trace.StringAttribute(e.Name, e.Value)) } + envVarAttr = append(envVarAttr, trace.Int64Attribute("Environment variable count", int64(len(envVarAttr)))) containerConfigSpan.Annotate(envVarAttr, "Environment variables added to container") config.Envs = envs From d0ebbec082cf70cb055a3ef89f172fdeff355d2c Mon Sep 17 00:00:00 2001 From: Sam Naser Date: Thu, 25 Oct 2018 12:26:56 -0700 Subject: [PATCH 2/5] Add pod added Span and pod transitioned to running Span --- pkg/kubelet/kubelet.go | 21 +++++++++++++++++++++ pkg/kubelet/status/status_manager.go | 22 ++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 27c0a835aa1c9..ae89b89d41969 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -35,6 +35,7 @@ import ( cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" + "go.opencensus.io/trace" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -75,6 +76,7 @@ import ( "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/images" "k8s.io/kubernetes/pkg/kubelet/kubeletconfig" + "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log" "k8s.io/kubernetes/pkg/kubelet/kuberuntime" "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/logs" @@ -112,6 +114,7 @@ import ( "k8s.io/kubernetes/pkg/util/mount" nodeutil "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/util/oom" + "k8s.io/kubernetes/pkg/util/trace" "k8s.io/kubernetes/pkg/volume" "k8s.io/kubernetes/pkg/volume/csi" utilexec "k8s.io/utils/exec" @@ -2052,9 +2055,27 @@ func (kl *Kubelet) HandlePodAdditions(pods []*v1.Pod) { continue } } + + // Create an register a OpenCensus + // Stackdriver Trace exporter. + exporter, err := traceutil.DefaultExporter() + if err != nil { + log.Errorf("could not register default exporter in kubelet") + } + + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + trace.RegisterExporter(exporter) + + _, remoteSpan, err := traceutil.SpanFromPodEncodedContext(pod, "Kubelet: handle pod addition") + if err != nil { + trace.ApplyConfig(trace.Config{DefaultSampler: trace.NeverSample()}) + } + mirrorPod, _ := kl.podManager.GetMirrorPodByPod(pod) kl.dispatchWork(pod, kubetypes.SyncPodCreate, mirrorPod, start) kl.probeManager.AddPod(pod) + + remoteSpan.End() } } diff --git a/pkg/kubelet/status/status_manager.go b/pkg/kubelet/status/status_manager.go index f184003117c82..bdadd7760b0d6 100644 --- a/pkg/kubelet/status/status_manager.go +++ b/pkg/kubelet/status/status_manager.go @@ -22,9 +22,8 @@ import ( "sync" "time" - clientset "k8s.io/client-go/kubernetes" - "github.com/golang/glog" + "go.opencensus.io/trace" "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" @@ -32,12 +31,14 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" podutil "k8s.io/kubernetes/pkg/api/v1/pod" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" kubepod "k8s.io/kubernetes/pkg/kubelet/pod" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/kubelet/util/format" statusutil "k8s.io/kubernetes/pkg/util/pod" + "k8s.io/kubernetes/pkg/util/trace" ) // A wrapper around v1.PodStatus that includes a version to enforce that stale pod statuses are @@ -503,6 +504,23 @@ func (m *manager) syncPod(uid types.UID, status versionedPodStatus) { } pod = newPod + //If transitioned from Pending to Running, then trace it + if newPod.Status.Phase == "Running" && oldStatus.Phase == "Pending" { + // Create an register a OpenCensus + // Stackdriver Trace exporter. + exporter, _ := traceutil.DefaultExporter() + + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + trace.RegisterExporter(exporter) + + _, podRunningSpan, err := traceutil.SpanFromPodEncodedContext(newPod, "Status manager: pod transitioned from pending to running") + if err != nil { + trace.ApplyConfig(trace.Config{DefaultSampler: trace.NeverSample()}) + } + + podRunningSpan.End() + } + glog.V(3).Infof("Status for pod %q updated successfully: (%d, %+v)", format.Pod(pod), status.version, status.status) m.apiStatusVersions[kubetypes.MirrorPodUID(pod.UID)] = status.version From 3a272358d6fd304f3476028954d3cc731b8ac4c2 Mon Sep 17 00:00:00 2001 From: Sam Naser Date: Fri, 2 Nov 2018 16:17:03 -0700 Subject: [PATCH 3/5] Add downward API modifications for TraceContext, pass into container through manifest --- .../downward-application/Dockerfile | 8 +++ .../downward-application/main.go | 57 +++++++++++++++++++ .../tracedPod.yaml | 18 ++++++ pkg/apis/core/pods/helpers.go | 1 + pkg/apis/core/v1/conversion.go | 1 + pkg/apis/core/validation/validation.go | 1 + pkg/fieldpath/fieldpath.go | 5 ++ .../apimachinery/pkg/apis/meta/v1/meta.go | 2 + .../apis/meta/v1/unstructured/unstructured.go | 4 ++ 9 files changed, 97 insertions(+) create mode 100644 downward-trace-experiment/downward-application/Dockerfile create mode 100755 downward-trace-experiment/downward-application/main.go create mode 100644 downward-trace-experiment/downwardTraceTestCustomConfig/tracedPod.yaml diff --git a/downward-trace-experiment/downward-application/Dockerfile b/downward-trace-experiment/downward-application/Dockerfile new file mode 100644 index 0000000000000..514591d373471 --- /dev/null +++ b/downward-trace-experiment/downward-application/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu +RUN apt-get update +RUN apt-get install -y ca-certificates +ADD ./main /usr/bin/main +ADD ./creds.json /usr/bin/creds.json +EXPOSE 6060 +ENV GOOGLE_APPLICATION_CREDENTIALS /usr/bin/creds.json +ENTRYPOINT ["/usr/bin/main", "-logtostderr"] diff --git a/downward-trace-experiment/downward-application/main.go b/downward-trace-experiment/downward-application/main.go new file mode 100755 index 0000000000000..fc819dc2ec535 --- /dev/null +++ b/downward-trace-experiment/downward-application/main.go @@ -0,0 +1,57 @@ +//mple trace_quickstart creates traces incoming and outgoing requests. +package main + +import ( + "context" + "encoding/base64" + "log" + "os" + "time" + + "contrib.go.opencensus.io/exporter/stackdriver" + "go.opencensus.io/trace" + "go.opencensus.io/trace/propagation" +) + +func main() { + + log.Println("Execution begin...") + traceContext := os.Getenv("KUBERNETES_TRACE_CONTEXT") + log.Println("Downward API passed trace context: ", traceContext) + log.Println("Another test") + + // Create an register a OpenCensus + // Stackdriver Trace exporter. + exporter, err := stackdriver.NewExporter(stackdriver.Options{ + ProjectID: "samnaser-gke-dev-217421", + }) + if err != nil { + log.Fatal(err) + } + + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + trace.RegisterExporter(exporter) + + log.Println("Stackdriver exporter created.") + + decodedContextBytes, err := base64.StdEncoding.DecodeString(traceContext) + if err != nil { + log.Fatal(err) + } + + log.Println("Decoded context.") + + spanContext, ok := propagation.FromBinary(decodedContextBytes) + if !ok { + log.Fatalf("could not convert raw bytes to trace") + } + + log.Println("Trace ID: ", spanContext.TraceID) + _, span := trace.StartSpanWithRemoteParent(context.Background(), "Deep roots are not reached by the frost", spanContext) + time.Sleep(2 * time.Second) + span.End() + + log.Println("Span ended.") + time.Sleep(time.Minute * 2) + +} diff --git a/downward-trace-experiment/downwardTraceTestCustomConfig/tracedPod.yaml b/downward-trace-experiment/downwardTraceTestCustomConfig/tracedPod.yaml new file mode 100644 index 0000000000000..ce2629b8a272a --- /dev/null +++ b/downward-trace-experiment/downwardTraceTestCustomConfig/tracedPod.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: traced-pod +spec: + containers: + - name: traced-pod + image: localhost:5000/traced-pod:v10 + env: + - name: KUBERNETES_TRACE_CONTEXT + valueFrom: + fieldRef: + fieldPath: metadata.traceContext + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + restartPolicy: Never diff --git a/pkg/apis/core/pods/helpers.go b/pkg/apis/core/pods/helpers.go index cf199cee73755..5ae424b515c91 100644 --- a/pkg/apis/core/pods/helpers.go +++ b/pkg/apis/core/pods/helpers.go @@ -46,6 +46,7 @@ func ConvertDownwardAPIFieldLabel(version, label, value string) (string, string, "metadata.name", "metadata.namespace", "metadata.uid", + "metadata.traceContext", "spec.nodeName", "spec.restartPolicy", "spec.serviceAccountName", diff --git a/pkg/apis/core/v1/conversion.go b/pkg/apis/core/v1/conversion.go index bab07f1950c77..4cd4064efcbe4 100644 --- a/pkg/apis/core/v1/conversion.go +++ b/pkg/apis/core/v1/conversion.go @@ -60,6 +60,7 @@ func addConversionFuncs(scheme *runtime.Scheme) error { switch label { case "metadata.name", "metadata.namespace", + "metadata.traceContext", "spec.nodeName", "spec.restartPolicy", "spec.schedulerName", diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go index db06578dd5449..b481fcfcd4dc5 100644 --- a/pkg/apis/core/validation/validation.go +++ b/pkg/apis/core/validation/validation.go @@ -2008,6 +2008,7 @@ var validEnvDownwardAPIFieldPathExpressions = sets.NewString( "metadata.name", "metadata.namespace", "metadata.uid", + "metadata.traceContext", "spec.nodeName", "spec.serviceAccountName", "status.hostIP", diff --git a/pkg/fieldpath/fieldpath.go b/pkg/fieldpath/fieldpath.go index b997751ec81fa..a8c858b7e3e79 100644 --- a/pkg/fieldpath/fieldpath.go +++ b/pkg/fieldpath/fieldpath.go @@ -20,6 +20,8 @@ import ( "fmt" "strings" + "github.com/golang/glog" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation" @@ -67,6 +69,9 @@ func ExtractFieldPathAsString(obj interface{}, fieldPath string) (string, error) } switch fieldPath { + case "metadata.traceContext": + glog.Errorf("Here we are!") + return accessor.GetTraceContext(), nil case "metadata.annotations": return FormatMap(accessor.GetAnnotations()), nil case "metadata.labels": diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/meta.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/meta.go index ee1447541fcdd..abda8200f8246 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/meta.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/meta.go @@ -31,6 +31,7 @@ type ObjectMetaAccessor interface { // not support that field (Name, UID, Namespace on lists) will be a no-op and return // a default value. type Object interface { + GetTraceContext() string GetNamespace() string SetNamespace(namespace string) GetName() string @@ -128,6 +129,7 @@ func (obj *ObjectMeta) GetObjectMeta() Object { return obj } // Namespace implements metav1.Object for any object with an ObjectMeta typed field. Allows // fast, direct access to metadata fields for API objects. +func (meta *ObjectMeta) GetTraceContext() string { return meta.TraceContext } func (meta *ObjectMeta) GetNamespace() string { return meta.Namespace } func (meta *ObjectMeta) SetNamespace(namespace string) { meta.Namespace = namespace } func (meta *ObjectMeta) GetName() string { return meta.Name } diff --git a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructured.go b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructured.go index 781469ec2657e..16e16336a22bc 100644 --- a/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructured.go +++ b/staging/src/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructured.go @@ -196,6 +196,10 @@ func (u *Unstructured) SetOwnerReferences(references []metav1.OwnerReference) { u.setNestedField(newReferences, "metadata", "ownerReferences") } +func (u *Unstructured) GetTraceContext() string { + return getNestedString(u.Object, "metadata", "traceContext") +} + func (u *Unstructured) GetAPIVersion() string { return getNestedString(u.Object, "apiVersion") } From bdf8e1efda3a4a9ddc53d97b4d942d00aabadc61 Mon Sep 17 00:00:00 2001 From: Sam Naser Date: Mon, 5 Nov 2018 15:08:53 -0800 Subject: [PATCH 4/5] Remove blank pod name checks from trace utilities, edit release shell script to add certs to all images (for trace purposes) --- build/lib/release.sh | 9 +++++++-- pkg/util/trace/traceutil.go | 14 +------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/build/lib/release.sh b/build/lib/release.sh index 1275e6e1d966c..4d6dfcd86fb94 100644 --- a/build/lib/release.sh +++ b/build/lib/release.sh @@ -355,8 +355,13 @@ function kube::release::create_docker_images_for_server() { rm -rf "${docker_build_path}" mkdir -p "${docker_build_path}" ln "${binary_dir}/${binary_name}" "${docker_build_path}/${binary_name}" - printf " FROM ${base_image} \n ADD ${binary_name} /usr/local/bin/${binary_name}\n" > "${docker_file_path}" - + cat < "${docker_file_path}" +FROM alpine:latest as certs +RUN apk --update add ca-certificates +FROM ${base_image} +COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY ${binary_name} /usr/local/bin/${binary_name} +EOF "${DOCKER[@]}" build --pull -q -t "${docker_image_tag}" "${docker_build_path}" >/dev/null "${DOCKER[@]}" tag "${docker_image_tag}" "${deprecated_image_tag}" >/dev/null "${DOCKER[@]}" save "${docker_image_tag}" "${deprecated_image_tag}" > "${binary_dir}/${binary_name}.tar" diff --git a/pkg/util/trace/traceutil.go b/pkg/util/trace/traceutil.go index ce4d582b5cb8e..358b7fcfd4504 100644 --- a/pkg/util/trace/traceutil.go +++ b/pkg/util/trace/traceutil.go @@ -16,11 +16,6 @@ import ( // SpanContextFromPodEncodedContext takes a pod to extract a SpanContext from and returns the decoded SpanContext func SpanContextFromPodEncodedContext(pod *v1.Pod) (spanContext trace.SpanContext, err error) { - // If there is no context encoded in the pod, error out - if pod.TraceContext == "" { - return trace.SpanContext{}, errors.New("could not extract trace context from given pod object") - } - decodedContextBytes, err := base64.StdEncoding.DecodeString(pod.TraceContext) if err != nil { return trace.SpanContext{}, err @@ -53,11 +48,6 @@ func SpanFromPodEncodedContext(pod *v1.Pod, name string) (ctx context.Context, r // Base64 encodes the wire format for the SpanContext, and puts it in the pod's TraceContext field func EncodeSpanContextIntoPod(pod *core.Pod, spanContext trace.SpanContext) error { - if string(pod.Name) == "" { - pod.TraceContext = "" - return errors.New("will not encode span into pod without name") - } - rawContextBytes := propagation.Binary(spanContext) encodedContext := base64.StdEncoding.EncodeToString(rawContextBytes) pod.TraceContext = encodedContext @@ -70,9 +60,7 @@ func EncodeSpanContextIntoPod(pod *core.Pod, spanContext trace.SpanContext) erro func DefaultExporter() (exporter trace.Exporter, err error) { // Create an register a OpenCensus // Stackdriver Trace exporter. - exporter, err = stackdriver.NewExporter(stackdriver.Options{ - ProjectID: "samnaser-gke-dev-217421", - }) + exporter, err = stackdriver.NewExporter(stackdriver.Options{}) return exporter, err } From fcaa8da72e668474474079d16443c1e0c2e08e67 Mon Sep 17 00:00:00 2001 From: Sam Naser Date: Mon, 5 Nov 2018 15:18:21 -0800 Subject: [PATCH 5/5] Implementation for utilizing the KUBERNETES_TRACE_CONTEXT downward API parameter to link application level traces to pod startup traces --- .../downward-application/main.go | 14 +++++++++++++- .../downwardTraceTestCustomConfig/tracedPod.yaml | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/downward-trace-experiment/downward-application/main.go b/downward-trace-experiment/downward-application/main.go index fc819dc2ec535..1503b23078f8e 100755 --- a/downward-trace-experiment/downward-application/main.go +++ b/downward-trace-experiment/downward-application/main.go @@ -47,7 +47,19 @@ func main() { } log.Println("Trace ID: ", spanContext.TraceID) - _, span := trace.StartSpanWithRemoteParent(context.Background(), "Deep roots are not reached by the frost", spanContext) + + _, span := trace.StartSpan(context.Background(), "ApplicationLevelTrace") + + link := trace.Link{ + TraceID: spanContext.TraceID, + SpanID: spanContext.SpanID, + Type: trace.LinkTypeChild, + } + + log.Println("Linking to span with TraceID -> SpanID: ", link.TraceID, link.SpanID) + + span.AddLink(link) + time.Sleep(2 * time.Second) span.End() diff --git a/downward-trace-experiment/downwardTraceTestCustomConfig/tracedPod.yaml b/downward-trace-experiment/downwardTraceTestCustomConfig/tracedPod.yaml index ce2629b8a272a..74fee12eee745 100644 --- a/downward-trace-experiment/downwardTraceTestCustomConfig/tracedPod.yaml +++ b/downward-trace-experiment/downwardTraceTestCustomConfig/tracedPod.yaml @@ -5,7 +5,7 @@ metadata: spec: containers: - name: traced-pod - image: localhost:5000/traced-pod:v10 + image: gcr.io/samnaser-gke-dev-217421/traced-pod:v2 env: - name: KUBERNETES_TRACE_CONTEXT valueFrom: