feat(scheduledsparkapplication): configurable timestampPrecision (nanos|micros|millis|seconds|minutes)

Add optional spec.timestampPrecision to configure the precision of the timestamp suffix
appended to generated SparkApplication names for scheduled runs. Default remains 'nanos'
for backward compatibility. Adds 'minutes' option to match CronJob granularity and keep
generated names short.

Includes helper function, unit tests and optional chart value.

Fixes: #2602

Signed-off-by: rahul810050 <rahul810050@gmail.com>

Author: Rahul Naskar

api/v1beta2/scheduledsparkapplication_types.go

@@ -23,6 +23,7 @@ import (
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
+// ScheduledSparkApplicationSpec defines the desired state of ScheduledSparkApplication.
 // ScheduledSparkApplicationSpec defines the desired state of ScheduledSparkApplication.
 type ScheduledSparkApplicationSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
@@ -38,6 +39,16 @@ type ScheduledSparkApplicationSpec struct {
 	TimeZone string `json:"timeZone,omitempty"`
 	// Template is a template from which SparkApplication instances can be created.
 	Template SparkApplicationSpec `json:"template"`
+	// TimestampPrecision controls the precision of the timestamp appended to generated
+	// SparkApplication names for scheduled runs.
+	//
+	// Allowed values: "nanos", "micros", "millis", "seconds", "minutes"
+	// +kubebuilder:validation:Enum=nanos;micros;millis;seconds;minutes
+	// +optional
+	// +kubebuilder:default:=nanos
+	// Defaults to "nanos" to preserve current behavior.
+	TimestampPrecision string `json:"timestampPrecision,omitempty"`
+
 	// Suspend is a flag telling the controller to suspend subsequent runs of the application if set to true.
 	// +optional
 	// Defaults to false.

charts/spark-operator-chart/templates/controller/deployment.yaml

@@ -135,10 +135,15 @@ spec:
           containerPort: {{ .Values.prometheus.metrics.port }}
         {{- end }}
         {{- end }}
-        {{- with .Values.controller.env }}
+
+        # --- Global scheduled-spark-app timestamp precision env var (default from values.yaml)
         env:
-        {{- toYaml . | nindent 8 }}
+        - name: SCHEDULED_SA_TIMESTAMP_PRECISION
+          value: {{ .Values.controller.scheduledSparkApplication.timestampPrecision | quote }}
+        {{- with .Values.controller.env }}
+{{ toYaml . | nindent 8 }}
         {{- end }}
+
         {{- with .Values.controller.envFrom }}
         envFrom:
         {{- toYaml . | nindent 8 }}

charts/spark-operator-chart/values.yaml

@@ -57,6 +57,10 @@ hook:
     tag: ""
 
 controller:
+  # -- default precision for ScheduledSparkApplication timestamp suffix
+  scheduledSparkApplication:
+    timestampPrecision: "nanos"
+
   # -- Number of replicas of controller.
   replicas: 1
 

config/crd/bases/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -60,8 +60,9 @@ spec:
           metadata:
             type: object
           spec:
-            description: ScheduledSparkApplicationSpec defines the desired state of
-              ScheduledSparkApplication.
+            description: |-
+              ScheduledSparkApplicationSpec defines the desired state of ScheduledSparkApplication.
+              ScheduledSparkApplicationSpec defines the desired state of ScheduledSparkApplication.
             properties:
               concurrencyPolicy:
                 description: ConcurrencyPolicy is the policy governing concurrent
@@ -12418,6 +12419,21 @@ spec:
                   or a valid IANA location name e.g. "America/New_York".
                   Defaults to "Local".
                 type: string
+              timestampPrecision:
+                default: nanos
+                description: |-
+                  TimestampPrecision controls the precision of the timestamp appended to generated
+                  SparkApplication names for scheduled runs.
+
+                  Allowed values: "nanos", "micros", "millis", "seconds", "minutes"
+                  Defaults to "nanos" to preserve current behavior.
+                enum:
+                - nanos
+                - micros
+                - millis
+                - seconds
+                - minutes
+                type: string
             required:
             - schedule
             - template

internal/controller/scheduledsparkapplication/controller.go

@@ -42,6 +42,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
+	"os"
+	"strconv"
+
 	"github.com/kubeflow/spark-operator/v2/api/v1beta2"
 	"github.com/kubeflow/spark-operator/v2/pkg/common"
 )
@@ -236,9 +239,24 @@ func (r *Reconciler) createSparkApplication(
 	for key, value := range scheduledApp.Labels {
 		labels[key] = value
 	}
+
+	// Determine timestamp precision with precedence:
+	// 1) controller-wide env var SCHEDULED_SA_TIMESTAMP_PRECISION (if set and non-empty)
+	// 2) per-app scheduledApp.Spec.TimestampPrecision (if set)
+	// 3) default "nanos" for backward compatibility
+	precision := "nanos" // fallback default
+
+	if envPrecision, ok := os.LookupEnv("SCHEDULED_SA_TIMESTAMP_PRECISION"); ok && strings.TrimSpace(envPrecision) != "" {
+		precision = strings.TrimSpace(envPrecision)
+	} else if strings.TrimSpace(scheduledApp.Spec.TimestampPrecision) != "" {
+		precision = strings.TrimSpace(scheduledApp.Spec.TimestampPrecision)
+	}
+
+	suffix := formatTimestamp(precision, t)
+
 	app := &v1beta2.SparkApplication{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-%d", scheduledApp.Name, t.UnixNano()),
+			Name:      fmt.Sprintf("%s-%s", scheduledApp.Name, suffix),
 			Namespace: scheduledApp.Namespace,
 			Labels:    labels,
 			OwnerReferences: []metav1.OwnerReference{{
@@ -257,6 +275,28 @@ func (r *Reconciler) createSparkApplication(
 	return app, nil
 }
 
+// formatTimestamp returns a decimal timestamp string according to the requested precision.
+// Allowed precisions: "nanos", "micros", "millis", "seconds", "minutes".
+// If precision is empty or unrecognized, defaults to "nanos" (current behavior).
+func formatTimestamp(precision string, t time.Time) string {
+	switch precision {
+	case "minutes":
+		// Use Unix epoch minutes. matches CronJob approach which is per-minute granularity.
+		return strconv.FormatInt(t.Unix()/60, 10)
+	case "seconds":
+		return strconv.FormatInt(t.Unix(), 10)
+	case "millis":
+		// Use UnixNano()/1e6 for compatibility with older Go versions.
+		return strconv.FormatInt(t.UnixNano()/1e6, 10)
+	case "micros":
+		return strconv.FormatInt(t.UnixNano()/1e3, 10)
+	case "nanos":
+		fallthrough
+	default:
+		return strconv.FormatInt(t.UnixNano(), 10)
+	}
+}
+
 // shouldStartNextRun checks if the next run should be started.
 func (r *Reconciler) shouldStartNextRun(scheduledApp *v1beta2.ScheduledSparkApplication) (bool, error) {
 	apps, err := r.listSparkApplications(scheduledApp)

internal/controller/scheduledsparkapplication/format_timestamp_test.go

@@ -0,0 +1,26 @@
+package scheduledsparkapplication
+
+import (
+	"testing"
+	"time"
+)
+
+func TestFormatTimestampLengths(t *testing.T) {
+	// deterministic time so tests are stable
+	now := time.Unix(1700000000, 123456789) // arbitrary fixed timestamp
+
+	cases := map[string]int{
+		"minutes": 8,  // 1700000000 / 60 ~= 28333333 (8 digits)
+		"seconds": 10, // 1700000000 -> 10 digits
+		"millis":  13,
+		"micros":  16,
+		"nanos":   19,
+	}
+
+	for precision, wantLen := range cases {
+		s := formatTimestamp(precision, now)
+		if len(s) != wantLen {
+			t.Fatalf("precision=%s: got len %d, want %d (value=%s)", precision, len(s), wantLen, s)
+		}
+	}
+}

scripts/setup-envtest-binaries.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# scripts/setup-envtest-binaries.sh
+# Download kube-apiserver and etcd for envtest to the location expected by tests.
+# Usage: from repo root: bash scripts/setup-envtest-binaries.sh
+# (Make executable with chmod +x scripts/setup-envtest-binaries.sh)
+
+set -euo pipefail
+
+# --- Configuration: edit if your tests expect a different K8S version
+K8S_VERSION="v1.32.0"            # kube-apiserver/kube-controller-manager version envtest expects
+ETCD_VERSION="v3.5.11"           # etcd version (3.5.x recommended)
+TARGET_DIR="./bin/k8s/${K8S_VERSION}-linux-amd64"  # target where envtest looked for binaries
+
+echo
+echo "==> Setting up envtest binaries"
+echo "Kubernetes version: ${K8S_VERSION}"
+echo "etcd version:       ${ETCD_VERSION}"
+echo "Target directory:   ${TARGET_DIR}"
+echo
+
+mkdir -p "${TARGET_DIR}"
+
+# Helper to check for curl or wget
+download() {
+  local url="$1" out="$2"
+  if command -v curl >/dev/null 2>&1; then
+    curl -fsSL -o "${out}" "${url}"
+  elif command -v wget >/dev/null 2>&1; then
+    wget -qO "${out}" "${url}"
+  else
+    echo "Error: need curl or wget to download files." >&2
+    exit 2
+  fi
+}
+
+# Download Kubernetes server tarball and extract kube-apiserver
+K8S_TAR="/tmp/k8s-server-${K8S_VERSION}.tar.gz"
+echo "Downloading Kubernetes server ${K8S_VERSION} -> ${K8S_TAR} ..."
+download "https://dl.k8s.io/${K8S_VERSION}/kubernetes-server-linux-amd64.tar.gz" "${K8S_TAR}"
+
+echo "Extracting kube-apiserver from ${K8S_TAR}..."
+rm -rf /tmp/kubernetes || true
+tar -C /tmp -xzf "${K8S_TAR}"
+
+KUBE_APISERVER_SRC="/tmp/kubernetes/server/bin/kube-apiserver"
+if [ ! -f "${KUBE_APISERVER_SRC}" ]; then
+  echo "Error: kube-apiserver not found at ${KUBE_APISERVER_SRC}" >&2
+  echo "Listing /tmp/kubernetes contents:"
+  ls -la /tmp/kubernetes || true
+  exit 3
+fi
+
+cp -v "${KUBE_APISERVER_SRC}" "${TARGET_DIR}/"
+
+# Download etcd tarball and extract etcd binary
+ETCD_TAR="/tmp/etcd-${ETCD_VERSION}-linux-amd64.tar.gz"
+echo "Downloading etcd ${ETCD_VERSION} -> ${ETCD_TAR} ..."
+download "https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" "${ETCD_TAR}"
+
+echo "Extracting etcd from ${ETCD_TAR}..."
+rm -rf "/tmp/etcd-${ETCD_VERSION}-linux-amd64" || true
+tar -C /tmp -xzf "${ETCD_TAR}"
+
+ETCD_SRC="/tmp/etcd-${ETCD_VERSION}-linux-amd64/etcd"
+if [ -f "${ETCD_SRC}" ]; then
+  cp -v "${ETCD_SRC}" "${TARGET_DIR}/"
+else
+  # fallback: search extracted tmp for etcd binary
+  found_etcd=$(find /tmp -type f -name etcd -print -quit || true)
+  if [ -n "${found_etcd}" ]; then
+    cp -v "${found_etcd}" "${TARGET_DIR}/"
+  else
+    echo "ERROR: etcd binary not found inside extracted etcd tarball." >&2
+    exit 4
+  fi
+fi
+
+# Make everything executable
+chmod +x "${TARGET_DIR}"/*
+
+# Export KUBEBUILDER_ASSETS for current shell user (print instruction)
+ABS_TARGET_DIR="$(cd "$(dirname "${TARGET_DIR}")" && pwd)/$(basename "${TARGET_DIR}")"
+echo
+echo "Binaries installed in: ${ABS_TARGET_DIR}"
+echo
+echo "To use these binaries for running tests in this shell, run:"
+echo "  export KUBEBUILDER_ASSETS=\"${ABS_TARGET_DIR}\""
+echo
+echo "You can also add that export line to your shell profile (e.g. ~/.bashrc) if you want it persistent."
+echo
+
+# Add bin/ to .gitignore if not present (local convenience)
+if ! rg -q "^bin/" .gitignore 2>/dev/null || [ ! -f .gitignore ]; then
+  echo "Adding 'bin/' to .gitignore (local convenience)"
+  echo "bin/" >> .gitignore
+  echo "Note: bin/ contains large local binaries; do not commit them."
+fi
+
+echo "Done. Now run the tests that failed, e.g.:"
+echo "  go test ./internal/controller/scheduledsparkapplication -v"
+echo "or to run all controller tests:"
+echo "  go test ./internal/controller/... -v"