[wip] jsaf line-based generator, no-histograms

Dockerfile

@@ -9,52 +9,28 @@ RUN cargo chef prepare --recipe-path recipe.json
 
 # Stage 1: Cacher - Build dependencies only
 FROM docker.io/rust:1.90.0-slim-bookworm AS cacher
-ARG SCCACHE_BUCKET
-ARG SCCACHE_REGION
-ARG AWS_ACCESS_KEY_ID
-ARG AWS_SECRET_ACCESS_KEY
-ARG AWS_SESSION_TOKEN
-ENV CARGO_INCREMENTAL=0
+ENV JEMALLOC_SYS_WITH_PROFILING=1
 WORKDIR /app
 RUN apt-get update && apt-get install -y \
+    build-essential \
     pkg-config=1.8.1-1 \
     libssl-dev=3.0.17-1~deb12u3 \
     protobuf-compiler=3.21.12-3 \
     fuse3=3.14.0-4 \
     libfuse3-dev=3.14.0-4 \
     curl \
     && rm -rf /var/lib/apt/lists/*
-# Download pre-built sccache binary
-RUN case "$(uname -m)" in \
-        x86_64) ARCH=x86_64-unknown-linux-musl ;; \
-        aarch64) ARCH=aarch64-unknown-linux-musl ;; \
-        *) echo "Unsupported architecture" && exit 1 ;; \
-    esac && \
-    curl -L https://github.com/mozilla/sccache/releases/download/v0.8.2/sccache-v0.8.2-${ARCH}.tar.gz | tar xz && \
-    mv sccache-v0.8.2-${ARCH}/sccache /usr/local/cargo/bin/ && \
-    rm -rf sccache-v0.8.2-${ARCH}
 RUN cargo install cargo-chef --version 0.1.73
 COPY --from=planner /app/recipe.json recipe.json
 # This layer is cached until Cargo.toml/Cargo.lock change
-# Use BuildKit secrets to pass AWS credentials securely (not exposed in image metadata)
-RUN --mount=type=secret,id=aws_access_key_id \
-    --mount=type=secret,id=aws_secret_access_key \
-    --mount=type=secret,id=aws_session_token \
-    export AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) && \
-    export AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) && \
-    export AWS_SESSION_TOKEN=$(cat /run/secrets/aws_session_token) && \
-    export RUSTC_WRAPPER=sccache && \
-    cargo chef cook --release --locked --features logrotate_fs --recipe-path recipe.json
+RUN cargo chef cook --release --locked --features "logrotate_fs jemalloc_profiling" --recipe-path recipe.json
 
 # Stage 2: Builder - Build source code
 FROM docker.io/rust:1.90.0-slim-bookworm AS builder
-ARG SCCACHE_BUCKET
-ARG SCCACHE_REGION
-ENV CARGO_INCREMENTAL=0
-ENV SCCACHE_BUCKET=${SCCACHE_BUCKET}
-ENV SCCACHE_REGION=${SCCACHE_REGION}
+ENV JEMALLOC_SYS_WITH_PROFILING=1
 WORKDIR /app
 RUN apt-get update && apt-get install -y \
+    build-essential \
     pkg-config=1.8.1-1 \
     libssl-dev=3.0.17-1~deb12u3 \
     protobuf-compiler=3.21.12-3 \
@@ -68,23 +44,130 @@ COPY --from=cacher /usr/local/cargo /usr/local/cargo
 COPY . .
 # Build binary - reuses cached dependencies + sccache
 # Use BuildKit secrets to pass AWS credentials securely (not exposed in image metadata)
-RUN --mount=type=secret,id=aws_access_key_id \
-    --mount=type=secret,id=aws_secret_access_key \
-    --mount=type=secret,id=aws_session_token \
-    export AWS_ACCESS_KEY_ID=$(cat /run/secrets/aws_access_key_id) && \
-    export AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_access_key) && \
-    export AWS_SESSION_TOKEN=$(cat /run/secrets/aws_session_token) && \
-    export RUSTC_WRAPPER=sccache && \
-    cargo build --release --locked --bin lading --features logrotate_fs
+RUN cargo build --release --locked --bin lading --features "logrotate_fs jemalloc_profiling"
 
 # Stage 3: Runtime
 FROM docker.io/debian:bookworm-20241202-slim
 RUN apt-get update && apt-get install -y \
     libfuse3-dev=3.14.0-4 \
     fuse3=3.14.0-4 \
+    curl \
+    ca-certificates \
+    procps \
+    gawk \
+    perl \
+    libjemalloc2 \
     && rm -rf /var/lib/apt/lists/*
+RUN ARCH="$(dpkg --print-architecture)" && \
+    url_release="https://github.com/DataDog/ddprof/releases/latest/download/ddprof-${ARCH}" && \
+    curl -L -o /usr/local/bin/ddprof "${url_release}" && \
+    chmod 755 /usr/local/bin/ddprof
+# Lightweight page-table/RSS sampler helper
+RUN cat > /usr/local/bin/pt-sampler <<'EOF' && chmod +x /usr/local/bin/pt-sampler
+#!/bin/sh
+# Usage: pt-sampler <pid> [interval_seconds]
+pid="$1"
+interval="${2:-5}"
+if [ -z "$pid" ]; then
+  echo "usage: pt-sampler <pid> [interval_seconds]" >&2
+  exit 1
+fi
+while :; do
+  ts="$(date -Is)"
+  pt_kb="$(awk '/^PageTables:/ {sum+=$2} END {print sum+0}' /proc/"$pid"/smaps 2>/dev/null)"
+  rss_kb="$(awk '/^VmRSS:/ {print $2}' /proc/"$pid"/status 2>/dev/null)"
+  echo "$ts rss_kb=${rss_kb:-0} pagetables_kb=${pt_kb:-0}"
+  sleep "$interval"
+done
+EOF
+
+# Simple smaps rollup helper to find largest RSS mappings
+RUN cat > /usr/local/bin/smaps-top <<'EOF' && chmod +x /usr/local/bin/smaps-top
+#!/bin/sh
+# Usage: smaps-top <pid> [topN]
+pid="$1"
+top="${2:-5}"
+if [ -z "$pid" ]; then
+  echo "usage: smaps-top <pid> [topN]" >&2
+  exit 1
+fi
+gawk -v top="$top" '
+BEGIN{RS=""; OFS=""; c=0}
+{
+  rss=0; size=0;
+  if (match($0,/Rss:[[:space:]]+([0-9]+)/,a)) rss=a[1];
+  if (match($0,/Size:[[:space:]]+([0-9]+)/,b)) size=b[1];
+  hdr=$0; sub(/\n.*/,"",hdr);
+  blocks[c,"rss"]=rss; blocks[c,"size"]=size; blocks[c,"hdr"]=hdr; blocks[c,"blk"]=$0; c++;
+}
+END{
+  n=c;
+  for(i=0;i<n;i++){
+    max=i;
+    for(j=i+1;j<n;j++){
+      if(blocks[j,"rss"]>blocks[max,"rss"]) max=j;
+    }
+    if(max!=i){
+      tmp_r=blocks[i,"rss"]; tmp_s=blocks[i,"size"]; tmp_h=blocks[i,"hdr"]; tmp_b=blocks[i,"blk"];
+      blocks[i,"rss"]=blocks[max,"rss"]; blocks[i,"size"]=blocks[max,"size"]; blocks[i,"hdr"]=blocks[max,"hdr"]; blocks[i,"blk"]=blocks[max,"blk"];
+      blocks[max,"rss"]=tmp_r; blocks[max,"size"]=tmp_s; blocks[max,"hdr"]=tmp_h; blocks[max,"blk"]=tmp_b;
+    }
+  }
+  limit=(top<n)?top:n;
+  for(i=0;i<limit;i++){
+    print "---- Block ", i+1, " ----";
+    print "Rss=", blocks[i,"rss"], " kB  Size=", blocks[i,"size"], " kB";
+    print blocks[i,"hdr"];
+    print blocks[i,"blk"];
+  }
+}
+' /proc/"$pid"/smaps
+EOF
+
 COPY --from=builder /app/target/release/lading /usr/bin/lading
+# ENV DD_TRACE_AGENT_URL=http://host.docker.internal:8126 \
+#     DD_SERVICE=jsaf-lading \
+#     DD_ENV=local \
+#     DD_VERSION=dev
+
+# Lightweight jeprof helper script for analyzing jemalloc heap dumps.
+# Fetch the upstream script (templated perl) and replace configure placeholders.
+RUN curl -fsSL https://raw.githubusercontent.com/jemalloc/jemalloc/5.3.0/bin/jeprof.in \
+    | sed \
+    -e 's|@PERL@|/usr/bin/env perl|' \
+    -e 's|@jemalloc_version@|5.3.0|' \
+    -e 's|@prefix@|/usr/local|' \
+    -e 's|@bindir@|/usr/local/bin|' \
+    > /usr/local/bin/jeprof && \
+    chmod +x /usr/local/bin/jeprof
+
+# Default jemalloc profiling configuration: enabled, active, 1MiB sample, dumps under /tmp.
+# Note: tikv-jemalloc-sys uses _RJEM_MALLOC_CONF due to the _rjem_ symbol prefix.
+# We set both for compatibility.
+ENV MALLOC_CONF=prof:true,prof_active:true,lg_prof_sample:20,prof_prefix:/tmp/lading-heap \
+    _RJEM_MALLOC_CONF=prof:true,prof_active:true,lg_prof_sample:20,prof_prefix:/tmp/lading-heap
+
+# Entrypoint wrapper: ensures MALLOC_CONF defaults are present and dump dir exists.
+RUN cat > /usr/local/bin/lading-entrypoint.sh <<'EOF' && chmod +x /usr/local/bin/lading-entrypoint.sh
+#!/bin/sh
+set -e
+
+# Allow override; if not set, use baked-in default
+# tikv-jemalloc-sys uses _RJEM_MALLOC_CONF due to symbol prefix, set both for compatibility
+: "${MALLOC_CONF:=prof:true,prof_active:true,lg_prof_sample:20,prof_prefix:/tmp/lading-heap}"
+: "${_RJEM_MALLOC_CONF:=$MALLOC_CONF}"
+export MALLOC_CONF _RJEM_MALLOC_CONF
+
+# Best-effort create dump directory from prof_prefix
+prefix="$(printf '%s' "$MALLOC_CONF" | tr ',' '\n' | sed -n 's/^prof_prefix://p' | head -n1)"
+if [ -n "$prefix" ]; then
+  dir="$(dirname "$prefix")"
+  mkdir -p "$dir" 2>/dev/null || true
+fi
+
+exec /usr/bin/lading "$@"
+EOF
 
 # Smoke test
 RUN ["/usr/bin/lading", "--help"]
-ENTRYPOINT ["/usr/bin/lading"]
+ENTRYPOINT ["/usr/local/bin/lading-entrypoint.sh"]

examples/lading-logrotatefs.yaml

@@ -9,7 +9,9 @@ generator:
         max_depth: 0
         variant: "ascii"
         load_profile:
-          constant: 1.3MiB
+          constant:
+            rate:
+              bytes_per_second: 1.3MiB
         maximum_prebuild_cache_size_bytes: 1GiB
         mount_point: /tmp/logrotate
 

lading/Cargo.toml

@@ -96,6 +96,10 @@ tonic-prost = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { version = "0.3", features = ["std", "env-filter"] }
 zstd = "0.13.3"
+signal-hook = { version = "0.3", optional = true }
+tikv-jemalloc-ctl = { version = "0.6", optional = true, features = ["profiling"] }
+tikv-jemallocator = { version = "0.6", optional = true, features = ["profiling"] }
+tikv-jemalloc-sys = { version = "0.6", optional = true, features = ["profiling"] }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { version = "0.18", default-features = false, features = [] }
@@ -111,8 +115,14 @@ prost-build = { workspace = true }
 tonic-prost-build = { workspace = true }
 
 [features]
-default = []
+default = ["jemalloc_profiling"]
 logrotate_fs = ["fuser"]
+jemalloc_profiling = [
+  "tikv-jemallocator/profiling",
+  "tikv-jemalloc-ctl/profiling",
+  "tikv-jemalloc-sys/profiling",
+  "signal-hook",
+]
 
 [lib]
 doctest = false