diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/bench_command_latency.py b/benchmarks/bench_command_latency.py new file mode 100644 index 0000000..98ed3fd --- /dev/null +++ b/benchmarks/bench_command_latency.py @@ -0,0 +1,143 @@ +"""Benchmark: per-command SSH round-trip latency. +Measures the overhead of vm.run() vs native subprocess. +""" + +from __future__ import annotations + +import subprocess +import sys + +try: + from .helpers import ( + format_stats, + is_sandbox_exec_available, + overhead_str, + print_header, + print_result, + print_subheader, + run_sandboxed, + stats_summary, + time_call_n, + ) +except ImportError: + from helpers import ( # type: ignore[no-redef] + format_stats, + is_sandbox_exec_available, + overhead_str, + print_header, + print_result, + print_subheader, + run_sandboxed, + stats_summary, + time_call_n, + ) + +ITERATIONS = 100 +WARMUP = 2 + + +def _host_true() -> None: + subprocess.run(["true"], capture_output=True) + + +def _host_echo() -> None: + result = subprocess.run(["echo", "hello"], capture_output=True, text=True) + assert "hello" in result.stdout + + +def _sandboxed_true() -> None: + run_sandboxed("true") + + +def _sandboxed_echo() -> None: + result = run_sandboxed("echo hello") + assert "hello" in result.stdout + + +def run_benchmark() -> dict: + """Run all command latency benchmarks and return results.""" + from smolvm import SmolVM + + results = {} + + print_header("Command Latency Benchmark") + print(f" Iterations: {ITERATIONS}, Warmup: {WARMUP}") + + # Host baseline + print_subheader("Host: subprocess.run(['true'])") + host_true_times = time_call_n(_host_true, ITERATIONS, warmup=WARMUP) + host_true_stats = stats_summary(host_true_times) + print_result("Stats", format_stats(host_true_stats)) + results["host_true"] = host_true_stats + + print_subheader("Host: subprocess.run(['echo', 'hello'])") + host_echo_times = time_call_n(_host_echo, ITERATIONS, warmup=WARMUP) + host_echo_stats = stats_summary(host_echo_times) + print_result("Stats", format_stats(host_echo_stats)) + results["host_echo"] = host_echo_stats + + # Sandbox baseline + if is_sandbox_exec_available(): + print_subheader("Sandbox: sandbox-exec 'true'") + sandbox_true_times = time_call_n(_sandboxed_true, ITERATIONS, warmup=WARMUP) + sandbox_true_stats = stats_summary(sandbox_true_times) + print_result("Stats", format_stats(sandbox_true_stats)) + results["sandbox_true"] = sandbox_true_stats + + print_subheader("Sandbox: sandbox-exec 'echo hello'") + sandbox_echo_times = time_call_n(_sandboxed_echo, ITERATIONS, warmup=WARMUP) + sandbox_echo_stats = stats_summary(sandbox_echo_times) + print_result("Stats", format_stats(sandbox_echo_stats)) + results["sandbox_echo"] = sandbox_echo_stats + else: + print("\n sandbox-exec not available (non-macOS), skipping sandbox baseline") + results["sandbox_true"] = None + results["sandbox_echo"] = None + + # SmolVM + print_subheader("SmolVM: vm.run('true')") + print(" Starting VM...") + with SmolVM() as vm: + # Sanity check + sanity = vm.run("echo smolvm_sanity") + assert "smolvm_sanity" in sanity.stdout, f"Sanity failed: {sanity.stdout}" + + def _vm_true(): + vm.run("true") + + def _vm_echo(): + result = vm.run("echo hello") + assert "hello" in result.stdout + + vm_true_times = time_call_n(_vm_true, ITERATIONS, warmup=WARMUP) + vm_true_stats = stats_summary(vm_true_times) + print_result("Stats", format_stats(vm_true_stats)) + results["vm_true"] = vm_true_stats + + print_subheader("SmolVM: vm.run('echo hello')") + vm_echo_times = time_call_n(_vm_echo, ITERATIONS, warmup=WARMUP) + vm_echo_stats = stats_summary(vm_echo_times) + print_result("Stats", format_stats(vm_echo_stats)) + results["vm_echo"] = vm_echo_stats + + # domparison + print_subheader("Comparison (p50)") + print_result( + "vm.run('true') vs host", + overhead_str(host_true_stats["p50"], vm_true_stats["p50"]), + ) + print_result( + "vm.run('echo') vs host", + overhead_str(host_echo_stats["p50"], vm_echo_stats["p50"]), + ) + if results.get("sandbox_true"): + print_result( + "vm.run('true') vs sandbox", + overhead_str(results["sandbox_true"]["p50"], vm_true_stats["p50"]), + ) + + return results + + +if __name__ == "__main__": + run_benchmark() diff --git a/benchmarks/bench_file_io.py b/benchmarks/bench_file_io.py new file mode 100644 index 0000000..cdb67c5 --- /dev/null +++ b/benchmarks/bench_file_io.py @@ -0,0 +1,223 @@ +"""Benchmark: file read/write throughput inside the VM. + +Simulates ralph's JSONL event logging, JSON job files, and session logs. +File IO inside the VM is native ext4 on virtio-blk, but getting results +out requires reading via vm.run("cat ..."). +""" + +from __future__ import annotations + +import os +import sys +import tempfile + +try: + from .helpers import ( + format_stats, + overhead_str, + print_header, + print_result, + print_subheader, + stats_summary, + time_call, + ) +except ImportError: + from helpers import ( # type: ignore[no-redef] + format_stats, + overhead_str, + print_header, + print_result, + print_subheader, + stats_summary, + time_call, + ) + +# A 1KB line simulating a JSONL event +_1KB_LINE = '{"ts":1234567890,"event":"prompt","session":"abc123","data":' + '"x' * 470 + '"}\n' +assert len(_1KB_LINE) >= 1000 + +_10MB_DATA_LINES = 10240 # ~10MB at 1KB per line + + +def _host_sequential_writes(n: int = 1000) -> float: + """Append n x 1KB lines to a file on the host.""" + + def _do(): + with tempfile.NamedTemporaryFile(mode="w", delete=True, suffix=".jsonl") as f: + for _ in range(n): + f.write(_1KB_LINE) + f.flush() + + return time_call(_do) + + +def _host_bulk_write() -> float: + """Write a single ~10MB file on the host.""" + data = _1KB_LINE * _10MB_DATA_LINES + + def _do(): + with tempfile.NamedTemporaryFile(mode="w", delete=True) as f: + f.write(data) + f.flush() + + return time_call(_do) + + +def _host_bulk_read() -> float: + """Write then read back a ~10MB file on the host.""" + with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".dat") as f: + f.write(_1KB_LINE * _10MB_DATA_LINES) + path = f.name + + def _do(): + with open(path) as f: + _ = f.read() + + t = time_call(_do) + os.unlink(path) + return t + + +def _host_many_small_files(n: int = 100) -> float: + """Create n x 1KB files on the host.""" + + def _do(): + tmpdir = tempfile.mkdtemp() + for i in range(n): + with open(os.path.join(tmpdir, f"job_{i}.json"), "w") as f: + f.write(_1KB_LINE) + # Cleanup + import shutil + + shutil.rmtree(tmpdir) + + return time_call(_do) + + +def run_benchmark() -> dict: + """Run all file IO benchmarks and return results.""" + from smolvm import SmolVM + + results = {} + + print_header("File IO Benchmark") + + # baslines on host + print_subheader("Host: Sequential small writes (1000 x 1KB)") + host_seq_times = [_host_sequential_writes() for _ in range(5)] + host_seq_stats = stats_summary(host_seq_times) + print_result("Stats", format_stats(host_seq_stats)) + results["host_seq_write"] = host_seq_stats + + print_subheader("Host: Bulk write (~10MB)") + host_bulk_w_times = [_host_bulk_write() for _ in range(5)] + host_bulk_w_stats = stats_summary(host_bulk_w_times) + print_result("Stats", format_stats(host_bulk_w_stats)) + results["host_bulk_write"] = host_bulk_w_stats + + print_subheader("Host: Bulk read (~10MB)") + host_bulk_r_times = [_host_bulk_read() for _ in range(5)] + host_bulk_r_stats = stats_summary(host_bulk_r_times) + print_result("Stats", format_stats(host_bulk_r_stats)) + results["host_bulk_read"] = host_bulk_r_stats + + print_subheader("Host: Many small files (100 x 1KB)") + host_many_times = [_host_many_small_files() for _ in range(5)] + host_many_stats = stats_summary(host_many_times) + print_result("Stats", format_stats(host_many_stats)) + results["host_many_files"] = host_many_stats + + # SmolVM + print_subheader("SmolVM: Starting VM...") + with SmolVM() as vm: + # warmup with a sanity check + sanity = vm.run("echo smolvm_ready") + assert "smolvm_ready" in sanity.stdout + + # sequential small writes + print_subheader("SmolVM: Sequential small writes (1000 x 1KB)") + # We write via a single vm.run command using a shell loop + write_cmd = ( + "rm -f /tmp/bench.jsonl; " + "i=0; while [ $i -lt 1000 ]; do " + "printf '%s\\n' '" + _1KB_LINE.strip().replace("'", "'\\''") + "' >> /tmp/bench.jsonl; " + "i=$((i+1)); done" + ) + vm_seq_times = [] + for _ in range(3): + t = time_call(lambda: vm.run(write_cmd, timeout=120)) + vm_seq_times.append(t) + vm_seq_stats = stats_summary(vm_seq_times) + print_result("Stats", format_stats(vm_seq_stats)) + results["vm_seq_write"] = vm_seq_stats + + # Bulk write + print_subheader("SmolVM: Bulk write (~10MB)") + # Use dd to write 10MB of data + bulk_write_cmd = "dd if=/dev/zero of=/tmp/bench_bulk.dat bs=1024 count=10240 2>&1" + vm_bulk_w_times = [] + for _ in range(3): + t = time_call(lambda: vm.run(bulk_write_cmd, timeout=60)) + vm_bulk_w_times.append(t) + vm_bulk_w_stats = stats_summary(vm_bulk_w_times) + print_result("Stats", format_stats(vm_bulk_w_stats)) + results["vm_bulk_write"] = vm_bulk_w_stats + + # Bulk read: use dd to /dev/null so only guest disk I/O is measured, + # not SSH/stdout transport overhead (cat would send ~10 MB through the + # command channel, distorting the result vs the host baseline). + print_subheader("SmolVM: Bulk read (~10MB, guest disk only)") + # First ensure the file exists + vm.run("dd if=/dev/zero of=/tmp/bench_read.dat bs=1024 count=10240 2>/dev/null", timeout=60) + vm_bulk_r_times = [] + for _ in range(3): + t = time_call(lambda: vm.run("dd if=/tmp/bench_read.dat of=/dev/null bs=1024 2>/dev/null", timeout=60)) + vm_bulk_r_times.append(t) + vm_bulk_r_stats = stats_summary(vm_bulk_r_times) + print_result("Stats", format_stats(vm_bulk_r_stats)) + results["vm_bulk_read"] = vm_bulk_r_stats + + # Many small files + print_subheader("SmolVM: Many small files (100 x 1KB)") + escaped_line = _1KB_LINE.strip().replace("'", "'\\''") + many_files_cmd = ( + "rm -rf /tmp/bench_jobs; mkdir -p /tmp/bench_jobs; " + "i=0; while [ $i -lt 100 ]; do " + "printf '%s' '" + escaped_line + "'" + " > /tmp/bench_jobs/job_$i.json; " + "i=$((i+1)); done" + ) + vm_many_times = [] + for _ in range(3): + t = time_call(lambda: vm.run(many_files_cmd, timeout=120)) + vm_many_times.append(t) + vm_many_stats = stats_summary(vm_many_times) + print_result("Stats", format_stats(vm_many_stats)) + results["vm_many_files"] = vm_many_stats + + # compare + print_subheader("Comparison (p50)") + print_result( + "Sequential writes: VM vs Host", + overhead_str(host_seq_stats["p50"], vm_seq_stats["p50"]), + ) + print_result( + "Bulk write: VM vs Host", + overhead_str(host_bulk_w_stats["p50"], vm_bulk_w_stats["p50"]), + ) + print_result( + "Bulk read: VM vs Host", + overhead_str(host_bulk_r_stats["p50"], vm_bulk_r_stats["p50"]), + ) + print_result( + "Many files: VM vs Host", + overhead_str(host_many_stats["p50"], vm_many_stats["p50"]), + ) + + return results + + +if __name__ == "__main__": + if "benchmarks" in __file__: + sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent)) + run_benchmark() diff --git a/benchmarks/bench_network_io.py b/benchmarks/bench_network_io.py new file mode 100644 index 0000000..acf95f9 --- /dev/null +++ b/benchmarks/bench_network_io.py @@ -0,0 +1,253 @@ +"""Benchmark: network IO between VM and host. + +Measures HTTP round-trip latency and SSE streaming throughput +""" + +from __future__ import annotations + +import http.server +import json +import socket +import subprocess +import sys +import threading +import time + +try: + from .helpers import ( + format_stats, + overhead_str, + print_header, + print_result, + print_subheader, + stats_summary, + time_call, + timer, + ) +except ImportError: + from helpers import ( # type: ignore[no-redef] + format_stats, + overhead_str, + print_header, + print_result, + print_subheader, + stats_summary, + time_call, + timer, + ) + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +class _BenchHandler(http.server.BaseHTTPRequestHandler): + """Simple HTTP handler for benchmarking.""" + + def do_GET(self): + if self.path == "/health": + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"ok") + elif self.path == "/stream": + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.end_headers() + # send SSE events for about 3 seconds + start = time.time() + event_count = 0 + try: + while time.time() - start < 3.0: + data = json.dumps({"event": event_count, "ts": time.time()}) + self.wfile.write(f"data: {data}\n\n".encode()) + self.wfile.flush() + event_count += 1 + time.sleep(0.01) + except (BrokenPipeError, ConnectionResetError): + pass + else: + self.send_response(404) + self.end_headers() + + def log_message(self, format, *args): + pass # Suppress request logging + + +def _start_http_server(port: int) -> http.server.HTTPServer: + """Start a background HTTP server on the given port.""" + server = http.server.HTTPServer(("0.0.0.0", port), _BenchHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server + + +def run_benchmark() -> dict: + """Run all network IO benchmarks and return results.""" + from smolvm import SmolVM + + results = {} + + print_header("Network IO Benchmark") + + # Start HTTP server on host + host_port = _find_free_port() + server = _start_http_server(host_port) + print(f" Test HTTP server running on port {host_port}") + + try: + ## here we set up a baseline on tyhe host + print_subheader("Host: HTTP request to localhost") + host_http_times = [] + for _ in range(100): + with timer() as t: + proc = subprocess.run( + ["curl", "-s", f"http://127.0.0.1:{host_port}/health"], + capture_output=True, + timeout=5, + ) + if proc.returncode == 0 and proc.stdout: + host_http_times.append(t.elapsed_ms) + else: + print(f" [warn] curl failed (rc={proc.returncode}), skipping sample") + host_http_stats = stats_summary(host_http_times) + print_result("Stats (100 iters)", format_stats(host_http_stats)) + results["host_http"] = host_http_stats + + # SSE streaming + print_subheader("Host: SSE streaming (~3s)") + with timer() as t: + result = subprocess.run( + ["curl", "-s", "-N", f"http://127.0.0.1:{host_port}/stream"], + capture_output=True, + text=True, + timeout=10, + ) + host_sse_lines = result.stdout.count("data:") + if result.returncode == 0 and host_sse_lines > 0: + print_result("Duration", f"{t.elapsed_ms:.0f}ms") + print_result("Events received", str(host_sse_lines)) + results["host_sse_ms"] = t.elapsed_ms + results["host_sse_events"] = host_sse_lines + else: + print(f" [warn] SSE curl failed (rc={result.returncode}, events={host_sse_lines}), skipping") + results["host_sse_ms"] = None + results["host_sse_events"] = host_sse_lines + + # now we bench SmolVM + print_subheader("SmolVM: Starting VM start up") + with SmolVM() as vm: + # warmup with a sanity check + sanity = vm.run("echo smolvm_ready") + assert "smolvm_ready" in sanity.stdout + + curl_check = vm.run("which curl 2>/dev/null || which wget 2>/dev/null || echo 'none'") + has_curl = "curl" in curl_check.stdout + has_wget = "wget" in curl_check.stdout and "none" not in curl_check.stdout + + if not has_curl and not has_wget: + # Install curl in the VM + print(" Installing curl in VM...") + vm.run("apk add --no-cache curl 2>/dev/null || true", timeout=60) + curl_check = vm.run("which curl 2>/dev/null || which wget 2>/dev/null || echo 'none'") + has_curl = "curl" in curl_check.stdout and "none" not in curl_check.stdout + has_wget = "wget" in curl_check.stdout and "none" not in curl_check.stdout + + if not has_curl and not has_wget: + raise RuntimeError( + "VM has no HTTP client (curl or wget) and apk install failed; " + "cannot run network benchmark." + ) + + # Get the VM's gateway IP + gateway_result = vm.run("ip route | grep default | awk '{print $3}'") + gateway_ip = gateway_result.output.strip() + assert gateway_ip, "Failed to get gateway IP from VM" + print(f" VM gateway IP: {gateway_ip}") + # Port exposure + print_subheader("SmolVM: expose_local() overhead") + # Start a simple server inside the VM for port exposure test + vm.run( + "nohup sh -c 'while true; do " + 'echo -e "HTTP/1.1 200 OK\\r\\nContent-Length: 2\\r\\n\\r\\nok"' + " | nc -l -p 8080 2>/dev/null; done' > /dev/null 2>&1 &" + ) + time.sleep(0.5) + + expose_times = [] + for _ in range(3): + t = time_call(lambda: vm.expose_local(8080)) + expose_times.append(t) + # Clean up for next iteration by unexposing + break # ther is only one meaningful measurement since it caches + expose_stats = stats_summary(expose_times) + print_result("Stats", format_stats(expose_stats)) + results["vm_expose_local"] = expose_stats + + # HTTP requests from VM to host + if has_curl: + fetch_cmd = f"curl -s http://{gateway_ip}:{host_port}/health" + else: + fetch_cmd = f"wget -qO- http://{gateway_ip}:{host_port}/health" + + print_subheader("SmolVM: HTTP request (VM -> host)") + vm_http_times = [] + for _ in range(100): + with timer() as t: + r = vm.run(fetch_cmd, timeout=10) + if r.returncode == 0 and r.stdout.strip(): + vm_http_times.append(t.elapsed_ms) + else: + print(f" [warn] VM HTTP request failed (rc={r.returncode}), skipping sample") + vm_http_stats = stats_summary(vm_http_times) + print_result("Stats (100 iters)", format_stats(vm_http_stats)) + results["vm_http"] = vm_http_stats + + # SSE streaming from VM + if has_curl: + print_subheader("SmolVM: SSE streaming (3s, VM -> host)") + with timer() as t: + sse_result = vm.run( + f"curl -s -N --max-time 5 http://{gateway_ip}:{host_port}/stream", + timeout=15, + ) + vm_sse_lines = sse_result.stdout.count("data:") + if sse_result.returncode == 0 and vm_sse_lines > 0: + print_result("Duration", f"{t.elapsed_ms:.0f}ms") + print_result("Events received", str(vm_sse_lines)) + results["vm_sse_ms"] = t.elapsed_ms + results["vm_sse_events"] = vm_sse_lines + else: + print(f" [warn] VM SSE failed (rc={sse_result.returncode}, events={vm_sse_lines}), skipping") + results["vm_sse_ms"] = None + results["vm_sse_events"] = vm_sse_lines + else: + print("\n Skipping SSE test (curl not available in VM)") + results["vm_sse_ms"] = None + results["vm_sse_events"] = None + + # compare + print_subheader("Comparison (p50)") + print_result( + "HTTP roundtrip: VM vs Host", + overhead_str(host_http_stats["p50"], vm_http_stats["p50"]), + ) + if results.get("vm_sse_ms") and results.get("host_sse_ms"): + print_result( + "SSE throughput: VM events vs Host events", + f"{results['vm_sse_events']} vs {results['host_sse_events']}", + ) + + finally: + server.shutdown() + + return results + + +if __name__ == "__main__": + if "benchmarks" in __file__: + sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent)) + run_benchmark() diff --git a/benchmarks/density_ramp.py b/benchmarks/density_ramp.py new file mode 100644 index 0000000..68b91ca --- /dev/null +++ b/benchmarks/density_ramp.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 +""" +SmolVM Density Ramp Test + +Ramps VMs until failure, measuring boot latency and host resource consumption. +Tiers: tiny / small / med. Sustain check runs across ALL VMs concurrently after +peak density is reached, not inline per-VM. + +Usage: + pip install smolvm psutil + python density_ramp.py --tier tiny --max-attempts 500 + python density_ramp.py --tier small --shared-disk --parallel 4 +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +import uuid +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +import psutil + +try: + from smolvm import SSH_BOOT_ARGS, ImageBuilder, SmolVM, VMConfig + from smolvm.utils import ensure_ssh_key # internal, stable across versions +except ImportError: + print("ERROR: pip install smolvm") + sys.exit(1) + +# Hard ceiling from SmolVM's IP pool (172.16.0.2–172.16.0.254) +SMOLVM_MAX_VMS = 253 + + +# --------------------------------------------------------------------------- +# Metrics +# --------------------------------------------------------------------------- + + +@dataclass +class Metrics: + timestamp: float + vms_running: int + host_cpu_pct: float + host_mem_used_gb: float + host_disk_used_gb: float + tap_count: int = 0 + firecracker_rss_mb: float = 0 + fc_socket_count: int = 0 # live Firecracker sockets in socket_dir + sqlite_db_kb: float = 0 + + @classmethod + def snapshot(cls, vms_running: int, socket_dir: Path) -> "Metrics": + ts = time.time() + cpu = psutil.cpu_percent(interval=0.1) + mem = psutil.virtual_memory().used / (1024**3) + # Measure the filesystem that actually holds SmolVM disk clones and images. + # On EC2 this is /mnt/nvme (symlinked from ~/.local/state/smolvm); fall back + # to the home filesystem when the SmolVM state dir doesn't exist yet. + _smolvm_state = Path.home() / ".local" / "state" / "smolvm" + _disk_probe = _smolvm_state if _smolvm_state.exists() else Path.home() + disk = psutil.disk_usage(str(_disk_probe.resolve())).used / (1024**3) + + # Count live Firecracker API sockets (accurate VM count proxy) + fc_sockets = len(list(socket_dir.glob("fc-*.sock"))) + + # TAP device count + try: + ip_out = subprocess.run( + ["ip", "-o", "link", "show"], + capture_output=True, + text=True, + timeout=2, + ).stdout + tap_count = sum(1 for line in ip_out.splitlines() if ": tap" in line) + except Exception: + tap_count = 0 + + # Firecracker aggregate RSS + try: + fc_rss = sum( + p.memory_info().rss / 1024**2 + for p in psutil.process_iter(["name"]) + if "firecracker" in (p.info["name"] or "").lower() + ) + except Exception: + fc_rss = 0.0 + + # SmolVM SQLite state DB size + db_path = Path.home() / ".local" / "state" / "smolvm" / "smolvm.db" + sqlite_kb = db_path.stat().st_size / 1024 if db_path.exists() else 0.0 + + return cls(ts, vms_running, cpu, mem, disk, tap_count, fc_rss, fc_sockets, sqlite_kb) + + +# --------------------------------------------------------------------------- +# VM lifecycle helpers +# --------------------------------------------------------------------------- + +running_vms: list[SmolVM] = [] + + +def cleanup() -> float: + """Delete all running VMs; return elapsed teardown seconds.""" + start = time.time() + for vm in running_vms: + try: + vm.delete() + except Exception as exc: + print(f"WARNING: failed to delete VM {getattr(vm, 'vm_id', '?')}: {exc}") + return time.time() - start + + +def build_image( + tier_config: dict[str, int], + backend: str | None, +) -> tuple[Path, Path, str]: + """Pre-build (or load from cache) the Alpine SSH image. + + Returns (kernel_path, rootfs_path, ssh_key_path). + """ + print("Building/loading image (cached after first run)...") + priv_key, pub_key = ensure_ssh_key() + + disk_size_mib = tier_config["disk_size_mib"] + image_name = "alpine-ssh-key" + if disk_size_mib != 512: + image_name = f"{image_name}-{disk_size_mib}m" + + builder = ImageBuilder() + kernel, rootfs = builder.build_alpine_ssh_key( + pub_key, + name=image_name, + rootfs_size_mb=disk_size_mib, + ) + print(f"Image ready: kernel={kernel.name} rootfs={rootfs.name}") + return kernel, rootfs, str(priv_key) + + +def _read_vm_log(vm: SmolVM, tail_lines: int = 60) -> str: + """Read the Firecracker/QEMU log for a VM (last N lines).""" + try: + log_path = vm.data_dir / f"{vm.vm_id}.log" + if log_path.exists(): + lines = log_path.read_text(errors="replace").splitlines() + return "\n".join(lines[-tail_lines:]) if lines else "(log file is empty)" + return f"(log not found: {log_path})" + except Exception as exc: + return f"(could not read log: {exc})" + + +def boot_one( + seq: int, + kernel: Path, + rootfs: Path, + ssh_key_path: str, + tier_config: dict[str, int], + disk_mode: str, + backend: str | None, + socket_dir: Path, +) -> tuple[int, SmolVM | None, float, dict[str, Any]]: + """Boot a single VM. Returns (seq, vm_or_None, boot_seconds, error_dict).""" + start_t = time.time() + vm: SmolVM | None = None + try: + config = VMConfig( + vm_id=f"vm-{uuid.uuid4().hex[:8]}", + vcpu_count=1, + mem_size_mib=tier_config["mem_size_mib"], + kernel_path=kernel, + rootfs_path=rootfs, + boot_args=SSH_BOOT_ARGS, + disk_mode=disk_mode, + backend=backend, + ) + vm = SmolVM(config, ssh_key_path=ssh_key_path, socket_dir=socket_dir) + vm.start() + vm.wait_for_ssh(timeout=30.0) + return seq, vm, time.time() - start_t, {} + except Exception as e: + fc_log = _read_vm_log(vm) if vm is not None else "(VM object not created)" + err: dict[str, Any] = {"error": str(e), "fc_log": fc_log} + if vm is not None: + try: + vm.delete() + except Exception as cleanup_exc: + err["cleanup_error"] = str(cleanup_exc) + return seq, None, time.time() - start_t, err + + +def check_vm_alive(vm: SmolVM, seq: int) -> tuple[int, bool, str]: + """Run a health check on one VM. Returns (seq, alive, error_msg).""" + try: + result = vm.run("uptime", timeout=10) + return seq, result.exit_code == 0, "" + except Exception as e: + return seq, False, str(e) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="SmolVM Density Ramp") + parser.add_argument("--tier", choices=["tiny", "small", "med"], default="tiny") + parser.add_argument("--max-attempts", type=int, default=500) + parser.add_argument( + "--sustain-sec", + type=int, + default=60, + help="Idle duration (seconds) after peak density before health-checking all VMs", + ) + parser.add_argument( + "--parallel", + type=int, + default=1, + help="Number of VMs to boot concurrently per batch", + ) + parser.add_argument( + "--shared-disk", + action="store_true", + help="Use disk_mode=shared (no per-VM rootfs clone); saves disk, less isolation", + ) + parser.add_argument( + "--backend", + default=None, + help="Backend override: firecracker, qemu, or auto", + ) + parser.add_argument( + "--socket-dir", + type=Path, + default=Path("/tmp"), + help="Firecracker socket directory (must match SmolVM runtime default)", + ) + parser.add_argument("--output", default="density.json") + args = parser.parse_args() + + tiers = { + "tiny": dict(mem_size_mib=128, disk_size_mib=512), + "small": dict(mem_size_mib=512, disk_size_mib=1024), + "med": dict(mem_size_mib=2048, disk_size_mib=4096), + } + tier_config = tiers[args.tier] + disk_mode = "shared" if args.shared_disk else "isolated" + + # Guard: SmolVM IP pool hard ceiling + if args.max_attempts > SMOLVM_MAX_VMS: + print( + f"WARNING: SmolVM IP pool caps at {SMOLVM_MAX_VMS} VMs. " + f"Clamping --max-attempts from {args.max_attempts} to {SMOLVM_MAX_VMS}." + ) + args.max_attempts = SMOLVM_MAX_VMS + + # Warn about disk usage in isolated mode + if disk_mode == "isolated": + est_disk_gb = (args.max_attempts * tier_config["disk_size_mib"]) / 1024 + _smolvm_state = Path.home() / ".local" / "state" / "smolvm" + _disk_probe = _smolvm_state if _smolvm_state.exists() else Path.home() + free_gb = psutil.disk_usage(str(_disk_probe.resolve())).free / (1024**3) + print( + f"INFO: isolated mode — up to {est_disk_gb:.0f} GB disk needed; " + f"{free_gb:.1f} GB free on home filesystem." + ) + if est_disk_gb > free_gb * 0.8: + print( + "WARNING: Risk of disk exhaustion before memory. " + "Consider --shared-disk to eliminate per-VM clones." + ) + + kernel, rootfs, ssh_key_path = build_image(tier_config, args.backend) + + results: list[dict[str, Any]] = [] + _diag_printed = False # Print Firecracker log only for the first failure + + try: + i = 0 + while i < args.max_attempts: + batch_size = min(args.parallel, args.max_attempts - i) + + # Boot one batch (serial or parallel). + # Each result is a 5-tuple (seq, vm, boot_t, err, snap) where snap + # is captured immediately when the boot completes — not post-batch — + # so parallel runs record per-VM resource state accurately. + # Results are sorted by seq before processing to preserve launch order. + if batch_size == 1: + seq_r, vm, boot_t, err = boot_one( + i + 1, + kernel, + rootfs, + ssh_key_path, + tier_config, + disk_mode, + args.backend, + args.socket_dir, + ) + density = len(running_vms) + (1 if vm is not None else 0) + batch_results = [(seq_r, vm, boot_t, err, Metrics.snapshot(density, args.socket_dir))] + else: + with ThreadPoolExecutor(max_workers=batch_size) as ex: + futures = [ + ex.submit( + boot_one, + i + j + 1, + kernel, + rootfs, + ssh_key_path, + tier_config, + disk_mode, + args.backend, + args.socket_dir, + ) + for j in range(batch_size) + ] + batch_results = [] + batch_ok_so_far = 0 + for f in as_completed(futures): + seq_r, vm, boot_t, err = f.result() + if vm is not None: + batch_ok_so_far += 1 + density = len(running_vms) + batch_ok_so_far + snap = Metrics.snapshot(density, args.socket_dir) + batch_results.append((seq_r, vm, boot_t, err, snap)) + + failed_this_batch = 0 + for seq_r, vm, boot_t, err, snap in sorted(batch_results): + i += 1 + + if vm is not None: + running_vms.append(vm) + print( + f"VM {seq_r}: boot={boot_t:.2f}s | " + f"mem={snap.host_mem_used_gb:.2f}GB " + f"disk={snap.host_disk_used_gb:.2f}GB " + f"tap={snap.tap_count} " + f"fc_rss={snap.firecracker_rss_mb:.0f}MB" + ) + results.append( + { + "event": "boot_ok", + "vm_seq": seq_r, + "boot_time_s": round(boot_t, 3), + "density_at_boot": len(running_vms), + **asdict(snap), + } + ) + else: + print(f"FAIL @ VM {seq_r}: {err.get('error', '?')}") + if not _diag_printed and err.get("fc_log"): + _diag_printed = True + print("--- Firecracker log (first failure) ---") + print(err["fc_log"]) + print("--- End Firecracker log ---") + results.append( + { + "event": "boot_fail", + "vm_seq": seq_r, + **err, + **asdict(snap), + } + ) + failed_this_batch += 1 + + if failed_this_batch == batch_size: + print("Entire batch failed — stopping ramp.") + break + + peak = len(running_vms) + print(f"\nPeak density: {peak} VMs (tier={args.tier}, disk_mode={disk_mode})") + + # ------------------------------------------------------------------ + # Sustain phase: idle all VMs together, then health-check concurrently + # ------------------------------------------------------------------ + if peak > 0 and args.sustain_sec > 0: + print(f"Sustaining {peak} VMs for {args.sustain_sec}s...") + time.sleep(args.sustain_sec) + + print(f"Running concurrent health checks across all {peak} VMs...") + workers = min(peak, 64) + with ThreadPoolExecutor(max_workers=workers) as ex: + futures = { + ex.submit(check_vm_alive, vm, idx + 1): idx + for idx, vm in enumerate(running_vms) + } + alive_count = 0 + dead_vms: list[int] = [] + for future in as_completed(futures): + seq, alive, err_msg = future.result() + if alive: + alive_count += 1 + else: + dead_vms.append(seq) + if err_msg: + print(f" VM {seq} dead: {err_msg}") + + snap = Metrics.snapshot(peak, args.socket_dir) + print( + f"Sustain result: {alive_count}/{peak} VMs alive " + f"({len(dead_vms)} dead: {dead_vms[:10]}{'...' if len(dead_vms) > 10 else ''})" + ) + results.append( + { + "event": "sustain_check", + "sustain_sec": args.sustain_sec, + "vms_alive": alive_count, + "vms_dead": dead_vms, + "vms_checked": peak, + **asdict(snap), + } + ) + + finally: + print("Tearing down VMs...") + teardown_s = cleanup() + snap = Metrics.snapshot(0, args.socket_dir) + print( + f"Teardown: {teardown_s:.1f}s | " + f"disk after cleanup={snap.host_disk_used_gb:.2f}GB " + f"(delta verifies rootfs clones released)" + ) + results.append( + { + "event": "teardown", + "teardown_time_s": round(teardown_s, 2), + **asdict(snap), + } + ) + + with open(args.output, "w") as f: + json.dump(results, f, indent=2) + print(f"Results written to {args.output}") diff --git a/benchmarks/docs/README_DENSITY_RAMP.md b/benchmarks/docs/README_DENSITY_RAMP.md new file mode 100644 index 0000000..419c1ce --- /dev/null +++ b/benchmarks/docs/README_DENSITY_RAMP.md @@ -0,0 +1,271 @@ +# Density Ramp Benchmark + +## Overview + +`density_ramp.py` is a stress test that measures SmolVM's maximum capacity by progressively creating and booting VMs until system resource exhaustion. It collects detailed metrics on host performance and individual VM boot times to characterize the limits of the Firecracker-based runtime. + +## Purpose + +This benchmark helps answer key questions about SmolVM: + +- **Maximum concurrent VMs**: How many VMs can run simultaneously on a given system? +- **Performance degradation**: How do boot times and system metrics change as density increases? +- **Resource utilization**: What are the per-VM resource costs (CPU, memory)? +- **Failure modes**: What resource limits cause deployments to fail? + +## Running on EC2 (Recommended) + +The easiest way to run this benchmark is with the provided `run_density_ramp_ec2.sh` script, which handles provisioning, setup, execution, and cleanup automatically. + +### Prerequisites + +- AWS CLI configured (`aws configure` or IAM role) +- An EC2 key pair with the `.pem` file accessible locally +- The key pair name set in the script or via `KEY_NAME` environment variable + +### Usage + +```bash +cd benchmarks/ +./run_density_ramp_ec2.sh [--tier tiny|small|med] [--max-attempts N] \ + [--sustain-sec N] [--parallel N] [--shared-disk] \ + [--output FILE] +``` + +The script will: + +1. Launch a `c5d.metal` instance (96 vCPUs, 192 GB RAM, 2x 900 GB NVMe) +2. Install all dependencies (Python 3.11, Docker, nftables) +3. Mount the NVMe drive and redirect all SmolVM data to it +4. Upload and run the benchmark +5. Download results to `./density_YYYYMMDD_HHMMSS.json` +6. Print a quick summary +7. Terminate the instance and clean up the security group + +### Environment Variable Overrides + +| Variable | Default | Description | +| -------- | ------- | ----------- | +| `KEY_NAME` | `smolvm-benchmark` | EC2 key pair name | +| `KEY_PATH` | `~/.ssh/${KEY_NAME}.pem` | Path to `.pem` file | +| `INSTANCE_TYPE` | `c5d.metal` | EC2 instance type | +| `REGION` | `us-east-2` | AWS region | +| `SECURITY_GROUP` | _(auto-created)_ | Existing security group ID | +| `SUBNET_ID` | _(default VPC)_ | Subnet ID | +| `AMI_ID` | _(latest AL2023)_ | AMI to use | + +### Instance Type Notes + +The default `c5d.metal` provides: + +- 96 vCPUs, 192 GB RAM +- **2x 900 GB NVMe instance store** — the NVMe is required for disk-intensive runs (isolated mode uses up to 126 GB for 253 tiny VMs) + +To override to a different instance type: `INSTANCE_TYPE=c5.metal ./run_density_ramp_ec2.sh` — but you will need to add a large EBS volume separately or use `--shared-disk`. + +--- + +## Running Locally + +### Basic Invocation + +```bash +python density_ramp.py --tier tiny --max-attempts 500 +``` + +### Arguments + +| Argument | Type | Default | Description | +| -------- | ---- | ------- | ----------- | +| `--tier` | `{tiny, small, med}` | `tiny` | VM size configuration tier | +| `--max-attempts` | int | `500` | Maximum number of VMs to launch | +| `--sustain-sec` | int | `60` | Idle duration (seconds) after peak density before health-checking all VMs | +| `--parallel` | int | `1` | Number of VMs to boot concurrently per batch | +| `--shared-disk` | flag | off | Use `disk_mode=shared` (no per-VM rootfs clone); saves disk, less isolation | +| `--backend` | string | `auto` | Backend override: `firecracker`, `qemu`, or `auto` | +| `--socket-dir` | path | `/tmp` | Firecracker socket directory | +| `--output` | string | `density.json` | Output file path for results (JSON format) | + +### VM Tiers + +Each tier defines VM memory and disk size: + +| Tier | Memory | Disk | Max disk (isolated, 253 VMs) | +| ---- | ------ | ---- | ---------------------------- | +| `tiny` | 128 MiB | 512 MiB | ~126 GB | +| `small` | 512 MiB | 1 GiB | ~253 GB | +| `med` | 2048 MiB | 4 GiB | ~1 TB | + +> **Tip**: Use `--shared-disk` to eliminate per-VM disk clones entirely when disk space is limited. + +## How It Works + +1. **Boot Loop**: For each iteration (up to `--max-attempts`): + - Create a new VM with the specified tier configuration + - Boot it and wait for SSH to become ready (up to 30 s) + - Record boot time (includes SSH readiness — this is the true "ready to serve" latency) + - Keep it running with the SSH connection pre-established (append to active VM list) + - Capture host metrics (CPU, memory, disk, Firecracker RSS, TAP count, socket count) + +2. **Sustain Check**: After the ramp completes (or hits `--max-attempts`), wait `--sustain-sec` seconds then run a liveness check (`uptime`) on all surviving VMs. Because SSH connections are pre-established during boot, the sustain check reuses existing paramiko transports rather than opening 64+ new connections concurrently. + +3. **Failure Handling**: If an **entire batch** fails to boot, the ramp stops. A single failure within a larger batch does not halt the run. + +4. **Cleanup**: On exit (success, failure, or interrupt), all VMs are deleted gracefully. + +## Output Format + +Results are written to a JSON file (default: `density.json`). Each entry represents an event: + +```json +[ + { + "event": "boot_ok", + "vm_seq": 1, + "boot_time_s": 0.45, + "density_at_boot": 1, + "timestamp": 1699564800.123, + "vms_running": 1, + "host_cpu_pct": 12.5, + "host_mem_used_gb": 2.3, + "host_disk_used_gb": 2.1, + "tap_count": 1, + "firecracker_rss_mb": 46.9, + "fc_socket_count": 1, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_fail", + "vm_seq": 245, + "error": "Out of memory", + "timestamp": 1699564900.456 + }, + { + "event": "sustain_check", + "sustain_sec": 60, + "vms_checked": 244, + "vms_alive": 244, + "vms_dead": [], + "timestamp": 1699564960.789 + }, + { + "event": "teardown", + "teardown_time_s": 45.2, + "timestamp": 1699565010.0 + } +] +``` + +## Prerequisites (Local) + +- Linux system with KVM support (`/dev/kvm` accessible) +- Python 3.10+ +- SmolVM installed: `pip install smolvm[benchmarks]` +- Docker (for building the Alpine SSH image on first run) +- `nftables` (`nft` command) for VM networking +- Firecracker binary (downloaded automatically via `smolvm.host.HostManager().install_firecracker()`) +- Sufficient disk space (see tier table above; use `--shared-disk` to reduce requirements) + +## Example Runs + +### Tiny VMs (Max Capacity Test) + +```bash +python density_ramp.py --tier tiny --max-attempts 253 --sustain-sec 30 --output tiny_density.json +``` + +Good for finding absolute maximum concurrent VM count with minimal overhead. + +### Tiny VMs, Parallel Boot + +```bash +python density_ramp.py --tier tiny --max-attempts 253 --parallel 10 --sustain-sec 30 +``` + +Boot 10 VMs at a time to reduce ramp-up time. + +### Small VMs (Realistic Mixed Workload) + +```bash +python density_ramp.py --tier small --max-attempts 100 --sustain-sec 60 --output small_density.json +``` + +Realistic for typical application workloads. + +### Medium VMs (Headroom Test) + +```bash +python density_ramp.py --tier med --max-attempts 50 --sustain-sec 60 --output med_density.json +``` + +Tests system behavior under heavier individual VM loads. + +### Shared Disk (Disk-Constrained Systems) + +```bash +python density_ramp.py --tier tiny --max-attempts 253 --shared-disk +``` + +All VMs boot from the same rootfs — no per-VM clone needed. + +## Analysis & Interpretation + +After running the benchmark: + +1. **Look at the final record** to identify the failure point and surrounding metrics +2. **Boot time trend**: Is boot time degrading as VMs accumulate? +3. **Memory usage**: How does host memory consumption scale with VM count? +4. **Firecracker RSS**: Individual Firecracker process overhead +5. **CPU contention**: Does CPU utilization increase as density rises? + +Example analysis (Python): + +```python +import json + +with open('density.json') as f: + data = json.load(f) + +boots = [d for d in data if d.get('event') == 'boot_ok'] +fails = [d for d in data if d.get('event') == 'boot_fail'] +sustain = next((d for d in data if d.get('event') == 'sustain_check'), None) + +print(f"Peak VMs : {len(boots)}") +print(f"Failed boots : {len(fails)}") +if boots: + times = [d['boot_time_s'] for d in boots] + print(f"Avg boot time : {sum(times)/len(times):.2f}s") + print(f"Max boot time : {max(times):.2f}s") + print(f"Peak mem used : {boots[-1]['host_mem_used_gb']:.2f} GB") +if sustain: + print(f"Sustain alive : {sustain['vms_alive']}/{sustain['vms_checked']} VMs") +``` + +## Notes + +- This is a "first draft" benchmark maintained in "skepticism mode" — results should be interpreted carefully and may vary based on host configuration +- `boot_time_s` includes the time to SSH readiness (not just Firecracker start), making it a true end-to-end "ready to serve" latency +- SSH connections are pre-established during the boot loop; the sustain check reuses these connections via persistent paramiko transports +- The liveness check (`uptime`) ensures VMs remain responsive, not just that they booted +- Firecracker RSS includes all Firecracker processes, not per-VM attribution +- The test does not perform VM cleanup between iterations — all VMs remain running until the end +- On macOS, this test will not run (KVM is Linux-only); use the EC2 script or a Linux system + +## Troubleshooting + +| Issue | Cause | Solution | +| ----- | ----- | -------- | +| `smolvm` not found | Package not installed | `pip3.11 install smolvm psutil` | +| `nft: command not found` | nftables not installed | `sudo dnf install -y nftables` | +| `Docker is required` | Docker not installed or not running | `sudo dnf install -y docker && sudo systemctl start docker` | +| `Firecracker binary not found` | Firecracker not downloaded | `python3 -c "from smolvm.host import HostManager; HostManager().install_firecracker()"` | +| Quick failure (VM 1) | KVM permissions | `sudo chmod 666 /dev/kvm` and add user to `kvm` group | +| Disk exhaustion warning | Isolated mode fills disk | Use `--shared-disk` or provision more disk (NVMe on `c5d.metal`) | +| All VMs dead in sustain check: `SSH did not become ready` | 64+ concurrent SSH connections opened simultaneously overwhelm paramiko or nftables DNAT under load | Fixed: SSH is now pre-established during the boot loop | +| Inconsistent results | System load variation | Run with minimal background processes | + +## See Also + +- [SmolVM Documentation](../README.md) +- [EC2 Benchmark Script](../run_density_ramp_ec2.sh) +- [Other Benchmarks](../) diff --git a/benchmarks/docs/v0_density_estimation_benchmark.md b/benchmarks/docs/v0_density_estimation_benchmark.md new file mode 100644 index 0000000..12721a0 --- /dev/null +++ b/benchmarks/docs/v0_density_estimation_benchmark.md @@ -0,0 +1,93 @@ +SmolVM Benchmarking Plan 🧪 + +Repo cloned (workspace/SmolVM-benchmarks). + +Existing benches: command latency, file I/O (ralph JSONL sim), net I/O. + +No density script—custom needed. + +SmolVM: Firecracker/KVM backend, ~572ms boot, low overhead. Firecracker hits 1000s/host in Lambda-scale, but per-instance varies (e.g., OCI BM > VM nested). + +1. Density (Max VMs) +Local Test First (your host equiv Ryzen 7): +```python +from smolvm import SmolVM +vms = [] +i = 0 +try: + while True: + try: + vm = SmolVM(mem_size_mib=128, disk_size_mib=512) # Minimal + vm.start() + vms.append(vm) + i += 1 + print(f"VM {i} OK") + except Exception as e: + print(f"Max: {i}, Fail: {e}") + break +finally: + for vm in vms: + try: + vm.delete() + except Exception: + pass +``` +Ramp until OOM/KVM slots (~host vCPUs). + +Metrics: ps -o rss,vsz %cpu per PID, /sys/kernel/debug/kvm slots. + + +| Instance | vCPU | Mem (GiB) | Est Max VMs | +|----------|------|----------|------------| +| t4g.nano | 2 | 0.5 | ~10-20 | +| t3.micro | 2 | 1 | ~20-50 | +| m5.large | 2 | 8 | ~100-200 | +| c5.xlarge | 4 | 8 | ~150-300 | + +Automate: Boto3 launch, SSM run test, CloudWatch mem/CPU. + +GCE: Similar (e2-micro/standard-2/4), gcloud compute. + + +Skepticism Mode Review: SmolVM Benchmarking Plan 🧐 + +(Embodying Skepticism Mode: Questioning assumptions, probing unverified claims, prioritizing precision/evidence over optimism. "So What?" test: Does this measure real value for AI agent workloads? Sources: memory/project-smolvm.md; repo benches; Firecracker docs.) + +1. Core Assumptions Flagged (Unproven/Fragile) + - Minimal Config Realism: 128MB mem/512MB disk "minimal" VMs—fine for boot tests, but ignores real workloads. AI agents run LLMs/tools: Need 1-4GB+ for pip install torch, API calls, or even curl. Assumption: Toy VMs proxy production? No—measure tiered: tiny (idle), small (Python script), med (LLM inference). + + - Firecracker Equivalence: SmolVM wraps Firecracker. Density = SmolVM overhead + Firecracker base. Plan doesn't isolate: What's raw Firecracker max? (Lambda claims ~4k/node, but EC2 varies 100-1k/instance per blogs/arXiv.) + + - Cloud Nested Virt Penalty: EC2/GCE VMs = nested KVM. Firecracker docs warn 20-50% perf hit vs bare-metal. t3.micro (lite) lacks dedicated KVM accel. Verify baseline on metal/EC2 bare-metal (i3.metal). + + - OOM/KVM as Sole Limiters: Ignores host limits: /proc/sys/kernel/threads-max, netif (TAP FD exhaustion >500 VMs), IRQ saturation, cgroup v2 memsw. + +1. Missing Metrics (What You're Not Measuring—Critical Gaps) + + +| Category | Plan Has | Missing (Why Matters) | Suggested Add | +|----------|----------|----------------------|---------------| +| Density | Max concurrent boot | Sustained uptime/crash rate (e.g., 1hr idle/active); Ramp-up time (boot 100 VMs serially? Parallel?); Teardown speed (leakage?); Per-VM breakdown (firecracker PID RSS/VSZ/CPU; guest mem via free; KVM slots via ls /dev/kvm-vm*; cat /sys/kernel/debug/kvm/max_vms). | htop --filter=firecracker; kvm stats; aggregate psutil script. | +| Performance | Boot/cmd/file/net I/O | CPU-bound (sysbench/prime95 guest); Multi-VM contention (contention @50%/90% density); Tail latency (p99 cmd exec @100 VMs); Power/$$ (cloud: CPU credits/cost/hour at density); Net saturation (iperf3 multi-stream @ density). | fio (rand/seq RW); stress-ng CPU; perf top (host KVM irq). | +| I/O Depth | File (JSONL sim) | Block device (virtio vs NVMe passthru); Multi-VM I/O storm (all VMs randwrite sim); Host cache effects (fio --direct=1); Guest fs (ext4 vs tmpfs). | fio --name=randrw --bs=4k --iodepth=32; blktrace host. | +| Comparisons | Vs Firecracker/KVM | Raw baselines (no SmolVM wrapper); Containers (Docker/podman for context); gVisor (kernel sim). | Fork Firecracker repo; docker stats. | +| Stability/Safety | N/A | Attack surface (escape tests?); Resource leaks (post-kill mem?); Host impact (temp rise, net flood). | falco/auditd guest-host; valgrind firecracker? | + + +1. Risks/Edge Cases Overlooked + - Fork Drift: tensor-ninja vs CelestoAI—bench both? Latest commits? + - Host Tuning: No /etc/sysctl (e.g., net.core.somaxconn, vm.swappiness=0)? Firecracker needs modprobe vhost_vsock. + - Reproducibility: Single-run? No stats (mean/std/p95)? No CI (GitHub Actions matrix)? + - Scale Reality: 300 VMs/c5.xlarge optimistic—Lambda tunes kernel specially. EC2 spot kills mid-test. + - "So What?" Fail: Max density cool, but throughput/$$ per VM matters more for agents (e.g., VMs/hour * success rate). + +1. Revised Plan Skeleton (Precision Fixes) +Phases: + 1. Local Baseline: Tune host → Run repo benches → Density ramp (3 tiers) → Stats table. + 2. Raw Firecracker: Build/run equivalent VMs. + 3. Cloud: Bare-metal → Nested → Automate (terraform + boto3/gcloud). + +- Output: CSV (inst|density|max_mem|boot_p95|io_mb/s|cost/hr) + graphs. +- Cost Guardrail: t3.micro spot ~$0.002/hr; cap 10 instances. + +- Verdict: Plan 6/10—good start (local/EC2 targets), but toy-focused, no contention/realism, weak baselines. Fix gaps → actionable. \ No newline at end of file diff --git a/benchmarks/docs/v0_review_density_ramp_report.md b/benchmarks/docs/v0_review_density_ramp_report.md new file mode 100644 index 0000000..92b0037 --- /dev/null +++ b/benchmarks/docs/v0_review_density_ramp_report.md @@ -0,0 +1,283 @@ +# SmolVM Density Ramp — Test Harness Report + +## Overview + +`density_ramp.py` is a stress test that answers one core question: **how many SmolVM +microVMs can a given host run simultaneously before something breaks?** It ramps up VMs +one-by-one (or in parallel batches), captures host resource state at each step, sustains +the peak density for a configurable idle window, then tears down cleanly and verifies +resource release. + +--- + +## What It Tests + +### 1. Boot Density Ramp + +The test progressively launches microVMs with a fixed configuration tier until one of +three terminal conditions is reached: + +- **An entire boot batch fails** — the host can no longer start new VMs. +- **`--max-attempts` is reached** — the configured ceiling (capped at 253, the hard + limit of SmolVM's IP pool) is hit before failure. +- **The user interrupts** — `finally` ensures cleanup regardless. + +Each boot is timed end-to-end: from `SmolVM(config)` construction through `vm.start()` +and `vm.wait_for_ssh(timeout=30.0)`. `boot_time_s` therefore represents the true +"ready to serve" latency — the time until SSH is responsive, not just until the +Firecracker process starts. The sequence number at which each VM was booted +(`density_at_boot`) is recorded alongside the boot time, so a plot of `boot_time_s` vs +`density_at_boot` shows whether boot latency degrades under load. + +The three tiers define the per-VM footprint: + +| Tier | Guest RAM | Rootfs Size | Realistic use case | +|-------|-----------|-------------|----------------------------------| +| tiny | 128 MiB | 512 MiB | Idle sandbox, trivial scripts | +| small | 512 MiB | 1 GiB | Python tooling, pip installs | +| med | 2 GiB | 4 GiB | Light inference, larger installs | + +### 2. Sustain Phase + +After the ramp exhausts, the full set of running VMs is held for `--sustain-sec` +(default 60 s). Then `vm.run("uptime", timeout=10)` is dispatched concurrently across +all VMs (up to 64 threads). A VM is alive if and only if `exit_code == 0` is returned +within the timeout. Silently crashed or OOM-killed VMs will surface here. + +### 3. Teardown and Resource Release + +`cleanup()` calls `vm.stop()` on every running VM. The post-cleanup `Metrics.snapshot()` +is written to the output, so the disk usage delta (pre- vs post-cleanup) can be compared +against the expected per-VM clone size. A mismatch indicates leaked rootfs clones. + +--- + +## Metrics Collected + +At every boot event and at the sustain/teardown checkpoints, the following are recorded: + +| Field | Source | What it tells you | +|----------------------|---------------------------------------|------------------------------------------------------------------------| +| `boot_time_s` | Wall clock around `vm.start()` | Per-VM boot latency; should be flat; degradation indicates I/O or scheduler saturation | +| `density_at_boot` | `len(running_vms)` at success | X-axis for the boot latency curve | +| `host_mem_used_gb` | `psutil.virtual_memory().used` | Actual host RAM consumed; slope = overhead per VM | +| `host_disk_used_gb` | `psutil.disk_usage(home).used` | Tracks rootfs clone growth in isolated mode; should recover after teardown | +| `host_cpu_pct` | `psutil.cpu_percent(interval=0.1)` | Transient boot-time CPU; sustained high values indicate host saturation | +| `tap_count` | `ip -o link show \| grep tap` | Each Firecracker VM owns one TAP device; should equal `vms_running` | +| `fc_socket_count` | `glob("/fc-*.sock")` | Live FC API sockets; cross-checks `vms_running`. Dir: `--socket-dir` | +| `firecracker_rss_mb` | Sum of RSS for all `firecracker` PIDs | Hypervisor overhead per VM independent of guest memory allocation | +| `sqlite_db_kb` | `stat(~/.local/state/smolvm/smolvm.db)` | State DB grows with each VM record; large size risks lock contention | + +The output JSON groups events by type: `boot_ok`, `boot_fail`, `sustain_check`, +`teardown`. + +--- + +## Why These Tests Are Relevant + +SmolVM's target workload is **on-demand AI agent sandboxes**: an agent task arrives, +a fresh microVM boots, the task executes, the VM is discarded. The density ramp +directly exercises the two properties that determine whether this model is economical +at scale. + +### Capacity per dollar + +The primary output — peak VM count — directly determines cost efficiency. If a +`c5.2xlarge` ($0.34/hr) sustains 60 simultaneous tiny-tier VMs, and each agent task +takes 2 minutes on average, the instance can process 1,800 tasks/hr at ~$0.0002/task. +Without the density number, any cost estimate is guesswork. + +### What the metrics reveal about SmolVM's architecture + +**`host_mem_used_gb` slope** measures true per-VM overhead. The naive expectation is +`128 MiB (guest) + ~20 MiB (Firecracker process)` = ~150 MiB per tiny-tier VM, yielding +~50 VMs on 8 GiB. If the measured slope is steeper — say 200 MiB/VM — it indicates +additional overhead from TAP buffers, kernel per-process structures, or smolvm state. +This is actionable: it means the guest memory setting is not the binding constraint and +simply reducing `mem_size_mib` further will not improve density as much as expected. + +**`boot_time_s` vs `density_at_boot` curve** reveals I/O and scheduler behaviour. +Firecracker VMs boot by cloning a rootfs image (in isolated mode) and writing it to +disk. If boot time at VM #80 is 3× that at VM #1, the host NVMe or the kernel's +per-process file copy is saturating — even though there is still free RAM. This matters +for burst workloads: a sharp latency cliff means you cannot spin up 50 VMs quickly, +only sequentially. + +**`tap_count` and `fc_socket_count`** cross-check `vms_running`. If `fc_socket_count < +vms_running`, Firecracker processes have exited silently — the VM count reported by +SmolVM is stale. This would be a serious reliability bug. Conversely, if `tap_count > +vms_running` after teardown, TAP devices are leaking and will accumulate across test +runs until a reboot or manual cleanup. + +**`host_disk_used_gb` delta at teardown** validates that `disk_mode=isolated` cleans up +properly. In isolated mode SmolVM clones the rootfs image per VM to +`~/.local/state/smolvm/disks/{vm_id}.ext4`. The post-teardown disk reading should +return to approximately the pre-test baseline. A persistent increase indicates leaked +clone files, which will silently exhaust disk on long-running hosts. + +**`sustain_check` alive ratio** distinguishes between VMs that boot successfully but +later crash (OOM-killed by the host kernel, or KVM slot exhausted) versus VMs that were +never viable. A 100% boot success rate followed by a 70% sustain rate is a qualitatively +different failure mode than a ramp that stops at 70%. + +**`sqlite_db_kb`** is a canary for state management overhead. SmolVM tracks all VM +state in a single SQLite file with exclusive transactions. In serial operation this is +fine; under `--parallel` load the DB is accessed from multiple threads simultaneously. +A DB size that grows without bound (i.e., records are not cleaned up on stop/delete) +would eventually degrade all write operations. + +--- + +## How to Administer the Tests + +### Prerequisites + +- Linux host with KVM enabled: `ls /dev/kvm` must succeed. +- Running as a user with permission to create TAP devices and iptables rules, or as root. +- `pip install smolvm psutil` +- Enough disk for the chosen tier (see the pre-run warning printed by the script). + +### Recommended test sequence + +**Step 1 — Baseline, serial, shared-disk** + +Establishes the memory-bound ceiling without disk being a confounding variable. + +```bash +python density_ramp.py \ + --tier tiny \ + --shared-disk \ + --sustain-sec 60 \ + --output results_tiny_shared.json +``` + +**Step 2 — Isolated mode to measure disk behaviour** + +Run the same tier with `--max-attempts` set conservatively (e.g. 50) to avoid filling +the disk, then inspect the teardown delta. + +```bash +python density_ramp.py \ + --tier tiny \ + --max-attempts 50 \ + --sustain-sec 60 \ + --output results_tiny_isolated.json +``` + +**Step 3 — Parallel boot to stress concurrent allocation** + +This exercises the TAP device setup, iptables rule insertion, and SQLite state writes +under concurrency. Use a moderate parallelism — start at 4, not 32. + +```bash +python density_ramp.py \ + --tier tiny \ + --shared-disk \ + --parallel 4 \ + --sustain-sec 30 \ + --output results_tiny_parallel4.json +``` + +**Step 4 — Realistic tier** + +Repeat steps 1–3 with `--tier small` to model real agent workloads (Python installs, +API calls). The tiny tier is primarily useful for measuring hypervisor overhead in +isolation. + +### Reading the output + +The JSON output is a flat array of event records. Quick analysis: + +```python +import json, statistics + +records = json.load(open("results_tiny_shared.json")) +boots = [r for r in records if r["event"] == "boot_ok"] + +print("Peak density:", max(r["density_at_boot"] for r in boots)) +print("Boot time p50:", statistics.median(r["boot_time_s"] for r in boots)) +print("Boot time p95:", statistics.quantiles([r["boot_time_s"] for r in boots], n=20)[18]) + +mem_per_vm = (boots[-1]["host_mem_used_gb"] - boots[0]["host_mem_used_gb"]) / len(boots) +print(f"Measured mem overhead per VM: {mem_per_vm*1024:.0f} MiB") + +sustain = next(r for r in records if r["event"] == "sustain_check") +print(f"Sustain: {sustain['vms_alive']}/{sustain['vms_checked']} alive") + +teardown = next(r for r in records if r["event"] == "teardown") +print(f"Teardown time: {teardown['teardown_time_s']}s") +``` + +--- + +## Pitfalls and Things to Watch For + +### Disk exhaustion will abort the test before memory is exhausted + +In `isolated` mode (the default), SmolVM clones the rootfs image for every VM. A +standard cloud instance root volume of 8 GiB is exhausted by ~16 tiny-tier VMs or ~8 +small-tier VMs. The script prints a warning and estimated disk requirement before +starting. Use `--shared-disk` when you want to measure memory-bound density, not +disk-bound density. Use isolated mode when you want to validate that the cleanup path +actually releases clones. + +### The 253-VM ceiling is architectural, not a performance limit + +SmolVM allocates guest IPs from a fixed `/24` subnet (`172.16.0.2–172.16.0.254`), TAP +devices named `tap2–tap254`, and SSH host ports from `2200–2999`. The script clamps +`--max-attempts` to 253. Hitting this ceiling before running out of RAM is a valid and +informative result — it means the host could support more VMs if the networking layer +were extended, not that the hardware is exhausted. + +### Nested KVM on cloud instances inflates all timings + +EC2, GCE, and Azure VM instances run Firecracker under a second layer of KVM +virtualisation. This imposes a 20–50% overhead on boot times and CPU-bound operations +compared to bare metal. Results from nested environments are internally consistent and +useful for relative comparisons (tier vs tier, isolated vs shared) but should not be +treated as representative of what SmolVM can do on a bare-metal host. For absolute +density numbers, use a bare-metal instance or a local machine with direct KVM access. + +### ARM instances require matching kernel and rootfs images + +`t4g` (Graviton) and other ARM instances use `aarch64`. The Alpine SSH image +pre-built by `ImageBuilder` is compiled for the host architecture. If you copy results +files between x86 and ARM hosts, the images are not interchangeable. The script will +fail at image build time with a meaningful error if there is a mismatch, but be aware +that `--tier` timings are not comparable across architectures. + +### `--parallel` with high concurrency exposes SQLite contention + +SmolVM serialises all VM state writes through a single SQLite database with exclusive +transactions. With `--parallel 1` this is invisible. At `--parallel 8` or higher, boot +threads queue behind the DB lock and measured boot times will include wait time that is +not present in production single-VM usage. Watch `sqlite_db_kb` growing suspiciously +large or boot times clustering around a suspiciously regular interval as signs of +contention. If you see this, the finding is real and worth reporting: it means concurrent +agent spawning is slower than sequential spawning even when RAM is not the constraint. + +### Leaked resources from interrupted runs + +If the script is killed with `SIGKILL` (e.g. `kill -9`, OOM killer), the `finally` +block does not run. Firecracker processes continue running, TAP devices remain, and +rootfs clone files remain on disk. Before re-running after an interrupted test: + +```bash +smolvm cleanup --all # stops all tracked VMs and removes their state +ip link show | grep tap # verify no stale TAP devices +ls /tmp/fc-*.sock # verify no stale Firecracker sockets +``` + +### `--sustain-sec 0` skips the health check + +Setting `--sustain-sec 0` disables the sustain phase entirely. This is useful for +a pure throughput measurement but means you get no signal on whether VMs remain alive +under density pressure. Do not use `--sustain-sec 0` when trying to characterise +reliability — only when characterising raw boot throughput. + +### Single-run results have significant variance + +Boot times and peak density can vary noticeably between runs due to kernel page cache +state, TAP device setup timing, and SSH handshake jitter. Run each configuration at +least three times and report the median peak and p95 boot time rather than a single +result. diff --git a/benchmarks/helpers.py b/benchmarks/helpers.py new file mode 100644 index 0000000..42a0e8b --- /dev/null +++ b/benchmarks/helpers.py @@ -0,0 +1,273 @@ +"""Shared utilities for SmolVM benchmarks. + +Provides timing helpers, statistical functions, sandbox-exec wrappers, +and table formatting for benchmark reporting. +""" + +from __future__ import annotations + +import math +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable +from contextlib import contextmanager +from typing import Any + +# --------------------------------------------------------------------------- +# Timing helpers +# --------------------------------------------------------------------------- + + +@contextmanager +def timer(): + """Context manager that records elapsed wall-clock time in milliseconds. + + Usage:: + + with timer() as t: + do_something() + print(t.elapsed_ms) + """ + t = _TimerResult() + t.start = time.perf_counter() + try: + yield t + finally: + t.end = time.perf_counter() + t.elapsed_ms = (t.end - t.start) * 1000.0 + + +class _TimerResult: + start: float = 0.0 + end: float = 0.0 + elapsed_ms: float = 0.0 + + +def time_call(fn: Callable[[], Any]) -> float: + """Run *fn* and return the elapsed time in milliseconds.""" + start = time.perf_counter() + fn() + return (time.perf_counter() - start) * 1000.0 + + +def time_call_n(fn: Callable[[], Any], n: int, *, warmup: int = 0) -> list[float]: + """Run *fn* ``warmup + n`` times and return the last *n* timings in ms.""" + for _ in range(warmup): + fn() + return [time_call(fn) for _ in range(n)] + + +# --------------------------------------------------------------------------- +# Statistics +# --------------------------------------------------------------------------- + + +def percentile(data: list[float], p: float) -> float: + """Return the *p*-th percentile (0-100) of *data*.""" + if not data: + return 0.0 + sorted_data = sorted(data) + k = (p / 100.0) * (len(sorted_data) - 1) + f = math.floor(k) + c = math.ceil(k) + if f == c: + return sorted_data[int(k)] + d0 = sorted_data[f] * (c - k) + d1 = sorted_data[c] * (k - f) + return d0 + d1 + + +def stats_summary(timings: list[float]) -> dict[str, float]: + """Return min, p50, p95, p99, max, mean, and stdev for *timings* (ms).""" + if not timings: + return dict.fromkeys(("min", "p50", "p95", "p99", "max", "mean", "stdev"), 0.0) + n = len(timings) + mean = sum(timings) / n + variance = sum((x - mean) ** 2 for x in timings) / n if n > 1 else 0.0 + return { + "min": min(timings), + "p50": percentile(timings, 50), + "p95": percentile(timings, 95), + "p99": percentile(timings, 99), + "max": max(timings), + "mean": mean, + "stdev": math.sqrt(variance), + } + + +def format_stats(stats: dict[str, float], unit: str = "ms") -> str: + """Format a stats dict as a single readable line.""" + return ( + f"min={stats['min']:.1f}{unit} " + f"p50={stats['p50']:.1f}{unit} " + f"p95={stats['p95']:.1f}{unit} " + f"p99={stats['p99']:.1f}{unit} " + f"max={stats['max']:.1f}{unit} " + f"mean={stats['mean']:.1f}{unit} " + f"stdev={stats['stdev']:.1f}{unit}" + ) + + +# --------------------------------------------------------------------------- +# sandbox-exec wrapper (macOS only) +# --------------------------------------------------------------------------- + +# Minimal Seatbelt profile that allows most operations except network. +# Mirrors the basic approach from nightshift's sandbox.ts. +_SEATBELT_PROFILE = """\ +(version 1) +(allow default) +""" + + +def is_sandbox_exec_available() -> bool: + """Return True if sandbox-exec is available (macOS only).""" + if platform.system() != "Darwin": + return False + return shutil.which("sandbox-exec") is not None + + +def run_sandboxed(command: str, timeout: int = 30) -> subprocess.CompletedProcess[str]: + """Execute *command* via ``sandbox-exec`` with a permissive profile. + + Falls back to plain ``subprocess.run`` if sandbox-exec is unavailable. + """ + if is_sandbox_exec_available(): + # Write a temporary profile + with tempfile.NamedTemporaryFile( + mode="w", suffix=".sb", delete=False + ) as fp: + fp.write(_SEATBELT_PROFILE) + profile_path = fp.name + try: + return subprocess.run( + ["sandbox-exec", "-f", profile_path, "bash", "-c", command], + capture_output=True, + text=True, + timeout=timeout, + ) + finally: + os.unlink(profile_path) + else: + return subprocess.run( + ["bash", "-c", command], + capture_output=True, + text=True, + timeout=timeout, + ) + + +def time_sandboxed(command: str, timeout: int = 30) -> float: + """Run a command via sandbox-exec and return elapsed time in ms.""" + start = time.perf_counter() + run_sandboxed(command, timeout=timeout) + return (time.perf_counter() - start) * 1000.0 + + +def time_host(command: str, timeout: int = 30) -> float: + """Run a command directly on the host and return elapsed time in ms.""" + start = time.perf_counter() + subprocess.run( + ["bash", "-c", command], + capture_output=True, + text=True, + timeout=timeout, + ) + return (time.perf_counter() - start) * 1000.0 + + +# --------------------------------------------------------------------------- +# Table formatting +# --------------------------------------------------------------------------- + + +def format_table( + headers: list[str], + rows: list[list[str]], + *, + align: str | None = None, +) -> str: + """Format a simple ASCII table. + + Args: + headers: Column header strings. + rows: List of rows, each a list of cell strings. + align: Optional alignment string ('l'=left, 'r'=right) per column. + Defaults to left-align for the first column, right for the rest. + """ + if not rows: + return "" + + col_count = len(headers) + if align is None: + align = "l" + "r" * (col_count - 1) + + widths = [len(h) for h in headers] + for row in rows: + for i, cell in enumerate(row): + widths[i] = max(widths[i], len(cell)) + + def _fmt_row(cells: list[str]) -> str: + parts = [] + for i, cell in enumerate(cells): + w = widths[i] + if i < len(align) and align[i] == "r": + parts.append(cell.rjust(w)) + else: + parts.append(cell.ljust(w)) + return "| " + " | ".join(parts) + " |" + + sep = "|" + "|".join("-" * (w + 2) for w in widths) + "|" + lines = [_fmt_row(headers), sep] + lines.extend(_fmt_row(row) for row in rows) + return "\n".join(lines) + + +def overhead_str(base_ms: float, test_ms: float) -> str: + """Return a human-readable overhead string like '2.5x' or '+150ms'.""" + if base_ms <= 0: + return "N/A" + ratio = test_ms / base_ms + diff = test_ms - base_ms + return f"{ratio:.1f}x (+{diff:.0f}ms)" + + +# --------------------------------------------------------------------------- +# Platform info +# --------------------------------------------------------------------------- + + +def platform_info() -> str: + """Return a one-line platform description.""" + system = platform.system() + machine = platform.machine() + version = platform.release() + py_version = sys.version.split()[0] + return f"{system} {machine} (kernel {version}), Python {py_version}" + + +# --------------------------------------------------------------------------- +# Printing helpers +# --------------------------------------------------------------------------- + + +def print_header(title: str) -> None: + """Print a section header.""" + print(f"\n{'=' * 60}") + print(f" {title}") + print(f"{'=' * 60}") + + +def print_subheader(title: str) -> None: + """Print a sub-section header.""" + print(f"\n--- {title} ---") + + +def print_result(label: str, value: str) -> None: + """Print a labelled result.""" + print(f" {label}: {value}") diff --git a/benchmarks/results/README.md b/benchmarks/results/README.md new file mode 100644 index 0000000..304457f --- /dev/null +++ b/benchmarks/results/README.md @@ -0,0 +1,194 @@ +# Density Ramp Results + +This directory contains JSON output files from `density_ramp.py` benchmark runs. Files are named `density_YYYYMMDD_HHMMSS.json` and timestamped at the start of the run. + +## File Structure + +Each file is a JSON array of event objects written sequentially during the run. Records appear in chronological order: one `boot_ok` or `boot_fail` per VM launch attempt, followed by a single `sustain_check`, and finally a `teardown` record. + +### Event Types + +#### `boot_ok` — successful VM boot + +```json +{ + "event": "boot_ok", + "vm_seq": 42, + "boot_time_s": 2.961, + "density_at_boot": 42, + "timestamp": 1773030151.45, + "vms_running": 42, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 4.82, + "host_disk_used_gb": 2.05, + "tap_count": 42, + "firecracker_rss_mb": 1968.5, + "fc_socket_count": 42, + "sqlite_db_kb": 120.0 +} +``` + +| Field | Description | +| ----- | ----------- | +| `vm_seq` | Sequential index of this VM (1-based launch order) | +| `boot_time_s` | Seconds from VM creation to SSH readiness — the true end-to-end "ready to serve" latency | +| `density_at_boot` | Number of concurrently running VMs at the moment this VM became ready (equals `vms_running` for sequential runs) | +| `timestamp` | Unix epoch at the moment boot was confirmed | +| `vms_running` | Total VMs alive on the host when metrics were sampled | +| `host_cpu_pct` | Host CPU utilization (%) at time of sampling | +| `host_mem_used_gb` | Host physical memory in use (GB) — includes OS, SmolVM process, and all Firecracker processes | +| `host_disk_used_gb` | Disk space consumed under the SmolVM data directory (GB) — grows with each per-VM rootfs clone in isolated mode | +| `tap_count` | Number of active tap network interfaces — should equal `vms_running` | +| `firecracker_rss_mb` | Aggregate RSS (resident set size) of all Firecracker processes (MB) | +| `fc_socket_count` | Number of Firecracker API sockets present — should equal `vms_running` | +| `sqlite_db_kb` | Size of the SmolVM SQLite state database (KB) — grows as VM records accumulate | + +#### `boot_fail` — VM failed to boot or reach SSH + +```json +{ + "event": "boot_fail", + "vm_seq": 254, + "error": "SSH did not become ready within 30s", + "timestamp": 1773031000.0, + "vms_running": 253, + "host_cpu_pct": 95.0, + "host_mem_used_gb": 191.5, + ... +} +``` + +Same host-metrics fields as `boot_ok` plus `error` (string describing the failure). The ramp stops after the first failure. + +#### `sustain_check` — liveness check after peak density + +```json +{ + "event": "sustain_check", + "sustain_sec": 60, + "vms_alive": 253, + "vms_dead": [], + "vms_checked": 253, + "timestamp": 1773030998.82, + "vms_running": 253, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.54, + "host_disk_used_gb": 2.07, + "tap_count": 253, + "firecracker_rss_mb": 11949.96, + "fc_socket_count": 253, + "sqlite_db_kb": 308.0 +} +``` + +| Field | Description | +| ----- | ----------- | +| `sustain_sec` | Seconds the benchmark idled at peak density before running this check | +| `vms_alive` | VMs that responded to the liveness command (`uptime`) | +| `vms_dead` | List of VM identifiers that failed the liveness check | +| `vms_checked` | Total VMs checked | + +Host metrics fields are the same as `boot_ok` and reflect system state at the time of the check — useful for spotting memory growth or CPU drift after a long soak. + +#### `teardown` — post-cleanup snapshot + +```json +{ + "event": "teardown", + "teardown_time_s": 127.57, + "timestamp": 1773031126.57, + "vms_running": 0, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.08, + "host_disk_used_gb": 2.07, + "tap_count": 253, + "firecracker_rss_mb": 0.0, + "fc_socket_count": 0, + "sqlite_db_kb": 308.0 +} +``` + +| Field | Description | +| ----- | ----------- | +| `teardown_time_s` | Seconds to stop all VMs gracefully | +| `host_disk_used_gb` | Disk after cleanup — delta vs. peak verifies per-VM rootfs clones were released | +| `tap_count` | Note: tap interfaces may linger after teardown until the OS reclaims them | +| `firecracker_rss_mb` | Should be 0.0 — confirms all Firecracker processes exited | + +--- + +## How to Interpret Results + +### Did the benchmark succeed? + +A successful run reaches `--max-attempts` VMs without any `boot_fail`, and the `sustain_check` shows `vms_alive == vms_checked`. A failed run ends with a `boot_fail` record; the `error` field and surrounding host metrics indicate the limiting resource. + +### What was the peak capacity? + +Count `boot_ok` records, or read `vms_running` from the `sustain_check`. + +### What was the per-VM memory cost? + +Compare `host_mem_used_gb` between early and late `boot_ok` records: + +```python +per_vm_mem_mb = (boots[-1]["host_mem_used_gb"] - boots[0]["host_mem_used_gb"]) \ + / (len(boots) - 1) * 1024 +``` + +For tiny VMs (128 MiB configured), expect roughly 48–55 MB actual RSS — Firecracker's overhead is well below the guest's configured memory. + +### What was the Firecracker process overhead per VM? + +```python +rss_per_vm_mb = boots[-1]["firecracker_rss_mb"] / len(boots) +``` + +This is the average RSS of a single Firecracker process. It should be roughly constant across density levels on a healthy run. + +### Was boot time stable under load? + +Plot `boot_time_s` against `vm_seq`. A flat line means boot latency doesn't degrade with density. An upward trend indicates CPU or I/O contention affecting startup. + +### What did teardown cost? + +`teardown_time_s` divided by `vms_checked` gives average stop-time per VM. Check that `host_disk_used_gb` in the teardown record is close to the value in the first `boot_ok` — a large residual indicates rootfs clones were not fully cleaned up. + +--- + +## Quick Analysis Script + +```python +import json + +with open("density_20260309_001743.json") as f: + data = json.load(f) + +boots = [d for d in data if d["event"] == "boot_ok"] +fails = [d for d in data if d["event"] == "boot_fail"] +sustain = next((d for d in data if d["event"] == "sustain_check"), None) +teardown = next((d for d in data if d["event"] == "teardown"), None) + +print(f"Peak VMs : {len(boots)}") +print(f"Failed boots : {len(fails)}") +if boots: + times = [d["boot_time_s"] for d in boots] + print(f"Avg boot time : {sum(times)/len(times):.2f}s") + print(f"Max boot time : {max(times):.2f}s") + print(f"Peak mem used : {boots[-1]['host_mem_used_gb']:.2f} GB") + print(f"RSS per VM : {boots[-1]['firecracker_rss_mb'] / len(boots):.1f} MB") +else: + if fails: + print(f"First failure : {fails[0].get('error', '?')}") +if sustain: + print(f"Sustain result : {sustain['vms_alive']}/{sustain['vms_checked']} alive") +if teardown: + print(f"Teardown time : {teardown['teardown_time_s']:.1f}s") + print(f"Disk after : {teardown['host_disk_used_gb']:.3f} GB") +``` + +--- + +## File Naming + +Files generated by `run_density_ramp_ec2.sh` are named `density_YYYYMMDD_HHMMSS.json` where the timestamp is the local time on the machine that launched the EC2 script (not the EC2 instance). Files generated by running `density_ramp.py` directly use whatever `--output` argument was passed (default: `density.json`). diff --git a/benchmarks/results/density_20260309_001743.json b/benchmarks/results/density_20260309_001743.json new file mode 100644 index 0000000..d31c445 --- /dev/null +++ b/benchmarks/results/density_20260309_001743.json @@ -0,0 +1,3826 @@ +[ + { + "event": "boot_ok", + "vm_seq": 1, + "boot_time_s": 3.476, + "density_at_boot": 1, + "timestamp": 1773030151.4499836, + "vms_running": 1, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 2.686443328857422, + "host_disk_used_gb": 2.0351104736328125, + "tap_count": 1, + "firecracker_rss_mb": 46.92578125, + "fc_socket_count": 1, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_ok", + "vm_seq": 2, + "boot_time_s": 3.017, + "density_at_boot": 2, + "timestamp": 1773030154.6498094, + "vms_running": 2, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 2.7395286560058594, + "host_disk_used_gb": 2.0351104736328125, + "tap_count": 2, + "firecracker_rss_mb": 93.59375, + "fc_socket_count": 2, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_ok", + "vm_seq": 3, + "boot_time_s": 2.115, + "density_at_boot": 3, + "timestamp": 1773030156.9101038, + "vms_running": 3, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 2.7911529541015625, + "host_disk_used_gb": 2.0351715087890625, + "tap_count": 3, + "firecracker_rss_mb": 141.04296875, + "fc_socket_count": 3, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_ok", + "vm_seq": 4, + "boot_time_s": 2.257, + "density_at_boot": 4, + "timestamp": 1773030159.3113978, + "vms_running": 4, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 2.849151611328125, + "host_disk_used_gb": 2.0351715087890625, + "tap_count": 4, + "firecracker_rss_mb": 188.74609375, + "fc_socket_count": 4, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_ok", + "vm_seq": 5, + "boot_time_s": 3.03, + "density_at_boot": 5, + "timestamp": 1773030162.488072, + "vms_running": 5, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 2.9040756225585938, + "host_disk_used_gb": 2.0352325439453125, + "tap_count": 5, + "firecracker_rss_mb": 235.671875, + "fc_socket_count": 5, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_ok", + "vm_seq": 6, + "boot_time_s": 2.979, + "density_at_boot": 6, + "timestamp": 1773030165.6119142, + "vms_running": 6, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 2.958332061767578, + "host_disk_used_gb": 2.0352325439453125, + "tap_count": 6, + "firecracker_rss_mb": 282.59765625, + "fc_socket_count": 6, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_ok", + "vm_seq": 7, + "boot_time_s": 2.93, + "density_at_boot": 7, + "timestamp": 1773030168.6872692, + "vms_running": 7, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 3.009899139404297, + "host_disk_used_gb": 2.0352935791015625, + "tap_count": 7, + "firecracker_rss_mb": 329.01171875, + "fc_socket_count": 7, + "sqlite_db_kb": 60.0 + }, + { + "event": "boot_ok", + "vm_seq": 8, + "boot_time_s": 3.018, + "density_at_boot": 8, + "timestamp": 1773030171.8506584, + "vms_running": 8, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.0595245361328125, + "host_disk_used_gb": 2.0352935791015625, + "tap_count": 8, + "firecracker_rss_mb": 375.9375, + "fc_socket_count": 8, + "sqlite_db_kb": 60.0 + }, + { + "event": "boot_ok", + "vm_seq": 9, + "boot_time_s": 2.972, + "density_at_boot": 9, + "timestamp": 1773030174.9678557, + "vms_running": 9, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.119335174560547, + "host_disk_used_gb": 2.0353546142578125, + "tap_count": 9, + "firecracker_rss_mb": 422.8671875, + "fc_socket_count": 9, + "sqlite_db_kb": 60.0 + }, + { + "event": "boot_ok", + "vm_seq": 10, + "boot_time_s": 2.933, + "density_at_boot": 10, + "timestamp": 1773030178.0462797, + "vms_running": 10, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.1758041381835938, + "host_disk_used_gb": 2.0353546142578125, + "tap_count": 10, + "firecracker_rss_mb": 469.80078125, + "fc_socket_count": 10, + "sqlite_db_kb": 60.0 + }, + { + "event": "boot_ok", + "vm_seq": 11, + "boot_time_s": 3.016, + "density_at_boot": 11, + "timestamp": 1773030181.2088463, + "vms_running": 11, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.226818084716797, + "host_disk_used_gb": 2.0354156494140625, + "tap_count": 11, + "firecracker_rss_mb": 516.21484375, + "fc_socket_count": 11, + "sqlite_db_kb": 60.0 + }, + { + "event": "boot_ok", + "vm_seq": 12, + "boot_time_s": 2.973, + "density_at_boot": 12, + "timestamp": 1773030184.3311768, + "vms_running": 12, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.2716827392578125, + "host_disk_used_gb": 2.0354156494140625, + "tap_count": 12, + "firecracker_rss_mb": 563.66015625, + "fc_socket_count": 12, + "sqlite_db_kb": 60.0 + }, + { + "event": "boot_ok", + "vm_seq": 13, + "boot_time_s": 3.052, + "density_at_boot": 13, + "timestamp": 1773030187.5301793, + "vms_running": 13, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 3.3329925537109375, + "host_disk_used_gb": 2.0359039306640625, + "tap_count": 13, + "firecracker_rss_mb": 610.328125, + "fc_socket_count": 13, + "sqlite_db_kb": 64.0 + }, + { + "event": "boot_ok", + "vm_seq": 14, + "boot_time_s": 2.929, + "density_at_boot": 14, + "timestamp": 1773030190.6060467, + "vms_running": 14, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 3.3888587951660156, + "host_disk_used_gb": 2.0359039306640625, + "tap_count": 14, + "firecracker_rss_mb": 658.03125, + "fc_socket_count": 14, + "sqlite_db_kb": 64.0 + }, + { + "event": "boot_ok", + "vm_seq": 15, + "boot_time_s": 3.019, + "density_at_boot": 15, + "timestamp": 1773030193.7698424, + "vms_running": 15, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.4387664794921875, + "host_disk_used_gb": 2.0359039306640625, + "tap_count": 15, + "firecracker_rss_mb": 705.73046875, + "fc_socket_count": 15, + "sqlite_db_kb": 64.0 + }, + { + "event": "boot_ok", + "vm_seq": 16, + "boot_time_s": 2.932, + "density_at_boot": 16, + "timestamp": 1773030196.8478305, + "vms_running": 16, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.4878005981445312, + "host_disk_used_gb": 2.0359039306640625, + "tap_count": 16, + "firecracker_rss_mb": 752.6640625, + "fc_socket_count": 16, + "sqlite_db_kb": 64.0 + }, + { + "event": "boot_ok", + "vm_seq": 17, + "boot_time_s": 3.018, + "density_at_boot": 17, + "timestamp": 1773030200.0110958, + "vms_running": 17, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.545806884765625, + "host_disk_used_gb": 2.0359039306640625, + "tap_count": 17, + "firecracker_rss_mb": 799.3359375, + "fc_socket_count": 17, + "sqlite_db_kb": 64.0 + }, + { + "event": "boot_ok", + "vm_seq": 18, + "boot_time_s": 2.974, + "density_at_boot": 18, + "timestamp": 1773030203.13211, + "vms_running": 18, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.587860107421875, + "host_disk_used_gb": 2.0359039306640625, + "tap_count": 18, + "firecracker_rss_mb": 846.5234375, + "fc_socket_count": 18, + "sqlite_db_kb": 64.0 + }, + { + "event": "boot_ok", + "vm_seq": 19, + "boot_time_s": 2.056, + "density_at_boot": 19, + "timestamp": 1773030205.3356764, + "vms_running": 19, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.651355743408203, + "host_disk_used_gb": 2.0359039306640625, + "tap_count": 19, + "firecracker_rss_mb": 893.96484375, + "fc_socket_count": 19, + "sqlite_db_kb": 68.0 + }, + { + "event": "boot_ok", + "vm_seq": 20, + "boot_time_s": 2.974, + "density_at_boot": 20, + "timestamp": 1773030208.4570715, + "vms_running": 20, + "host_cpu_pct": 0.6, + "host_mem_used_gb": 3.710704803466797, + "host_disk_used_gb": 2.0359039306640625, + "tap_count": 20, + "firecracker_rss_mb": 941.6640625, + "fc_socket_count": 20, + "sqlite_db_kb": 68.0 + }, + { + "event": "boot_ok", + "vm_seq": 21, + "boot_time_s": 3.009, + "density_at_boot": 21, + "timestamp": 1773030211.6118374, + "vms_running": 21, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.7615509033203125, + "host_disk_used_gb": 2.0359039306640625, + "tap_count": 21, + "firecracker_rss_mb": 989.109375, + "fc_socket_count": 21, + "sqlite_db_kb": 68.0 + }, + { + "event": "boot_ok", + "vm_seq": 22, + "boot_time_s": 2.069, + "density_at_boot": 22, + "timestamp": 1773030213.8284674, + "vms_running": 22, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 3.8169288635253906, + "host_disk_used_gb": 2.0359268188476562, + "tap_count": 22, + "firecracker_rss_mb": 1036.80078125, + "fc_socket_count": 22, + "sqlite_db_kb": 68.0 + }, + { + "event": "boot_ok", + "vm_seq": 23, + "boot_time_s": 3.078, + "density_at_boot": 23, + "timestamp": 1773030217.0511816, + "vms_running": 23, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.870014190673828, + "host_disk_used_gb": 2.0358848571777344, + "tap_count": 23, + "firecracker_rss_mb": 1083.46875, + "fc_socket_count": 23, + "sqlite_db_kb": 68.0 + }, + { + "event": "boot_ok", + "vm_seq": 24, + "boot_time_s": 2.93, + "density_at_boot": 24, + "timestamp": 1773030220.1277292, + "vms_running": 24, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 3.9227828979492188, + "host_disk_used_gb": 2.0358848571777344, + "tap_count": 24, + "firecracker_rss_mb": 1130.65625, + "fc_socket_count": 24, + "sqlite_db_kb": 68.0 + }, + { + "event": "boot_ok", + "vm_seq": 25, + "boot_time_s": 2.235, + "density_at_boot": 25, + "timestamp": 1773030222.5083845, + "vms_running": 25, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.972370147705078, + "host_disk_used_gb": 2.0358848571777344, + "tap_count": 25, + "firecracker_rss_mb": 1177.3203125, + "fc_socket_count": 25, + "sqlite_db_kb": 72.0 + }, + { + "event": "boot_ok", + "vm_seq": 26, + "boot_time_s": 2.99, + "density_at_boot": 26, + "timestamp": 1773030225.6468253, + "vms_running": 26, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 4.025539398193359, + "host_disk_used_gb": 2.0358848571777344, + "tap_count": 26, + "firecracker_rss_mb": 1224.765625, + "fc_socket_count": 26, + "sqlite_db_kb": 72.0 + }, + { + "event": "boot_ok", + "vm_seq": 27, + "boot_time_s": 3.097, + "density_at_boot": 27, + "timestamp": 1773030228.8900096, + "vms_running": 27, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.0738067626953125, + "host_disk_used_gb": 2.0358657836914062, + "tap_count": 27, + "firecracker_rss_mb": 1271.17578125, + "fc_socket_count": 27, + "sqlite_db_kb": 72.0 + }, + { + "event": "boot_ok", + "vm_seq": 28, + "boot_time_s": 3.141, + "density_at_boot": 28, + "timestamp": 1773030232.1799617, + "vms_running": 28, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 4.126220703125, + "host_disk_used_gb": 2.0358657836914062, + "tap_count": 28, + "firecracker_rss_mb": 1318.1015625, + "fc_socket_count": 28, + "sqlite_db_kb": 72.0 + }, + { + "event": "boot_ok", + "vm_seq": 29, + "boot_time_s": 3.047, + "density_at_boot": 29, + "timestamp": 1773030235.3704205, + "vms_running": 29, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.184684753417969, + "host_disk_used_gb": 2.0368423461914062, + "tap_count": 29, + "firecracker_rss_mb": 1365.546875, + "fc_socket_count": 29, + "sqlite_db_kb": 72.0 + }, + { + "event": "boot_ok", + "vm_seq": 30, + "boot_time_s": 2.931, + "density_at_boot": 30, + "timestamp": 1773030238.448102, + "vms_running": 30, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 4.227451324462891, + "host_disk_used_gb": 2.0368423461914062, + "tap_count": 30, + "firecracker_rss_mb": 1413.25, + "fc_socket_count": 30, + "sqlite_db_kb": 72.0 + }, + { + "event": "boot_ok", + "vm_seq": 31, + "boot_time_s": 3.055, + "density_at_boot": 31, + "timestamp": 1773030241.648236, + "vms_running": 31, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.283435821533203, + "host_disk_used_gb": 2.036846160888672, + "tap_count": 31, + "firecracker_rss_mb": 1460.17578125, + "fc_socket_count": 31, + "sqlite_db_kb": 76.0 + }, + { + "event": "boot_ok", + "vm_seq": 32, + "boot_time_s": 2.971, + "density_at_boot": 32, + "timestamp": 1773030244.7659106, + "vms_running": 32, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.325809478759766, + "host_disk_used_gb": 2.036846160888672, + "tap_count": 32, + "firecracker_rss_mb": 1507.87890625, + "fc_socket_count": 32, + "sqlite_db_kb": 76.0 + }, + { + "event": "boot_ok", + "vm_seq": 33, + "boot_time_s": 3.019, + "density_at_boot": 33, + "timestamp": 1773030247.929167, + "vms_running": 33, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.375846862792969, + "host_disk_used_gb": 2.036846160888672, + "tap_count": 33, + "firecracker_rss_mb": 1554.8046875, + "fc_socket_count": 33, + "sqlite_db_kb": 76.0 + }, + { + "event": "boot_ok", + "vm_seq": 34, + "boot_time_s": 3.011, + "density_at_boot": 34, + "timestamp": 1773030251.087229, + "vms_running": 34, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 4.419551849365234, + "host_disk_used_gb": 2.036846160888672, + "tap_count": 34, + "firecracker_rss_mb": 1601.7265625, + "fc_socket_count": 34, + "sqlite_db_kb": 76.0 + }, + { + "event": "boot_ok", + "vm_seq": 35, + "boot_time_s": 3.015, + "density_at_boot": 35, + "timestamp": 1773030254.2502804, + "vms_running": 35, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.469638824462891, + "host_disk_used_gb": 2.036846160888672, + "tap_count": 35, + "firecracker_rss_mb": 1648.140625, + "fc_socket_count": 35, + "sqlite_db_kb": 76.0 + }, + { + "event": "boot_ok", + "vm_seq": 36, + "boot_time_s": 2.972, + "density_at_boot": 36, + "timestamp": 1773030257.3696184, + "vms_running": 36, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.522861480712891, + "host_disk_used_gb": 2.036846160888672, + "tap_count": 36, + "firecracker_rss_mb": 1694.80859375, + "fc_socket_count": 36, + "sqlite_db_kb": 76.0 + }, + { + "event": "boot_ok", + "vm_seq": 37, + "boot_time_s": 2.931, + "density_at_boot": 37, + "timestamp": 1773030260.4480922, + "vms_running": 37, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.574893951416016, + "host_disk_used_gb": 2.036846160888672, + "tap_count": 37, + "firecracker_rss_mb": 1742.0, + "fc_socket_count": 37, + "sqlite_db_kb": 80.0 + }, + { + "event": "boot_ok", + "vm_seq": 38, + "boot_time_s": 3.013, + "density_at_boot": 38, + "timestamp": 1773030263.6088576, + "vms_running": 38, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.616920471191406, + "host_disk_used_gb": 2.036846160888672, + "tap_count": 38, + "firecracker_rss_mb": 1788.671875, + "fc_socket_count": 38, + "sqlite_db_kb": 80.0 + }, + { + "event": "boot_ok", + "vm_seq": 39, + "boot_time_s": 2.112, + "density_at_boot": 39, + "timestamp": 1773030265.8692183, + "vms_running": 39, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.669769287109375, + "host_disk_used_gb": 2.044658660888672, + "tap_count": 39, + "firecracker_rss_mb": 1835.08203125, + "fc_socket_count": 39, + "sqlite_db_kb": 80.0 + }, + { + "event": "boot_ok", + "vm_seq": 40, + "boot_time_s": 3.031, + "density_at_boot": 40, + "timestamp": 1773030269.0514448, + "vms_running": 40, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.712181091308594, + "host_disk_used_gb": 2.044658660888672, + "tap_count": 40, + "firecracker_rss_mb": 1882.52734375, + "fc_socket_count": 40, + "sqlite_db_kb": 80.0 + }, + { + "event": "boot_ok", + "vm_seq": 41, + "boot_time_s": 2.11, + "density_at_boot": 41, + "timestamp": 1773030271.3120732, + "vms_running": 41, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.763164520263672, + "host_disk_used_gb": 2.044658660888672, + "tap_count": 41, + "firecracker_rss_mb": 1929.45703125, + "fc_socket_count": 41, + "sqlite_db_kb": 80.0 + }, + { + "event": "boot_ok", + "vm_seq": 42, + "boot_time_s": 2.054, + "density_at_boot": 42, + "timestamp": 1773030273.51748, + "vms_running": 42, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 4.8074798583984375, + "host_disk_used_gb": 2.044658660888672, + "tap_count": 42, + "firecracker_rss_mb": 1976.90234375, + "fc_socket_count": 42, + "sqlite_db_kb": 80.0 + }, + { + "event": "boot_ok", + "vm_seq": 43, + "boot_time_s": 3.143, + "density_at_boot": 43, + "timestamp": 1773030276.8104312, + "vms_running": 43, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.8608245849609375, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 43, + "firecracker_rss_mb": 2023.3125, + "fc_socket_count": 43, + "sqlite_db_kb": 84.0 + }, + { + "event": "boot_ok", + "vm_seq": 44, + "boot_time_s": 3.047, + "density_at_boot": 44, + "timestamp": 1773030280.0094585, + "vms_running": 44, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 4.910457611083984, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 44, + "firecracker_rss_mb": 2070.75390625, + "fc_socket_count": 44, + "sqlite_db_kb": 84.0 + }, + { + "event": "boot_ok", + "vm_seq": 45, + "boot_time_s": 3.133, + "density_at_boot": 45, + "timestamp": 1773030283.289295, + "vms_running": 45, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 4.958141326904297, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 45, + "firecracker_rss_mb": 2117.42578125, + "fc_socket_count": 45, + "sqlite_db_kb": 84.0 + }, + { + "event": "boot_ok", + "vm_seq": 46, + "boot_time_s": 3.168, + "density_at_boot": 46, + "timestamp": 1773030286.6071842, + "vms_running": 46, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 5.001457214355469, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 46, + "firecracker_rss_mb": 2164.609375, + "fc_socket_count": 46, + "sqlite_db_kb": 84.0 + }, + { + "event": "boot_ok", + "vm_seq": 47, + "boot_time_s": 3.012, + "density_at_boot": 47, + "timestamp": 1773030289.7695093, + "vms_running": 47, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.0518341064453125, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 47, + "firecracker_rss_mb": 2211.53515625, + "fc_socket_count": 47, + "sqlite_db_kb": 84.0 + }, + { + "event": "boot_ok", + "vm_seq": 48, + "boot_time_s": 3.049, + "density_at_boot": 48, + "timestamp": 1773030292.96944, + "vms_running": 48, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.094669342041016, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 48, + "firecracker_rss_mb": 2258.20703125, + "fc_socket_count": 48, + "sqlite_db_kb": 84.0 + }, + { + "event": "boot_ok", + "vm_seq": 49, + "boot_time_s": 3.051, + "density_at_boot": 49, + "timestamp": 1773030296.1715212, + "vms_running": 49, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 5.140800476074219, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 49, + "firecracker_rss_mb": 2305.13671875, + "fc_socket_count": 49, + "sqlite_db_kb": 88.0 + }, + { + "event": "boot_ok", + "vm_seq": 50, + "boot_time_s": 3.05, + "density_at_boot": 50, + "timestamp": 1773030299.372849, + "vms_running": 50, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 5.188892364501953, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 50, + "firecracker_rss_mb": 2353.09375, + "fc_socket_count": 50, + "sqlite_db_kb": 88.0 + }, + { + "event": "boot_ok", + "vm_seq": 51, + "boot_time_s": 3.046, + "density_at_boot": 51, + "timestamp": 1773030302.5714993, + "vms_running": 51, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 5.234947204589844, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 51, + "firecracker_rss_mb": 2400.27734375, + "fc_socket_count": 51, + "sqlite_db_kb": 88.0 + }, + { + "event": "boot_ok", + "vm_seq": 52, + "boot_time_s": 2.969, + "density_at_boot": 52, + "timestamp": 1773030305.691596, + "vms_running": 52, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.279937744140625, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 52, + "firecracker_rss_mb": 2447.203125, + "fc_socket_count": 52, + "sqlite_db_kb": 88.0 + }, + { + "event": "boot_ok", + "vm_seq": 53, + "boot_time_s": 2.106, + "density_at_boot": 53, + "timestamp": 1773030307.9493294, + "vms_running": 53, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 5.32672119140625, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 53, + "firecracker_rss_mb": 2493.875, + "fc_socket_count": 53, + "sqlite_db_kb": 88.0 + }, + { + "event": "boot_ok", + "vm_seq": 54, + "boot_time_s": 2.986, + "density_at_boot": 54, + "timestamp": 1773030311.0870476, + "vms_running": 54, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 5.3830413818359375, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 54, + "firecracker_rss_mb": 2541.57421875, + "fc_socket_count": 54, + "sqlite_db_kb": 88.0 + }, + { + "event": "boot_ok", + "vm_seq": 55, + "boot_time_s": 3.013, + "density_at_boot": 55, + "timestamp": 1773030314.2510355, + "vms_running": 55, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 5.427848815917969, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 55, + "firecracker_rss_mb": 2588.5, + "fc_socket_count": 55, + "sqlite_db_kb": 92.0 + }, + { + "event": "boot_ok", + "vm_seq": 56, + "boot_time_s": 3.048, + "density_at_boot": 56, + "timestamp": 1773030317.4508066, + "vms_running": 56, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.4666900634765625, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 56, + "firecracker_rss_mb": 2635.68359375, + "fc_socket_count": 56, + "sqlite_db_kb": 92.0 + }, + { + "event": "boot_ok", + "vm_seq": 57, + "boot_time_s": 2.968, + "density_at_boot": 57, + "timestamp": 1773030320.5722733, + "vms_running": 57, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 5.512035369873047, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 57, + "firecracker_rss_mb": 2683.125, + "fc_socket_count": 57, + "sqlite_db_kb": 92.0 + }, + { + "event": "boot_ok", + "vm_seq": 58, + "boot_time_s": 3.045, + "density_at_boot": 58, + "timestamp": 1773030323.770579, + "vms_running": 58, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.568767547607422, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 58, + "firecracker_rss_mb": 2730.5625, + "fc_socket_count": 58, + "sqlite_db_kb": 92.0 + }, + { + "event": "boot_ok", + "vm_seq": 59, + "boot_time_s": 3.046, + "density_at_boot": 59, + "timestamp": 1773030326.9697428, + "vms_running": 59, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 5.609596252441406, + "host_disk_used_gb": 2.0446548461914062, + "tap_count": 59, + "firecracker_rss_mb": 2777.234375, + "fc_socket_count": 59, + "sqlite_db_kb": 92.0 + }, + { + "event": "boot_ok", + "vm_seq": 60, + "boot_time_s": 3.128, + "density_at_boot": 60, + "timestamp": 1773030330.2500215, + "vms_running": 60, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.655048370361328, + "host_disk_used_gb": 2.0466079711914062, + "tap_count": 60, + "firecracker_rss_mb": 2824.1640625, + "fc_socket_count": 60, + "sqlite_db_kb": 92.0 + }, + { + "event": "boot_ok", + "vm_seq": 61, + "boot_time_s": 3.045, + "density_at_boot": 61, + "timestamp": 1773030333.4491975, + "vms_running": 61, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.709114074707031, + "host_disk_used_gb": 2.0466079711914062, + "tap_count": 61, + "firecracker_rss_mb": 2871.08984375, + "fc_socket_count": 61, + "sqlite_db_kb": 104.0 + }, + { + "event": "boot_ok", + "vm_seq": 62, + "boot_time_s": 2.133, + "density_at_boot": 62, + "timestamp": 1773030335.737336, + "vms_running": 62, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 5.753002166748047, + "host_disk_used_gb": 2.0466079711914062, + "tap_count": 62, + "firecracker_rss_mb": 2917.5, + "fc_socket_count": 62, + "sqlite_db_kb": 104.0 + }, + { + "event": "boot_ok", + "vm_seq": 63, + "boot_time_s": 3.082, + "density_at_boot": 63, + "timestamp": 1773030338.9716988, + "vms_running": 63, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.805698394775391, + "host_disk_used_gb": 2.0466079711914062, + "tap_count": 63, + "firecracker_rss_mb": 2964.6875, + "fc_socket_count": 63, + "sqlite_db_kb": 104.0 + }, + { + "event": "boot_ok", + "vm_seq": 64, + "boot_time_s": 3.122, + "density_at_boot": 64, + "timestamp": 1773030342.249478, + "vms_running": 64, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.852519989013672, + "host_disk_used_gb": 2.046611785888672, + "tap_count": 64, + "firecracker_rss_mb": 3011.6171875, + "fc_socket_count": 64, + "sqlite_db_kb": 104.0 + }, + { + "event": "boot_ok", + "vm_seq": 65, + "boot_time_s": 3.046, + "density_at_boot": 65, + "timestamp": 1773030345.4513075, + "vms_running": 65, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.89459228515625, + "host_disk_used_gb": 2.0466346740722656, + "tap_count": 65, + "firecracker_rss_mb": 3058.5390625, + "fc_socket_count": 65, + "sqlite_db_kb": 104.0 + }, + { + "event": "boot_ok", + "vm_seq": 66, + "boot_time_s": 2.161, + "density_at_boot": 66, + "timestamp": 1773030347.7697043, + "vms_running": 66, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.9460906982421875, + "host_disk_used_gb": 2.0466346740722656, + "tap_count": 66, + "firecracker_rss_mb": 3105.7265625, + "fc_socket_count": 66, + "sqlite_db_kb": 104.0 + }, + { + "event": "boot_ok", + "vm_seq": 67, + "boot_time_s": 3.206, + "density_at_boot": 67, + "timestamp": 1773030351.1298063, + "vms_running": 67, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 5.993007659912109, + "host_disk_used_gb": 2.0466346740722656, + "tap_count": 67, + "firecracker_rss_mb": 3152.9140625, + "fc_socket_count": 67, + "sqlite_db_kb": 108.0 + }, + { + "event": "boot_ok", + "vm_seq": 68, + "boot_time_s": 2.084, + "density_at_boot": 68, + "timestamp": 1773030353.371058, + "vms_running": 68, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.039974212646484, + "host_disk_used_gb": 2.0466346740722656, + "tap_count": 68, + "firecracker_rss_mb": 3200.1015625, + "fc_socket_count": 68, + "sqlite_db_kb": 108.0 + }, + { + "event": "boot_ok", + "vm_seq": 69, + "boot_time_s": 2.048, + "density_at_boot": 69, + "timestamp": 1773030355.576164, + "vms_running": 69, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.083442687988281, + "host_disk_used_gb": 2.0466346740722656, + "tap_count": 69, + "firecracker_rss_mb": 3246.76953125, + "fc_socket_count": 69, + "sqlite_db_kb": 108.0 + }, + { + "event": "boot_ok", + "vm_seq": 70, + "boot_time_s": 3.08, + "density_at_boot": 70, + "timestamp": 1773030358.8102856, + "vms_running": 70, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.127902984619141, + "host_disk_used_gb": 2.0466156005859375, + "tap_count": 70, + "firecracker_rss_mb": 3293.703125, + "fc_socket_count": 70, + "sqlite_db_kb": 108.0 + }, + { + "event": "boot_ok", + "vm_seq": 71, + "boot_time_s": 3.043, + "density_at_boot": 71, + "timestamp": 1773030362.0093975, + "vms_running": 71, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.174079895019531, + "host_disk_used_gb": 2.0466156005859375, + "tap_count": 71, + "firecracker_rss_mb": 3340.890625, + "fc_socket_count": 71, + "sqlite_db_kb": 108.0 + }, + { + "event": "boot_ok", + "vm_seq": 72, + "boot_time_s": 2.106, + "density_at_boot": 72, + "timestamp": 1773030364.2694788, + "vms_running": 72, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 6.221134185791016, + "host_disk_used_gb": 2.0466156005859375, + "tap_count": 72, + "firecracker_rss_mb": 3388.078125, + "fc_socket_count": 72, + "sqlite_db_kb": 108.0 + }, + { + "event": "boot_ok", + "vm_seq": 73, + "boot_time_s": 3.027, + "density_at_boot": 73, + "timestamp": 1773030367.4498286, + "vms_running": 73, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.259243011474609, + "host_disk_used_gb": 2.0466156005859375, + "tap_count": 73, + "firecracker_rss_mb": 3434.75, + "fc_socket_count": 73, + "sqlite_db_kb": 112.0 + }, + { + "event": "boot_ok", + "vm_seq": 74, + "boot_time_s": 3.003, + "density_at_boot": 74, + "timestamp": 1773030370.60756, + "vms_running": 74, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 6.309257507324219, + "host_disk_used_gb": 2.0466156005859375, + "tap_count": 74, + "firecracker_rss_mb": 3482.70703125, + "fc_socket_count": 74, + "sqlite_db_kb": 112.0 + }, + { + "event": "boot_ok", + "vm_seq": 75, + "boot_time_s": 3.091, + "density_at_boot": 75, + "timestamp": 1773030373.8524258, + "vms_running": 75, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.356410980224609, + "host_disk_used_gb": 2.0466156005859375, + "tap_count": 75, + "firecracker_rss_mb": 3529.6328125, + "fc_socket_count": 75, + "sqlite_db_kb": 120.0 + }, + { + "event": "boot_ok", + "vm_seq": 76, + "boot_time_s": 3.042, + "density_at_boot": 76, + "timestamp": 1773030377.0512536, + "vms_running": 76, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.398593902587891, + "host_disk_used_gb": 2.0466156005859375, + "tap_count": 76, + "firecracker_rss_mb": 3576.30078125, + "fc_socket_count": 76, + "sqlite_db_kb": 120.0 + }, + { + "event": "boot_ok", + "vm_seq": 77, + "boot_time_s": 3.042, + "density_at_boot": 77, + "timestamp": 1773030380.2496924, + "vms_running": 77, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.446514129638672, + "host_disk_used_gb": 2.0466156005859375, + "tap_count": 77, + "firecracker_rss_mb": 3623.48828125, + "fc_socket_count": 77, + "sqlite_db_kb": 120.0 + }, + { + "event": "boot_ok", + "vm_seq": 78, + "boot_time_s": 3.045, + "density_at_boot": 78, + "timestamp": 1773030383.449578, + "vms_running": 78, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.494434356689453, + "host_disk_used_gb": 2.0466156005859375, + "tap_count": 78, + "firecracker_rss_mb": 3670.67578125, + "fc_socket_count": 78, + "sqlite_db_kb": 120.0 + }, + { + "event": "boot_ok", + "vm_seq": 79, + "boot_time_s": 3.044, + "density_at_boot": 79, + "timestamp": 1773030386.6508563, + "vms_running": 79, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.528438568115234, + "host_disk_used_gb": 2.0466156005859375, + "tap_count": 79, + "firecracker_rss_mb": 3717.6015625, + "fc_socket_count": 79, + "sqlite_db_kb": 124.0 + }, + { + "event": "boot_ok", + "vm_seq": 80, + "boot_time_s": 3.043, + "density_at_boot": 80, + "timestamp": 1773030389.8509417, + "vms_running": 80, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.581905364990234, + "host_disk_used_gb": 2.0466156005859375, + "tap_count": 80, + "firecracker_rss_mb": 3764.01171875, + "fc_socket_count": 80, + "sqlite_db_kb": 124.0 + }, + { + "event": "boot_ok", + "vm_seq": 81, + "boot_time_s": 2.997, + "density_at_boot": 81, + "timestamp": 1773030393.0078764, + "vms_running": 81, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.63055419921875, + "host_disk_used_gb": 2.0466384887695312, + "tap_count": 81, + "firecracker_rss_mb": 3810.6796875, + "fc_socket_count": 81, + "sqlite_db_kb": 124.0 + }, + { + "event": "boot_ok", + "vm_seq": 82, + "boot_time_s": 2.126, + "density_at_boot": 82, + "timestamp": 1773030395.2907133, + "vms_running": 82, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.671733856201172, + "host_disk_used_gb": 2.0466384887695312, + "tap_count": 82, + "firecracker_rss_mb": 3857.08984375, + "fc_socket_count": 82, + "sqlite_db_kb": 124.0 + }, + { + "event": "boot_ok", + "vm_seq": 83, + "boot_time_s": 3.122, + "density_at_boot": 83, + "timestamp": 1773030398.5717978, + "vms_running": 83, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.717853546142578, + "host_disk_used_gb": 2.0466384887695312, + "tap_count": 83, + "firecracker_rss_mb": 3903.5, + "fc_socket_count": 83, + "sqlite_db_kb": 124.0 + }, + { + "event": "boot_ok", + "vm_seq": 84, + "boot_time_s": 3.042, + "density_at_boot": 84, + "timestamp": 1773030401.7711225, + "vms_running": 84, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.760826110839844, + "host_disk_used_gb": 2.0466384887695312, + "tap_count": 84, + "firecracker_rss_mb": 3950.68359375, + "fc_socket_count": 84, + "sqlite_db_kb": 124.0 + }, + { + "event": "boot_ok", + "vm_seq": 85, + "boot_time_s": 3.043, + "density_at_boot": 85, + "timestamp": 1773030404.9720445, + "vms_running": 85, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.809974670410156, + "host_disk_used_gb": 2.046642303466797, + "tap_count": 85, + "firecracker_rss_mb": 3997.35546875, + "fc_socket_count": 85, + "sqlite_db_kb": 128.0 + }, + { + "event": "boot_ok", + "vm_seq": 86, + "boot_time_s": 2.079, + "density_at_boot": 86, + "timestamp": 1773030407.2089183, + "vms_running": 86, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.854888916015625, + "host_disk_used_gb": 2.046642303466797, + "tap_count": 86, + "firecracker_rss_mb": 4043.765625, + "fc_socket_count": 86, + "sqlite_db_kb": 128.0 + }, + { + "event": "boot_ok", + "vm_seq": 87, + "boot_time_s": 3.121, + "density_at_boot": 87, + "timestamp": 1773030410.490603, + "vms_running": 87, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.8976898193359375, + "host_disk_used_gb": 2.046642303466797, + "tap_count": 87, + "firecracker_rss_mb": 4090.6953125, + "fc_socket_count": 87, + "sqlite_db_kb": 128.0 + }, + { + "event": "boot_ok", + "vm_seq": 88, + "boot_time_s": 3.204, + "density_at_boot": 88, + "timestamp": 1773030413.8518996, + "vms_running": 88, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.943305969238281, + "host_disk_used_gb": 2.046642303466797, + "tap_count": 88, + "firecracker_rss_mb": 4137.62109375, + "fc_socket_count": 88, + "sqlite_db_kb": 128.0 + }, + { + "event": "boot_ok", + "vm_seq": 89, + "boot_time_s": 3.115, + "density_at_boot": 89, + "timestamp": 1773030417.1286414, + "vms_running": 89, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.985912322998047, + "host_disk_used_gb": 2.046642303466797, + "tap_count": 89, + "firecracker_rss_mb": 4184.29296875, + "fc_socket_count": 89, + "sqlite_db_kb": 128.0 + }, + { + "event": "boot_ok", + "vm_seq": 90, + "boot_time_s": 3.122, + "density_at_boot": 90, + "timestamp": 1773030420.4100835, + "vms_running": 90, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 7.030841827392578, + "host_disk_used_gb": 2.046642303466797, + "tap_count": 90, + "firecracker_rss_mb": 4230.70703125, + "fc_socket_count": 90, + "sqlite_db_kb": 128.0 + }, + { + "event": "boot_ok", + "vm_seq": 91, + "boot_time_s": 3.043, + "density_at_boot": 91, + "timestamp": 1773030423.6119673, + "vms_running": 91, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 7.0834808349609375, + "host_disk_used_gb": 2.0466232299804688, + "tap_count": 91, + "firecracker_rss_mb": 4277.375, + "fc_socket_count": 91, + "sqlite_db_kb": 132.0 + }, + { + "event": "boot_ok", + "vm_seq": 92, + "boot_time_s": 2.263, + "density_at_boot": 92, + "timestamp": 1773030426.0323734, + "vms_running": 92, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.128719329833984, + "host_disk_used_gb": 2.0466232299804688, + "tap_count": 92, + "firecracker_rss_mb": 4324.81640625, + "fc_socket_count": 92, + "sqlite_db_kb": 132.0 + }, + { + "event": "boot_ok", + "vm_seq": 93, + "boot_time_s": 2.977, + "density_at_boot": 93, + "timestamp": 1773030429.1687467, + "vms_running": 93, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.167278289794922, + "host_disk_used_gb": 2.0466232299804688, + "tap_count": 93, + "firecracker_rss_mb": 4371.48828125, + "fc_socket_count": 93, + "sqlite_db_kb": 132.0 + }, + { + "event": "boot_ok", + "vm_seq": 94, + "boot_time_s": 3.164, + "density_at_boot": 94, + "timestamp": 1773030432.4911287, + "vms_running": 94, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.213123321533203, + "host_disk_used_gb": 2.0466232299804688, + "tap_count": 94, + "firecracker_rss_mb": 4418.67578125, + "fc_socket_count": 94, + "sqlite_db_kb": 132.0 + }, + { + "event": "boot_ok", + "vm_seq": 95, + "boot_time_s": 3.04, + "density_at_boot": 95, + "timestamp": 1773030435.6898105, + "vms_running": 95, + "host_cpu_pct": 0.6, + "host_mem_used_gb": 7.2559356689453125, + "host_disk_used_gb": 2.0466232299804688, + "tap_count": 95, + "firecracker_rss_mb": 4465.34375, + "fc_socket_count": 95, + "sqlite_db_kb": 132.0 + }, + { + "event": "boot_ok", + "vm_seq": 96, + "boot_time_s": 3.204, + "density_at_boot": 96, + "timestamp": 1773030439.0509996, + "vms_running": 96, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.304416656494141, + "host_disk_used_gb": 2.0466232299804688, + "tap_count": 96, + "firecracker_rss_mb": 4512.015625, + "fc_socket_count": 96, + "sqlite_db_kb": 132.0 + }, + { + "event": "boot_ok", + "vm_seq": 97, + "boot_time_s": 3.048, + "density_at_boot": 97, + "timestamp": 1773030442.2581456, + "vms_running": 97, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 7.358158111572266, + "host_disk_used_gb": 2.0466461181640625, + "tap_count": 97, + "firecracker_rss_mb": 4558.94140625, + "fc_socket_count": 97, + "sqlite_db_kb": 136.0 + }, + { + "event": "boot_ok", + "vm_seq": 98, + "boot_time_s": 3.115, + "density_at_boot": 98, + "timestamp": 1773030445.5299532, + "vms_running": 98, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.401889801025391, + "host_disk_used_gb": 2.0466461181640625, + "tap_count": 98, + "firecracker_rss_mb": 4606.38671875, + "fc_socket_count": 98, + "sqlite_db_kb": 136.0 + }, + { + "event": "boot_ok", + "vm_seq": 99, + "boot_time_s": 3.046, + "density_at_boot": 99, + "timestamp": 1773030448.7335317, + "vms_running": 99, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.446315765380859, + "host_disk_used_gb": 2.0466461181640625, + "tap_count": 99, + "firecracker_rss_mb": 4653.83203125, + "fc_socket_count": 99, + "sqlite_db_kb": 136.0 + }, + { + "event": "boot_ok", + "vm_seq": 100, + "boot_time_s": 3.041, + "density_at_boot": 100, + "timestamp": 1773030451.9317963, + "vms_running": 100, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.496425628662109, + "host_disk_used_gb": 2.0466461181640625, + "tap_count": 100, + "firecracker_rss_mb": 4700.5, + "fc_socket_count": 100, + "sqlite_db_kb": 136.0 + }, + { + "event": "boot_ok", + "vm_seq": 101, + "boot_time_s": 2.138, + "density_at_boot": 101, + "timestamp": 1773030454.2258134, + "vms_running": 101, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 7.546222686767578, + "host_disk_used_gb": 2.0466461181640625, + "tap_count": 101, + "firecracker_rss_mb": 4748.4609375, + "fc_socket_count": 101, + "sqlite_db_kb": 136.0 + }, + { + "event": "boot_ok", + "vm_seq": 102, + "boot_time_s": 3.073, + "density_at_boot": 102, + "timestamp": 1773030457.451235, + "vms_running": 102, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 7.5894927978515625, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 102, + "firecracker_rss_mb": 4794.87109375, + "fc_socket_count": 102, + "sqlite_db_kb": 136.0 + }, + { + "event": "boot_ok", + "vm_seq": 103, + "boot_time_s": 2.194, + "density_at_boot": 103, + "timestamp": 1773030459.801489, + "vms_running": 103, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 7.636894226074219, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 103, + "firecracker_rss_mb": 4842.0546875, + "fc_socket_count": 103, + "sqlite_db_kb": 140.0 + }, + { + "event": "boot_ok", + "vm_seq": 104, + "boot_time_s": 3.172, + "density_at_boot": 104, + "timestamp": 1773030463.1279225, + "vms_running": 104, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.676593780517578, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 104, + "firecracker_rss_mb": 4888.72265625, + "fc_socket_count": 104, + "sqlite_db_kb": 140.0 + }, + { + "event": "boot_ok", + "vm_seq": 105, + "boot_time_s": 3.003, + "density_at_boot": 105, + "timestamp": 1773030466.287735, + "vms_running": 105, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 7.724208831787109, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 105, + "firecracker_rss_mb": 4936.42578125, + "fc_socket_count": 105, + "sqlite_db_kb": 140.0 + }, + { + "event": "boot_ok", + "vm_seq": 106, + "boot_time_s": 3.087, + "density_at_boot": 106, + "timestamp": 1773030469.5299685, + "vms_running": 106, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 7.775714874267578, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 106, + "firecracker_rss_mb": 4983.3515625, + "fc_socket_count": 106, + "sqlite_db_kb": 140.0 + }, + { + "event": "boot_ok", + "vm_seq": 107, + "boot_time_s": 3.123, + "density_at_boot": 107, + "timestamp": 1773030472.8123348, + "vms_running": 107, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 7.826240539550781, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 107, + "firecracker_rss_mb": 5030.27734375, + "fc_socket_count": 107, + "sqlite_db_kb": 140.0 + }, + { + "event": "boot_ok", + "vm_seq": 108, + "boot_time_s": 3.202, + "density_at_boot": 108, + "timestamp": 1773030476.1711428, + "vms_running": 108, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 7.882148742675781, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 108, + "firecracker_rss_mb": 5076.69140625, + "fc_socket_count": 108, + "sqlite_db_kb": 140.0 + }, + { + "event": "boot_ok", + "vm_seq": 109, + "boot_time_s": 2.184, + "density_at_boot": 109, + "timestamp": 1773030478.512836, + "vms_running": 109, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 7.934665679931641, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 109, + "firecracker_rss_mb": 5124.13671875, + "fc_socket_count": 109, + "sqlite_db_kb": 144.0 + }, + { + "event": "boot_ok", + "vm_seq": 110, + "boot_time_s": 2.977, + "density_at_boot": 110, + "timestamp": 1773030481.6478655, + "vms_running": 110, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 7.981426239013672, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 110, + "firecracker_rss_mb": 5171.0625, + "fc_socket_count": 110, + "sqlite_db_kb": 144.0 + }, + { + "event": "boot_ok", + "vm_seq": 111, + "boot_time_s": 3.081, + "density_at_boot": 111, + "timestamp": 1773030484.8901343, + "vms_running": 111, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.023605346679688, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 111, + "firecracker_rss_mb": 5218.24609375, + "fc_socket_count": 111, + "sqlite_db_kb": 144.0 + }, + { + "event": "boot_ok", + "vm_seq": 112, + "boot_time_s": 3.117, + "density_at_boot": 112, + "timestamp": 1773030488.168403, + "vms_running": 112, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 8.06637954711914, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 112, + "firecracker_rss_mb": 5265.42578125, + "fc_socket_count": 112, + "sqlite_db_kb": 144.0 + }, + { + "event": "boot_ok", + "vm_seq": 113, + "boot_time_s": 3.122, + "density_at_boot": 113, + "timestamp": 1773030491.4480917, + "vms_running": 113, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 8.119144439697266, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 113, + "firecracker_rss_mb": 5312.3515625, + "fc_socket_count": 113, + "sqlite_db_kb": 144.0 + }, + { + "event": "boot_ok", + "vm_seq": 114, + "boot_time_s": 3.122, + "density_at_boot": 114, + "timestamp": 1773030494.7311392, + "vms_running": 114, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.159713745117188, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 114, + "firecracker_rss_mb": 5358.7578125, + "fc_socket_count": 114, + "sqlite_db_kb": 144.0 + }, + { + "event": "boot_ok", + "vm_seq": 115, + "boot_time_s": 3.281, + "density_at_boot": 115, + "timestamp": 1773030498.170372, + "vms_running": 115, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 8.206192016601562, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 115, + "firecracker_rss_mb": 5405.171875, + "fc_socket_count": 115, + "sqlite_db_kb": 148.0 + }, + { + "event": "boot_ok", + "vm_seq": 116, + "boot_time_s": 2.136, + "density_at_boot": 116, + "timestamp": 1773030500.4652536, + "vms_running": 116, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.25857162475586, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 116, + "firecracker_rss_mb": 5452.10546875, + "fc_socket_count": 116, + "sqlite_db_kb": 148.0 + }, + { + "event": "boot_ok", + "vm_seq": 117, + "boot_time_s": 3.227, + "density_at_boot": 117, + "timestamp": 1773030503.8517723, + "vms_running": 117, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.298316955566406, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 117, + "firecracker_rss_mb": 5499.03515625, + "fc_socket_count": 117, + "sqlite_db_kb": 148.0 + }, + { + "event": "boot_ok", + "vm_seq": 118, + "boot_time_s": 3.118, + "density_at_boot": 118, + "timestamp": 1773030507.1285613, + "vms_running": 118, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 8.353229522705078, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 118, + "firecracker_rss_mb": 5546.22265625, + "fc_socket_count": 118, + "sqlite_db_kb": 148.0 + }, + { + "event": "boot_ok", + "vm_seq": 119, + "boot_time_s": 3.284, + "density_at_boot": 119, + "timestamp": 1773030510.5695388, + "vms_running": 119, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.397003173828125, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 119, + "firecracker_rss_mb": 5592.6328125, + "fc_socket_count": 119, + "sqlite_db_kb": 148.0 + }, + { + "event": "boot_ok", + "vm_seq": 120, + "boot_time_s": 3.043, + "density_at_boot": 120, + "timestamp": 1773030513.7708027, + "vms_running": 120, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 8.439407348632812, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 120, + "firecracker_rss_mb": 5639.5546875, + "fc_socket_count": 120, + "sqlite_db_kb": 152.0 + }, + { + "event": "boot_ok", + "vm_seq": 121, + "boot_time_s": 2.183, + "density_at_boot": 121, + "timestamp": 1773030516.1103702, + "vms_running": 121, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.481975555419922, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 121, + "firecracker_rss_mb": 5686.2265625, + "fc_socket_count": 121, + "sqlite_db_kb": 156.0 + }, + { + "event": "boot_ok", + "vm_seq": 122, + "boot_time_s": 3.1, + "density_at_boot": 122, + "timestamp": 1773030519.3698952, + "vms_running": 122, + "host_cpu_pct": 0.6, + "host_mem_used_gb": 8.528865814208984, + "host_disk_used_gb": 2.0466270446777344, + "tap_count": 122, + "firecracker_rss_mb": 5733.15625, + "fc_socket_count": 122, + "sqlite_db_kb": 156.0 + }, + { + "event": "boot_ok", + "vm_seq": 123, + "boot_time_s": 3.122, + "density_at_boot": 123, + "timestamp": 1773030522.651457, + "vms_running": 123, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 8.57718276977539, + "host_disk_used_gb": 2.0505332946777344, + "tap_count": 123, + "firecracker_rss_mb": 5779.828125, + "fc_socket_count": 123, + "sqlite_db_kb": 156.0 + }, + { + "event": "boot_ok", + "vm_seq": 124, + "boot_time_s": 3.197, + "density_at_boot": 124, + "timestamp": 1773030526.0088286, + "vms_running": 124, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 8.63067626953125, + "host_disk_used_gb": 2.0505332946777344, + "tap_count": 124, + "firecracker_rss_mb": 5826.75390625, + "fc_socket_count": 124, + "sqlite_db_kb": 156.0 + }, + { + "event": "boot_ok", + "vm_seq": 125, + "boot_time_s": 2.181, + "density_at_boot": 125, + "timestamp": 1773030528.3499966, + "vms_running": 125, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.678535461425781, + "host_disk_used_gb": 2.0505332946777344, + "tap_count": 125, + "firecracker_rss_mb": 5873.67578125, + "fc_socket_count": 125, + "sqlite_db_kb": 156.0 + }, + { + "event": "boot_ok", + "vm_seq": 126, + "boot_time_s": 3.1, + "density_at_boot": 126, + "timestamp": 1773030531.6112616, + "vms_running": 126, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 8.732891082763672, + "host_disk_used_gb": 2.0505332946777344, + "tap_count": 126, + "firecracker_rss_mb": 5920.60546875, + "fc_socket_count": 126, + "sqlite_db_kb": 156.0 + }, + { + "event": "boot_ok", + "vm_seq": 127, + "boot_time_s": 3.12, + "density_at_boot": 127, + "timestamp": 1773030534.8902926, + "vms_running": 127, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.7703857421875, + "host_disk_used_gb": 2.0505332946777344, + "tap_count": 127, + "firecracker_rss_mb": 5967.01953125, + "fc_socket_count": 127, + "sqlite_db_kb": 160.0 + }, + { + "event": "boot_ok", + "vm_seq": 128, + "boot_time_s": 3.118, + "density_at_boot": 128, + "timestamp": 1773030538.1708665, + "vms_running": 128, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.829139709472656, + "host_disk_used_gb": 2.0505332946777344, + "tap_count": 128, + "firecracker_rss_mb": 6014.20703125, + "fc_socket_count": 128, + "sqlite_db_kb": 160.0 + }, + { + "event": "boot_ok", + "vm_seq": 129, + "boot_time_s": 3.277, + "density_at_boot": 129, + "timestamp": 1773030541.6091838, + "vms_running": 129, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.878833770751953, + "host_disk_used_gb": 2.0505332946777344, + "tap_count": 129, + "firecracker_rss_mb": 6060.87890625, + "fc_socket_count": 129, + "sqlite_db_kb": 160.0 + }, + { + "event": "boot_ok", + "vm_seq": 130, + "boot_time_s": 3.119, + "density_at_boot": 130, + "timestamp": 1773030544.8907666, + "vms_running": 130, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.921592712402344, + "host_disk_used_gb": 2.0505332946777344, + "tap_count": 130, + "firecracker_rss_mb": 6108.0625, + "fc_socket_count": 130, + "sqlite_db_kb": 160.0 + }, + { + "event": "boot_ok", + "vm_seq": 131, + "boot_time_s": 2.18, + "density_at_boot": 131, + "timestamp": 1773030547.231964, + "vms_running": 131, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 8.96993637084961, + "host_disk_used_gb": 2.0505332946777344, + "tap_count": 131, + "firecracker_rss_mb": 6155.76171875, + "fc_socket_count": 131, + "sqlite_db_kb": 160.0 + }, + { + "event": "boot_ok", + "vm_seq": 132, + "boot_time_s": 3.097, + "density_at_boot": 132, + "timestamp": 1773030550.4896605, + "vms_running": 132, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.014835357666016, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 132, + "firecracker_rss_mb": 6203.20703125, + "fc_socket_count": 132, + "sqlite_db_kb": 160.0 + }, + { + "event": "boot_ok", + "vm_seq": 133, + "boot_time_s": 3.121, + "density_at_boot": 133, + "timestamp": 1773030553.7719686, + "vms_running": 133, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 9.057540893554688, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 133, + "firecracker_rss_mb": 6250.13671875, + "fc_socket_count": 133, + "sqlite_db_kb": 164.0 + }, + { + "event": "boot_ok", + "vm_seq": 134, + "boot_time_s": 3.116, + "density_at_boot": 134, + "timestamp": 1773030557.049676, + "vms_running": 134, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.09420394897461, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 134, + "firecracker_rss_mb": 6296.80859375, + "fc_socket_count": 134, + "sqlite_db_kb": 164.0 + }, + { + "event": "boot_ok", + "vm_seq": 135, + "boot_time_s": 3.119, + "density_at_boot": 135, + "timestamp": 1773030560.3302941, + "vms_running": 135, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 9.1478271484375, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 135, + "firecracker_rss_mb": 6343.21875, + "fc_socket_count": 135, + "sqlite_db_kb": 164.0 + }, + { + "event": "boot_ok", + "vm_seq": 136, + "boot_time_s": 3.28, + "density_at_boot": 136, + "timestamp": 1773030563.7712762, + "vms_running": 136, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.195480346679688, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 136, + "firecracker_rss_mb": 6390.1484375, + "fc_socket_count": 136, + "sqlite_db_kb": 164.0 + }, + { + "event": "boot_ok", + "vm_seq": 137, + "boot_time_s": 3.112, + "density_at_boot": 137, + "timestamp": 1773030567.0495977, + "vms_running": 137, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 9.255233764648438, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 137, + "firecracker_rss_mb": 6437.33203125, + "fc_socket_count": 137, + "sqlite_db_kb": 164.0 + }, + { + "event": "boot_ok", + "vm_seq": 138, + "boot_time_s": 3.12, + "density_at_boot": 138, + "timestamp": 1773030570.3295374, + "vms_running": 138, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.305980682373047, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 138, + "firecracker_rss_mb": 6484.515625, + "fc_socket_count": 138, + "sqlite_db_kb": 164.0 + }, + { + "event": "boot_ok", + "vm_seq": 139, + "boot_time_s": 3.117, + "density_at_boot": 139, + "timestamp": 1773030573.6100335, + "vms_running": 139, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 9.334781646728516, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 139, + "firecracker_rss_mb": 6531.43359375, + "fc_socket_count": 139, + "sqlite_db_kb": 168.0 + }, + { + "event": "boot_ok", + "vm_seq": 140, + "boot_time_s": 3.071, + "density_at_boot": 140, + "timestamp": 1773030576.8478024, + "vms_running": 140, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 9.384021759033203, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 140, + "firecracker_rss_mb": 6578.1015625, + "fc_socket_count": 140, + "sqlite_db_kb": 168.0 + }, + { + "event": "boot_ok", + "vm_seq": 141, + "boot_time_s": 2.175, + "density_at_boot": 141, + "timestamp": 1773030579.1875234, + "vms_running": 141, + "host_cpu_pct": 0.7, + "host_mem_used_gb": 9.426738739013672, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 141, + "firecracker_rss_mb": 6625.8046875, + "fc_socket_count": 141, + "sqlite_db_kb": 168.0 + }, + { + "event": "boot_ok", + "vm_seq": 142, + "boot_time_s": 3.145, + "density_at_boot": 142, + "timestamp": 1773030582.4925492, + "vms_running": 142, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 9.477222442626953, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 142, + "firecracker_rss_mb": 6672.4765625, + "fc_socket_count": 142, + "sqlite_db_kb": 168.0 + }, + { + "event": "boot_ok", + "vm_seq": 143, + "boot_time_s": 3.114, + "density_at_boot": 143, + "timestamp": 1773030585.7700963, + "vms_running": 143, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.525901794433594, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 143, + "firecracker_rss_mb": 6719.66015625, + "fc_socket_count": 143, + "sqlite_db_kb": 168.0 + }, + { + "event": "boot_ok", + "vm_seq": 144, + "boot_time_s": 3.117, + "density_at_boot": 144, + "timestamp": 1773030589.0509582, + "vms_running": 144, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.575244903564453, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 144, + "firecracker_rss_mb": 6766.328125, + "fc_socket_count": 144, + "sqlite_db_kb": 168.0 + }, + { + "event": "boot_ok", + "vm_seq": 145, + "boot_time_s": 3.115, + "density_at_boot": 145, + "timestamp": 1773030592.3305056, + "vms_running": 145, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.611865997314453, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 145, + "firecracker_rss_mb": 6814.02734375, + "fc_socket_count": 145, + "sqlite_db_kb": 172.0 + }, + { + "event": "boot_ok", + "vm_seq": 146, + "boot_time_s": 3.118, + "density_at_boot": 146, + "timestamp": 1773030595.6105, + "vms_running": 146, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.659114837646484, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 146, + "firecracker_rss_mb": 6860.6953125, + "fc_socket_count": 146, + "sqlite_db_kb": 172.0 + }, + { + "event": "boot_ok", + "vm_seq": 147, + "boot_time_s": 2.258, + "density_at_boot": 147, + "timestamp": 1773030598.0313594, + "vms_running": 147, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 9.711494445800781, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 147, + "firecracker_rss_mb": 6907.87890625, + "fc_socket_count": 147, + "sqlite_db_kb": 172.0 + }, + { + "event": "boot_ok", + "vm_seq": 148, + "boot_time_s": 2.237, + "density_at_boot": 148, + "timestamp": 1773030600.4328067, + "vms_running": 148, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 9.750930786132812, + "host_disk_used_gb": 2.0583457946777344, + "tap_count": 148, + "firecracker_rss_mb": 6954.55078125, + "fc_socket_count": 148, + "sqlite_db_kb": 176.0 + }, + { + "event": "boot_ok", + "vm_seq": 149, + "boot_time_s": 3.254, + "density_at_boot": 149, + "timestamp": 1773030603.8505654, + "vms_running": 149, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.808307647705078, + "host_disk_used_gb": 2.0583953857421875, + "tap_count": 149, + "firecracker_rss_mb": 7001.734375, + "fc_socket_count": 149, + "sqlite_db_kb": 176.0 + }, + { + "event": "boot_ok", + "vm_seq": 150, + "boot_time_s": 2.41, + "density_at_boot": 150, + "timestamp": 1773030606.4302688, + "vms_running": 150, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 9.865447998046875, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 150, + "firecracker_rss_mb": 7048.40234375, + "fc_socket_count": 150, + "sqlite_db_kb": 176.0 + }, + { + "event": "boot_ok", + "vm_seq": 151, + "boot_time_s": 3.09, + "density_at_boot": 151, + "timestamp": 1773030609.688327, + "vms_running": 151, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.906211853027344, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 151, + "firecracker_rss_mb": 7095.84375, + "fc_socket_count": 151, + "sqlite_db_kb": 180.0 + }, + { + "event": "boot_ok", + "vm_seq": 152, + "boot_time_s": 3.112, + "density_at_boot": 152, + "timestamp": 1773030612.9691172, + "vms_running": 152, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 9.950942993164062, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 152, + "firecracker_rss_mb": 7143.01953125, + "fc_socket_count": 152, + "sqlite_db_kb": 180.0 + }, + { + "event": "boot_ok", + "vm_seq": 153, + "boot_time_s": 3.114, + "density_at_boot": 153, + "timestamp": 1773030616.2511256, + "vms_running": 153, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 10.000373840332031, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 153, + "firecracker_rss_mb": 7189.9453125, + "fc_socket_count": 153, + "sqlite_db_kb": 180.0 + }, + { + "event": "boot_ok", + "vm_seq": 154, + "boot_time_s": 3.072, + "density_at_boot": 154, + "timestamp": 1773030619.4881804, + "vms_running": 154, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.039505004882812, + "host_disk_used_gb": 2.0583724975585938, + "tap_count": 154, + "firecracker_rss_mb": 7236.87109375, + "fc_socket_count": 154, + "sqlite_db_kb": 180.0 + }, + { + "event": "boot_ok", + "vm_seq": 155, + "boot_time_s": 3.155, + "density_at_boot": 155, + "timestamp": 1773030622.8121285, + "vms_running": 155, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.089401245117188, + "host_disk_used_gb": 2.0583724975585938, + "tap_count": 155, + "firecracker_rss_mb": 7283.28125, + "fc_socket_count": 155, + "sqlite_db_kb": 180.0 + }, + { + "event": "boot_ok", + "vm_seq": 156, + "boot_time_s": 3.109, + "density_at_boot": 156, + "timestamp": 1773030626.0889082, + "vms_running": 156, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.142635345458984, + "host_disk_used_gb": 2.0583724975585938, + "tap_count": 156, + "firecracker_rss_mb": 7330.98046875, + "fc_socket_count": 156, + "sqlite_db_kb": 180.0 + }, + { + "event": "boot_ok", + "vm_seq": 157, + "boot_time_s": 2.337, + "density_at_boot": 157, + "timestamp": 1773030628.5903733, + "vms_running": 157, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 10.189945220947266, + "host_disk_used_gb": 2.0583724975585938, + "tap_count": 157, + "firecracker_rss_mb": 7378.16796875, + "fc_socket_count": 157, + "sqlite_db_kb": 184.0 + }, + { + "event": "boot_ok", + "vm_seq": 158, + "boot_time_s": 3.094, + "density_at_boot": 158, + "timestamp": 1773030631.8515127, + "vms_running": 158, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.231304168701172, + "host_disk_used_gb": 2.0583724975585938, + "tap_count": 158, + "firecracker_rss_mb": 7424.58203125, + "fc_socket_count": 158, + "sqlite_db_kb": 184.0 + }, + { + "event": "boot_ok", + "vm_seq": 159, + "boot_time_s": 3.197, + "density_at_boot": 159, + "timestamp": 1773030635.2117321, + "vms_running": 159, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.280147552490234, + "host_disk_used_gb": 2.0583953857421875, + "tap_count": 159, + "firecracker_rss_mb": 7471.25, + "fc_socket_count": 159, + "sqlite_db_kb": 184.0 + }, + { + "event": "boot_ok", + "vm_seq": 160, + "boot_time_s": 3.107, + "density_at_boot": 160, + "timestamp": 1773030638.4903946, + "vms_running": 160, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 10.326778411865234, + "host_disk_used_gb": 2.0583953857421875, + "tap_count": 160, + "firecracker_rss_mb": 7517.91796875, + "fc_socket_count": 160, + "sqlite_db_kb": 184.0 + }, + { + "event": "boot_ok", + "vm_seq": 161, + "boot_time_s": 2.258, + "density_at_boot": 161, + "timestamp": 1773030640.9121716, + "vms_running": 161, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.386829376220703, + "host_disk_used_gb": 2.0583953857421875, + "tap_count": 161, + "firecracker_rss_mb": 7565.10546875, + "fc_socket_count": 161, + "sqlite_db_kb": 184.0 + }, + { + "event": "boot_ok", + "vm_seq": 162, + "boot_time_s": 3.167, + "density_at_boot": 162, + "timestamp": 1773030644.2489805, + "vms_running": 162, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 10.427810668945312, + "host_disk_used_gb": 2.0583953857421875, + "tap_count": 162, + "firecracker_rss_mb": 7612.8046875, + "fc_socket_count": 162, + "sqlite_db_kb": 184.0 + }, + { + "event": "boot_ok", + "vm_seq": 163, + "boot_time_s": 3.197, + "density_at_boot": 163, + "timestamp": 1773030647.610271, + "vms_running": 163, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 10.476997375488281, + "host_disk_used_gb": 2.0583953857421875, + "tap_count": 163, + "firecracker_rss_mb": 7659.21484375, + "fc_socket_count": 163, + "sqlite_db_kb": 188.0 + }, + { + "event": "boot_ok", + "vm_seq": 164, + "boot_time_s": 3.112, + "density_at_boot": 164, + "timestamp": 1773030650.890383, + "vms_running": 164, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.52157974243164, + "host_disk_used_gb": 2.0583953857421875, + "tap_count": 164, + "firecracker_rss_mb": 7706.39453125, + "fc_socket_count": 164, + "sqlite_db_kb": 188.0 + }, + { + "event": "boot_ok", + "vm_seq": 165, + "boot_time_s": 3.111, + "density_at_boot": 165, + "timestamp": 1773030654.1697454, + "vms_running": 165, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 10.563072204589844, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 165, + "firecracker_rss_mb": 7753.0625, + "fc_socket_count": 165, + "sqlite_db_kb": 188.0 + }, + { + "event": "boot_ok", + "vm_seq": 166, + "boot_time_s": 3.112, + "density_at_boot": 166, + "timestamp": 1773030657.4499469, + "vms_running": 166, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.615184783935547, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 166, + "firecracker_rss_mb": 7799.98828125, + "fc_socket_count": 166, + "sqlite_db_kb": 188.0 + }, + { + "event": "boot_ok", + "vm_seq": 167, + "boot_time_s": 3.109, + "density_at_boot": 167, + "timestamp": 1773030660.7303357, + "vms_running": 167, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 10.67031478881836, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 167, + "firecracker_rss_mb": 7846.91796875, + "fc_socket_count": 167, + "sqlite_db_kb": 188.0 + }, + { + "event": "boot_ok", + "vm_seq": 168, + "boot_time_s": 3.27, + "density_at_boot": 168, + "timestamp": 1773030664.1703522, + "vms_running": 168, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.707141876220703, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 168, + "firecracker_rss_mb": 7893.84765625, + "fc_socket_count": 168, + "sqlite_db_kb": 188.0 + }, + { + "event": "boot_ok", + "vm_seq": 169, + "boot_time_s": 3.19, + "density_at_boot": 169, + "timestamp": 1773030667.531072, + "vms_running": 169, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 10.751598358154297, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 169, + "firecracker_rss_mb": 7940.7734375, + "fc_socket_count": 169, + "sqlite_db_kb": 192.0 + }, + { + "event": "boot_ok", + "vm_seq": 170, + "boot_time_s": 3.193, + "density_at_boot": 170, + "timestamp": 1773030670.8925266, + "vms_running": 170, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.796680450439453, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 170, + "firecracker_rss_mb": 7987.69921875, + "fc_socket_count": 170, + "sqlite_db_kb": 192.0 + }, + { + "event": "boot_ok", + "vm_seq": 171, + "boot_time_s": 3.27, + "density_at_boot": 171, + "timestamp": 1773030674.331295, + "vms_running": 171, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.846931457519531, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 171, + "firecracker_rss_mb": 8035.140625, + "fc_socket_count": 171, + "sqlite_db_kb": 192.0 + }, + { + "event": "boot_ok", + "vm_seq": 172, + "boot_time_s": 3.108, + "density_at_boot": 172, + "timestamp": 1773030677.6099017, + "vms_running": 172, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 10.904075622558594, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 172, + "firecracker_rss_mb": 8081.8125, + "fc_socket_count": 172, + "sqlite_db_kb": 192.0 + }, + { + "event": "boot_ok", + "vm_seq": 173, + "boot_time_s": 3.113, + "density_at_boot": 173, + "timestamp": 1773030680.8904607, + "vms_running": 173, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.955322265625, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 173, + "firecracker_rss_mb": 8129.0, + "fc_socket_count": 173, + "sqlite_db_kb": 192.0 + }, + { + "event": "boot_ok", + "vm_seq": 174, + "boot_time_s": 3.108, + "density_at_boot": 174, + "timestamp": 1773030684.1715791, + "vms_running": 174, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.999347686767578, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 174, + "firecracker_rss_mb": 8175.671875, + "fc_socket_count": 174, + "sqlite_db_kb": 192.0 + }, + { + "event": "boot_ok", + "vm_seq": 175, + "boot_time_s": 2.159, + "density_at_boot": 175, + "timestamp": 1773030686.5001981, + "vms_running": 175, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 11.050048828125, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 175, + "firecracker_rss_mb": 8222.59765625, + "fc_socket_count": 175, + "sqlite_db_kb": 196.0 + }, + { + "event": "boot_ok", + "vm_seq": 176, + "boot_time_s": 2.245, + "density_at_boot": 176, + "timestamp": 1773030688.9127882, + "vms_running": 176, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.09844970703125, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 176, + "firecracker_rss_mb": 8269.78515625, + "fc_socket_count": 176, + "sqlite_db_kb": 196.0 + }, + { + "event": "boot_ok", + "vm_seq": 177, + "boot_time_s": 2.306, + "density_at_boot": 177, + "timestamp": 1773030691.3898923, + "vms_running": 177, + "host_cpu_pct": 0.5, + "host_mem_used_gb": 11.14117431640625, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 177, + "firecracker_rss_mb": 8317.7421875, + "fc_socket_count": 177, + "sqlite_db_kb": 200.0 + }, + { + "event": "boot_ok", + "vm_seq": 178, + "boot_time_s": 2.394, + "density_at_boot": 178, + "timestamp": 1773030693.9516737, + "vms_running": 178, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.173641204833984, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 178, + "firecracker_rss_mb": 8364.67578125, + "fc_socket_count": 178, + "sqlite_db_kb": 200.0 + }, + { + "event": "boot_ok", + "vm_seq": 179, + "boot_time_s": 3.123, + "density_at_boot": 179, + "timestamp": 1773030697.2473683, + "vms_running": 179, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 11.224136352539062, + "host_disk_used_gb": 2.0583763122558594, + "tap_count": 179, + "firecracker_rss_mb": 8411.59765625, + "fc_socket_count": 179, + "sqlite_db_kb": 200.0 + }, + { + "event": "boot_ok", + "vm_seq": 180, + "boot_time_s": 3.112, + "density_at_boot": 180, + "timestamp": 1773030700.5280075, + "vms_running": 180, + "host_cpu_pct": 0.7, + "host_mem_used_gb": 11.270156860351562, + "host_disk_used_gb": 2.058399200439453, + "tap_count": 180, + "firecracker_rss_mb": 8458.265625, + "fc_socket_count": 180, + "sqlite_db_kb": 200.0 + }, + { + "event": "boot_ok", + "vm_seq": 181, + "boot_time_s": 3.107, + "density_at_boot": 181, + "timestamp": 1773030703.8073387, + "vms_running": 181, + "host_cpu_pct": 0.6, + "host_mem_used_gb": 11.326152801513672, + "host_disk_used_gb": 2.058399200439453, + "tap_count": 181, + "firecracker_rss_mb": 8504.67578125, + "fc_socket_count": 181, + "sqlite_db_kb": 204.0 + }, + { + "event": "boot_ok", + "vm_seq": 182, + "boot_time_s": 3.155, + "density_at_boot": 182, + "timestamp": 1773030707.1316395, + "vms_running": 182, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 11.362808227539062, + "host_disk_used_gb": 2.058399200439453, + "tap_count": 182, + "firecracker_rss_mb": 8551.34375, + "fc_socket_count": 182, + "sqlite_db_kb": 204.0 + }, + { + "event": "boot_ok", + "vm_seq": 183, + "boot_time_s": 3.185, + "density_at_boot": 183, + "timestamp": 1773030710.4908752, + "vms_running": 183, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 11.415206909179688, + "host_disk_used_gb": 2.058399200439453, + "tap_count": 183, + "firecracker_rss_mb": 8597.75390625, + "fc_socket_count": 183, + "sqlite_db_kb": 204.0 + }, + { + "event": "boot_ok", + "vm_seq": 184, + "boot_time_s": 3.105, + "density_at_boot": 184, + "timestamp": 1773030713.7688148, + "vms_running": 184, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 11.45949935913086, + "host_disk_used_gb": 2.058399200439453, + "tap_count": 184, + "firecracker_rss_mb": 8644.42578125, + "fc_socket_count": 184, + "sqlite_db_kb": 204.0 + }, + { + "event": "boot_ok", + "vm_seq": 185, + "boot_time_s": 2.253, + "density_at_boot": 185, + "timestamp": 1773030716.1917515, + "vms_running": 185, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.509101867675781, + "host_disk_used_gb": 2.066211700439453, + "tap_count": 185, + "firecracker_rss_mb": 8691.09765625, + "fc_socket_count": 185, + "sqlite_db_kb": 204.0 + }, + { + "event": "boot_ok", + "vm_seq": 186, + "boot_time_s": 3.088, + "density_at_boot": 186, + "timestamp": 1773030719.4508097, + "vms_running": 186, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 11.55441665649414, + "host_disk_used_gb": 2.066211700439453, + "tap_count": 186, + "firecracker_rss_mb": 8738.0234375, + "fc_socket_count": 186, + "sqlite_db_kb": 204.0 + }, + { + "event": "boot_ok", + "vm_seq": 187, + "boot_time_s": 3.187, + "density_at_boot": 187, + "timestamp": 1773030722.8107624, + "vms_running": 187, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 11.595779418945312, + "host_disk_used_gb": 2.0662155151367188, + "tap_count": 187, + "firecracker_rss_mb": 8785.46875, + "fc_socket_count": 187, + "sqlite_db_kb": 208.0 + }, + { + "event": "boot_ok", + "vm_seq": 188, + "boot_time_s": 3.183, + "density_at_boot": 188, + "timestamp": 1773030726.1698601, + "vms_running": 188, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 11.638084411621094, + "host_disk_used_gb": 2.0662155151367188, + "tap_count": 188, + "firecracker_rss_mb": 8832.140625, + "fc_socket_count": 188, + "sqlite_db_kb": 208.0 + }, + { + "event": "boot_ok", + "vm_seq": 189, + "boot_time_s": 3.188, + "density_at_boot": 189, + "timestamp": 1773030729.5314174, + "vms_running": 189, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 11.685455322265625, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 189, + "firecracker_rss_mb": 8878.546875, + "fc_socket_count": 189, + "sqlite_db_kb": 208.0 + }, + { + "event": "boot_ok", + "vm_seq": 190, + "boot_time_s": 3.186, + "density_at_boot": 190, + "timestamp": 1773030732.8922422, + "vms_running": 190, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.731220245361328, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 190, + "firecracker_rss_mb": 8925.21875, + "fc_socket_count": 190, + "sqlite_db_kb": 208.0 + }, + { + "event": "boot_ok", + "vm_seq": 191, + "boot_time_s": 2.566, + "density_at_boot": 191, + "timestamp": 1773030735.6304379, + "vms_running": 191, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.782596588134766, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 191, + "firecracker_rss_mb": 8972.14453125, + "fc_socket_count": 191, + "sqlite_db_kb": 208.0 + }, + { + "event": "boot_ok", + "vm_seq": 192, + "boot_time_s": 3.119, + "density_at_boot": 192, + "timestamp": 1773030738.926491, + "vms_running": 192, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 11.833518981933594, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 192, + "firecracker_rss_mb": 9018.55859375, + "fc_socket_count": 192, + "sqlite_db_kb": 208.0 + }, + { + "event": "boot_ok", + "vm_seq": 193, + "boot_time_s": 3.237, + "density_at_boot": 193, + "timestamp": 1773030742.3309171, + "vms_running": 193, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 11.877479553222656, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 193, + "firecracker_rss_mb": 9065.2265625, + "fc_socket_count": 193, + "sqlite_db_kb": 212.0 + }, + { + "event": "boot_ok", + "vm_seq": 194, + "boot_time_s": 3.184, + "density_at_boot": 194, + "timestamp": 1773030745.690347, + "vms_running": 194, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.925689697265625, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 194, + "firecracker_rss_mb": 9111.63671875, + "fc_socket_count": 194, + "sqlite_db_kb": 212.0 + }, + { + "event": "boot_ok", + "vm_seq": 195, + "boot_time_s": 3.183, + "density_at_boot": 195, + "timestamp": 1773030749.0497975, + "vms_running": 195, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.97934341430664, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 195, + "firecracker_rss_mb": 9158.8203125, + "fc_socket_count": 195, + "sqlite_db_kb": 212.0 + }, + { + "event": "boot_ok", + "vm_seq": 196, + "boot_time_s": 3.186, + "density_at_boot": 196, + "timestamp": 1773030752.4114738, + "vms_running": 196, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.023429870605469, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 196, + "firecracker_rss_mb": 9206.265625, + "fc_socket_count": 196, + "sqlite_db_kb": 212.0 + }, + { + "event": "boot_ok", + "vm_seq": 197, + "boot_time_s": 3.187, + "density_at_boot": 197, + "timestamp": 1773030755.7730057, + "vms_running": 197, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.065494537353516, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 197, + "firecracker_rss_mb": 9253.19921875, + "fc_socket_count": 197, + "sqlite_db_kb": 212.0 + }, + { + "event": "boot_ok", + "vm_seq": 198, + "boot_time_s": 3.185, + "density_at_boot": 198, + "timestamp": 1773030759.1315176, + "vms_running": 198, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.100284576416016, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 198, + "firecracker_rss_mb": 9300.125, + "fc_socket_count": 198, + "sqlite_db_kb": 212.0 + }, + { + "event": "boot_ok", + "vm_seq": 199, + "boot_time_s": 3.186, + "density_at_boot": 199, + "timestamp": 1773030762.4906874, + "vms_running": 199, + "host_cpu_pct": 0.5, + "host_mem_used_gb": 12.152381896972656, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 199, + "firecracker_rss_mb": 9346.79296875, + "fc_socket_count": 199, + "sqlite_db_kb": 216.0 + }, + { + "event": "boot_ok", + "vm_seq": 200, + "boot_time_s": 3.19, + "density_at_boot": 200, + "timestamp": 1773030765.8521874, + "vms_running": 200, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.194412231445312, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 200, + "firecracker_rss_mb": 9393.203125, + "fc_socket_count": 200, + "sqlite_db_kb": 216.0 + }, + { + "event": "boot_ok", + "vm_seq": 201, + "boot_time_s": 3.182, + "density_at_boot": 201, + "timestamp": 1773030769.2110555, + "vms_running": 201, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.248680114746094, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 201, + "firecracker_rss_mb": 9440.38671875, + "fc_socket_count": 201, + "sqlite_db_kb": 216.0 + }, + { + "event": "boot_ok", + "vm_seq": 202, + "boot_time_s": 3.187, + "density_at_boot": 202, + "timestamp": 1773030772.5721247, + "vms_running": 202, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.292709350585938, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 202, + "firecracker_rss_mb": 9487.31640625, + "fc_socket_count": 202, + "sqlite_db_kb": 216.0 + }, + { + "event": "boot_ok", + "vm_seq": 203, + "boot_time_s": 3.184, + "density_at_boot": 203, + "timestamp": 1773030775.9301012, + "vms_running": 203, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.340782165527344, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 203, + "firecracker_rss_mb": 9534.5, + "fc_socket_count": 203, + "sqlite_db_kb": 216.0 + }, + { + "event": "boot_ok", + "vm_seq": 204, + "boot_time_s": 3.186, + "density_at_boot": 204, + "timestamp": 1773030779.2917938, + "vms_running": 204, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.385353088378906, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 204, + "firecracker_rss_mb": 9581.42578125, + "fc_socket_count": 204, + "sqlite_db_kb": 216.0 + }, + { + "event": "boot_ok", + "vm_seq": 205, + "boot_time_s": 2.228, + "density_at_boot": 205, + "timestamp": 1773030781.699019, + "vms_running": 205, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 12.443012237548828, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 205, + "firecracker_rss_mb": 9628.09375, + "fc_socket_count": 205, + "sqlite_db_kb": 220.0 + }, + { + "event": "boot_ok", + "vm_seq": 206, + "boot_time_s": 3.258, + "density_at_boot": 206, + "timestamp": 1773030785.1319654, + "vms_running": 206, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.488906860351562, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 206, + "firecracker_rss_mb": 9674.765625, + "fc_socket_count": 206, + "sqlite_db_kb": 220.0 + }, + { + "event": "boot_ok", + "vm_seq": 207, + "boot_time_s": 3.183, + "density_at_boot": 207, + "timestamp": 1773030788.4889982, + "vms_running": 207, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 12.53817367553711, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 207, + "firecracker_rss_mb": 9720.91796875, + "fc_socket_count": 207, + "sqlite_db_kb": 220.0 + }, + { + "event": "boot_ok", + "vm_seq": 208, + "boot_time_s": 3.182, + "density_at_boot": 208, + "timestamp": 1773030791.851776, + "vms_running": 208, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 12.58414077758789, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 208, + "firecracker_rss_mb": 9767.58984375, + "fc_socket_count": 208, + "sqlite_db_kb": 220.0 + }, + { + "event": "boot_ok", + "vm_seq": 209, + "boot_time_s": 3.262, + "density_at_boot": 209, + "timestamp": 1773030795.2908702, + "vms_running": 209, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 12.634292602539062, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 209, + "firecracker_rss_mb": 9814.515625, + "fc_socket_count": 209, + "sqlite_db_kb": 220.0 + }, + { + "event": "boot_ok", + "vm_seq": 210, + "boot_time_s": 3.185, + "density_at_boot": 210, + "timestamp": 1773030798.651638, + "vms_running": 210, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.678108215332031, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 210, + "firecracker_rss_mb": 9861.44140625, + "fc_socket_count": 210, + "sqlite_db_kb": 220.0 + }, + { + "event": "boot_ok", + "vm_seq": 211, + "boot_time_s": 3.345, + "density_at_boot": 211, + "timestamp": 1773030802.1711235, + "vms_running": 211, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 12.741558074951172, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 211, + "firecracker_rss_mb": 9908.37109375, + "fc_socket_count": 211, + "sqlite_db_kb": 224.0 + }, + { + "event": "boot_ok", + "vm_seq": 212, + "boot_time_s": 2.33, + "density_at_boot": 212, + "timestamp": 1773030804.678945, + "vms_running": 212, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 12.78213119506836, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 212, + "firecracker_rss_mb": 9955.55078125, + "fc_socket_count": 212, + "sqlite_db_kb": 224.0 + }, + { + "event": "boot_ok", + "vm_seq": 213, + "boot_time_s": 3.317, + "density_at_boot": 213, + "timestamp": 1773030808.1695566, + "vms_running": 213, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.833099365234375, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 213, + "firecracker_rss_mb": 10002.4765625, + "fc_socket_count": 213, + "sqlite_db_kb": 224.0 + }, + { + "event": "boot_ok", + "vm_seq": 214, + "boot_time_s": 3.182, + "density_at_boot": 214, + "timestamp": 1773030811.5305533, + "vms_running": 214, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.866477966308594, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 214, + "firecracker_rss_mb": 10048.890625, + "fc_socket_count": 214, + "sqlite_db_kb": 224.0 + }, + { + "event": "boot_ok", + "vm_seq": 215, + "boot_time_s": 3.182, + "density_at_boot": 215, + "timestamp": 1773030814.8910744, + "vms_running": 215, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 12.90692138671875, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 215, + "firecracker_rss_mb": 10095.55859375, + "fc_socket_count": 215, + "sqlite_db_kb": 224.0 + }, + { + "event": "boot_ok", + "vm_seq": 216, + "boot_time_s": 3.262, + "density_at_boot": 216, + "timestamp": 1773030818.331957, + "vms_running": 216, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 12.956188201904297, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 216, + "firecracker_rss_mb": 10142.2265625, + "fc_socket_count": 216, + "sqlite_db_kb": 224.0 + }, + { + "event": "boot_ok", + "vm_seq": 217, + "boot_time_s": 3.177, + "density_at_boot": 217, + "timestamp": 1773030821.6885786, + "vms_running": 217, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 13.005245208740234, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 217, + "firecracker_rss_mb": 10189.15625, + "fc_socket_count": 217, + "sqlite_db_kb": 236.0 + }, + { + "event": "boot_ok", + "vm_seq": 218, + "boot_time_s": 3.192, + "density_at_boot": 218, + "timestamp": 1773030825.0542357, + "vms_running": 218, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 13.052791595458984, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 218, + "firecracker_rss_mb": 10235.5703125, + "fc_socket_count": 218, + "sqlite_db_kb": 236.0 + }, + { + "event": "boot_ok", + "vm_seq": 219, + "boot_time_s": 2.186, + "density_at_boot": 219, + "timestamp": 1773030827.4184048, + "vms_running": 219, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 13.102787017822266, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 219, + "firecracker_rss_mb": 10282.7578125, + "fc_socket_count": 219, + "sqlite_db_kb": 236.0 + }, + { + "event": "boot_ok", + "vm_seq": 220, + "boot_time_s": 3.22, + "density_at_boot": 220, + "timestamp": 1773030830.8120756, + "vms_running": 220, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 13.151496887207031, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 220, + "firecracker_rss_mb": 10329.9453125, + "fc_socket_count": 220, + "sqlite_db_kb": 236.0 + }, + { + "event": "boot_ok", + "vm_seq": 221, + "boot_time_s": 3.259, + "density_at_boot": 221, + "timestamp": 1773030834.2496157, + "vms_running": 221, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 13.200881958007812, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 221, + "firecracker_rss_mb": 10377.12890625, + "fc_socket_count": 221, + "sqlite_db_kb": 240.0 + }, + { + "event": "boot_ok", + "vm_seq": 222, + "boot_time_s": 3.138, + "density_at_boot": 222, + "timestamp": 1773030837.5682049, + "vms_running": 222, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 13.251548767089844, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 222, + "firecracker_rss_mb": 10423.5390625, + "fc_socket_count": 222, + "sqlite_db_kb": 280.0 + }, + { + "event": "boot_ok", + "vm_seq": 223, + "boot_time_s": 3.224, + "density_at_boot": 223, + "timestamp": 1773030840.9711266, + "vms_running": 223, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 13.295562744140625, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 223, + "firecracker_rss_mb": 10470.20703125, + "fc_socket_count": 223, + "sqlite_db_kb": 284.0 + }, + { + "event": "boot_ok", + "vm_seq": 224, + "boot_time_s": 3.177, + "density_at_boot": 224, + "timestamp": 1773030844.3304691, + "vms_running": 224, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 13.34567642211914, + "host_disk_used_gb": 2.0661964416503906, + "tap_count": 224, + "firecracker_rss_mb": 10517.13671875, + "fc_socket_count": 224, + "sqlite_db_kb": 284.0 + }, + { + "event": "boot_ok", + "vm_seq": 225, + "boot_time_s": 2.279, + "density_at_boot": 225, + "timestamp": 1773030846.7891102, + "vms_running": 225, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 13.394973754882812, + "host_disk_used_gb": 2.0662193298339844, + "tap_count": 225, + "firecracker_rss_mb": 10564.06640625, + "fc_socket_count": 225, + "sqlite_db_kb": 284.0 + }, + { + "event": "boot_ok", + "vm_seq": 226, + "boot_time_s": 3.206, + "density_at_boot": 226, + "timestamp": 1773030850.1729743, + "vms_running": 226, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 13.445087432861328, + "host_disk_used_gb": 2.0662193298339844, + "tap_count": 226, + "firecracker_rss_mb": 10610.9921875, + "fc_socket_count": 226, + "sqlite_db_kb": 284.0 + }, + { + "event": "boot_ok", + "vm_seq": 227, + "boot_time_s": 3.255, + "density_at_boot": 227, + "timestamp": 1773030853.609889, + "vms_running": 227, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 13.492591857910156, + "host_disk_used_gb": 2.0662193298339844, + "tap_count": 227, + "firecracker_rss_mb": 10657.91796875, + "fc_socket_count": 227, + "sqlite_db_kb": 284.0 + }, + { + "event": "boot_ok", + "vm_seq": 228, + "boot_time_s": 3.217, + "density_at_boot": 228, + "timestamp": 1773030857.0059326, + "vms_running": 228, + "host_cpu_pct": 0.6, + "host_mem_used_gb": 13.540851593017578, + "host_disk_used_gb": 2.0662193298339844, + "tap_count": 228, + "firecracker_rss_mb": 10704.84765625, + "fc_socket_count": 228, + "sqlite_db_kb": 284.0 + }, + { + "event": "boot_ok", + "vm_seq": 229, + "boot_time_s": 3.232, + "density_at_boot": 229, + "timestamp": 1773030860.4121194, + "vms_running": 229, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 13.580379486083984, + "host_disk_used_gb": 2.0662002563476562, + "tap_count": 229, + "firecracker_rss_mb": 10751.77734375, + "fc_socket_count": 229, + "sqlite_db_kb": 288.0 + }, + { + "event": "boot_ok", + "vm_seq": 230, + "boot_time_s": 2.274, + "density_at_boot": 230, + "timestamp": 1773030862.8678098, + "vms_running": 230, + "host_cpu_pct": 0.5, + "host_mem_used_gb": 13.631404876708984, + "host_disk_used_gb": 2.0662002563476562, + "tap_count": 230, + "firecracker_rss_mb": 10799.4765625, + "fc_socket_count": 230, + "sqlite_db_kb": 288.0 + }, + { + "event": "boot_ok", + "vm_seq": 231, + "boot_time_s": 3.333, + "density_at_boot": 231, + "timestamp": 1773030866.3781908, + "vms_running": 231, + "host_cpu_pct": 0.8, + "host_mem_used_gb": 13.685527801513672, + "host_disk_used_gb": 2.0662002563476562, + "tap_count": 231, + "firecracker_rss_mb": 10846.66015625, + "fc_socket_count": 231, + "sqlite_db_kb": 288.0 + }, + { + "event": "boot_ok", + "vm_seq": 232, + "boot_time_s": 3.287, + "density_at_boot": 232, + "timestamp": 1773030869.849167, + "vms_running": 232, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 13.733783721923828, + "host_disk_used_gb": 2.0662002563476562, + "tap_count": 232, + "firecracker_rss_mb": 10893.5859375, + "fc_socket_count": 232, + "sqlite_db_kb": 288.0 + }, + { + "event": "boot_ok", + "vm_seq": 233, + "boot_time_s": 3.192, + "density_at_boot": 233, + "timestamp": 1773030873.2213478, + "vms_running": 233, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 13.771747589111328, + "host_disk_used_gb": 2.0740127563476562, + "tap_count": 233, + "firecracker_rss_mb": 10940.51953125, + "fc_socket_count": 233, + "sqlite_db_kb": 288.0 + }, + { + "event": "boot_ok", + "vm_seq": 234, + "boot_time_s": 3.244, + "density_at_boot": 234, + "timestamp": 1773030876.6512637, + "vms_running": 234, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 13.816898345947266, + "host_disk_used_gb": 2.0740127563476562, + "tap_count": 234, + "firecracker_rss_mb": 10987.96484375, + "fc_socket_count": 234, + "sqlite_db_kb": 292.0 + }, + { + "event": "boot_ok", + "vm_seq": 235, + "boot_time_s": 3.18, + "density_at_boot": 235, + "timestamp": 1773030880.011383, + "vms_running": 235, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 13.860759735107422, + "host_disk_used_gb": 2.07403564453125, + "tap_count": 235, + "firecracker_rss_mb": 11034.62890625, + "fc_socket_count": 235, + "sqlite_db_kb": 296.0 + }, + { + "event": "boot_ok", + "vm_seq": 236, + "boot_time_s": 3.335, + "density_at_boot": 236, + "timestamp": 1773030883.5314436, + "vms_running": 236, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 13.902542114257812, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 236, + "firecracker_rss_mb": 11081.55859375, + "fc_socket_count": 236, + "sqlite_db_kb": 296.0 + }, + { + "event": "boot_ok", + "vm_seq": 237, + "boot_time_s": 2.319, + "density_at_boot": 237, + "timestamp": 1773030886.0321264, + "vms_running": 237, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 13.950122833251953, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 237, + "firecracker_rss_mb": 11128.48828125, + "fc_socket_count": 237, + "sqlite_db_kb": 296.0 + }, + { + "event": "boot_ok", + "vm_seq": 238, + "boot_time_s": 3.318, + "density_at_boot": 238, + "timestamp": 1773030889.532275, + "vms_running": 238, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 14.003963470458984, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 238, + "firecracker_rss_mb": 11175.16015625, + "fc_socket_count": 238, + "sqlite_db_kb": 296.0 + }, + { + "event": "boot_ok", + "vm_seq": 239, + "boot_time_s": 3.254, + "density_at_boot": 239, + "timestamp": 1773030892.9688118, + "vms_running": 239, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 14.04220962524414, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 239, + "firecracker_rss_mb": 11222.0859375, + "fc_socket_count": 239, + "sqlite_db_kb": 296.0 + }, + { + "event": "boot_ok", + "vm_seq": 240, + "boot_time_s": 3.259, + "density_at_boot": 240, + "timestamp": 1773030896.410509, + "vms_running": 240, + "host_cpu_pct": 0.6, + "host_mem_used_gb": 14.089305877685547, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 240, + "firecracker_rss_mb": 11268.7578125, + "fc_socket_count": 240, + "sqlite_db_kb": 296.0 + }, + { + "event": "boot_ok", + "vm_seq": 241, + "boot_time_s": 2.32, + "density_at_boot": 241, + "timestamp": 1773030898.912195, + "vms_running": 241, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.13222885131836, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 241, + "firecracker_rss_mb": 11315.68359375, + "fc_socket_count": 241, + "sqlite_db_kb": 300.0 + }, + { + "event": "boot_ok", + "vm_seq": 242, + "boot_time_s": 3.16, + "density_at_boot": 242, + "timestamp": 1773030902.252483, + "vms_running": 242, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.179145812988281, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 242, + "firecracker_rss_mb": 11362.3515625, + "fc_socket_count": 242, + "sqlite_db_kb": 300.0 + }, + { + "event": "boot_ok", + "vm_seq": 243, + "boot_time_s": 3.177, + "density_at_boot": 243, + "timestamp": 1773030905.6109297, + "vms_running": 243, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 14.226566314697266, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 243, + "firecracker_rss_mb": 11409.53125, + "fc_socket_count": 243, + "sqlite_db_kb": 300.0 + }, + { + "event": "boot_ok", + "vm_seq": 244, + "boot_time_s": 3.176, + "density_at_boot": 244, + "timestamp": 1773030908.969251, + "vms_running": 244, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.275192260742188, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 244, + "firecracker_rss_mb": 11456.19921875, + "fc_socket_count": 244, + "sqlite_db_kb": 300.0 + }, + { + "event": "boot_ok", + "vm_seq": 245, + "boot_time_s": 2.396, + "density_at_boot": 245, + "timestamp": 1773030911.5503266, + "vms_running": 245, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 14.319305419921875, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 245, + "firecracker_rss_mb": 11503.12890625, + "fc_socket_count": 245, + "sqlite_db_kb": 300.0 + }, + { + "event": "boot_ok", + "vm_seq": 246, + "boot_time_s": 3.239, + "density_at_boot": 246, + "timestamp": 1773030914.9717393, + "vms_running": 246, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 14.369342803955078, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 246, + "firecracker_rss_mb": 11549.5390625, + "fc_socket_count": 246, + "sqlite_db_kb": 300.0 + }, + { + "event": "boot_ok", + "vm_seq": 247, + "boot_time_s": 3.256, + "density_at_boot": 247, + "timestamp": 1773030918.4093506, + "vms_running": 247, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.412841796875, + "host_disk_used_gb": 2.074016571044922, + "tap_count": 247, + "firecracker_rss_mb": 11596.46875, + "fc_socket_count": 247, + "sqlite_db_kb": 304.0 + }, + { + "event": "boot_ok", + "vm_seq": 248, + "boot_time_s": 3.257, + "density_at_boot": 248, + "timestamp": 1773030921.849107, + "vms_running": 248, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 14.468254089355469, + "host_disk_used_gb": 2.0740699768066406, + "tap_count": 248, + "firecracker_rss_mb": 11643.39453125, + "fc_socket_count": 248, + "sqlite_db_kb": 304.0 + }, + { + "event": "boot_ok", + "vm_seq": 249, + "boot_time_s": 2.335, + "density_at_boot": 249, + "timestamp": 1773030924.3612962, + "vms_running": 249, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 14.515098571777344, + "host_disk_used_gb": 2.0740814208984375, + "tap_count": 249, + "firecracker_rss_mb": 11690.06640625, + "fc_socket_count": 249, + "sqlite_db_kb": 304.0 + }, + { + "event": "boot_ok", + "vm_seq": 250, + "boot_time_s": 3.342, + "density_at_boot": 250, + "timestamp": 1773030927.8892078, + "vms_running": 250, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.576744079589844, + "host_disk_used_gb": 2.0741424560546875, + "tap_count": 250, + "firecracker_rss_mb": 11737.24609375, + "fc_socket_count": 250, + "sqlite_db_kb": 304.0 + }, + { + "event": "boot_ok", + "vm_seq": 251, + "boot_time_s": 3.222, + "density_at_boot": 251, + "timestamp": 1773030931.2900705, + "vms_running": 251, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 14.61867904663086, + "host_disk_used_gb": 2.0741424560546875, + "tap_count": 251, + "firecracker_rss_mb": 11783.65625, + "fc_socket_count": 251, + "sqlite_db_kb": 304.0 + }, + { + "event": "boot_ok", + "vm_seq": 252, + "boot_time_s": 3.333, + "density_at_boot": 252, + "timestamp": 1773030934.8102415, + "vms_running": 252, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 14.672557830810547, + "host_disk_used_gb": 2.0742034912109375, + "tap_count": 252, + "firecracker_rss_mb": 11830.5859375, + "fc_socket_count": 252, + "sqlite_db_kb": 304.0 + }, + { + "event": "boot_ok", + "vm_seq": 253, + "boot_time_s": 3.253, + "density_at_boot": 253, + "timestamp": 1773030938.2493396, + "vms_running": 253, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 14.717056274414062, + "host_disk_used_gb": 2.0742034912109375, + "tap_count": 253, + "firecracker_rss_mb": 11877.26171875, + "fc_socket_count": 253, + "sqlite_db_kb": 308.0 + }, + { + "event": "sustain_check", + "sustain_sec": 60, + "vms_alive": 253, + "vms_dead": [], + "vms_checked": 253, + "timestamp": 1773030998.823995, + "vms_running": 253, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.539192199707031, + "host_disk_used_gb": 2.074188232421875, + "tap_count": 253, + "firecracker_rss_mb": 11949.96484375, + "fc_socket_count": 253, + "sqlite_db_kb": 308.0 + }, + { + "event": "teardown", + "teardown_time_s": 127.57, + "timestamp": 1773031126.5712855, + "vms_running": 0, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.0837554931640625, + "host_disk_used_gb": 2.0742149353027344, + "tap_count": 253, + "firecracker_rss_mb": 0.0, + "fc_socket_count": 0, + "sqlite_db_kb": 308.0 + } +] \ No newline at end of file diff --git a/benchmarks/results/density_20260309_010721.json b/benchmarks/results/density_20260309_010721.json new file mode 100644 index 0000000..8d948cf --- /dev/null +++ b/benchmarks/results/density_20260309_010721.json @@ -0,0 +1,3826 @@ +[ + { + "event": "boot_ok", + "vm_seq": 1, + "boot_time_s": 3.557, + "density_at_boot": 1, + "timestamp": 1773033027.8439739, + "vms_running": 1, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 2.7269515991210938, + "host_disk_used_gb": 2.0351943969726562, + "tap_count": 1, + "firecracker_rss_mb": 47.4375, + "fc_socket_count": 1, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_ok", + "vm_seq": 2, + "boot_time_s": 3.012, + "density_at_boot": 2, + "timestamp": 1773033031.0415125, + "vms_running": 2, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 2.7830047607421875, + "host_disk_used_gb": 2.0351943969726562, + "tap_count": 2, + "firecracker_rss_mb": 94.62109375, + "fc_socket_count": 2, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_ok", + "vm_seq": 3, + "boot_time_s": 2.976, + "density_at_boot": 3, + "timestamp": 1773033034.1608803, + "vms_running": 3, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 2.8303489685058594, + "host_disk_used_gb": 2.0352554321289062, + "tap_count": 3, + "firecracker_rss_mb": 141.55078125, + "fc_socket_count": 3, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_ok", + "vm_seq": 4, + "boot_time_s": 3.058, + "density_at_boot": 4, + "timestamp": 1773033037.361834, + "vms_running": 4, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 2.877735137939453, + "host_disk_used_gb": 2.0352554321289062, + "tap_count": 4, + "firecracker_rss_mb": 188.73828125, + "fc_socket_count": 4, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_ok", + "vm_seq": 5, + "boot_time_s": 3.218, + "density_at_boot": 5, + "timestamp": 1773033040.722996, + "vms_running": 5, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 2.936298370361328, + "host_disk_used_gb": 2.0353164672851562, + "tap_count": 5, + "firecracker_rss_mb": 235.14453125, + "fc_socket_count": 5, + "sqlite_db_kb": 52.0 + }, + { + "event": "boot_ok", + "vm_seq": 6, + "boot_time_s": 2.977, + "density_at_boot": 6, + "timestamp": 1773033043.8448167, + "vms_running": 6, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 2.993610382080078, + "host_disk_used_gb": 2.0353164672851562, + "tap_count": 6, + "firecracker_rss_mb": 282.84765625, + "fc_socket_count": 6, + "sqlite_db_kb": 60.0 + }, + { + "event": "boot_ok", + "vm_seq": 7, + "boot_time_s": 2.973, + "density_at_boot": 7, + "timestamp": 1773033046.962404, + "vms_running": 7, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.0497703552246094, + "host_disk_used_gb": 2.0353775024414062, + "tap_count": 7, + "firecracker_rss_mb": 330.29296875, + "fc_socket_count": 7, + "sqlite_db_kb": 60.0 + }, + { + "event": "boot_ok", + "vm_seq": 8, + "boot_time_s": 2.117, + "density_at_boot": 8, + "timestamp": 1773033049.2229445, + "vms_running": 8, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.100330352783203, + "host_disk_used_gb": 2.0353775024414062, + "tap_count": 8, + "firecracker_rss_mb": 377.22265625, + "fc_socket_count": 8, + "sqlite_db_kb": 60.0 + }, + { + "event": "boot_ok", + "vm_seq": 9, + "boot_time_s": 3.035, + "density_at_boot": 9, + "timestamp": 1773033052.4030461, + "vms_running": 9, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.1607666015625, + "host_disk_used_gb": 2.0354385375976562, + "tap_count": 9, + "firecracker_rss_mb": 424.40625, + "fc_socket_count": 9, + "sqlite_db_kb": 60.0 + }, + { + "event": "boot_ok", + "vm_seq": 10, + "boot_time_s": 3.054, + "density_at_boot": 10, + "timestamp": 1773033055.6033888, + "vms_running": 10, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 3.2168922424316406, + "host_disk_used_gb": 2.0354385375976562, + "tap_count": 10, + "firecracker_rss_mb": 470.81640625, + "fc_socket_count": 10, + "sqlite_db_kb": 60.0 + }, + { + "event": "boot_ok", + "vm_seq": 11, + "boot_time_s": 2.973, + "density_at_boot": 11, + "timestamp": 1773033058.7229776, + "vms_running": 11, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.2730178833007812, + "host_disk_used_gb": 2.0354995727539062, + "tap_count": 11, + "firecracker_rss_mb": 517.48046875, + "fc_socket_count": 11, + "sqlite_db_kb": 64.0 + }, + { + "event": "boot_ok", + "vm_seq": 12, + "boot_time_s": 2.97, + "density_at_boot": 12, + "timestamp": 1773033061.8406522, + "vms_running": 12, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.3240890502929688, + "host_disk_used_gb": 2.0354995727539062, + "tap_count": 12, + "firecracker_rss_mb": 564.671875, + "fc_socket_count": 12, + "sqlite_db_kb": 64.0 + }, + { + "event": "boot_ok", + "vm_seq": 13, + "boot_time_s": 2.975, + "density_at_boot": 13, + "timestamp": 1773033064.9625804, + "vms_running": 13, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.38525390625, + "host_disk_used_gb": 2.0359878540039062, + "tap_count": 13, + "firecracker_rss_mb": 612.37109375, + "fc_socket_count": 13, + "sqlite_db_kb": 64.0 + }, + { + "event": "boot_ok", + "vm_seq": 14, + "boot_time_s": 2.053, + "density_at_boot": 14, + "timestamp": 1773033067.1609247, + "vms_running": 14, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.4327239990234375, + "host_disk_used_gb": 2.0359878540039062, + "tap_count": 14, + "firecracker_rss_mb": 660.078125, + "fc_socket_count": 14, + "sqlite_db_kb": 64.0 + }, + { + "event": "boot_ok", + "vm_seq": 15, + "boot_time_s": 2.075, + "density_at_boot": 15, + "timestamp": 1773033069.3810976, + "vms_running": 15, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.485340118408203, + "host_disk_used_gb": 2.0359878540039062, + "tap_count": 15, + "firecracker_rss_mb": 707.0, + "fc_socket_count": 15, + "sqlite_db_kb": 64.0 + }, + { + "event": "boot_ok", + "vm_seq": 16, + "boot_time_s": 1.971, + "density_at_boot": 16, + "timestamp": 1773033071.4986103, + "vms_running": 16, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 3.5328598022460938, + "host_disk_used_gb": 2.0359878540039062, + "tap_count": 16, + "firecracker_rss_mb": 754.95703125, + "fc_socket_count": 16, + "sqlite_db_kb": 68.0 + }, + { + "event": "boot_ok", + "vm_seq": 17, + "boot_time_s": 2.139, + "density_at_boot": 17, + "timestamp": 1773033073.7816103, + "vms_running": 17, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.590251922607422, + "host_disk_used_gb": 2.0359878540039062, + "tap_count": 17, + "firecracker_rss_mb": 801.8828125, + "fc_socket_count": 17, + "sqlite_db_kb": 68.0 + }, + { + "event": "boot_ok", + "vm_seq": 18, + "boot_time_s": 2.993, + "density_at_boot": 18, + "timestamp": 1773033076.9206767, + "vms_running": 18, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.6424293518066406, + "host_disk_used_gb": 2.0359878540039062, + "tap_count": 18, + "firecracker_rss_mb": 848.80859375, + "fc_socket_count": 18, + "sqlite_db_kb": 68.0 + }, + { + "event": "boot_ok", + "vm_seq": 19, + "boot_time_s": 3.095, + "density_at_boot": 19, + "timestamp": 1773033080.1619146, + "vms_running": 19, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.6910667419433594, + "host_disk_used_gb": 2.0359878540039062, + "tap_count": 19, + "firecracker_rss_mb": 895.73828125, + "fc_socket_count": 19, + "sqlite_db_kb": 68.0 + }, + { + "event": "boot_ok", + "vm_seq": 20, + "boot_time_s": 2.93, + "density_at_boot": 20, + "timestamp": 1773033083.2386947, + "vms_running": 20, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 3.746997833251953, + "host_disk_used_gb": 2.0359878540039062, + "tap_count": 20, + "firecracker_rss_mb": 943.69921875, + "fc_socket_count": 20, + "sqlite_db_kb": 68.0 + }, + { + "event": "boot_ok", + "vm_seq": 21, + "boot_time_s": 2.239, + "density_at_boot": 21, + "timestamp": 1773033085.62172, + "vms_running": 21, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 3.7973709106445312, + "host_disk_used_gb": 2.0359878540039062, + "tap_count": 21, + "firecracker_rss_mb": 990.62890625, + "fc_socket_count": 21, + "sqlite_db_kb": 72.0 + }, + { + "event": "boot_ok", + "vm_seq": 22, + "boot_time_s": 3.034, + "density_at_boot": 22, + "timestamp": 1773033088.8028262, + "vms_running": 22, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 3.8506126403808594, + "host_disk_used_gb": 2.0359878540039062, + "tap_count": 22, + "firecracker_rss_mb": 1037.55078125, + "fc_socket_count": 22, + "sqlite_db_kb": 72.0 + }, + { + "event": "boot_ok", + "vm_seq": 23, + "boot_time_s": 2.17, + "density_at_boot": 23, + "timestamp": 1773033091.1226404, + "vms_running": 23, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.9020309448242188, + "host_disk_used_gb": 2.0359878540039062, + "tap_count": 23, + "firecracker_rss_mb": 1084.47265625, + "fc_socket_count": 23, + "sqlite_db_kb": 72.0 + }, + { + "event": "boot_ok", + "vm_seq": 24, + "boot_time_s": 3.054, + "density_at_boot": 24, + "timestamp": 1773033094.3238337, + "vms_running": 24, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.951507568359375, + "host_disk_used_gb": 2.0360107421875, + "tap_count": 24, + "firecracker_rss_mb": 1131.40234375, + "fc_socket_count": 24, + "sqlite_db_kb": 72.0 + }, + { + "event": "boot_ok", + "vm_seq": 25, + "boot_time_s": 3.051, + "density_at_boot": 25, + "timestamp": 1773033097.5225387, + "vms_running": 25, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 4.002567291259766, + "host_disk_used_gb": 2.035991668701172, + "tap_count": 25, + "firecracker_rss_mb": 1178.328125, + "fc_socket_count": 25, + "sqlite_db_kb": 72.0 + }, + { + "event": "boot_ok", + "vm_seq": 26, + "boot_time_s": 2.231, + "density_at_boot": 26, + "timestamp": 1773033099.9013627, + "vms_running": 26, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.061363220214844, + "host_disk_used_gb": 2.035991668701172, + "tap_count": 26, + "firecracker_rss_mb": 1225.76953125, + "fc_socket_count": 26, + "sqlite_db_kb": 76.0 + }, + { + "event": "boot_ok", + "vm_seq": 27, + "boot_time_s": 3.073, + "density_at_boot": 27, + "timestamp": 1773033103.1235607, + "vms_running": 27, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.113433837890625, + "host_disk_used_gb": 2.035991668701172, + "tap_count": 27, + "firecracker_rss_mb": 1272.69921875, + "fc_socket_count": 27, + "sqlite_db_kb": 76.0 + }, + { + "event": "boot_ok", + "vm_seq": 28, + "boot_time_s": 3.128, + "density_at_boot": 28, + "timestamp": 1773033106.4006462, + "vms_running": 28, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.165180206298828, + "host_disk_used_gb": 2.035991668701172, + "tap_count": 28, + "firecracker_rss_mb": 1319.3671875, + "fc_socket_count": 28, + "sqlite_db_kb": 76.0 + }, + { + "event": "boot_ok", + "vm_seq": 29, + "boot_time_s": 3.056, + "density_at_boot": 29, + "timestamp": 1773033109.6045015, + "vms_running": 29, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.213230133056641, + "host_disk_used_gb": 2.036968231201172, + "tap_count": 29, + "firecracker_rss_mb": 1366.80859375, + "fc_socket_count": 29, + "sqlite_db_kb": 76.0 + }, + { + "event": "boot_ok", + "vm_seq": 30, + "boot_time_s": 3.051, + "density_at_boot": 30, + "timestamp": 1773033112.804648, + "vms_running": 30, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.257366180419922, + "host_disk_used_gb": 2.036968231201172, + "tap_count": 30, + "firecracker_rss_mb": 1413.48046875, + "fc_socket_count": 30, + "sqlite_db_kb": 76.0 + }, + { + "event": "boot_ok", + "vm_seq": 31, + "boot_time_s": 3.05, + "density_at_boot": 31, + "timestamp": 1773033116.0029683, + "vms_running": 31, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.305568695068359, + "host_disk_used_gb": 2.036968231201172, + "tap_count": 31, + "firecracker_rss_mb": 1460.40234375, + "fc_socket_count": 31, + "sqlite_db_kb": 80.0 + }, + { + "event": "boot_ok", + "vm_seq": 32, + "boot_time_s": 3.052, + "density_at_boot": 32, + "timestamp": 1773033119.2038808, + "vms_running": 32, + "host_cpu_pct": 0.5, + "host_mem_used_gb": 4.353710174560547, + "host_disk_used_gb": 2.0369720458984375, + "tap_count": 32, + "firecracker_rss_mb": 1507.328125, + "fc_socket_count": 32, + "sqlite_db_kb": 80.0 + }, + { + "event": "boot_ok", + "vm_seq": 33, + "boot_time_s": 3.13, + "density_at_boot": 33, + "timestamp": 1773033122.4816763, + "vms_running": 33, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 4.400493621826172, + "host_disk_used_gb": 2.0369720458984375, + "tap_count": 33, + "firecracker_rss_mb": 1554.2578125, + "fc_socket_count": 33, + "sqlite_db_kb": 80.0 + }, + { + "event": "boot_ok", + "vm_seq": 34, + "boot_time_s": 3.006, + "density_at_boot": 34, + "timestamp": 1773033125.6367514, + "vms_running": 34, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 4.442150115966797, + "host_disk_used_gb": 2.0369720458984375, + "tap_count": 34, + "firecracker_rss_mb": 1601.703125, + "fc_socket_count": 34, + "sqlite_db_kb": 80.0 + }, + { + "event": "boot_ok", + "vm_seq": 35, + "boot_time_s": 3.019, + "density_at_boot": 35, + "timestamp": 1773033128.801706, + "vms_running": 35, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.484050750732422, + "host_disk_used_gb": 2.0369720458984375, + "tap_count": 35, + "firecracker_rss_mb": 1648.6328125, + "fc_socket_count": 35, + "sqlite_db_kb": 80.0 + }, + { + "event": "boot_ok", + "vm_seq": 36, + "boot_time_s": 3.05, + "density_at_boot": 36, + "timestamp": 1773033132.0019503, + "vms_running": 36, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.5365753173828125, + "host_disk_used_gb": 2.0369720458984375, + "tap_count": 36, + "firecracker_rss_mb": 1695.5625, + "fc_socket_count": 36, + "sqlite_db_kb": 84.0 + }, + { + "event": "boot_ok", + "vm_seq": 37, + "boot_time_s": 3.048, + "density_at_boot": 37, + "timestamp": 1773033135.201699, + "vms_running": 37, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 4.581813812255859, + "host_disk_used_gb": 2.0369720458984375, + "tap_count": 37, + "firecracker_rss_mb": 1742.7421875, + "fc_socket_count": 37, + "sqlite_db_kb": 84.0 + }, + { + "event": "boot_ok", + "vm_seq": 38, + "boot_time_s": 3.051, + "density_at_boot": 38, + "timestamp": 1773033138.4028268, + "vms_running": 38, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.623851776123047, + "host_disk_used_gb": 2.0369720458984375, + "tap_count": 38, + "firecracker_rss_mb": 1789.671875, + "fc_socket_count": 38, + "sqlite_db_kb": 84.0 + }, + { + "event": "boot_ok", + "vm_seq": 39, + "boot_time_s": 3.051, + "density_at_boot": 39, + "timestamp": 1773033141.6038764, + "vms_running": 39, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.667224884033203, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 39, + "firecracker_rss_mb": 1836.34375, + "fc_socket_count": 39, + "sqlite_db_kb": 84.0 + }, + { + "event": "boot_ok", + "vm_seq": 40, + "boot_time_s": 3.047, + "density_at_boot": 40, + "timestamp": 1773033144.8007727, + "vms_running": 40, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.7087860107421875, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 40, + "firecracker_rss_mb": 1883.5234375, + "fc_socket_count": 40, + "sqlite_db_kb": 84.0 + }, + { + "event": "boot_ok", + "vm_seq": 41, + "boot_time_s": 3.051, + "density_at_boot": 41, + "timestamp": 1773033148.0039015, + "vms_running": 41, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 4.762592315673828, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 41, + "firecracker_rss_mb": 1930.19140625, + "fc_socket_count": 41, + "sqlite_db_kb": 88.0 + }, + { + "event": "boot_ok", + "vm_seq": 42, + "boot_time_s": 3.047, + "density_at_boot": 42, + "timestamp": 1773033151.2032633, + "vms_running": 42, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.809150695800781, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 42, + "firecracker_rss_mb": 1977.37890625, + "fc_socket_count": 42, + "sqlite_db_kb": 88.0 + }, + { + "event": "boot_ok", + "vm_seq": 43, + "boot_time_s": 3.048, + "density_at_boot": 43, + "timestamp": 1773033154.4018452, + "vms_running": 43, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.859760284423828, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 43, + "firecracker_rss_mb": 2024.8203125, + "fc_socket_count": 43, + "sqlite_db_kb": 88.0 + }, + { + "event": "boot_ok", + "vm_seq": 44, + "boot_time_s": 3.051, + "density_at_boot": 44, + "timestamp": 1773033157.601361, + "vms_running": 44, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.906074523925781, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 44, + "firecracker_rss_mb": 2071.4921875, + "fc_socket_count": 44, + "sqlite_db_kb": 88.0 + }, + { + "event": "boot_ok", + "vm_seq": 45, + "boot_time_s": 3.009, + "density_at_boot": 45, + "timestamp": 1773033160.7611601, + "vms_running": 45, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 4.957592010498047, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 45, + "firecracker_rss_mb": 2118.421875, + "fc_socket_count": 45, + "sqlite_db_kb": 88.0 + }, + { + "event": "boot_ok", + "vm_seq": 46, + "boot_time_s": 3.171, + "density_at_boot": 46, + "timestamp": 1773033164.0836096, + "vms_running": 46, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.002906799316406, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 46, + "firecracker_rss_mb": 2165.34765625, + "fc_socket_count": 46, + "sqlite_db_kb": 92.0 + }, + { + "event": "boot_ok", + "vm_seq": 47, + "boot_time_s": 3.046, + "density_at_boot": 47, + "timestamp": 1773033167.2820804, + "vms_running": 47, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.047607421875, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 47, + "firecracker_rss_mb": 2212.53515625, + "fc_socket_count": 47, + "sqlite_db_kb": 92.0 + }, + { + "event": "boot_ok", + "vm_seq": 48, + "boot_time_s": 3.047, + "density_at_boot": 48, + "timestamp": 1773033170.4816763, + "vms_running": 48, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.085933685302734, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 48, + "firecracker_rss_mb": 2259.46484375, + "fc_socket_count": 48, + "sqlite_db_kb": 92.0 + }, + { + "event": "boot_ok", + "vm_seq": 49, + "boot_time_s": 3.128, + "density_at_boot": 49, + "timestamp": 1773033173.761677, + "vms_running": 49, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.1317138671875, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 49, + "firecracker_rss_mb": 2306.13671875, + "fc_socket_count": 49, + "sqlite_db_kb": 92.0 + }, + { + "event": "boot_ok", + "vm_seq": 50, + "boot_time_s": 3.05, + "density_at_boot": 50, + "timestamp": 1773033176.9637046, + "vms_running": 50, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 5.177833557128906, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 50, + "firecracker_rss_mb": 2353.83984375, + "fc_socket_count": 50, + "sqlite_db_kb": 92.0 + }, + { + "event": "boot_ok", + "vm_seq": 51, + "boot_time_s": 3.046, + "density_at_boot": 51, + "timestamp": 1773033180.1613538, + "vms_running": 51, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 5.221309661865234, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 51, + "firecracker_rss_mb": 2400.51171875, + "fc_socket_count": 51, + "sqlite_db_kb": 96.0 + }, + { + "event": "boot_ok", + "vm_seq": 52, + "boot_time_s": 3.05, + "density_at_boot": 52, + "timestamp": 1773033183.3628714, + "vms_running": 52, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 5.270954132080078, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 52, + "firecracker_rss_mb": 2447.18359375, + "fc_socket_count": 52, + "sqlite_db_kb": 96.0 + }, + { + "event": "boot_ok", + "vm_seq": 53, + "boot_time_s": 2.111, + "density_at_boot": 53, + "timestamp": 1773033185.6233401, + "vms_running": 53, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.3211517333984375, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 53, + "firecracker_rss_mb": 2494.11328125, + "fc_socket_count": 53, + "sqlite_db_kb": 96.0 + }, + { + "event": "boot_ok", + "vm_seq": 54, + "boot_time_s": 3.024, + "density_at_boot": 54, + "timestamp": 1773033188.8009796, + "vms_running": 54, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 5.3712005615234375, + "host_disk_used_gb": 2.0447845458984375, + "tap_count": 54, + "firecracker_rss_mb": 2541.296875, + "fc_socket_count": 54, + "sqlite_db_kb": 96.0 + }, + { + "event": "boot_ok", + "vm_seq": 55, + "boot_time_s": 3.133, + "density_at_boot": 55, + "timestamp": 1773033192.0828273, + "vms_running": 55, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.407588958740234, + "host_disk_used_gb": 2.0448074340820312, + "tap_count": 55, + "firecracker_rss_mb": 2587.96484375, + "fc_socket_count": 55, + "sqlite_db_kb": 96.0 + }, + { + "event": "boot_ok", + "vm_seq": 56, + "boot_time_s": 3.007, + "density_at_boot": 56, + "timestamp": 1773033195.2397757, + "vms_running": 56, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.461029052734375, + "host_disk_used_gb": 2.0447463989257812, + "tap_count": 56, + "firecracker_rss_mb": 2635.40625, + "fc_socket_count": 56, + "sqlite_db_kb": 100.0 + }, + { + "event": "boot_ok", + "vm_seq": 57, + "boot_time_s": 3.012, + "density_at_boot": 57, + "timestamp": 1773033198.4022074, + "vms_running": 57, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.516471862792969, + "host_disk_used_gb": 2.0447463989257812, + "tap_count": 57, + "firecracker_rss_mb": 2682.5859375, + "fc_socket_count": 57, + "sqlite_db_kb": 100.0 + }, + { + "event": "boot_ok", + "vm_seq": 58, + "boot_time_s": 2.065, + "density_at_boot": 58, + "timestamp": 1773033200.6183062, + "vms_running": 58, + "host_cpu_pct": 0.5, + "host_mem_used_gb": 5.566684722900391, + "host_disk_used_gb": 2.0447463989257812, + "tap_count": 58, + "firecracker_rss_mb": 2730.546875, + "fc_socket_count": 58, + "sqlite_db_kb": 100.0 + }, + { + "event": "boot_ok", + "vm_seq": 59, + "boot_time_s": 2.995, + "density_at_boot": 59, + "timestamp": 1773033203.7625341, + "vms_running": 59, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.616291046142578, + "host_disk_used_gb": 2.0447463989257812, + "tap_count": 59, + "firecracker_rss_mb": 2777.98828125, + "fc_socket_count": 59, + "sqlite_db_kb": 100.0 + }, + { + "event": "boot_ok", + "vm_seq": 60, + "boot_time_s": 3.005, + "density_at_boot": 60, + "timestamp": 1773033206.9203105, + "vms_running": 60, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.656620025634766, + "host_disk_used_gb": 2.0466995239257812, + "tap_count": 60, + "firecracker_rss_mb": 2824.65234375, + "fc_socket_count": 60, + "sqlite_db_kb": 100.0 + }, + { + "event": "boot_ok", + "vm_seq": 61, + "boot_time_s": 2.968, + "density_at_boot": 61, + "timestamp": 1773033210.0405595, + "vms_running": 61, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.709175109863281, + "host_disk_used_gb": 2.0466995239257812, + "tap_count": 61, + "firecracker_rss_mb": 2871.06640625, + "fc_socket_count": 61, + "sqlite_db_kb": 112.0 + }, + { + "event": "boot_ok", + "vm_seq": 62, + "boot_time_s": 2.968, + "density_at_boot": 62, + "timestamp": 1773033213.160054, + "vms_running": 62, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.756771087646484, + "host_disk_used_gb": 2.0466995239257812, + "tap_count": 62, + "firecracker_rss_mb": 2917.48046875, + "fc_socket_count": 62, + "sqlite_db_kb": 112.0 + }, + { + "event": "boot_ok", + "vm_seq": 63, + "boot_time_s": 3.09, + "density_at_boot": 63, + "timestamp": 1773033216.403268, + "vms_running": 63, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 5.798877716064453, + "host_disk_used_gb": 2.0466995239257812, + "tap_count": 63, + "firecracker_rss_mb": 2964.92578125, + "fc_socket_count": 63, + "sqlite_db_kb": 112.0 + }, + { + "event": "boot_ok", + "vm_seq": 64, + "boot_time_s": 2.064, + "density_at_boot": 64, + "timestamp": 1773033218.6201143, + "vms_running": 64, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 5.846565246582031, + "host_disk_used_gb": 2.0466995239257812, + "tap_count": 64, + "firecracker_rss_mb": 3012.11328125, + "fc_socket_count": 64, + "sqlite_db_kb": 112.0 + }, + { + "event": "boot_ok", + "vm_seq": 65, + "boot_time_s": 2.039, + "density_at_boot": 65, + "timestamp": 1773033220.8116317, + "vms_running": 65, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.889556884765625, + "host_disk_used_gb": 2.0466995239257812, + "tap_count": 65, + "firecracker_rss_mb": 3058.78515625, + "fc_socket_count": 65, + "sqlite_db_kb": 112.0 + }, + { + "event": "boot_ok", + "vm_seq": 66, + "boot_time_s": 3.036, + "density_at_boot": 66, + "timestamp": 1773033224.000127, + "vms_running": 66, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 5.9334716796875, + "host_disk_used_gb": 2.0466995239257812, + "tap_count": 66, + "firecracker_rss_mb": 3105.72265625, + "fc_socket_count": 66, + "sqlite_db_kb": 116.0 + }, + { + "event": "boot_ok", + "vm_seq": 67, + "boot_time_s": 2.274, + "density_at_boot": 67, + "timestamp": 1773033226.4246364, + "vms_running": 67, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 5.988681793212891, + "host_disk_used_gb": 2.0466995239257812, + "tap_count": 67, + "firecracker_rss_mb": 3152.6484375, + "fc_socket_count": 67, + "sqlite_db_kb": 116.0 + }, + { + "event": "boot_ok", + "vm_seq": 68, + "boot_time_s": 3.103, + "density_at_boot": 68, + "timestamp": 1773033229.6829405, + "vms_running": 68, + "host_cpu_pct": 0.5, + "host_mem_used_gb": 6.028797149658203, + "host_disk_used_gb": 2.0466995239257812, + "tap_count": 68, + "firecracker_rss_mb": 3199.83203125, + "fc_socket_count": 68, + "sqlite_db_kb": 116.0 + }, + { + "event": "boot_ok", + "vm_seq": 69, + "boot_time_s": 3.05, + "density_at_boot": 69, + "timestamp": 1773033232.883963, + "vms_running": 69, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.079540252685547, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 69, + "firecracker_rss_mb": 3247.015625, + "fc_socket_count": 69, + "sqlite_db_kb": 116.0 + }, + { + "event": "boot_ok", + "vm_seq": 70, + "boot_time_s": 3.047, + "density_at_boot": 70, + "timestamp": 1773033236.083304, + "vms_running": 70, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.1288909912109375, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 70, + "firecracker_rss_mb": 3293.94140625, + "fc_socket_count": 70, + "sqlite_db_kb": 116.0 + }, + { + "event": "boot_ok", + "vm_seq": 71, + "boot_time_s": 3.045, + "density_at_boot": 71, + "timestamp": 1773033239.2812428, + "vms_running": 71, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.164848327636719, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 71, + "firecracker_rss_mb": 3340.8671875, + "fc_socket_count": 71, + "sqlite_db_kb": 120.0 + }, + { + "event": "boot_ok", + "vm_seq": 72, + "boot_time_s": 3.007, + "density_at_boot": 72, + "timestamp": 1773033242.4412746, + "vms_running": 72, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.211551666259766, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 72, + "firecracker_rss_mb": 3388.05078125, + "fc_socket_count": 72, + "sqlite_db_kb": 120.0 + }, + { + "event": "boot_ok", + "vm_seq": 73, + "boot_time_s": 3.088, + "density_at_boot": 73, + "timestamp": 1773033245.683507, + "vms_running": 73, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.2582550048828125, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 73, + "firecracker_rss_mb": 3435.23828125, + "fc_socket_count": 73, + "sqlite_db_kb": 120.0 + }, + { + "event": "boot_ok", + "vm_seq": 74, + "boot_time_s": 2.142, + "density_at_boot": 74, + "timestamp": 1773033247.978695, + "vms_running": 74, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.303688049316406, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 74, + "firecracker_rss_mb": 3481.90625, + "fc_socket_count": 74, + "sqlite_db_kb": 120.0 + }, + { + "event": "boot_ok", + "vm_seq": 75, + "boot_time_s": 2.031, + "density_at_boot": 75, + "timestamp": 1773033250.163482, + "vms_running": 75, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.348236083984375, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 75, + "firecracker_rss_mb": 3528.8359375, + "fc_socket_count": 75, + "sqlite_db_kb": 128.0 + }, + { + "event": "boot_ok", + "vm_seq": 76, + "boot_time_s": 2.186, + "density_at_boot": 76, + "timestamp": 1773033252.5023098, + "vms_running": 76, + "host_cpu_pct": 0.5, + "host_mem_used_gb": 6.395847320556641, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 76, + "firecracker_rss_mb": 3576.01953125, + "fc_socket_count": 76, + "sqlite_db_kb": 132.0 + }, + { + "event": "boot_ok", + "vm_seq": 77, + "boot_time_s": 3.112, + "density_at_boot": 77, + "timestamp": 1773033255.7646894, + "vms_running": 77, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.447135925292969, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 77, + "firecracker_rss_mb": 3622.69140625, + "fc_socket_count": 77, + "sqlite_db_kb": 132.0 + }, + { + "event": "boot_ok", + "vm_seq": 78, + "boot_time_s": 2.997, + "density_at_boot": 78, + "timestamp": 1773033258.9184716, + "vms_running": 78, + "host_cpu_pct": 0.6, + "host_mem_used_gb": 6.4872283935546875, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 78, + "firecracker_rss_mb": 3670.39453125, + "fc_socket_count": 78, + "sqlite_db_kb": 132.0 + }, + { + "event": "boot_ok", + "vm_seq": 79, + "boot_time_s": 3.011, + "density_at_boot": 79, + "timestamp": 1773033262.0819917, + "vms_running": 79, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.534366607666016, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 79, + "firecracker_rss_mb": 3717.3203125, + "fc_socket_count": 79, + "sqlite_db_kb": 132.0 + }, + { + "event": "boot_ok", + "vm_seq": 80, + "boot_time_s": 3.082, + "density_at_boot": 80, + "timestamp": 1773033265.3196058, + "vms_running": 80, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.580863952636719, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 80, + "firecracker_rss_mb": 3763.73046875, + "fc_socket_count": 80, + "sqlite_db_kb": 132.0 + }, + { + "event": "boot_ok", + "vm_seq": 81, + "boot_time_s": 2.965, + "density_at_boot": 81, + "timestamp": 1773033268.439817, + "vms_running": 81, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.624134063720703, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 81, + "firecracker_rss_mb": 3810.14453125, + "fc_socket_count": 81, + "sqlite_db_kb": 136.0 + }, + { + "event": "boot_ok", + "vm_seq": 82, + "boot_time_s": 3.007, + "density_at_boot": 82, + "timestamp": 1773033271.6033342, + "vms_running": 82, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 6.6730499267578125, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 82, + "firecracker_rss_mb": 3856.5546875, + "fc_socket_count": 82, + "sqlite_db_kb": 136.0 + }, + { + "event": "boot_ok", + "vm_seq": 83, + "boot_time_s": 2.999, + "density_at_boot": 83, + "timestamp": 1773033274.7591214, + "vms_running": 83, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.721355438232422, + "host_disk_used_gb": 2.045482635498047, + "tap_count": 83, + "firecracker_rss_mb": 3902.96875, + "fc_socket_count": 83, + "sqlite_db_kb": 136.0 + }, + { + "event": "boot_ok", + "vm_seq": 84, + "boot_time_s": 3.009, + "density_at_boot": 84, + "timestamp": 1773033277.9239209, + "vms_running": 84, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.7634429931640625, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 84, + "firecracker_rss_mb": 3950.15234375, + "fc_socket_count": 84, + "sqlite_db_kb": 136.0 + }, + { + "event": "boot_ok", + "vm_seq": 85, + "boot_time_s": 3.122, + "density_at_boot": 85, + "timestamp": 1773033281.2036185, + "vms_running": 85, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 6.805477142333984, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 85, + "firecracker_rss_mb": 3996.56640625, + "fc_socket_count": 85, + "sqlite_db_kb": 136.0 + }, + { + "event": "boot_ok", + "vm_seq": 86, + "boot_time_s": 3.124, + "density_at_boot": 86, + "timestamp": 1773033284.4832091, + "vms_running": 86, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 6.858394622802734, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 86, + "firecracker_rss_mb": 4042.98046875, + "fc_socket_count": 86, + "sqlite_db_kb": 140.0 + }, + { + "event": "boot_ok", + "vm_seq": 87, + "boot_time_s": 2.996, + "density_at_boot": 87, + "timestamp": 1773033287.63918, + "vms_running": 87, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 6.907325744628906, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 87, + "firecracker_rss_mb": 4090.42578125, + "fc_socket_count": 87, + "sqlite_db_kb": 140.0 + }, + { + "event": "boot_ok", + "vm_seq": 88, + "boot_time_s": 3.25, + "density_at_boot": 88, + "timestamp": 1773033291.0429933, + "vms_running": 88, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 6.96649169921875, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 88, + "firecracker_rss_mb": 4136.8359375, + "fc_socket_count": 88, + "sqlite_db_kb": 140.0 + }, + { + "event": "boot_ok", + "vm_seq": 89, + "boot_time_s": 3.04, + "density_at_boot": 89, + "timestamp": 1773033294.2414389, + "vms_running": 89, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.016696929931641, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 89, + "firecracker_rss_mb": 4184.28125, + "fc_socket_count": 89, + "sqlite_db_kb": 140.0 + }, + { + "event": "boot_ok", + "vm_seq": 90, + "boot_time_s": 3.127, + "density_at_boot": 90, + "timestamp": 1773033297.5242565, + "vms_running": 90, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.061672210693359, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 90, + "firecracker_rss_mb": 4230.953125, + "fc_socket_count": 90, + "sqlite_db_kb": 140.0 + }, + { + "event": "boot_ok", + "vm_seq": 91, + "boot_time_s": 3.121, + "density_at_boot": 91, + "timestamp": 1773033300.8026967, + "vms_running": 91, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 7.102863311767578, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 91, + "firecracker_rss_mb": 4278.13671875, + "fc_socket_count": 91, + "sqlite_db_kb": 144.0 + }, + { + "event": "boot_ok", + "vm_seq": 92, + "boot_time_s": 3.125, + "density_at_boot": 92, + "timestamp": 1773033304.0850515, + "vms_running": 92, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.147693634033203, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 92, + "firecracker_rss_mb": 4325.83984375, + "fc_socket_count": 92, + "sqlite_db_kb": 144.0 + }, + { + "event": "boot_ok", + "vm_seq": 93, + "boot_time_s": 2.18, + "density_at_boot": 93, + "timestamp": 1773033306.4212089, + "vms_running": 93, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 7.193912506103516, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 93, + "firecracker_rss_mb": 4372.25, + "fc_socket_count": 93, + "sqlite_db_kb": 144.0 + }, + { + "event": "boot_ok", + "vm_seq": 94, + "boot_time_s": 3.104, + "density_at_boot": 94, + "timestamp": 1773033309.6850111, + "vms_running": 94, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.243099212646484, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 94, + "firecracker_rss_mb": 4418.40625, + "fc_socket_count": 94, + "sqlite_db_kb": 144.0 + }, + { + "event": "boot_ok", + "vm_seq": 95, + "boot_time_s": 2.189, + "density_at_boot": 95, + "timestamp": 1773033312.0331566, + "vms_running": 95, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.286735534667969, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 95, + "firecracker_rss_mb": 4465.3359375, + "fc_socket_count": 95, + "sqlite_db_kb": 144.0 + }, + { + "event": "boot_ok", + "vm_seq": 96, + "boot_time_s": 3.088, + "density_at_boot": 96, + "timestamp": 1773033315.280903, + "vms_running": 96, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.325340270996094, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 96, + "firecracker_rss_mb": 4512.265625, + "fc_socket_count": 96, + "sqlite_db_kb": 148.0 + }, + { + "event": "boot_ok", + "vm_seq": 97, + "boot_time_s": 3.044, + "density_at_boot": 97, + "timestamp": 1773033318.4851127, + "vms_running": 97, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 7.373432159423828, + "host_disk_used_gb": 2.046703338623047, + "tap_count": 97, + "firecracker_rss_mb": 4559.19921875, + "fc_socket_count": 97, + "sqlite_db_kb": 148.0 + }, + { + "event": "boot_ok", + "vm_seq": 98, + "boot_time_s": 3.04, + "density_at_boot": 98, + "timestamp": 1773033321.6831727, + "vms_running": 98, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 7.423236846923828, + "host_disk_used_gb": 2.0467262268066406, + "tap_count": 98, + "firecracker_rss_mb": 4606.38671875, + "fc_socket_count": 98, + "sqlite_db_kb": 148.0 + }, + { + "event": "boot_ok", + "vm_seq": 99, + "boot_time_s": 3.124, + "density_at_boot": 99, + "timestamp": 1773033324.964652, + "vms_running": 99, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.467609405517578, + "host_disk_used_gb": 2.0467071533203125, + "tap_count": 99, + "firecracker_rss_mb": 4653.05859375, + "fc_socket_count": 99, + "sqlite_db_kb": 148.0 + }, + { + "event": "boot_ok", + "vm_seq": 100, + "boot_time_s": 3.122, + "density_at_boot": 100, + "timestamp": 1773033328.2430675, + "vms_running": 100, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 7.511039733886719, + "host_disk_used_gb": 2.0467071533203125, + "tap_count": 100, + "firecracker_rss_mb": 4699.46875, + "fc_socket_count": 100, + "sqlite_db_kb": 148.0 + }, + { + "event": "boot_ok", + "vm_seq": 101, + "boot_time_s": 3.127, + "density_at_boot": 101, + "timestamp": 1773033331.522668, + "vms_running": 101, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 7.559967041015625, + "host_disk_used_gb": 2.0467071533203125, + "tap_count": 101, + "firecracker_rss_mb": 4746.3984375, + "fc_socket_count": 101, + "sqlite_db_kb": 152.0 + }, + { + "event": "boot_ok", + "vm_seq": 102, + "boot_time_s": 2.18, + "density_at_boot": 102, + "timestamp": 1773033333.8610601, + "vms_running": 102, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 7.608421325683594, + "host_disk_used_gb": 2.0467071533203125, + "tap_count": 102, + "firecracker_rss_mb": 4793.0703125, + "fc_socket_count": 102, + "sqlite_db_kb": 152.0 + }, + { + "event": "boot_ok", + "vm_seq": 103, + "boot_time_s": 3.025, + "density_at_boot": 103, + "timestamp": 1773033337.0418603, + "vms_running": 103, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.656368255615234, + "host_disk_used_gb": 2.0467071533203125, + "tap_count": 103, + "firecracker_rss_mb": 4840.76953125, + "fc_socket_count": 103, + "sqlite_db_kb": 152.0 + }, + { + "event": "boot_ok", + "vm_seq": 104, + "boot_time_s": 2.184, + "density_at_boot": 104, + "timestamp": 1773033339.383744, + "vms_running": 104, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 7.709503173828125, + "host_disk_used_gb": 2.0467071533203125, + "tap_count": 104, + "firecracker_rss_mb": 4887.95703125, + "fc_socket_count": 104, + "sqlite_db_kb": 152.0 + }, + { + "event": "boot_ok", + "vm_seq": 105, + "boot_time_s": 3.184, + "density_at_boot": 105, + "timestamp": 1773033342.721289, + "vms_running": 105, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 7.748188018798828, + "host_disk_used_gb": 2.0467071533203125, + "tap_count": 105, + "firecracker_rss_mb": 4934.625, + "fc_socket_count": 105, + "sqlite_db_kb": 152.0 + }, + { + "event": "boot_ok", + "vm_seq": 106, + "boot_time_s": 3.08, + "density_at_boot": 106, + "timestamp": 1773033345.959677, + "vms_running": 106, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 7.791416168212891, + "host_disk_used_gb": 2.0467071533203125, + "tap_count": 106, + "firecracker_rss_mb": 4982.06640625, + "fc_socket_count": 106, + "sqlite_db_kb": 156.0 + }, + { + "event": "boot_ok", + "vm_seq": 107, + "boot_time_s": 2.106, + "density_at_boot": 107, + "timestamp": 1773033348.2202184, + "vms_running": 107, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 7.840679168701172, + "host_disk_used_gb": 2.0467071533203125, + "tap_count": 107, + "firecracker_rss_mb": 5028.4765625, + "fc_socket_count": 107, + "sqlite_db_kb": 156.0 + }, + { + "event": "boot_ok", + "vm_seq": 108, + "boot_time_s": 3.064, + "density_at_boot": 108, + "timestamp": 1773033351.4419806, + "vms_running": 108, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 7.8821258544921875, + "host_disk_used_gb": 2.0467071533203125, + "tap_count": 108, + "firecracker_rss_mb": 5075.40234375, + "fc_socket_count": 108, + "sqlite_db_kb": 156.0 + }, + { + "event": "boot_ok", + "vm_seq": 109, + "boot_time_s": 3.285, + "density_at_boot": 109, + "timestamp": 1773033354.881558, + "vms_running": 109, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 7.931728363037109, + "host_disk_used_gb": 2.0467300415039062, + "tap_count": 109, + "firecracker_rss_mb": 5122.328125, + "fc_socket_count": 109, + "sqlite_db_kb": 156.0 + }, + { + "event": "boot_ok", + "vm_seq": 110, + "boot_time_s": 2.181, + "density_at_boot": 110, + "timestamp": 1773033357.2215796, + "vms_running": 110, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 7.985015869140625, + "host_disk_used_gb": 2.0467300415039062, + "tap_count": 110, + "firecracker_rss_mb": 5169.25, + "fc_socket_count": 110, + "sqlite_db_kb": 156.0 + }, + { + "event": "boot_ok", + "vm_seq": 111, + "boot_time_s": 3.026, + "density_at_boot": 111, + "timestamp": 1773033360.4034083, + "vms_running": 111, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 8.026233673095703, + "host_disk_used_gb": 2.0467300415039062, + "tap_count": 111, + "firecracker_rss_mb": 5215.921875, + "fc_socket_count": 111, + "sqlite_db_kb": 160.0 + }, + { + "event": "boot_ok", + "vm_seq": 112, + "boot_time_s": 3.039, + "density_at_boot": 112, + "timestamp": 1773033363.60248, + "vms_running": 112, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 8.06964111328125, + "host_disk_used_gb": 2.0467300415039062, + "tap_count": 112, + "firecracker_rss_mb": 5262.58984375, + "fc_socket_count": 112, + "sqlite_db_kb": 160.0 + }, + { + "event": "boot_ok", + "vm_seq": 113, + "boot_time_s": 3.121, + "density_at_boot": 113, + "timestamp": 1773033366.8814447, + "vms_running": 113, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.123348236083984, + "host_disk_used_gb": 2.0467300415039062, + "tap_count": 113, + "firecracker_rss_mb": 5309.26171875, + "fc_socket_count": 113, + "sqlite_db_kb": 160.0 + }, + { + "event": "boot_ok", + "vm_seq": 114, + "boot_time_s": 3.135, + "density_at_boot": 114, + "timestamp": 1773033370.1746638, + "vms_running": 114, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 8.168418884277344, + "host_disk_used_gb": 2.0467300415039062, + "tap_count": 114, + "firecracker_rss_mb": 5356.19140625, + "fc_socket_count": 114, + "sqlite_db_kb": 160.0 + }, + { + "event": "boot_ok", + "vm_seq": 115, + "boot_time_s": 3.112, + "density_at_boot": 115, + "timestamp": 1773033373.4441922, + "vms_running": 115, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 8.210132598876953, + "host_disk_used_gb": 2.0467300415039062, + "tap_count": 115, + "firecracker_rss_mb": 5403.37890625, + "fc_socket_count": 115, + "sqlite_db_kb": 160.0 + }, + { + "event": "boot_ok", + "vm_seq": 116, + "boot_time_s": 3.121, + "density_at_boot": 116, + "timestamp": 1773033376.7231078, + "vms_running": 116, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.251083374023438, + "host_disk_used_gb": 2.0467300415039062, + "tap_count": 116, + "firecracker_rss_mb": 5450.046875, + "fc_socket_count": 116, + "sqlite_db_kb": 164.0 + }, + { + "event": "boot_ok", + "vm_seq": 117, + "boot_time_s": 3.122, + "density_at_boot": 117, + "timestamp": 1773033380.0035434, + "vms_running": 117, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 8.304367065429688, + "host_disk_used_gb": 2.0467300415039062, + "tap_count": 117, + "firecracker_rss_mb": 5496.9765625, + "fc_socket_count": 117, + "sqlite_db_kb": 164.0 + }, + { + "event": "boot_ok", + "vm_seq": 118, + "boot_time_s": 3.202, + "density_at_boot": 118, + "timestamp": 1773033383.3633437, + "vms_running": 118, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.354461669921875, + "host_disk_used_gb": 2.046710968017578, + "tap_count": 118, + "firecracker_rss_mb": 5543.8984375, + "fc_socket_count": 118, + "sqlite_db_kb": 164.0 + }, + { + "event": "boot_ok", + "vm_seq": 119, + "boot_time_s": 3.117, + "density_at_boot": 119, + "timestamp": 1773033386.6418316, + "vms_running": 119, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 8.395759582519531, + "host_disk_used_gb": 2.046710968017578, + "tap_count": 119, + "firecracker_rss_mb": 5590.56640625, + "fc_socket_count": 119, + "sqlite_db_kb": 164.0 + }, + { + "event": "boot_ok", + "vm_seq": 120, + "boot_time_s": 3.123, + "density_at_boot": 120, + "timestamp": 1773033389.9224994, + "vms_running": 120, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.447036743164062, + "host_disk_used_gb": 2.046710968017578, + "tap_count": 120, + "firecracker_rss_mb": 5636.97265625, + "fc_socket_count": 120, + "sqlite_db_kb": 168.0 + }, + { + "event": "boot_ok", + "vm_seq": 121, + "boot_time_s": 3.076, + "density_at_boot": 121, + "timestamp": 1773033393.1599932, + "vms_running": 121, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.49530029296875, + "host_disk_used_gb": 2.046710968017578, + "tap_count": 121, + "firecracker_rss_mb": 5684.15234375, + "fc_socket_count": 121, + "sqlite_db_kb": 172.0 + }, + { + "event": "boot_ok", + "vm_seq": 122, + "boot_time_s": 3.082, + "density_at_boot": 122, + "timestamp": 1773033396.4016879, + "vms_running": 122, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 8.541004180908203, + "host_disk_used_gb": 2.046710968017578, + "tap_count": 122, + "firecracker_rss_mb": 5731.33984375, + "fc_socket_count": 122, + "sqlite_db_kb": 172.0 + }, + { + "event": "boot_ok", + "vm_seq": 123, + "boot_time_s": 3.04, + "density_at_boot": 123, + "timestamp": 1773033399.601394, + "vms_running": 123, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 8.587997436523438, + "host_disk_used_gb": 2.048664093017578, + "tap_count": 123, + "firecracker_rss_mb": 5778.00390625, + "fc_socket_count": 123, + "sqlite_db_kb": 172.0 + }, + { + "event": "boot_ok", + "vm_seq": 124, + "boot_time_s": 3.201, + "density_at_boot": 124, + "timestamp": 1773033402.96306, + "vms_running": 124, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.630805969238281, + "host_disk_used_gb": 2.048664093017578, + "tap_count": 124, + "firecracker_rss_mb": 5824.671875, + "fc_socket_count": 124, + "sqlite_db_kb": 172.0 + }, + { + "event": "boot_ok", + "vm_seq": 125, + "boot_time_s": 3.117, + "density_at_boot": 125, + "timestamp": 1773033406.2411644, + "vms_running": 125, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 8.680839538574219, + "host_disk_used_gb": 2.048664093017578, + "tap_count": 125, + "firecracker_rss_mb": 5871.0859375, + "fc_socket_count": 125, + "sqlite_db_kb": 172.0 + }, + { + "event": "boot_ok", + "vm_seq": 126, + "boot_time_s": 3.08, + "density_at_boot": 126, + "timestamp": 1773033409.4800408, + "vms_running": 126, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 8.728694915771484, + "host_disk_used_gb": 2.048664093017578, + "tap_count": 126, + "firecracker_rss_mb": 5918.53125, + "fc_socket_count": 126, + "sqlite_db_kb": 176.0 + }, + { + "event": "boot_ok", + "vm_seq": 127, + "boot_time_s": 3.088, + "density_at_boot": 127, + "timestamp": 1773033412.7238917, + "vms_running": 127, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 8.772705078125, + "host_disk_used_gb": 2.048664093017578, + "tap_count": 127, + "firecracker_rss_mb": 5965.71484375, + "fc_socket_count": 127, + "sqlite_db_kb": 176.0 + }, + { + "event": "boot_ok", + "vm_seq": 128, + "boot_time_s": 3.117, + "density_at_boot": 128, + "timestamp": 1773033416.000301, + "vms_running": 128, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.832965850830078, + "host_disk_used_gb": 2.0486679077148438, + "tap_count": 128, + "firecracker_rss_mb": 6012.640625, + "fc_socket_count": 128, + "sqlite_db_kb": 176.0 + }, + { + "event": "boot_ok", + "vm_seq": 129, + "boot_time_s": 3.119, + "density_at_boot": 129, + "timestamp": 1773033419.281909, + "vms_running": 129, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 8.882610321044922, + "host_disk_used_gb": 2.0486679077148438, + "tap_count": 129, + "firecracker_rss_mb": 6059.56640625, + "fc_socket_count": 129, + "sqlite_db_kb": 176.0 + }, + { + "event": "boot_ok", + "vm_seq": 130, + "boot_time_s": 3.117, + "density_at_boot": 130, + "timestamp": 1773033422.562466, + "vms_running": 130, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 8.926074981689453, + "host_disk_used_gb": 2.0486679077148438, + "tap_count": 130, + "firecracker_rss_mb": 6107.0078125, + "fc_socket_count": 130, + "sqlite_db_kb": 176.0 + }, + { + "event": "boot_ok", + "vm_seq": 131, + "boot_time_s": 3.121, + "density_at_boot": 131, + "timestamp": 1773033425.844859, + "vms_running": 131, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 8.970325469970703, + "host_disk_used_gb": 2.0486679077148438, + "tap_count": 131, + "firecracker_rss_mb": 6153.421875, + "fc_socket_count": 131, + "sqlite_db_kb": 180.0 + }, + { + "event": "boot_ok", + "vm_seq": 132, + "boot_time_s": 2.254, + "density_at_boot": 132, + "timestamp": 1773033428.2608886, + "vms_running": 132, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 9.01382064819336, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 132, + "firecracker_rss_mb": 6200.08984375, + "fc_socket_count": 132, + "sqlite_db_kb": 180.0 + }, + { + "event": "boot_ok", + "vm_seq": 133, + "boot_time_s": 3.099, + "density_at_boot": 133, + "timestamp": 1773033431.521233, + "vms_running": 133, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 9.0545654296875, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 133, + "firecracker_rss_mb": 6246.50390625, + "fc_socket_count": 133, + "sqlite_db_kb": 180.0 + }, + { + "event": "boot_ok", + "vm_seq": 134, + "boot_time_s": 3.121, + "density_at_boot": 134, + "timestamp": 1773033434.803238, + "vms_running": 134, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 9.105091094970703, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 134, + "firecracker_rss_mb": 6293.42578125, + "fc_socket_count": 134, + "sqlite_db_kb": 180.0 + }, + { + "event": "boot_ok", + "vm_seq": 135, + "boot_time_s": 3.118, + "density_at_boot": 135, + "timestamp": 1773033438.0817564, + "vms_running": 135, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 9.156326293945312, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 135, + "firecracker_rss_mb": 6340.34765625, + "fc_socket_count": 135, + "sqlite_db_kb": 180.0 + }, + { + "event": "boot_ok", + "vm_seq": 136, + "boot_time_s": 3.123, + "density_at_boot": 136, + "timestamp": 1773033441.3631392, + "vms_running": 136, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.193977355957031, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 136, + "firecracker_rss_mb": 6387.27734375, + "fc_socket_count": 136, + "sqlite_db_kb": 184.0 + }, + { + "event": "boot_ok", + "vm_seq": 137, + "boot_time_s": 3.119, + "density_at_boot": 137, + "timestamp": 1773033444.6434557, + "vms_running": 137, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 9.247077941894531, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 137, + "firecracker_rss_mb": 6433.9453125, + "fc_socket_count": 137, + "sqlite_db_kb": 184.0 + }, + { + "event": "boot_ok", + "vm_seq": 138, + "boot_time_s": 3.075, + "density_at_boot": 138, + "timestamp": 1773033447.87982, + "vms_running": 138, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 9.296798706054688, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 138, + "firecracker_rss_mb": 6480.35546875, + "fc_socket_count": 138, + "sqlite_db_kb": 184.0 + }, + { + "event": "boot_ok", + "vm_seq": 139, + "boot_time_s": 2.222, + "density_at_boot": 139, + "timestamp": 1773033450.2640986, + "vms_running": 139, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.344768524169922, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 139, + "firecracker_rss_mb": 6527.54296875, + "fc_socket_count": 139, + "sqlite_db_kb": 184.0 + }, + { + "event": "boot_ok", + "vm_seq": 140, + "boot_time_s": 3.1, + "density_at_boot": 140, + "timestamp": 1773033453.524573, + "vms_running": 140, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.38943862915039, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 140, + "firecracker_rss_mb": 6574.21484375, + "fc_socket_count": 140, + "sqlite_db_kb": 184.0 + }, + { + "event": "boot_ok", + "vm_seq": 141, + "boot_time_s": 3.117, + "density_at_boot": 141, + "timestamp": 1773033456.803698, + "vms_running": 141, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 9.433296203613281, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 141, + "firecracker_rss_mb": 6622.17578125, + "fc_socket_count": 141, + "sqlite_db_kb": 188.0 + }, + { + "event": "boot_ok", + "vm_seq": 142, + "boot_time_s": 3.115, + "density_at_boot": 142, + "timestamp": 1773033460.0809586, + "vms_running": 142, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 9.485233306884766, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 142, + "firecracker_rss_mb": 6669.10546875, + "fc_socket_count": 142, + "sqlite_db_kb": 188.0 + }, + { + "event": "boot_ok", + "vm_seq": 143, + "boot_time_s": 2.257, + "density_at_boot": 143, + "timestamp": 1773033462.5032816, + "vms_running": 143, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.534984588623047, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 143, + "firecracker_rss_mb": 6715.7734375, + "fc_socket_count": 143, + "sqlite_db_kb": 188.0 + }, + { + "event": "boot_ok", + "vm_seq": 144, + "boot_time_s": 3.092, + "density_at_boot": 144, + "timestamp": 1773033465.7614236, + "vms_running": 144, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 9.581172943115234, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 144, + "firecracker_rss_mb": 6762.18359375, + "fc_socket_count": 144, + "sqlite_db_kb": 188.0 + }, + { + "event": "boot_ok", + "vm_seq": 145, + "boot_time_s": 2.259, + "density_at_boot": 145, + "timestamp": 1773033468.1824205, + "vms_running": 145, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.623035430908203, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 145, + "firecracker_rss_mb": 6808.59375, + "fc_socket_count": 145, + "sqlite_db_kb": 188.0 + }, + { + "event": "boot_ok", + "vm_seq": 146, + "boot_time_s": 2.237, + "density_at_boot": 146, + "timestamp": 1773033470.5823827, + "vms_running": 146, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.670127868652344, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 146, + "firecracker_rss_mb": 6855.7734375, + "fc_socket_count": 146, + "sqlite_db_kb": 192.0 + }, + { + "event": "boot_ok", + "vm_seq": 147, + "boot_time_s": 3.099, + "density_at_boot": 147, + "timestamp": 1773033473.8449368, + "vms_running": 147, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.725379943847656, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 147, + "firecracker_rss_mb": 6903.4765625, + "fc_socket_count": 147, + "sqlite_db_kb": 192.0 + }, + { + "event": "boot_ok", + "vm_seq": 148, + "boot_time_s": 3.113, + "density_at_boot": 148, + "timestamp": 1773033477.121257, + "vms_running": 148, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 9.766246795654297, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 148, + "firecracker_rss_mb": 6950.1484375, + "fc_socket_count": 148, + "sqlite_db_kb": 196.0 + }, + { + "event": "boot_ok", + "vm_seq": 149, + "boot_time_s": 2.34, + "density_at_boot": 149, + "timestamp": 1773033479.6245906, + "vms_running": 149, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 9.817317962646484, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 149, + "firecracker_rss_mb": 6996.8203125, + "fc_socket_count": 149, + "sqlite_db_kb": 196.0 + }, + { + "event": "boot_ok", + "vm_seq": 150, + "boot_time_s": 3.093, + "density_at_boot": 150, + "timestamp": 1773033482.8818614, + "vms_running": 150, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 9.861000061035156, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 150, + "firecracker_rss_mb": 7043.4921875, + "fc_socket_count": 150, + "sqlite_db_kb": 196.0 + }, + { + "event": "boot_ok", + "vm_seq": 151, + "boot_time_s": 3.116, + "density_at_boot": 151, + "timestamp": 1773033486.1640375, + "vms_running": 151, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.906044006347656, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 151, + "firecracker_rss_mb": 7090.15625, + "fc_socket_count": 151, + "sqlite_db_kb": 200.0 + }, + { + "event": "boot_ok", + "vm_seq": 152, + "boot_time_s": 3.193, + "density_at_boot": 152, + "timestamp": 1773033489.521806, + "vms_running": 152, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 9.950984954833984, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 152, + "firecracker_rss_mb": 7136.828125, + "fc_socket_count": 152, + "sqlite_db_kb": 200.0 + }, + { + "event": "boot_ok", + "vm_seq": 153, + "boot_time_s": 3.194, + "density_at_boot": 153, + "timestamp": 1773033492.8805733, + "vms_running": 153, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 9.998855590820312, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 153, + "firecracker_rss_mb": 7183.75390625, + "fc_socket_count": 153, + "sqlite_db_kb": 200.0 + }, + { + "event": "boot_ok", + "vm_seq": 154, + "boot_time_s": 2.252, + "density_at_boot": 154, + "timestamp": 1773033495.3022184, + "vms_running": 154, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.040069580078125, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 154, + "firecracker_rss_mb": 7230.42578125, + "fc_socket_count": 154, + "sqlite_db_kb": 200.0 + }, + { + "event": "boot_ok", + "vm_seq": 155, + "boot_time_s": 3.254, + "density_at_boot": 155, + "timestamp": 1773033498.7247493, + "vms_running": 155, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 10.091270446777344, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 155, + "firecracker_rss_mb": 7277.09375, + "fc_socket_count": 155, + "sqlite_db_kb": 200.0 + }, + { + "event": "boot_ok", + "vm_seq": 156, + "boot_time_s": 3.193, + "density_at_boot": 156, + "timestamp": 1773033502.0826788, + "vms_running": 156, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 10.131889343261719, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 156, + "firecracker_rss_mb": 7324.26953125, + "fc_socket_count": 156, + "sqlite_db_kb": 204.0 + }, + { + "event": "boot_ok", + "vm_seq": 157, + "boot_time_s": 3.119, + "density_at_boot": 157, + "timestamp": 1773033505.362828, + "vms_running": 157, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.181037902832031, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 157, + "firecracker_rss_mb": 7370.94140625, + "fc_socket_count": 157, + "sqlite_db_kb": 204.0 + }, + { + "event": "boot_ok", + "vm_seq": 158, + "boot_time_s": 3.112, + "density_at_boot": 158, + "timestamp": 1773033508.6444862, + "vms_running": 158, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.230438232421875, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 158, + "firecracker_rss_mb": 7417.87109375, + "fc_socket_count": 158, + "sqlite_db_kb": 204.0 + }, + { + "event": "boot_ok", + "vm_seq": 159, + "boot_time_s": 3.109, + "density_at_boot": 159, + "timestamp": 1773033511.9205093, + "vms_running": 159, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.284061431884766, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 159, + "firecracker_rss_mb": 7464.796875, + "fc_socket_count": 159, + "sqlite_db_kb": 204.0 + }, + { + "event": "boot_ok", + "vm_seq": 160, + "boot_time_s": 3.194, + "density_at_boot": 160, + "timestamp": 1773033515.2842944, + "vms_running": 160, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.324714660644531, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 160, + "firecracker_rss_mb": 7511.203125, + "fc_socket_count": 160, + "sqlite_db_kb": 204.0 + }, + { + "event": "boot_ok", + "vm_seq": 161, + "boot_time_s": 2.253, + "density_at_boot": 161, + "timestamp": 1773033517.7042155, + "vms_running": 161, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.376792907714844, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 161, + "firecracker_rss_mb": 7558.12890625, + "fc_socket_count": 161, + "sqlite_db_kb": 208.0 + }, + { + "event": "boot_ok", + "vm_seq": 162, + "boot_time_s": 3.094, + "density_at_boot": 162, + "timestamp": 1773033520.9644217, + "vms_running": 162, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.431446075439453, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 162, + "firecracker_rss_mb": 7605.0546875, + "fc_socket_count": 162, + "sqlite_db_kb": 208.0 + }, + { + "event": "boot_ok", + "vm_seq": 163, + "boot_time_s": 3.11, + "density_at_boot": 163, + "timestamp": 1773033524.243907, + "vms_running": 163, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.468460083007812, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 163, + "firecracker_rss_mb": 7651.98046875, + "fc_socket_count": 163, + "sqlite_db_kb": 208.0 + }, + { + "event": "boot_ok", + "vm_seq": 164, + "boot_time_s": 3.191, + "density_at_boot": 164, + "timestamp": 1773033527.6043, + "vms_running": 164, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.510318756103516, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 164, + "firecracker_rss_mb": 7698.90625, + "fc_socket_count": 164, + "sqlite_db_kb": 208.0 + }, + { + "event": "boot_ok", + "vm_seq": 165, + "boot_time_s": 3.108, + "density_at_boot": 165, + "timestamp": 1773033530.8813622, + "vms_running": 165, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.557815551757812, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 165, + "firecracker_rss_mb": 7745.8359375, + "fc_socket_count": 165, + "sqlite_db_kb": 208.0 + }, + { + "event": "boot_ok", + "vm_seq": 166, + "boot_time_s": 3.123, + "density_at_boot": 166, + "timestamp": 1773033534.1663275, + "vms_running": 166, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.608741760253906, + "host_disk_used_gb": 2.0564804077148438, + "tap_count": 166, + "firecracker_rss_mb": 7793.01953125, + "fc_socket_count": 166, + "sqlite_db_kb": 212.0 + }, + { + "event": "boot_ok", + "vm_seq": 167, + "boot_time_s": 3.189, + "density_at_boot": 167, + "timestamp": 1773033537.5240815, + "vms_running": 167, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.653961181640625, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 167, + "firecracker_rss_mb": 7839.94140625, + "fc_socket_count": 167, + "sqlite_db_kb": 212.0 + }, + { + "event": "boot_ok", + "vm_seq": 168, + "boot_time_s": 2.25, + "density_at_boot": 168, + "timestamp": 1773033539.9457948, + "vms_running": 168, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 10.698810577392578, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 168, + "firecracker_rss_mb": 7886.61328125, + "fc_socket_count": 168, + "sqlite_db_kb": 212.0 + }, + { + "event": "boot_ok", + "vm_seq": 169, + "boot_time_s": 3.168, + "density_at_boot": 169, + "timestamp": 1773033543.2839813, + "vms_running": 169, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 10.747535705566406, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 169, + "firecracker_rss_mb": 7933.5390625, + "fc_socket_count": 169, + "sqlite_db_kb": 212.0 + }, + { + "event": "boot_ok", + "vm_seq": 170, + "boot_time_s": 3.347, + "density_at_boot": 170, + "timestamp": 1773033546.8029087, + "vms_running": 170, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 10.788501739501953, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 170, + "firecracker_rss_mb": 7980.2109375, + "fc_socket_count": 170, + "sqlite_db_kb": 212.0 + }, + { + "event": "boot_ok", + "vm_seq": 171, + "boot_time_s": 3.111, + "density_at_boot": 171, + "timestamp": 1773033550.0837052, + "vms_running": 171, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 10.838539123535156, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 171, + "firecracker_rss_mb": 8026.625, + "fc_socket_count": 171, + "sqlite_db_kb": 216.0 + }, + { + "event": "boot_ok", + "vm_seq": 172, + "boot_time_s": 3.107, + "density_at_boot": 172, + "timestamp": 1773033553.3607836, + "vms_running": 172, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 10.891731262207031, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 172, + "firecracker_rss_mb": 8073.5546875, + "fc_socket_count": 172, + "sqlite_db_kb": 216.0 + }, + { + "event": "boot_ok", + "vm_seq": 173, + "boot_time_s": 3.191, + "density_at_boot": 173, + "timestamp": 1773033556.7229428, + "vms_running": 173, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 10.934406280517578, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 173, + "firecracker_rss_mb": 8119.96484375, + "fc_socket_count": 173, + "sqlite_db_kb": 216.0 + }, + { + "event": "boot_ok", + "vm_seq": 174, + "boot_time_s": 3.111, + "density_at_boot": 174, + "timestamp": 1773033560.0019217, + "vms_running": 174, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 10.979808807373047, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 174, + "firecracker_rss_mb": 8166.890625, + "fc_socket_count": 174, + "sqlite_db_kb": 216.0 + }, + { + "event": "boot_ok", + "vm_seq": 175, + "boot_time_s": 3.111, + "density_at_boot": 175, + "timestamp": 1773033563.2828329, + "vms_running": 175, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.024150848388672, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 175, + "firecracker_rss_mb": 8213.55859375, + "fc_socket_count": 175, + "sqlite_db_kb": 216.0 + }, + { + "event": "boot_ok", + "vm_seq": 176, + "boot_time_s": 3.068, + "density_at_boot": 176, + "timestamp": 1773033566.5202913, + "vms_running": 176, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 11.070877075195312, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 176, + "firecracker_rss_mb": 8259.96484375, + "fc_socket_count": 176, + "sqlite_db_kb": 220.0 + }, + { + "event": "boot_ok", + "vm_seq": 177, + "boot_time_s": 3.15, + "density_at_boot": 177, + "timestamp": 1773033569.841767, + "vms_running": 177, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.11007308959961, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 177, + "firecracker_rss_mb": 8306.89453125, + "fc_socket_count": 177, + "sqlite_db_kb": 224.0 + }, + { + "event": "boot_ok", + "vm_seq": 178, + "boot_time_s": 3.109, + "density_at_boot": 178, + "timestamp": 1773033573.1206295, + "vms_running": 178, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 11.160285949707031, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 178, + "firecracker_rss_mb": 8353.82421875, + "fc_socket_count": 178, + "sqlite_db_kb": 224.0 + }, + { + "event": "boot_ok", + "vm_seq": 179, + "boot_time_s": 2.253, + "density_at_boot": 179, + "timestamp": 1773033575.5447557, + "vms_running": 179, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 11.212104797363281, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 179, + "firecracker_rss_mb": 8400.75, + "fc_socket_count": 179, + "sqlite_db_kb": 224.0 + }, + { + "event": "boot_ok", + "vm_seq": 180, + "boot_time_s": 3.165, + "density_at_boot": 180, + "timestamp": 1773033578.8825028, + "vms_running": 180, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 11.261924743652344, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 180, + "firecracker_rss_mb": 8447.9375, + "fc_socket_count": 180, + "sqlite_db_kb": 224.0 + }, + { + "event": "boot_ok", + "vm_seq": 181, + "boot_time_s": 3.193, + "density_at_boot": 181, + "timestamp": 1773033582.2439106, + "vms_running": 181, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 11.309494018554688, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 181, + "firecracker_rss_mb": 8494.86328125, + "fc_socket_count": 181, + "sqlite_db_kb": 228.0 + }, + { + "event": "boot_ok", + "vm_seq": 182, + "boot_time_s": 3.188, + "density_at_boot": 182, + "timestamp": 1773033585.6021996, + "vms_running": 182, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 11.352672576904297, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 182, + "firecracker_rss_mb": 8541.2734375, + "fc_socket_count": 182, + "sqlite_db_kb": 228.0 + }, + { + "event": "boot_ok", + "vm_seq": 183, + "boot_time_s": 3.149, + "density_at_boot": 183, + "timestamp": 1773033588.9205325, + "vms_running": 183, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.397899627685547, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 183, + "firecracker_rss_mb": 8588.45703125, + "fc_socket_count": 183, + "sqlite_db_kb": 228.0 + }, + { + "event": "boot_ok", + "vm_seq": 184, + "boot_time_s": 3.157, + "density_at_boot": 184, + "timestamp": 1773033592.2446215, + "vms_running": 184, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 11.445987701416016, + "host_disk_used_gb": 2.056476593017578, + "tap_count": 184, + "firecracker_rss_mb": 8635.12890625, + "fc_socket_count": 184, + "sqlite_db_kb": 228.0 + }, + { + "event": "boot_ok", + "vm_seq": 185, + "boot_time_s": 3.19, + "density_at_boot": 185, + "timestamp": 1773033595.6052225, + "vms_running": 185, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 11.482105255126953, + "host_disk_used_gb": 2.064289093017578, + "tap_count": 185, + "firecracker_rss_mb": 8682.3125, + "fc_socket_count": 185, + "sqlite_db_kb": 228.0 + }, + { + "event": "boot_ok", + "vm_seq": 186, + "boot_time_s": 3.188, + "density_at_boot": 186, + "timestamp": 1773033598.9635153, + "vms_running": 186, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 11.541004180908203, + "host_disk_used_gb": 2.068195343017578, + "tap_count": 186, + "firecracker_rss_mb": 8729.23828125, + "fc_socket_count": 186, + "sqlite_db_kb": 232.0 + }, + { + "event": "boot_ok", + "vm_seq": 187, + "boot_time_s": 2.226, + "density_at_boot": 187, + "timestamp": 1773033601.3632512, + "vms_running": 187, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 11.587959289550781, + "host_disk_used_gb": 2.0681991577148438, + "tap_count": 187, + "firecracker_rss_mb": 8775.90625, + "fc_socket_count": 187, + "sqlite_db_kb": 232.0 + }, + { + "event": "boot_ok", + "vm_seq": 188, + "boot_time_s": 3.346, + "density_at_boot": 188, + "timestamp": 1773033604.8829737, + "vms_running": 188, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 11.639976501464844, + "host_disk_used_gb": 2.068256378173828, + "tap_count": 188, + "firecracker_rss_mb": 8822.578125, + "fc_socket_count": 188, + "sqlite_db_kb": 232.0 + }, + { + "event": "boot_ok", + "vm_seq": 189, + "boot_time_s": 3.187, + "density_at_boot": 189, + "timestamp": 1773033608.2422018, + "vms_running": 189, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.687469482421875, + "host_disk_used_gb": 2.0682373046875, + "tap_count": 189, + "firecracker_rss_mb": 8868.98828125, + "fc_socket_count": 189, + "sqlite_db_kb": 232.0 + }, + { + "event": "boot_ok", + "vm_seq": 190, + "boot_time_s": 3.271, + "density_at_boot": 190, + "timestamp": 1773033611.6846218, + "vms_running": 190, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.74105453491211, + "host_disk_used_gb": 2.0682373046875, + "tap_count": 190, + "firecracker_rss_mb": 8916.17578125, + "fc_socket_count": 190, + "sqlite_db_kb": 232.0 + }, + { + "event": "boot_ok", + "vm_seq": 191, + "boot_time_s": 2.209, + "density_at_boot": 191, + "timestamp": 1773033614.0609727, + "vms_running": 191, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 11.793479919433594, + "host_disk_used_gb": 2.0682601928710938, + "tap_count": 191, + "firecracker_rss_mb": 8963.1015625, + "fc_socket_count": 191, + "sqlite_db_kb": 236.0 + }, + { + "event": "boot_ok", + "vm_seq": 192, + "boot_time_s": 2.264, + "density_at_boot": 192, + "timestamp": 1773033616.501715, + "vms_running": 192, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 11.839092254638672, + "host_disk_used_gb": 2.0682601928710938, + "tap_count": 192, + "firecracker_rss_mb": 9009.2578125, + "fc_socket_count": 192, + "sqlite_db_kb": 236.0 + }, + { + "event": "boot_ok", + "vm_seq": 193, + "boot_time_s": 3.328, + "density_at_boot": 193, + "timestamp": 1773033620.0044794, + "vms_running": 193, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 11.886859893798828, + "host_disk_used_gb": 2.0682601928710938, + "tap_count": 193, + "firecracker_rss_mb": 9055.671875, + "fc_socket_count": 193, + "sqlite_db_kb": 236.0 + }, + { + "event": "boot_ok", + "vm_seq": 194, + "boot_time_s": 3.187, + "density_at_boot": 194, + "timestamp": 1773033623.3625393, + "vms_running": 194, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 11.928947448730469, + "host_disk_used_gb": 2.0682601928710938, + "tap_count": 194, + "firecracker_rss_mb": 9102.6015625, + "fc_socket_count": 194, + "sqlite_db_kb": 236.0 + }, + { + "event": "boot_ok", + "vm_seq": 195, + "boot_time_s": 3.194, + "density_at_boot": 195, + "timestamp": 1773033626.7250767, + "vms_running": 195, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 11.971641540527344, + "host_disk_used_gb": 2.0682830810546875, + "tap_count": 195, + "firecracker_rss_mb": 9149.26953125, + "fc_socket_count": 195, + "sqlite_db_kb": 236.0 + }, + { + "event": "boot_ok", + "vm_seq": 196, + "boot_time_s": 3.185, + "density_at_boot": 196, + "timestamp": 1773033630.0824122, + "vms_running": 196, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.018150329589844, + "host_disk_used_gb": 2.0682830810546875, + "tap_count": 196, + "firecracker_rss_mb": 9196.453125, + "fc_socket_count": 196, + "sqlite_db_kb": 240.0 + }, + { + "event": "boot_ok", + "vm_seq": 197, + "boot_time_s": 3.188, + "density_at_boot": 197, + "timestamp": 1773033633.4426615, + "vms_running": 197, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 12.065910339355469, + "host_disk_used_gb": 2.0682830810546875, + "tap_count": 197, + "firecracker_rss_mb": 9243.3828125, + "fc_socket_count": 197, + "sqlite_db_kb": 240.0 + }, + { + "event": "boot_ok", + "vm_seq": 198, + "boot_time_s": 3.187, + "density_at_boot": 198, + "timestamp": 1773033636.802098, + "vms_running": 198, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.115619659423828, + "host_disk_used_gb": 2.0682830810546875, + "tap_count": 198, + "firecracker_rss_mb": 9290.30859375, + "fc_socket_count": 198, + "sqlite_db_kb": 240.0 + }, + { + "event": "boot_ok", + "vm_seq": 199, + "boot_time_s": 3.141, + "density_at_boot": 199, + "timestamp": 1773033640.119371, + "vms_running": 199, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 12.157161712646484, + "host_disk_used_gb": 2.0682830810546875, + "tap_count": 199, + "firecracker_rss_mb": 9337.23828125, + "fc_socket_count": 199, + "sqlite_db_kb": 240.0 + }, + { + "event": "boot_ok", + "vm_seq": 200, + "boot_time_s": 3.15, + "density_at_boot": 200, + "timestamp": 1773033643.4410403, + "vms_running": 200, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.199546813964844, + "host_disk_used_gb": 2.0682830810546875, + "tap_count": 200, + "firecracker_rss_mb": 9383.65234375, + "fc_socket_count": 200, + "sqlite_db_kb": 240.0 + }, + { + "event": "boot_ok", + "vm_seq": 201, + "boot_time_s": 3.187, + "density_at_boot": 201, + "timestamp": 1773033646.8031938, + "vms_running": 201, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.243377685546875, + "host_disk_used_gb": 2.0682830810546875, + "tap_count": 201, + "firecracker_rss_mb": 9430.83984375, + "fc_socket_count": 201, + "sqlite_db_kb": 244.0 + }, + { + "event": "boot_ok", + "vm_seq": 202, + "boot_time_s": 2.21, + "density_at_boot": 202, + "timestamp": 1773033649.1886933, + "vms_running": 202, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.294654846191406, + "host_disk_used_gb": 2.0682640075683594, + "tap_count": 202, + "firecracker_rss_mb": 9477.51171875, + "fc_socket_count": 202, + "sqlite_db_kb": 244.0 + }, + { + "event": "boot_ok", + "vm_seq": 203, + "boot_time_s": 3.117, + "density_at_boot": 203, + "timestamp": 1773033652.4823601, + "vms_running": 203, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 12.346118927001953, + "host_disk_used_gb": 2.0682640075683594, + "tap_count": 203, + "firecracker_rss_mb": 9524.9609375, + "fc_socket_count": 203, + "sqlite_db_kb": 244.0 + }, + { + "event": "boot_ok", + "vm_seq": 204, + "boot_time_s": 3.184, + "density_at_boot": 204, + "timestamp": 1773033655.8429387, + "vms_running": 204, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.397769927978516, + "host_disk_used_gb": 2.0682640075683594, + "tap_count": 204, + "firecracker_rss_mb": 9571.6328125, + "fc_socket_count": 204, + "sqlite_db_kb": 244.0 + }, + { + "event": "boot_ok", + "vm_seq": 205, + "boot_time_s": 3.185, + "density_at_boot": 205, + "timestamp": 1773033659.2011416, + "vms_running": 205, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.447380065917969, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 205, + "firecracker_rss_mb": 9618.5546875, + "fc_socket_count": 205, + "sqlite_db_kb": 244.0 + }, + { + "event": "boot_ok", + "vm_seq": 206, + "boot_time_s": 3.141, + "density_at_boot": 206, + "timestamp": 1773033662.5197165, + "vms_running": 206, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 12.492847442626953, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 206, + "firecracker_rss_mb": 9665.484375, + "fc_socket_count": 206, + "sqlite_db_kb": 248.0 + }, + { + "event": "boot_ok", + "vm_seq": 207, + "boot_time_s": 3.183, + "density_at_boot": 207, + "timestamp": 1773033665.8796723, + "vms_running": 207, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 12.531898498535156, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 207, + "firecracker_rss_mb": 9712.1484375, + "fc_socket_count": 207, + "sqlite_db_kb": 248.0 + }, + { + "event": "boot_ok", + "vm_seq": 208, + "boot_time_s": 3.224, + "density_at_boot": 208, + "timestamp": 1773033669.2808225, + "vms_running": 208, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 12.575054168701172, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 208, + "firecracker_rss_mb": 9759.08203125, + "fc_socket_count": 208, + "sqlite_db_kb": 248.0 + }, + { + "event": "boot_ok", + "vm_seq": 209, + "boot_time_s": 3.191, + "density_at_boot": 209, + "timestamp": 1773033672.6431203, + "vms_running": 209, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.619369506835938, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 209, + "firecracker_rss_mb": 9805.75390625, + "fc_socket_count": 209, + "sqlite_db_kb": 248.0 + }, + { + "event": "boot_ok", + "vm_seq": 210, + "boot_time_s": 2.493, + "density_at_boot": 210, + "timestamp": 1773033675.3110383, + "vms_running": 210, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 12.677513122558594, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 210, + "firecracker_rss_mb": 9852.9375, + "fc_socket_count": 210, + "sqlite_db_kb": 248.0 + }, + { + "event": "boot_ok", + "vm_seq": 211, + "boot_time_s": 3.235, + "density_at_boot": 211, + "timestamp": 1773033678.7220268, + "vms_running": 211, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.724494934082031, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 211, + "firecracker_rss_mb": 9900.1171875, + "fc_socket_count": 211, + "sqlite_db_kb": 252.0 + }, + { + "event": "boot_ok", + "vm_seq": 212, + "boot_time_s": 3.18, + "density_at_boot": 212, + "timestamp": 1773033682.0824926, + "vms_running": 212, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.762870788574219, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 212, + "firecracker_rss_mb": 9946.78125, + "fc_socket_count": 212, + "sqlite_db_kb": 252.0 + }, + { + "event": "boot_ok", + "vm_seq": 213, + "boot_time_s": 3.265, + "density_at_boot": 213, + "timestamp": 1773033685.5220332, + "vms_running": 213, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.811355590820312, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 213, + "firecracker_rss_mb": 9993.70703125, + "fc_socket_count": 213, + "sqlite_db_kb": 252.0 + }, + { + "event": "boot_ok", + "vm_seq": 214, + "boot_time_s": 2.224, + "density_at_boot": 214, + "timestamp": 1773033687.9207778, + "vms_running": 214, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.849857330322266, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 214, + "firecracker_rss_mb": 10040.37890625, + "fc_socket_count": 214, + "sqlite_db_kb": 252.0 + }, + { + "event": "boot_ok", + "vm_seq": 215, + "boot_time_s": 3.185, + "density_at_boot": 215, + "timestamp": 1773033691.2841442, + "vms_running": 215, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.905830383300781, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 215, + "firecracker_rss_mb": 10087.30859375, + "fc_socket_count": 215, + "sqlite_db_kb": 252.0 + }, + { + "event": "boot_ok", + "vm_seq": 216, + "boot_time_s": 2.32, + "density_at_boot": 216, + "timestamp": 1773033693.7820256, + "vms_running": 216, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 12.955081939697266, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 216, + "firecracker_rss_mb": 10133.72265625, + "fc_socket_count": 216, + "sqlite_db_kb": 256.0 + }, + { + "event": "boot_ok", + "vm_seq": 217, + "boot_time_s": 3.163, + "density_at_boot": 217, + "timestamp": 1773033697.1216776, + "vms_running": 217, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 12.997241973876953, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 217, + "firecracker_rss_mb": 10180.6484375, + "fc_socket_count": 217, + "sqlite_db_kb": 264.0 + }, + { + "event": "boot_ok", + "vm_seq": 218, + "boot_time_s": 3.265, + "density_at_boot": 218, + "timestamp": 1773033700.5650856, + "vms_running": 218, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 13.045398712158203, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 218, + "firecracker_rss_mb": 10227.83203125, + "fc_socket_count": 218, + "sqlite_db_kb": 264.0 + }, + { + "event": "boot_ok", + "vm_seq": 219, + "boot_time_s": 2.279, + "density_at_boot": 219, + "timestamp": 1773033703.022103, + "vms_running": 219, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 13.093620300292969, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 219, + "firecracker_rss_mb": 10275.2734375, + "fc_socket_count": 219, + "sqlite_db_kb": 264.0 + }, + { + "event": "boot_ok", + "vm_seq": 220, + "boot_time_s": 2.433, + "density_at_boot": 220, + "timestamp": 1773033705.6311026, + "vms_running": 220, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 13.141223907470703, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 220, + "firecracker_rss_mb": 10322.19921875, + "fc_socket_count": 220, + "sqlite_db_kb": 264.0 + }, + { + "event": "boot_ok", + "vm_seq": 221, + "boot_time_s": 3.156, + "density_at_boot": 221, + "timestamp": 1773033708.9627576, + "vms_running": 221, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 13.182518005371094, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 221, + "firecracker_rss_mb": 10368.8671875, + "fc_socket_count": 221, + "sqlite_db_kb": 272.0 + }, + { + "event": "boot_ok", + "vm_seq": 222, + "boot_time_s": 2.405, + "density_at_boot": 222, + "timestamp": 1773033711.541123, + "vms_running": 222, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 13.22659683227539, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 222, + "firecracker_rss_mb": 10415.53515625, + "fc_socket_count": 222, + "sqlite_db_kb": 312.0 + }, + { + "event": "boot_ok", + "vm_seq": 223, + "boot_time_s": 2.305, + "density_at_boot": 223, + "timestamp": 1773033714.0210059, + "vms_running": 223, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 13.278278350830078, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 223, + "firecracker_rss_mb": 10462.4609375, + "fc_socket_count": 223, + "sqlite_db_kb": 312.0 + }, + { + "event": "boot_ok", + "vm_seq": 224, + "boot_time_s": 3.239, + "density_at_boot": 224, + "timestamp": 1773033717.4414132, + "vms_running": 224, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 13.334373474121094, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 224, + "firecracker_rss_mb": 10509.1328125, + "fc_socket_count": 224, + "sqlite_db_kb": 312.0 + }, + { + "event": "boot_ok", + "vm_seq": 225, + "boot_time_s": 2.319, + "density_at_boot": 225, + "timestamp": 1773033719.9409328, + "vms_running": 225, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 13.38070297241211, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 225, + "firecracker_rss_mb": 10555.8046875, + "fc_socket_count": 225, + "sqlite_db_kb": 312.0 + }, + { + "event": "boot_ok", + "vm_seq": 226, + "boot_time_s": 3.323, + "density_at_boot": 226, + "timestamp": 1773033723.4444726, + "vms_running": 226, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 13.42630386352539, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 226, + "firecracker_rss_mb": 10602.7265625, + "fc_socket_count": 226, + "sqlite_db_kb": 316.0 + }, + { + "event": "boot_ok", + "vm_seq": 227, + "boot_time_s": 3.176, + "density_at_boot": 227, + "timestamp": 1773033726.8020544, + "vms_running": 227, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 13.469928741455078, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 227, + "firecracker_rss_mb": 10649.390625, + "fc_socket_count": 227, + "sqlite_db_kb": 316.0 + }, + { + "event": "boot_ok", + "vm_seq": 228, + "boot_time_s": 3.187, + "density_at_boot": 228, + "timestamp": 1773033730.1644514, + "vms_running": 228, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 13.523738861083984, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 228, + "firecracker_rss_mb": 10696.3203125, + "fc_socket_count": 228, + "sqlite_db_kb": 316.0 + }, + { + "event": "boot_ok", + "vm_seq": 229, + "boot_time_s": 3.262, + "density_at_boot": 229, + "timestamp": 1773033733.6042023, + "vms_running": 229, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 13.56546401977539, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 229, + "firecracker_rss_mb": 10743.50390625, + "fc_socket_count": 229, + "sqlite_db_kb": 316.0 + }, + { + "event": "boot_ok", + "vm_seq": 230, + "boot_time_s": 3.259, + "density_at_boot": 230, + "timestamp": 1773033737.0411217, + "vms_running": 230, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 13.606185913085938, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 230, + "firecracker_rss_mb": 10790.16796875, + "fc_socket_count": 230, + "sqlite_db_kb": 316.0 + }, + { + "event": "boot_ok", + "vm_seq": 231, + "boot_time_s": 3.261, + "density_at_boot": 231, + "timestamp": 1773033740.4821775, + "vms_running": 231, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 13.671470642089844, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 231, + "firecracker_rss_mb": 10837.09765625, + "fc_socket_count": 231, + "sqlite_db_kb": 320.0 + }, + { + "event": "boot_ok", + "vm_seq": 232, + "boot_time_s": 3.175, + "density_at_boot": 232, + "timestamp": 1773033743.8407533, + "vms_running": 232, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 13.712615966796875, + "host_disk_used_gb": 2.0682449340820312, + "tap_count": 232, + "firecracker_rss_mb": 10884.0234375, + "fc_socket_count": 232, + "sqlite_db_kb": 320.0 + }, + { + "event": "boot_ok", + "vm_seq": 233, + "boot_time_s": 3.221, + "density_at_boot": 233, + "timestamp": 1773033747.2403154, + "vms_running": 233, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 13.756458282470703, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 233, + "firecracker_rss_mb": 10930.94921875, + "fc_socket_count": 233, + "sqlite_db_kb": 320.0 + }, + { + "event": "boot_ok", + "vm_seq": 234, + "boot_time_s": 2.28, + "density_at_boot": 234, + "timestamp": 1773033749.7022865, + "vms_running": 234, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 13.799877166748047, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 234, + "firecracker_rss_mb": 10978.13671875, + "fc_socket_count": 234, + "sqlite_db_kb": 324.0 + }, + { + "event": "boot_ok", + "vm_seq": 235, + "boot_time_s": 3.32, + "density_at_boot": 235, + "timestamp": 1773033753.2051058, + "vms_running": 235, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 13.846595764160156, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 235, + "firecracker_rss_mb": 11024.80859375, + "fc_socket_count": 235, + "sqlite_db_kb": 324.0 + }, + { + "event": "boot_ok", + "vm_seq": 236, + "boot_time_s": 3.259, + "density_at_boot": 236, + "timestamp": 1773033756.6450055, + "vms_running": 236, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 13.88980484008789, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 236, + "firecracker_rss_mb": 11071.734375, + "fc_socket_count": 236, + "sqlite_db_kb": 328.0 + }, + { + "event": "boot_ok", + "vm_seq": 237, + "boot_time_s": 3.262, + "density_at_boot": 237, + "timestamp": 1773033760.0850143, + "vms_running": 237, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 13.9407958984375, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 237, + "firecracker_rss_mb": 11118.40234375, + "fc_socket_count": 237, + "sqlite_db_kb": 328.0 + }, + { + "event": "boot_ok", + "vm_seq": 238, + "boot_time_s": 3.258, + "density_at_boot": 238, + "timestamp": 1773033763.5224333, + "vms_running": 238, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 13.986568450927734, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 238, + "firecracker_rss_mb": 11165.5859375, + "fc_socket_count": 238, + "sqlite_db_kb": 328.0 + }, + { + "event": "boot_ok", + "vm_seq": 239, + "boot_time_s": 3.257, + "density_at_boot": 239, + "timestamp": 1773033766.9620733, + "vms_running": 239, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.035202026367188, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 239, + "firecracker_rss_mb": 11212.25390625, + "fc_socket_count": 239, + "sqlite_db_kb": 328.0 + }, + { + "event": "boot_ok", + "vm_seq": 240, + "boot_time_s": 3.34, + "density_at_boot": 240, + "timestamp": 1773033770.482531, + "vms_running": 240, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 14.085208892822266, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 240, + "firecracker_rss_mb": 11259.18359375, + "fc_socket_count": 240, + "sqlite_db_kb": 328.0 + }, + { + "event": "boot_ok", + "vm_seq": 241, + "boot_time_s": 2.274, + "density_at_boot": 241, + "timestamp": 1773033772.939943, + "vms_running": 241, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 14.136260986328125, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 241, + "firecracker_rss_mb": 11305.6015625, + "fc_socket_count": 241, + "sqlite_db_kb": 332.0 + }, + { + "event": "boot_ok", + "vm_seq": 242, + "boot_time_s": 3.284, + "density_at_boot": 242, + "timestamp": 1773033776.4048007, + "vms_running": 242, + "host_cpu_pct": 0.6, + "host_mem_used_gb": 14.17977523803711, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 242, + "firecracker_rss_mb": 11352.2734375, + "fc_socket_count": 242, + "sqlite_db_kb": 332.0 + }, + { + "event": "boot_ok", + "vm_seq": 243, + "boot_time_s": 2.318, + "density_at_boot": 243, + "timestamp": 1773033778.9048815, + "vms_running": 243, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 14.225570678710938, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 243, + "firecracker_rss_mb": 11399.97265625, + "fc_socket_count": 243, + "sqlite_db_kb": 332.0 + }, + { + "event": "boot_ok", + "vm_seq": 244, + "boot_time_s": 3.241, + "density_at_boot": 244, + "timestamp": 1773033782.3236187, + "vms_running": 244, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 14.274162292480469, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 244, + "firecracker_rss_mb": 11446.90234375, + "fc_socket_count": 244, + "sqlite_db_kb": 332.0 + }, + { + "event": "boot_ok", + "vm_seq": 245, + "boot_time_s": 3.412, + "density_at_boot": 245, + "timestamp": 1773033785.9220474, + "vms_running": 245, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 14.318473815917969, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 245, + "firecracker_rss_mb": 11493.3125, + "fc_socket_count": 245, + "sqlite_db_kb": 332.0 + }, + { + "event": "boot_ok", + "vm_seq": 246, + "boot_time_s": 3.255, + "density_at_boot": 246, + "timestamp": 1773033789.3639004, + "vms_running": 246, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.37160873413086, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 246, + "firecracker_rss_mb": 11539.7265625, + "fc_socket_count": 246, + "sqlite_db_kb": 336.0 + }, + { + "event": "boot_ok", + "vm_seq": 247, + "boot_time_s": 3.259, + "density_at_boot": 247, + "timestamp": 1773033792.8039594, + "vms_running": 247, + "host_cpu_pct": 0.4, + "host_mem_used_gb": 14.427932739257812, + "host_disk_used_gb": 2.0760574340820312, + "tap_count": 247, + "firecracker_rss_mb": 11586.65234375, + "fc_socket_count": 247, + "sqlite_db_kb": 336.0 + }, + { + "event": "boot_ok", + "vm_seq": 248, + "boot_time_s": 3.26, + "density_at_boot": 248, + "timestamp": 1773033796.244873, + "vms_running": 248, + "host_cpu_pct": 0.1, + "host_mem_used_gb": 14.464420318603516, + "host_disk_used_gb": 2.074138641357422, + "tap_count": 248, + "firecracker_rss_mb": 11633.32421875, + "fc_socket_count": 248, + "sqlite_db_kb": 336.0 + }, + { + "event": "boot_ok", + "vm_seq": 249, + "boot_time_s": 3.26, + "density_at_boot": 249, + "timestamp": 1773033799.6849911, + "vms_running": 249, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.509002685546875, + "host_disk_used_gb": 2.074169158935547, + "tap_count": 249, + "firecracker_rss_mb": 11680.25, + "fc_socket_count": 249, + "sqlite_db_kb": 336.0 + }, + { + "event": "boot_ok", + "vm_seq": 250, + "boot_time_s": 3.131, + "density_at_boot": 250, + "timestamp": 1773033803.0008395, + "vms_running": 250, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 14.562793731689453, + "host_disk_used_gb": 2.074230194091797, + "tap_count": 250, + "firecracker_rss_mb": 11727.43359375, + "fc_socket_count": 250, + "sqlite_db_kb": 336.0 + }, + { + "event": "boot_ok", + "vm_seq": 251, + "boot_time_s": 3.217, + "density_at_boot": 251, + "timestamp": 1773033806.4029758, + "vms_running": 251, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.611839294433594, + "host_disk_used_gb": 2.0742530822753906, + "tap_count": 251, + "firecracker_rss_mb": 11774.62109375, + "fc_socket_count": 251, + "sqlite_db_kb": 340.0 + }, + { + "event": "boot_ok", + "vm_seq": 252, + "boot_time_s": 3.417, + "density_at_boot": 252, + "timestamp": 1773033810.0056808, + "vms_running": 252, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 14.651336669921875, + "host_disk_used_gb": 2.0743331909179688, + "tap_count": 252, + "firecracker_rss_mb": 11822.0625, + "fc_socket_count": 252, + "sqlite_db_kb": 340.0 + }, + { + "event": "boot_ok", + "vm_seq": 253, + "boot_time_s": 3.25, + "density_at_boot": 253, + "timestamp": 1773033813.440772, + "vms_running": 253, + "host_cpu_pct": 0.2, + "host_mem_used_gb": 14.702342987060547, + "host_disk_used_gb": 2.0743141174316406, + "tap_count": 253, + "firecracker_rss_mb": 11868.73046875, + "fc_socket_count": 253, + "sqlite_db_kb": 340.0 + }, + { + "event": "sustain_check", + "sustain_sec": 60, + "vms_alive": 253, + "vms_dead": [], + "vms_checked": 253, + "timestamp": 1773033874.0488198, + "vms_running": 253, + "host_cpu_pct": 0.3, + "host_mem_used_gb": 14.580455780029297, + "host_disk_used_gb": 2.074298858642578, + "tap_count": 253, + "firecracker_rss_mb": 11942.98046875, + "fc_socket_count": 253, + "sqlite_db_kb": 340.0 + }, + { + "event": "teardown", + "teardown_time_s": 127.52, + "timestamp": 1773034001.7450569, + "vms_running": 0, + "host_cpu_pct": 0.0, + "host_mem_used_gb": 3.0937042236328125, + "host_disk_used_gb": 2.0742874145507812, + "tap_count": 253, + "firecracker_rss_mb": 0.0, + "fc_socket_count": 0, + "sqlite_db_kb": 340.0 + } +] \ No newline at end of file diff --git a/benchmarks/run_density_ramp_ec2.sh b/benchmarks/run_density_ramp_ec2.sh new file mode 100755 index 0000000..92c6b5a --- /dev/null +++ b/benchmarks/run_density_ramp_ec2.sh @@ -0,0 +1,408 @@ +#!/usr/bin/env bash +# run_density_ramp_ec2.sh +# +# Launches an EC2 instance, deploys and runs the density_ramp benchmark, +# collects results, then terminates the instance. +# +# Prerequisites: +# - AWS CLI configured (aws configure or IAM role) +# - An existing EC2 key pair (set KEY_NAME below or via env) +# - The key's .pem file accessible at KEY_PATH +# +# Usage: +# ./run_density_ramp_ec2.sh [--tier tiny|small|med] [--max-attempts N] \ +# [--sustain-sec N] [--parallel N] [--shared-disk] \ +# [--output FILE] +# +# Environment variable overrides: +# KEY_NAME EC2 key pair name (required) +# KEY_PATH Path to .pem file (default: ~/.ssh/${KEY_NAME}.pem) +# INSTANCE_TYPE EC2 instance type (default: c5.metal) +# AMI_ID AMI to use (default: latest Amazon Linux 2023 in us-east-1) +# REGION AWS region (default: us-east-1) +# SECURITY_GROUP Security group ID (optional; creates a temporary one if unset) +# SUBNET_ID Subnet ID (optional; uses default VPC subnet if unset) + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Defaults (override via environment variables) +# --------------------------------------------------------------------------- +REGION="${REGION:-us-east-2}" +INSTANCE_TYPE="${INSTANCE_TYPE:-c5d.metal}" +KEY_NAME="${KEY_NAME:-smolvm-benchmark}" +KEY_PATH="${KEY_PATH:-}" +SECURITY_GROUP="${SECURITY_GROUP:-}" +SUBNET_ID="${SUBNET_ID:-}" +AMI_ID="${AMI_ID:-}" + +# Benchmark args (overridden by CLI flags below) +TIER="tiny" +MAX_ATTEMPTS="500" +SUSTAIN_SEC="60" +PARALLEL="1" +SHARED_DISK="" +OUTPUT_FILE="density_$(date +%Y%m%d_%H%M%S).json" + +# --------------------------------------------------------------------------- +# Parse CLI flags +# --------------------------------------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + --tier) TIER="$2"; shift 2 ;; + --max-attempts) MAX_ATTEMPTS="$2"; shift 2 ;; + --sustain-sec) SUSTAIN_SEC="$2"; shift 2 ;; + --parallel) PARALLEL="$2"; shift 2 ;; + --shared-disk) SHARED_DISK="--shared-disk"; shift ;; + --output) OUTPUT_FILE="$2"; shift 2 ;; + *) echo "Unknown argument: $1"; exit 1 ;; + esac +done + +# --------------------------------------------------------------------------- +# Validate prerequisites +# --------------------------------------------------------------------------- +if ! command -v aws &>/dev/null; then + echo "ERROR: AWS CLI not found. Install it: https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html" + exit 1 +fi + +if [[ -z "$KEY_NAME" ]]; then + echo "ERROR: KEY_NAME is not set. Export it or edit this script." + echo " export KEY_NAME=my-key-pair" + exit 1 +fi + +KEY_PATH="${KEY_PATH:-$HOME/.ssh/${KEY_NAME}.pem}" +if [[ ! -f "$KEY_PATH" ]]; then + echo "ERROR: Key file not found at $KEY_PATH" + echo " Set KEY_PATH to the correct location of your .pem file." + exit 1 +fi +chmod 600 "$KEY_PATH" + +# --------------------------------------------------------------------------- +# Resolve AMI (latest Amazon Linux 2023 x86_64) +# --------------------------------------------------------------------------- +if [[ -z "$AMI_ID" ]]; then + echo "Resolving latest Amazon Linux 2023 AMI in ${REGION}..." + AMI_ID=$(aws ec2 describe-images \ + --region "$REGION" \ + --owners amazon \ + --filters \ + "Name=name,Values=al2023-ami-2023*-x86_64" \ + "Name=state,Values=available" \ + --query "sort_by(Images, &CreationDate)[-1].ImageId" \ + --output text) + echo "Using AMI: $AMI_ID" +fi + +# --------------------------------------------------------------------------- +# Optionally create a temporary security group +# --------------------------------------------------------------------------- +CREATED_SG="" +if [[ -z "$SECURITY_GROUP" ]]; then + echo "Creating temporary security group..." + SG_NAME="smolvm-bench-$(date +%s)" + + # Get default VPC + DEFAULT_VPC=$(aws ec2 describe-vpcs \ + --region "$REGION" \ + --filters "Name=isDefault,Values=true" \ + --query "Vpcs[0].VpcId" \ + --output text) + + SECURITY_GROUP=$(aws ec2 create-security-group \ + --region "$REGION" \ + --group-name "$SG_NAME" \ + --description "Temporary SG for SmolVM density benchmark" \ + --vpc-id "$DEFAULT_VPC" \ + --query "GroupId" \ + --output text) + + # Allow SSH only from the caller's public IP + CALLER_IP=$(curl -sf https://checkip.amazonaws.com 2>/dev/null \ + || curl -sf https://ifconfig.me 2>/dev/null \ + || echo "") + if [[ -n "$CALLER_IP" ]]; then + SSH_CIDR="${CALLER_IP}/32" + else + echo "WARNING: could not determine public IP; opening SSH to 0.0.0.0/0" + SSH_CIDR="0.0.0.0/0" + fi + aws ec2 authorize-security-group-ingress \ + --region "$REGION" \ + --group-id "$SECURITY_GROUP" \ + --protocol tcp \ + --port 22 \ + --cidr "$SSH_CIDR" \ + >/dev/null + + CREATED_SG="$SECURITY_GROUP" + echo "Created security group: $SECURITY_GROUP" +fi + +# --------------------------------------------------------------------------- +# Launch EC2 instance +# --------------------------------------------------------------------------- +INSTANCE_ID="" + +cleanup() { + echo "" + echo "--- Cleanup ---" + + if [[ -n "$INSTANCE_ID" ]]; then + echo "Terminating instance $INSTANCE_ID..." + aws ec2 terminate-instances \ + --region "$REGION" \ + --instance-ids "$INSTANCE_ID" \ + >/dev/null + echo "Termination requested." + fi + + if [[ -n "$CREATED_SG" ]]; then + echo "Waiting for instance to terminate before deleting security group..." + aws ec2 wait instance-terminated \ + --region "$REGION" \ + --instance-ids "$INSTANCE_ID" 2>/dev/null || true + echo "Deleting security group $CREATED_SG..." + aws ec2 delete-security-group \ + --region "$REGION" \ + --group-id "$CREATED_SG" 2>/dev/null || true + fi +} +trap cleanup EXIT + +echo "" +echo "Launching $INSTANCE_TYPE instance..." + +LAUNCH_ARGS=( + --region "$REGION" + --image-id "$AMI_ID" + --instance-type "$INSTANCE_TYPE" + --key-name "$KEY_NAME" + --security-group-ids "$SECURITY_GROUP" + --count 1 + --instance-initiated-shutdown-behavior terminate + --query "Instances[0].InstanceId" + --output text +) +if [[ -n "$SUBNET_ID" ]]; then + LAUNCH_ARGS+=(--subnet-id "$SUBNET_ID") +fi + +INSTANCE_ID=$(aws ec2 run-instances "${LAUNCH_ARGS[@]}") +echo "Instance ID: $INSTANCE_ID" + +# --------------------------------------------------------------------------- +# Wait for instance to be running and pass status checks +# --------------------------------------------------------------------------- +echo "Waiting for instance to reach 'running' state..." +aws ec2 wait instance-running \ + --region "$REGION" \ + --instance-ids "$INSTANCE_ID" + +echo "Waiting for system status checks to pass..." +aws ec2 wait instance-status-ok \ + --region "$REGION" \ + --instance-ids "$INSTANCE_ID" + +PUBLIC_IP=$(aws ec2 describe-instances \ + --region "$REGION" \ + --instance-ids "$INSTANCE_ID" \ + --query "Reservations[0].Instances[0].PublicIpAddress" \ + --output text) +echo "Instance running at $PUBLIC_IP" + +SSH="ssh -i $KEY_PATH -o StrictHostKeyChecking=no -o ConnectTimeout=10 ec2-user@$PUBLIC_IP" + +# --------------------------------------------------------------------------- +# Wait for SSH to become available +# --------------------------------------------------------------------------- +echo "Waiting for SSH to become available..." +for i in $(seq 1 30); do + if $SSH "echo ok" &>/dev/null; then + echo "SSH is up." + break + fi + if [[ $i -eq 30 ]]; then + echo "ERROR: SSH never became available." + exit 1 + fi + sleep 10 +done + +# --------------------------------------------------------------------------- +# Package smolvm source for upload +# --------------------------------------------------------------------------- +echo "" +echo "--- Packaging smolvm source ---" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +SOURCE_TAR="$(mktemp -d)/smolvm-src.tar.gz" +(cd "$REPO_ROOT" && git archive --format=tar.gz HEAD -- src pyproject.toml README.md > "$SOURCE_TAR") +echo "Packaged source: $(basename "$SOURCE_TAR")" + +# --------------------------------------------------------------------------- +# Install dependencies on the instance +# --------------------------------------------------------------------------- +echo "" +echo "--- Installing dependencies ---" +$SSH bash <<'REMOTE' +set -euo pipefail + +# Enable KVM access +sudo usermod -aG kvm ec2-user || true +sudo chmod 666 /dev/kvm + +# Raise open-file and process limits for high-density VM runs +echo "ec2-user soft nofile 65536" | sudo tee -a /etc/security/limits.conf +echo "ec2-user hard nofile 65536" | sudo tee -a /etc/security/limits.conf +echo "ec2-user soft nproc 65536" | sudo tee -a /etc/security/limits.conf +echo "ec2-user hard nproc 65536" | sudo tee -a /etc/security/limits.conf +# Allow a large number of concurrent TCP connections +sudo sysctl -w net.ipv4.ip_local_port_range="1024 65535" >/dev/null +sudo sysctl -w net.core.somaxconn=65535 >/dev/null + +# Disable firewalld — it uses the nftables backend and can conflict with +# SmolVM's own nftables rules, blocking TAP device forwarding. +sudo systemctl stop firewalld 2>/dev/null || true +sudo systemctl disable firewalld 2>/dev/null || true + +# Install Python 3.11, pip, and Docker +sudo dnf install -y python3.11 python3.11-pip docker nftables --quiet +sudo systemctl start docker +sudo usermod -aG docker ec2-user + +# Mount NVMe instance store and redirect all smolvm data to it (if present) +if test -b /dev/nvme1n1; then + sudo mkfs.ext4 -F /dev/nvme1n1 + sudo mkdir -p /mnt/nvme + sudo mount /dev/nvme1n1 /mnt/nvme + sudo mkdir -p /mnt/nvme/smolvm-images /mnt/nvme/smolvm-state + sudo chown ec2-user:ec2-user /mnt/nvme /mnt/nvme/smolvm-images /mnt/nvme/smolvm-state + # Image cache (~/.smolvm/images/) → NVMe + ln -sfn /mnt/nvme/smolvm-images /home/ec2-user/.smolvm + # Disk clones (~/.local/state/smolvm/disks/) → NVMe + mkdir -p /home/ec2-user/.local/state + ln -sfn /mnt/nvme/smolvm-state /home/ec2-user/.local/state/smolvm +else + echo "WARNING: /dev/nvme1n1 not found; using root volume for SmolVM data." +fi +REMOTE + +scp -i "$KEY_PATH" -o StrictHostKeyChecking=no \ + "$SOURCE_TAR" \ + "ec2-user@$PUBLIC_IP:~/smolvm-src.tar.gz" + +$SSH bash <<'REMOTE' +set -euo pipefail + +# Install smolvm from source and psutil +pip3.11 install --quiet ~/smolvm-src.tar.gz psutil + +# Download Firecracker binary +python3.11 -c "from smolvm.host import HostManager; HostManager().install_firecracker()" +REMOTE + +rm -rf "$(dirname "$SOURCE_TAR")" +echo "Dependencies installed." + +# --------------------------------------------------------------------------- +# Upload benchmark script +# --------------------------------------------------------------------------- +echo "" +echo "--- Uploading benchmark ---" +scp -i "$KEY_PATH" -o StrictHostKeyChecking=no \ + "$SCRIPT_DIR/density_ramp.py" \ + "ec2-user@$PUBLIC_IP:~/density_ramp.py" +echo "Uploaded density_ramp.py" + +# --------------------------------------------------------------------------- +# Run benchmark +# --------------------------------------------------------------------------- +REMOTE_OUTPUT="density.json" +BENCHMARK_CMD="python3.11 ~/density_ramp.py \ + --tier $TIER \ + --max-attempts $MAX_ATTEMPTS \ + --sustain-sec $SUSTAIN_SEC \ + --parallel $PARALLEL \ + ${SHARED_DISK} \ + --output ~/$REMOTE_OUTPUT" + +echo "" +echo "--- Running benchmark ---" +echo " tier=$TIER max-attempts=$MAX_ATTEMPTS sustain-sec=$SUSTAIN_SEC parallel=$PARALLEL ${SHARED_DISK:+shared-disk}" +echo "" + +# Run with a pseudo-TTY so output streams live; failure here still triggers cleanup +$SSH -t "$BENCHMARK_CMD" || { + echo "" + echo "WARNING: Benchmark exited with a non-zero status (may be expected at resource limit)." +} + +# --------------------------------------------------------------------------- +# Post-run diagnostics (network state + first VM log) +# --------------------------------------------------------------------------- +echo "" +echo "--- Post-run diagnostics ---" +$SSH bash <<'DIAG' || true +set -uo pipefail + +echo "=== Network interfaces (TAP devices) ===" +ip -o link show | grep -E "tap|lo|eth" || true + +echo "" +echo "=== Routes ===" +ip route show + +echo "" +echo "=== nftables ruleset ===" +sudo nft list ruleset 2>/dev/null || echo "(nft list failed)" + +echo "" +echo "=== Most recent Firecracker VM log (last 60 lines) ===" +LOG=$(ls -t ~/.local/state/smolvm/vm-*.log 2>/dev/null | head -1) +if [ -n "$LOG" ]; then + echo "Log: $LOG" + tail -60 "$LOG" +else + echo "(no VM log files found under ~/.local/state/smolvm/)" +fi +DIAG + +# --------------------------------------------------------------------------- +# Collect results +# --------------------------------------------------------------------------- +echo "" +echo "--- Collecting results ---" +scp -i "$KEY_PATH" -o StrictHostKeyChecking=no \ + "ec2-user@$PUBLIC_IP:~/$REMOTE_OUTPUT" \ + "./$OUTPUT_FILE" + +echo "Results saved to: $OUTPUT_FILE" +echo "" +echo "Quick summary:" +python3 - "$OUTPUT_FILE" <<'PYEOF' +import json, sys + +with open(sys.argv[1]) as f: + data = json.load(f) + +boots = [d for d in data if d.get("event") == "boot_ok"] +fails = [d for d in data if d.get("event") == "boot_fail"] +sustain = next((d for d in data if d.get("event") == "sustain_check"), None) + +print(f" Successful boots : {len(boots)}") +print(f" Failed boots : {len(fails)}") +if boots: + times = [d["boot_time_s"] for d in boots] + print(f" Boot time avg : {sum(times)/len(times):.2f}s") + print(f" Boot time max : {max(times):.2f}s") + last = boots[-1] + print(f" Peak mem used : {last['host_mem_used_gb']:.2f} GB") +if sustain: + print(f" Sustain alive : {sustain['vms_alive']}/{sustain['vms_checked']} VMs") +PYEOF + +# Instance is terminated by the cleanup trap on EXIT diff --git a/benchmarks/run_density_ramp_gcp.sh b/benchmarks/run_density_ramp_gcp.sh new file mode 100755 index 0000000..5dbba31 --- /dev/null +++ b/benchmarks/run_density_ramp_gcp.sh @@ -0,0 +1,338 @@ +#!/usr/bin/env bash +# run_density_ramp_gcp.sh +# +# Launches a GCP instance, deploys and runs the density_ramp benchmark, +# collects results, then deletes the instance. +# +# Prerequisites: +# - gcloud CLI configured (gcloud auth login && gcloud config set project PROJECT) +# - A GCP project with the Compute Engine API enabled +# +# Usage: +# ./run_density_ramp_gcp.sh [--tier tiny|small|med] [--max-attempts N] \ +# [--sustain-sec N] [--parallel N] [--shared-disk] \ +# [--output FILE] +# +# Environment variable overrides: +# PROJECT GCP project ID (required if not set in gcloud config) +# ZONE GCP zone (default: us-central1-a) +# MACHINE_TYPE GCP machine type (default: n2-standard-16) +# IMAGE_FAMILY OS image family (default: debian-12) +# IMAGE_PROJECT OS image project (default: debian-cloud) +# FIREWALL_RULE Firewall rule name (optional; creates a temporary one if unset) + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Defaults (override via environment variables) +# --------------------------------------------------------------------------- +PROJECT="${PROJECT:-$(gcloud config get-value project 2>/dev/null || true)}" +ZONE="${ZONE:-us-central1-a}" +MACHINE_TYPE="${MACHINE_TYPE:-n2-standard-16}" +IMAGE_FAMILY="${IMAGE_FAMILY:-debian-12}" +IMAGE_PROJECT="${IMAGE_PROJECT:-debian-cloud}" +FIREWALL_RULE="${FIREWALL_RULE:-}" + +# Benchmark args (overridden by CLI flags below) +TIER="tiny" +MAX_ATTEMPTS="500" +SUSTAIN_SEC="60" +PARALLEL="1" +SHARED_DISK="" +OUTPUT_FILE="density_$(date +%Y%m%d_%H%M%S).json" + +# --------------------------------------------------------------------------- +# Parse CLI flags +# --------------------------------------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + --tier) TIER="$2"; shift 2 ;; + --max-attempts) MAX_ATTEMPTS="$2"; shift 2 ;; + --sustain-sec) SUSTAIN_SEC="$2"; shift 2 ;; + --parallel) PARALLEL="$2"; shift 2 ;; + --shared-disk) SHARED_DISK="--shared-disk"; shift ;; + --output) OUTPUT_FILE="$2"; shift 2 ;; + *) echo "Unknown argument: $1"; exit 1 ;; + esac +done + +# --------------------------------------------------------------------------- +# Validate prerequisites +# --------------------------------------------------------------------------- +if ! command -v gcloud &>/dev/null; then + echo "ERROR: gcloud CLI not found. Install it: https://cloud.google.com/sdk/docs/install" + exit 1 +fi + +if [[ -z "$PROJECT" ]]; then + echo "ERROR: GCP project is not set." + echo " export PROJECT=my-gcp-project" + echo " or: gcloud config set project my-gcp-project" + exit 1 +fi + +# --------------------------------------------------------------------------- +# Optionally create a temporary firewall rule allowing SSH +# --------------------------------------------------------------------------- +INSTANCE_NAME="smolvm-bench-$(date +%s)" +NETWORK_TAG="$INSTANCE_NAME" +CREATED_FIREWALL_RULE="" + +if [[ -z "$FIREWALL_RULE" ]]; then + FIREWALL_RULE="$NETWORK_TAG-ssh" + # Restrict SSH to the caller's public IP + CALLER_IP=$(curl -sf https://ifconfig.me 2>/dev/null \ + || curl -sf https://checkip.amazonaws.com 2>/dev/null \ + || echo "") + if [[ -n "$CALLER_IP" ]]; then + SSH_SOURCE_RANGE="${CALLER_IP}/32" + else + echo "WARNING: could not determine public IP; opening SSH to 0.0.0.0/0" + SSH_SOURCE_RANGE="0.0.0.0/0" + fi + echo "Creating temporary firewall rule $FIREWALL_RULE (source: $SSH_SOURCE_RANGE)..." + gcloud compute firewall-rules create "$FIREWALL_RULE" \ + --project="$PROJECT" \ + --direction=INGRESS \ + --action=ALLOW \ + --rules=tcp:22 \ + --source-ranges="$SSH_SOURCE_RANGE" \ + --target-tags="$NETWORK_TAG" \ + --quiet + CREATED_FIREWALL_RULE="$FIREWALL_RULE" + echo "Created firewall rule: $FIREWALL_RULE" +fi + +# --------------------------------------------------------------------------- +# Launch instance +# --------------------------------------------------------------------------- +INSTANCE_CREATED=0 + +cleanup() { + echo "" + echo "--- Cleanup ---" + + if [[ "$INSTANCE_CREATED" -eq 1 ]]; then + echo "Deleting instance $INSTANCE_NAME..." + gcloud compute instances delete "$INSTANCE_NAME" \ + --project="$PROJECT" \ + --zone="$ZONE" \ + --quiet 2>/dev/null || true + echo "Instance deleted." + fi + + if [[ -n "$CREATED_FIREWALL_RULE" ]]; then + echo "Deleting firewall rule $CREATED_FIREWALL_RULE..." + gcloud compute firewall-rules delete "$CREATED_FIREWALL_RULE" \ + --project="$PROJECT" \ + --quiet 2>/dev/null || true + fi +} +trap cleanup EXIT + +echo "" +echo "Launching $MACHINE_TYPE instance ($INSTANCE_NAME)..." + +# --enable-nested-virtualization is required for KVM on GCP +gcloud compute instances create "$INSTANCE_NAME" \ + --project="$PROJECT" \ + --zone="$ZONE" \ + --machine-type="$MACHINE_TYPE" \ + --image-family="$IMAGE_FAMILY" \ + --image-project="$IMAGE_PROJECT" \ + --boot-disk-size=100GB \ + --boot-disk-type=pd-ssd \ + --enable-nested-virtualization \ + --tags="$NETWORK_TAG" \ + --quiet + +INSTANCE_CREATED=1 +echo "Instance created: $INSTANCE_NAME" + +# --------------------------------------------------------------------------- +# Wait for instance to be running +# --------------------------------------------------------------------------- +echo "Waiting for instance to reach RUNNING state..." +for i in $(seq 1 30); do + STATUS=$(gcloud compute instances describe "$INSTANCE_NAME" \ + --project="$PROJECT" \ + --zone="$ZONE" \ + --format="value(status)") + if [[ "$STATUS" == "RUNNING" ]]; then + echo "Instance is running." + break + fi + if [[ $i -eq 30 ]]; then + echo "ERROR: Instance never reached RUNNING state (last status: $STATUS)." + exit 1 + fi + sleep 5 +done + +# --------------------------------------------------------------------------- +# Wait for SSH to become available +# --------------------------------------------------------------------------- +echo "Waiting for SSH to become available..." +for i in $(seq 1 30); do + if gcloud compute ssh "$INSTANCE_NAME" \ + --project="$PROJECT" \ + --zone="$ZONE" \ + --strict-host-key-checking=no \ + --command="echo ok" &>/dev/null; then + echo "SSH is up." + break + fi + if [[ $i -eq 30 ]]; then + echo "ERROR: SSH never became available." + exit 1 + fi + sleep 10 +done + +GSSH=(gcloud compute ssh "$INSTANCE_NAME" + --project="$PROJECT" + --zone="$ZONE" + --strict-host-key-checking=no +) +GSCP=(gcloud compute scp + --project="$PROJECT" + --zone="$ZONE" + --strict-host-key-checking=no +) + +# --------------------------------------------------------------------------- +# Install dependencies on the instance +# --------------------------------------------------------------------------- +echo "" +echo "--- Installing dependencies ---" +"${GSSH[@]}" --command="bash -s" <<'REMOTE' +set -euo pipefail + +# Enable KVM access +sudo usermod -aG kvm "$USER" || true +sudo setfacl -m u:"$USER":rw /dev/kvm 2>/dev/null || sudo chmod 666 /dev/kvm + +# Raise open-file and process limits for high-density VM runs +echo "$USER soft nofile 65536" | sudo tee -a /etc/security/limits.conf +echo "$USER hard nofile 65536" | sudo tee -a /etc/security/limits.conf +echo "$USER soft nproc 65536" | sudo tee -a /etc/security/limits.conf +echo "$USER hard nproc 65536" | sudo tee -a /etc/security/limits.conf +# Allow a large number of concurrent TCP connections +sudo sysctl -w net.ipv4.ip_local_port_range="1024 65535" >/dev/null +sudo sysctl -w net.core.somaxconn=65535 >/dev/null + +# Install Python, pip, Docker, and nftables +sudo apt-get update -qq +sudo apt-get install -y -qq python3 python3-pip python3-venv docker.io nftables +sudo systemctl start docker +sudo usermod -aG docker "$USER" + +# Install smolvm and psutil into a venv (avoids PEP 668 externally-managed-env error) +python3 -m venv ~/venv +~/venv/bin/pip install --quiet smolvm psutil + +# Trigger Firecracker binary download +~/venv/bin/smolvm demo list 2>/dev/null || true +REMOTE + +echo "Dependencies installed." + +# --------------------------------------------------------------------------- +# Upload benchmark script +# --------------------------------------------------------------------------- +echo "" +echo "--- Uploading benchmark ---" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +"${GSCP[@]}" \ + "$SCRIPT_DIR/density_ramp.py" \ + "${INSTANCE_NAME}:~/density_ramp.py" +echo "Uploaded density_ramp.py" + +# --------------------------------------------------------------------------- +# Run benchmark +# --------------------------------------------------------------------------- +REMOTE_OUTPUT="density.json" +BENCHMARK_CMD="~/venv/bin/python ~/density_ramp.py \ + --tier $TIER \ + --max-attempts $MAX_ATTEMPTS \ + --sustain-sec $SUSTAIN_SEC \ + --parallel $PARALLEL \ + ${SHARED_DISK} \ + --output ~/$REMOTE_OUTPUT" + +echo "" +echo "--- Running benchmark ---" +echo " tier=$TIER max-attempts=$MAX_ATTEMPTS sustain-sec=$SUSTAIN_SEC parallel=$PARALLEL ${SHARED_DISK:+shared-disk}" +echo "" + +# Run with a pseudo-TTY so output streams live; failure here still triggers cleanup +"${GSSH[@]}" --command="$BENCHMARK_CMD" -- -t || { + echo "" + echo "WARNING: Benchmark exited with a non-zero status (may be expected at resource limit)." +} + +# --------------------------------------------------------------------------- +# Post-run diagnostics (network state + first VM log) +# --------------------------------------------------------------------------- +echo "" +echo "--- Post-run diagnostics ---" +"${GSSH[@]}" --command="bash -s" <<'DIAG' || true +set -uo pipefail + +echo "=== Network interfaces (TAP devices) ===" +ip -o link show | grep -E "tap|lo|eth" || true + +echo "" +echo "=== Routes ===" +ip route show + +echo "" +echo "=== nftables ruleset ===" +sudo nft list ruleset 2>/dev/null || echo "(nft list failed)" + +echo "" +echo "=== Most recent Firecracker VM log (last 60 lines) ===" +LOG=$(ls -t ~/.local/state/smolvm/vm-*.log 2>/dev/null | head -1) +if [ -n "$LOG" ]; then + echo "Log: $LOG" + tail -60 "$LOG" +else + echo "(no VM log files found under ~/.local/state/smolvm/)" +fi +DIAG + +# --------------------------------------------------------------------------- +# Collect results +# --------------------------------------------------------------------------- +echo "" +echo "--- Collecting results ---" +"${GSCP[@]}" \ + "${INSTANCE_NAME}:~/$REMOTE_OUTPUT" \ + "./$OUTPUT_FILE" + +echo "Results saved to: $OUTPUT_FILE" +echo "" +echo "Quick summary:" +python3 - "$OUTPUT_FILE" <<'PYEOF' +import json, sys + +with open(sys.argv[1]) as f: + data = json.load(f) + +boots = [d for d in data if d.get("event") == "boot_ok"] +fails = [d for d in data if d.get("event") == "boot_fail"] +sustain = next((d for d in data if d.get("event") == "sustain_check"), None) + +print(f" Successful boots : {len(boots)}") +print(f" Failed boots : {len(fails)}") +if boots: + times = [d["boot_time_s"] for d in boots] + print(f" Boot time avg : {sum(times)/len(times):.2f}s") + print(f" Boot time max : {max(times):.2f}s") + last = boots[-1] + print(f" Peak mem used : {last['host_mem_used_gb']:.2f} GB") +if sustain: + print(f" Sustain alive : {sustain['vms_alive']}/{sustain['vms_checked']} VMs") +PYEOF + +# Instance is deleted by the cleanup trap on EXIT diff --git a/pyproject.toml b/pyproject.toml index 9cf1b1a..e479223 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,9 @@ dependencies = [ ] [project.optional-dependencies] +benchmarks = [ + "psutil>=7.2.2", +] dev = [ "pytest>=7.0", "pytest-cov>=4.0", diff --git a/src/smolvm/build.py b/src/smolvm/build.py index f512f92..5706c4c 100644 --- a/src/smolvm/build.py +++ b/src/smolvm/build.py @@ -20,6 +20,7 @@ import hashlib import json import logging +import os import platform import re import shlex @@ -974,9 +975,7 @@ def _create_ext4_with_docker( """ logger.info(" [3/4] Creating ext4 filesystem via Docker helper (%dMB)...", rootfs_size_mb) - rootfs_dir = tmpdir / "rootfs-dir" - rootfs_dir.mkdir() - + # Security check: scan tar member paths and link targets before Docker extracts them. with tarfile.open(tar_path, "r") as tar: for member in tar.getmembers(): member_path = Path(member.name) @@ -984,23 +983,41 @@ def _create_ext4_with_docker( raise ImageError( f"Refusing to extract suspicious tar path from docker export: {member.name}" ) - - # Python 3.14 defaults to a restrictive extraction filter that rejects - # absolute symlink targets commonly present in container rootfs archives. - # We already validated member names above, so trusted extraction is safe here. - try: - tar.extractall(path=rootfs_dir, filter="fully_trusted") - except TypeError: - # Python <3.12 does not support the 'filter' argument. - tar.extractall(path=rootfs_dir) - + if member.issym() or member.islnk(): + link_path = Path(member.linkname) + if link_path.is_absolute() or ".." in link_path.parts: + raise ImageError( + "Refusing to extract suspicious tar link from docker export: " + f"{member.name} -> {member.linkname}" + ) + + # Let Docker (running as root) extract the tarball and build the ext4 + # image in a single pass. This preserves the correct uid/gid for every + # file in the container image — Python's tarfile.extractall() silently + # drops root ownership when the host user is non-root, which causes sshd + # to reject /root/.ssh/authorized_keys and crash on /var/empty. + # + # We exclude ./dev to avoid device-node creation (CAP_MKNOD is not + # available without --privileged); the guest init mounts devtmpfs at + # boot so device nodes in the rootfs are not required. rootfs_path.unlink(missing_ok=True) rootfs_name = shlex.quote(rootfs_path.name) + tar_name = shlex.quote(tar_path.name) + uid, gid = os.getuid(), os.getgid() shell_cmd = ( "set -e; " "apk add --no-cache e2fsprogs >/dev/null; " + "mkdir /work/rootfs; " + f"tar xf /work/tar/{tar_name} -C /work/rootfs --exclude=./dev; " + "mkdir -p /work/rootfs/dev; " + # Defensive: guard against edge cases in the Alpine base image. + "chown root:root /work/rootfs/var/empty 2>/dev/null || true; " + "chmod 755 /work/rootfs/var/empty 2>/dev/null || true; " f"mke2fs -d /work/rootfs -t ext4 -F /work/out/{rootfs_name} " - f"{rootfs_size_mb}M >/dev/null" + f"{rootfs_size_mb}M >/dev/null; " + # Restore ownership of the output file so the calling (non-root) user + # can read, cache, and clean up the ext4 image on Linux with rootful Docker. + f"chown {uid}:{gid} /work/out/{rootfs_name} 2>/dev/null || true" ) try: @@ -1010,7 +1027,9 @@ def _create_ext4_with_docker( "run", "--rm", "-v", - f"{rootfs_dir.resolve()}:/work/rootfs:ro", + # tar_path lives inside tmpdir; mount tmpdir read-only so + # Docker can read the tarball without touching the host tree. + f"{tar_path.parent.resolve()}:/work/tar:ro", "-v", f"{rootfs_path.parent.resolve()}:/work/out", "alpine:3.19",