From 6634cc3829b1550f5537cc21868f91c2e1ae1b66 Mon Sep 17 00:00:00 2001 From: Taleh Ibrahimli Date: Sun, 29 Mar 2026 23:20:00 +0200 Subject: [PATCH] Add gRPC reflection support, Docker integration, documentation updates, and crash recovery improvements. --- .gitignore | 1 + Cargo.lock | 14 +++ Cargo.toml | 3 +- Dockerfile | 45 +++++++ README.md | 211 ++++++++++++++++----------------- build.rs | 11 +- docker-compose.yml | 14 +++ docs/Design.md | 2 + docs/benchmarks.md | 55 +++++++++ docs/grpc_examples.md | 121 +++++++++++++++++++ docs/operations_consistency.md | 94 +++++++++++++++ src/grpc/server.rs | 18 ++- tests/crash_recovery_test.rs | 51 +++++++- 13 files changed, 525 insertions(+), 115 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 docs/benchmarks.md create mode 100644 docs/grpc_examples.md create mode 100644 docs/operations_consistency.md diff --git a/.gitignore b/.gitignore index 6a54a9e..8caa9d8 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ target .idea/ **/wal.bin +**/snapshot.bin dev/ diff --git a/Cargo.lock b/Cargo.lock index b76bc04..59ce56e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1189,6 +1189,7 @@ dependencies = [ "tokio", "tonic", "tonic-build", + "tonic-reflection", ] [[package]] @@ -1475,6 +1476,19 @@ dependencies = [ "syn", ] +[[package]] +name = "tonic-reflection" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "878d81f52e7fcfd80026b7fdb6a9b578b3c3653ba987f87f0dce4b64043cba27" +dependencies = [ + "prost", + "prost-types", + "tokio", + "tokio-stream", + "tonic", +] + [[package]] name = "tower" version = "0.4.13" diff --git a/Cargo.toml b/Cargo.toml index 2aa5a94..c9af563 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ rand = "0.8" smallvec = { version = "1.13", features = ["serde"] } bitflags = { version = "2.8", features = ["serde"] } tonic = { version = "0.12", features = ["transport"], optional = true } +tonic-reflection = { version = "0.12", optional = true } prost = { version = "0.13", optional = true } [dev-dependencies] @@ -30,7 +31,7 @@ criterion = { version = "0.5", features = ["async_tokio"] } [features] default = [] -grpc = ["dep:tonic", "dep:prost", "dep:tonic-build"] +grpc = ["dep:tonic", "dep:prost", "dep:tonic-build", "dep:tonic-reflection"] [build-dependencies] tonic-build = { version = "0.12", optional = true } diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..057906a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +# Stage 1: Builder +FROM rust:latest AS builder + +# Install protobuf compiler for tonic-build +RUN apt-get update && apt-get install -y \ + protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/src/roda-ledger + +# Copy all files +COPY . . + +# Build the project with grpc feature in release mode +RUN cargo build --release --features grpc + +# Stage 2: Runtime +FROM debian:bookworm-slim + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy the binary from the builder stage +COPY --from=builder /usr/src/roda-ledger/target/release/roda-ledger /app/roda-ledger + +# Expose the gRPC port +EXPOSE 50051 + +# Set default environment variables +ENV RODA_GRPC_ADDR=0.0.0.0:50051 +ENV RODA_DATA_DIR=/data +ENV RODA_MAX_ACCOUNTS=1000000 +ENV RODA_SNAPSHOT_INTERVAL=600 +ENV RODA_IN_MEMORY=false + +# Create data directory and set volume +RUN mkdir -p /data +VOLUME /data + +# Run the binary +ENTRYPOINT ["/app/roda-ledger"] diff --git a/README.md b/README.md index 1e7f66b..3b351b9 100644 --- a/README.md +++ b/README.md @@ -1,147 +1,146 @@ # Roda-Ledger -A high-performance, durable, and crash-consistent financial ledger and transaction executor built in Rust. +### 🚀 Ultra-High Performance Ledger for Modern Finance -## Overview +A high-performance, durable, and crash-consistent financial ledger and transaction executor built in Rust, capable of processing **6.6 Million+ transactions per second** with **nanosecond-level latency**. -Roda-Ledger is designed for low-latency, high-throughput financial applications. It utilizes a pipelined architecture to -maximize hardware efficiency while ensuring strict transaction ordering and durability. It is built to handle millions -of transactions per second with microsecond-level latency. +--- -## Key Features +## What is Roda-Ledger? -- **High Performance:** Pipelined execution model optimized for modern CPU architectures. -- **Strict Durability:** Write-Ahead Logging (WAL) ensures every transaction is persisted before confirmation. -- **Crash Consistency:** Automatic state recovery via snapshot loading and WAL replay. -- **Customizable:** Use the built-in `Operation` enum for standard financial transactions or define multi-step `Complex` atomic operations. -- **Thread-Safe:** Lock-free communication between pipeline stages. +Roda-Ledger is a specialized database engine designed for recording and executing financial transactions with industry-leading performance. It serves as a high-performance "source of truth" for account balances, providing strict durability, deterministic execution, and the reliability required for mission-critical financial systems. -## Architecture: Pipelined Execution +### Target Use Cases +- **Core Banking**: High-volume retail and investment banking ledger systems. +- **Crypto-Exchanges**: Real-time matching engines and wallet management. +- **Payment Gateways**: High-volume transaction clearing and settlement. +- **Gaming Economies**: Managing in-game currencies and item trades at scale. +- **HFT Systems**: Low-latency risk management and position tracking. -Roda-Ledger uses a "Core-Per-Stage" model, utilizing asynchronous pipelining and lock-free `ArrayQueue`s to eliminate -thread contention and maximize mechanical sympathy. +--- -1. **Sequencer (Core 0):** The entry point. Assigns unique, monotonic IDs to incoming transactions and manages the - ingestion interface. -2. **Transactor (Core 1):** The deterministic engine. Performs all business logic, balance checks, and state - transitions. It maintains a hot cache of the ledger state in memory. -3. **WAL Storer (Core 2):** The durability layer. Responsible for writing transactions to the Write-Ahead Log (WAL) on - disk. -4. **Snapshotter (Core 3):** The archival layer. Periodically takes snapshots of the ledger state to optimize recovery - and manage WAL growth. +## Why Roda-Ledger? -For a deep dive into the architecture, see [Design.md](docs/Design.md) and our [Architectural Decision Records](docs/adr/). +Roda-Ledger is built for scale. While traditional databases struggle with the lock contention of high-frequency ledger updates, Roda-Ledger's architecture allows it to outpace everything in its class. -## Core Components +### The "Plus" (Strengths) +- **🚀 Industry-Leading Throughput**: Process over **6.6 Million transactions per second** (TPS) on CX33 server at Hetzner. +- **⏱️ Predictable Low Latency**: Nanosecond to microsecond level execution times, ensuring your system never bottlenecks. +- **💾 Strict Durability**: Every transaction is persisted via Write-Ahead Logging (WAL) before confirmation—zero data loss. +- **⚛️ Atomic Composite Operations**: Perform multi-step transfers (e.g., Transfer + Fee) as a single atomic unit. +- **🔄 Crash Consistency**: Automatic state recovery from snapshots and WAL replay after a crash. +- **🛠️ Flexible Integration**: Run as a standalone gRPC server or embed it as a Rust library. -### Ledger +### The "Minus" (Trade-offs) +- **Single-Node Focus**: Optimized for vertical scaling; currently operates as a single-leader instance. +- **Memory-Bound**: For peak performance, the "hot" state (account balances) should fit in RAM. +- **Specialized Engine**: Not a general-purpose database; designed specifically for financial ledger operations. -The `Ledger` is the primary interface. It coordinates the pipeline stages and provides methods for submitting -transactions and retrieving balances. +--- -```rust -use roda_ledger::ledger::{Ledger, LedgerConfig}; -use roda_ledger::transaction::Operation; +## ⚡ Quick Start: Server Mode (Docker) -let config = LedgerConfig { - location: Some("ledger_data".to_string()), - in_memory: false, - ..Default::default() -}; +The fastest way to deploy Roda-Ledger is using the official Docker image. -let mut ledger = Ledger::new(config); -ledger.start(); +### 1. Run the Server +```bash +docker run -p 50051:50051 -v $(pwd)/data:/data tislib/roda-ledger:latest +``` -// Submit a transaction (Deposit) -let account_id = 1; -let tx_id = ledger.submit(Operation::Deposit { - account: account_id, - amount: 100, - user_ref: 0 -}); +### 2. Interact via gRPC +Use `grpcurl` to submit operations: -// Wait for the transaction to be fully processed across all pipeline stages -ledger.wait_for_transaction(tx_id); +**Deposit Funds:** +```bash +grpcurl -plaintext -d '{"deposit": {"account": 1, "amount": "1000", "user_ref": "123"}}' \ + localhost:50051 roda.ledger.v1.Ledger/SubmitOperation +``` -// Retrieve the resulting balance -let balance = ledger.get_balance(account_id); +**Get Balance:** +```bash +grpcurl -plaintext -d '{"account_id": 1}' \ + localhost:50051 roda.ledger.v1.Ledger/GetBalance ``` -### Operations +--- -Roda-Ledger uses a concrete `Operation` enum for all interactions. It supports `Deposit`, `Withdrawal`, `Transfer`, `Composite`, and `Named` operations. +## 🦀 Quick Start: Library Mode (Rust) -```rust -use roda_ledger::transaction::{Operation, CompositeOperation, Step, CompositeOperationFlags}; -use smallvec::smallvec; - -// 1. Simple Deposit -ledger.submit(Operation::Deposit { account: 1, amount: 1000, user_ref: 0 }); - -// 2. Simple Transfer -ledger.submit(Operation::Transfer { from: 1, to: 2, amount: 500, user_ref: 0 }); - -// 3. Composite Atomic Operation (e.g., Transfer with a Fee) -ledger.submit(Operation::Composite(Box::new(CompositeOperation { - steps: smallvec![ - Step::Credit { account_id: 1, amount: 105 }, // Sender pays amount + fee - Step::Debit { account_id: 2, amount: 100 }, // Receiver gets amount - Step::Debit { account_id: 0, amount: 5 }, // System gets fee - ], - flags: CompositeOperationFlags::CHECK_NEGATIVE_BALANCE, - user_ref: 12345, -}))); +Integrate Roda-Ledger directly into your Rust application for maximum performance. + +### 1. Add Dependency +Add this to your `Cargo.toml`: +```toml +[dependencies] +roda-ledger = { git = "https://github.com/tislib/roda-ledger" } ``` -## Durability, Persistence, and Crash Recovery +### 2. Basic Usage +```rust +use roda_ledger::ledger::{Ledger, LedgerConfig}; +use roda_ledger::transaction::Operation; -Roda-Ledger is built to be "crash-safe" by design, ensuring that once a transaction is confirmed as committed, it will -never be lost. +fn main() { + // Initialize with default config + let mut ledger = Ledger::new(LedgerConfig::default()); + ledger.start(); + + // Submit a deposit + let tx_id = ledger.submit(Operation::Deposit { + account: 1, + amount: 1000, + user_ref: 0 + }); + + // Wait for the transaction to be persisted (WAL) + ledger.wait_for_transaction(tx_id); + + // Retrieve balance + let balance = ledger.get_balance(1); + println!("Account 1 balance: {}", balance); +} +``` -- **Write-Ahead Log (WAL):** Every transaction is appended to a persistent, append-only log on disk before it is marked - as `Committed`. -- **Snapshots:** To prevent the WAL from growing indefinitely and to speed up startup, the system periodically takes - snapshots of all account balances. -- **Persistence:** You can choose between full persistence (on-disk) or high-performance in-memory mode via - `LedgerConfig`. -- **Crash Recovery:** Upon restart, Roda-Ledger automatically: - 1. Identifies the latest consistent snapshot. - 2. Loads all balances from that snapshot. - 3. Replays all transactions from the WAL that occurred *after* the snapshot was taken. - 4. Resumes normal operation from the exact point where it left off. +--- -## Benchmarks +## Key Features -Roda-Ledger is highly optimized for throughput and latency. Below are the results from the `wallet_bench`: +- **Pipelined Architecture**: Separates Sequencing, Execution, Persistence, and Snapshotting into dedicated CPU cores to eliminate lock contention. +- **Zero-Sum Invariant**: Ensures that value is never created or destroyed; every credit is balanced by a corresponding debit. +- **Complex Operations**: Support for `Composite` operations allowing complex business logic within a single transaction. +- **Lock-Free Communication**: Uses high-speed `ArrayQueue`s for inter-stage communication. -| Operation | Latency (Avg) | Throughput (Avg) | -|:--------------------|:--------------|:----------------------| -| **Wallet Deposit** | 150.59 ns | **6.64 Million tx/s** | -| **Wallet Transfer** | 165.61 ns | **6.04 Million tx/s** | +## Architecture -*Benchmarks performed using Criterion.rs with persistence enabled (WAL active).* +Roda-Ledger utilizes a **Core-Per-Stage** model to maximize mechanical sympathy: +1. **Sequencer**: Assigns monotonic IDs and manages ingestion. +2. **Transactor**: Single-threaded deterministic execution engine (No locks needed!). +3. **WAL Storer**: Handles high-speed disk persistence. +4. **Snapshotter**: Periodically captures state for fast recovery. -## Stress Testing Performance +Detailed docs: [Benchmarks](docs/benchmarks.md) | [Architecture & Design](docs/Design.md) | [Consistency Model](docs/operations_consistency.md) -The system has been extensively stress-tested across various scenarios. Below are some highlights from the latest reports: +## Performance Benchmarks -- **Maximum Throughput:** Reached up to **5.5 Million TPS** during high-contention stress tests. -- **Ultra-Low Latency:** Achieved as low as **102ns** mean latency during load ramp-up. -- **Sustained Performance:** Maintains over **650K TPS** with nanosecond-level latency in long-running stability tests. +| Operation | Latency (Avg) | Throughput (Avg) | +| :--- | :--- | :--- | +| **Wallet Deposit** | 198 ns | **5.0 Million tx/s** | +| **Wallet Transfer** | 151 ns | **6.6 Million tx/s** | -For detailed reports and more scenarios, see the [Stress Testing Reports](docs/reporting/README.md). +*Benchmarks performed with persistence enabled (WAL active) on CX33 server at Hetzner. Full report: [docs/benchmarks.md](docs/benchmarks.md)* -## Getting Started +--- -Add Roda-Ledger to your `Cargo.toml`: +## The Road Ahead -```toml -[dependencies] -roda-ledger = { git = "https://github.com/tislib/roda-ledger" } -``` +Roda-Ledger is evolving from a high-performance core into a full-featured distributed financial infrastructure. -Check out the [examples](examples/) and [tests](tests/) directories for more examples of how to use the ledger and its features. +- **🌐 Distribution via Raft**: High availability and horizontal read scalability through a Raft-based cluster mode. +- **🏗️ Account Hierarchies**: Support for complex sub-account structures and parent-child balance aggregation. +- **⚡ WASM Runtime**: Sandbox for user-defined transaction logic, allowing custom business rules to run at native speeds. +- **🛡️ Extreme Resilience**: Continued hardening with advanced crash-simulations (OOMKill, MultiCrash) and checksum-validated recovery. -## License +--- -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. +## License +Apache License 2.0. See [LICENSE](LICENSE) for details. diff --git a/build.rs b/build.rs index a4c099f..ae1c897 100644 --- a/build.rs +++ b/build.rs @@ -1,4 +1,13 @@ fn main() { #[cfg(feature = "grpc")] - tonic_build::compile_protos("proto/ledger.proto").unwrap(); + { + use std::env; + use std::path::PathBuf; + + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + tonic_build::configure() + .file_descriptor_set_path(out_dir.join("ledger_descriptor.bin")) + .compile_protos(&["proto/ledger.proto"], &["proto"]) + .unwrap(); + } } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1ec534b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +services: + roda-ledger: + build: . + ports: + - "50051:50051" + environment: + - RODA_GRPC_ADDR=0.0.0.0:50051 + - RODA_DATA_DIR=/data + - RODA_MAX_ACCOUNTS=1000000 + - RODA_SNAPSHOT_INTERVAL=600 + - RODA_IN_MEMORY=false + volumes: + - ./data:/data + restart: always diff --git a/docs/Design.md b/docs/Design.md index 81556f0..51c76b2 100644 --- a/docs/Design.md +++ b/docs/Design.md @@ -4,6 +4,8 @@ Roda-Ledger is built on a "Core-Per-Stage" model, utilizing asynchronous pipelining and lock-free ArrayQueues to eliminate thread contention and maximize mechanical sympathy. +For detailed information on the consistency guarantees (serializability, linearizability) and the zero-sum invariant, see the [Operations & Consistency Model](operations_consistency.md). + ## 1. Pipeline Stages ### A. Core 0: The Sequencer diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000..d7afebb --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,55 @@ +# Performance Benchmarks + +This document contains the latest performance benchmarks for Roda-Ledger. + +**Date:** 2026-03-29 +**Environment:** CX33 server at Hetzner + +## Snapshot Performance + +| Operation | Time (Avg) | Throughput (Avg) | +| :--- | :--- | :--- | +| `snapshot/process` | 123.64 ns | 8.0879 Melem/s | + +## Transactor Performance + +| Operation | Time (Avg) | Throughput (Avg) | +| :--- | :--- | :--- | +| `transactor/process` | 284.16 ns | 3.5191 Melem/s | + +## WAL (Write-Ahead Log) Performance + +| Operation | Time (Avg) | Throughput (Avg) | +| :--- | :--- | :--- | +| `wal_in_memory/append` | 18.237 ns | 54.835 Melem/s | +| `wal_on_disk/append` | 40.873 ns | 24.466 Melem/s | + +## Ledger Operations Performance + +| Operation | Time (Avg) | Throughput (Avg) | +| :--- | :--- | :--- | +| `ledger/deposit` | 197.94 ns | 5.0520 Melem/s | +| `ledger/transfer_1000` | 172.18 ns | 5.8080 Melem/s | +| `ledger/transfer_1000000` | 150.53 ns | 6.6432 Melem/s | +| `ledger/transfer_10000000` | 206.21 ns | 4.8493 Melem/s | +| `ledger/transfer_50000000` | 219.65 ns | 4.5526 Melem/s | +| `ledger/complex_operation` | 538.06 ns | 1.8585 Melem/s | + +## gRPC Performance + +### Submit Operation (Unary) + +| Operation | Time (Avg) | +| :--- | :--- | +| `grpc_submit_operation/unary_deposit` | 62.881 µs | + +### Submit Batch + +| Batch Size | Time (Avg) | Throughput (Avg) | +| :--- | :--- | :--- | +| 10 | 6.7757 µs | 147.59 Kelem/s | +| 100 | 840.84 ns | 1.1893 Melem/s | +| 1000 | 298.70 ns | 3.3478 Melem/s | + +--- +*Benchmarks performed with persistence enabled (WAL active) on CX33 server at Hetzner.* diff --git a/docs/grpc_examples.md b/docs/grpc_examples.md new file mode 100644 index 0000000..bc0097e --- /dev/null +++ b/docs/grpc_examples.md @@ -0,0 +1,121 @@ +# gRPC API Examples + +This document provides concrete examples of how to interact with the Roda-Ledger gRPC API using `grpcurl`. + +## Prerequisites + +- [grpcurl](https://github.com/fullstorydev/grpcurl) installed. +- Roda-Ledger server running (e.g., via Docker on `localhost:50051`). + +## Operations + +### Submit a Deposit + +Deposits credit an account from the system source. + +```bash +grpcurl -plaintext -d '{ + "deposit": { + "account": 1, + "amount": "1000", + "user_ref": "100" + } +}' localhost:50051 roda.ledger.v1.Ledger/SubmitOperation +``` + +### Submit a Withdrawal + +Withdrawals debit an account to the system sink. Fails if balance is insufficient. + +```bash +grpcurl -plaintext -d '{ + "withdrawal": { + "account": 1, + "amount": "500", + "user_ref": "101" + } +}' localhost:50051 roda.ledger.v1.Ledger/SubmitOperation +``` + +### Submit a Transfer + +Atomic movement of funds between two accounts. + +```bash +grpcurl -plaintext -d '{ + "transfer": { + "from": 1, + "to": 2, + "amount": "250", + "user_ref": "102" + } +}' localhost:50051 roda.ledger.v1.Ledger/SubmitOperation +``` + +### Submit a Composite Operation + +A multi-step atomic operation. The following example performs a transfer with a fee, ensuring the sender doesn't go negative. + +```bash +grpcurl -plaintext -d '{ + "composite": { + "steps": [ + { "credit": { "account_id": 1, "amount": "105" } }, + { "debit": { "account_id": 2, "amount": "100" } }, + { "debit": { "account_id": 0, "amount": "5" } } + ], + "flags": "1", + "user_ref": "103" + } +}' localhost:50051 roda.ledger.v1.Ledger/SubmitOperation +``` +*Note: `flags: "1"` corresponds to `CHECK_NEGATIVE_BALANCE`.* + +### Submit a Batch of Operations + +```bash +grpcurl -plaintext -d '{ + "operations": [ + { "deposit": { "account": 3, "amount": "1000" } }, + { "deposit": { "account": 4, "amount": "1000" } } + ] +}' localhost:50051 roda.ledger.v1.Ledger/SubmitBatch +``` + +## Queries + +### Get Account Balance + +```bash +grpcurl -plaintext -d '{"account_id": 1}' localhost:50051 roda.ledger.v1.Ledger/GetBalance +``` + +### Get Multiple Balances + +```bash +grpcurl -plaintext -d '{"account_ids": [1, 2, 3]}' localhost:50051 roda.ledger.v1.Ledger/GetBalances +``` + +### Get Transaction Status + +Check the current stage of a transaction (PENDING, COMPUTED, COMMITTED, ON_SNAPSHOT, or ERROR). + +```bash +grpcurl -plaintext -d '{"transaction_id": "1"}' localhost:50051 roda.ledger.v1.Ledger/GetTransactionStatus +``` + +### Get Pipeline Indexes + +Monitor the progress of the entire pipeline. + +```bash +grpcurl -plaintext -d '{}' localhost:50051 roda.ledger.v1.Ledger/GetPipelineIndex +``` + +## Transaction Statuses + +- `PENDING` (0): Sequenced but not yet processed. +- `COMPUTED` (1): Logic executed, balance updated in memory. +- `COMMITTED` (2): Flushed to WAL, durable. +- `ON_SNAPSHOT` (3): Applied to balance cache, queryable via `GetBalance`. +- `ERROR` (4): Rejected (check `fail_reason`). diff --git a/docs/operations_consistency.md b/docs/operations_consistency.md new file mode 100644 index 0000000..3202983 --- /dev/null +++ b/docs/operations_consistency.md @@ -0,0 +1,94 @@ +# Operations and Consistency Model + +Roda-Ledger provides a high-performance, durable, and strictly ordered execution environment for financial transactions. This document details the consistency guarantees, isolation levels, and the underlying mechanisms that ensure the integrity of the ledger. + +## 1. Core Guarantees + +The system is designed around several key distributed systems and database properties: + +| Property | Guarantee | Mechanism | +| :--- | :--- | :--- | +| **Atomicity** | All-or-Nothing | Transactions are processed as single units; either all entries are applied or none are. | +| **Consistency** | Strong | The Zero-Sum Invariant ensures that no value is created or destroyed within the system. | +| **Isolation** | Serializability | A single-threaded deterministic Transactor ensures a total global order of all operations. | +| **Durability** | Strict | Write-Ahead Logging (WAL) ensures every transaction is persisted before it is considered "Committed". | +| **Linearizability** | Provided | Monotonic Transaction IDs assigned by a single Sequencer and polling for pipeline status enable linearizability. | + +--- + +## 2. Linearizability (Atomic Consistency) + +Linearizability is the strongest consistency model for single-object operations. In Roda-Ledger, every transaction is assigned a unique, monotonic `transaction_id` by the **Sequencer** (Core 0). + +- **Total Order**: Every operation that enters the system is placed in a single, global timeline by the Sequencer. This timeline is immutable; once a `transaction_id` is assigned, its relative order to all other transactions is fixed. +- **Real-time Guarantee**: If a client receives a confirmation for transaction A before they start submitting transaction B, A will always have a lower `transaction_id` than B. +- **Single Source of Truth**: The Sequencer is the only component that assigns IDs, preventing any race conditions or ordering conflicts at the ingestion layer. This is equivalent to a "single leader" model in distributed systems, but within a single multi-core machine. + +--- + +## 3. Serializability + +Roda-Ledger achieves **Serializability** — the highest level of isolation — through its "Core-Per-Stage" pipelined architecture. + +### The Single-Threaded Transactor +All business logic, balance checks, and state transitions happen in the **Transactor** (Core 1). Because the Transactor is single-threaded: +1. **No Concurrency Anomalies**: Issues like Dirty Reads, Non-Repeatable Reads, or Phantom Reads are architecturally impossible. +2. **Deterministic Execution**: Given the same initial state and the same sequence of transactions, the Transactor will always produce the same resulting state. +3. **Implicit Locking**: There are no database locks. The single-threaded nature of the stage acts as a global lock without the overhead of lock contention or deadlocks. + +--- + +## 4. Strong Consistency & Pipeline Stages + +Consistency in Roda-Ledger is viewed through the progress of a transaction through the pipeline. A transaction's status is defined by its position relative to three global indices: + +1. **Compute Index (`last_computed_id`)**: + - The Transactor has processed the transaction. + - The result is reflected in the Transactor's hot memory cache. + - **Guarantee**: Read-your-writes (if querying the Transactor's state). + +2. **Commit Index (`last_committed_id`)**: + - The WAL Storer has persisted the transaction (and its resulting entries) to disk. + - **Guarantee**: Durability. Even if the system crashes now, the transaction will be recovered. + +3. **Snapshot Index (`last_snapshot_id`)**: + - The Snapshotter has applied the entries to the persistent balance store. + - **Guarantee**: The transaction is now part of the "base state" used for fast recovery and balance queries. + +### Client-Side Consistency +Clients can choose their desired level of consistency by polling the transaction status: +- **Fast Path**: Assume success once `transaction_id` is received (Optimistic). +- **Safe Path**: Wait until `status == COMMITTED` (Durable). +- **Global Path**: Wait until `status == ON_SNAPSHOT` (Fully Integrated). + +--- + +## 5. The Zero-Sum Invariant + +A fundamental safety property of Roda-Ledger is the **Zero-Sum Invariant**: +> `sum(credits) == sum(debits)` for every transaction. + +- Every unit of value entering the system (Deposit) is balanced by a credit from a reserved `SYSTEM` account. +- Every unit leaving (Withdrawal) is balanced by a debit to a `SYSTEM` account. +- Transfers move value between accounts, naturally balancing to zero. +- **Enforcement**: The Transactor validates this invariant for every operation. Any operation that would violate it is rejected immediately and never reaches the WAL. + +--- + +## 6. gRPC and Consistency + +The gRPC interface (defined in [ADR-004](adr/0004-grpc-interface.md)) is designed to maintain these core guarantees for external clients: + +- **Unary Operations**: Each `SubmitOperation` request is synchronous and returns the `transaction_id` once the Sequencer has accepted it. +- **Polling for Finality**: Because gRPC is stateless and doesn't support server-push for every transaction, clients use `GetTransactionStatuses` or `GetPipelineIndex` to poll for `COMMITTED` status. +- **Batching and Ordering**: `SubmitBatch` guarantees that operations within the same batch are submitted to the Sequencer in the order they appear in the request. However, operations from other clients can be interleaved between these batch elements once they reach the Sequencer. +- **Consistency vs. Freshness**: `GetBalance` returns the `last_snapshot_tx_id`. If a client needs a more up-to-date balance, they must wait until the `last_snapshot_tx_id` is greater than or equal to their latest `transaction_id`. + +## 7. Recovery and Fault Tolerance + +In the event of a crash, the system restores its strongly consistent state: +1. **Snapshot Loading**: The latest snapshot is loaded into the Snapshotter and Transactor. +2. **WAL Replay**: All transactions in the WAL with an ID greater than the snapshot ID are replayed in exact order. +3. **Index Alignment**: The Compute, Commit, and Snapshot indices are restored to the last known valid state on disk. + +Because the execution is deterministic and the WAL contains physical entries (credits/debits) rather than logical operations, the recovery process is guaranteed to reconstruct the exact state prior to the crash. diff --git a/src/grpc/server.rs b/src/grpc/server.rs index 0f139b7..c66f71c 100644 --- a/src/grpc/server.rs +++ b/src/grpc/server.rs @@ -20,10 +20,20 @@ impl GrpcServer { println!("gRPC server listening on {}", self.addr); - Server::builder() - .add_service(LedgerServer::new(handler)) - .serve(self.addr) - .await?; + let mut builder = Server::builder().add_service(LedgerServer::new(handler)); + + #[cfg(feature = "grpc")] + { + let reflection_service = tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set(include_bytes!(concat!( + env!("OUT_DIR"), + "/ledger_descriptor.bin" + ))) + .build_v1()?; + builder = builder.add_service(reflection_service); + } + + builder.serve(self.addr).await?; Ok(()) } diff --git a/tests/crash_recovery_test.rs b/tests/crash_recovery_test.rs index d8e6b56..66a25d8 100644 --- a/tests/crash_recovery_test.rs +++ b/tests/crash_recovery_test.rs @@ -51,10 +51,49 @@ fn crash_recovery_test() { std::thread::yield_now(); } + // Wait for snapshot + std::thread::sleep(Duration::from_millis(150)); // Ledger is dropped here when it goes out of scope } - // Phase 2: Start again and verify + // Phase 2: Start again and add more transactions + { + let config = LedgerConfig { + queue_size: 1024, + location: Some(temp_dir.to_string()), + in_memory: false, + snapshot_interval: Duration::from_millis(100), + max_accounts: 1_000_000, + ..Default::default() + }; + + let mut ledger = Ledger::new(config); + ledger.start(); // This should trigger replay + + let mut last_tx_id = 0; + for _ in 0..num_transactions { + last_tx_id = ledger.submit(Operation::Deposit { + account: account_id, + amount: deposit_amount, + user_ref: 0, + }); + } + + // Wait until transactions reach WAL by checking status + loop { + let status = ledger.get_transaction_status(last_tx_id); + if status.is_committed() { + break; + } + std::thread::yield_now(); + } + + // Wait for snapshot + std::thread::sleep(Duration::from_millis(150)); + // Ledger is dropped here when it goes out of scope + } + + // Phase 3: Start again and verify { let config = LedgerConfig { queue_size: 1024, @@ -71,8 +110,14 @@ fn crash_recovery_test() { let balance = ledger.get_balance(account_id); // Verify balance - assert_eq!(balance, (num_transactions * deposit_amount) as i64); + assert_eq!(balance, (2 * num_transactions * deposit_amount) as i64); - // Final cleanup + // Wait for snapshot + std::thread::sleep(Duration::from_millis(150)); + } + + // Final cleanup + if Path::new(&temp_dir).exists() { + let _ = fs::remove_dir_all(&temp_dir); } }