Skip to content

Commit b0e335e

Browse files
committed
feat: initial support multi-raft
1 parent 7dabf9e commit b0e335e

File tree

37 files changed

+5668
-7
lines changed

37 files changed

+5668
-7
lines changed

.github/workflows/ci.yaml

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
matrix:
1515
include:
1616
- toolchain: 'nightly'
17-
features: 'bench,serde,bt,singlethreaded'
17+
features: 'bench,serde,bt,singlethreaded,multi-raft'
1818

1919
steps:
2020
- name: Setup | Checkout
@@ -258,6 +258,10 @@ jobs:
258258
- toolchain: 'nightly'
259259
features: 'serde,singlethreaded'
260260

261+
# Multi-Raft feature
262+
- toolchain: 'nightly'
263+
features: 'multi-raft,serde'
264+
261265
steps:
262266
- name: Setup | Checkout
263267
uses: actions/checkout@v4
@@ -374,6 +378,8 @@ jobs:
374378
- 'raft-kv-memstore-opendal-snapshot-data'
375379
- 'raft-kv-memstore-singlethreaded'
376380
- 'raft-kv-rocksdb'
381+
- 'multi-raft-kv'
382+
- 'multi-raft-sharding'
377383

378384
steps:
379385
- uses: actions/checkout@v4
@@ -401,13 +407,17 @@ jobs:
401407

402408
- name: Test demo script of examples/${{ matrix.example }}
403409
# The script is not meant for testing. Just to ensure it works but do not
404-
# rely on it.
410+
# rely on it. Skip if test-cluster.sh doesn't exist.
405411
if: ${{ matrix.toolchain == 'stable' }}
406412

407413
shell: bash
408414
run: |
409415
cd examples/${{ matrix.example }}
410-
./test-cluster.sh
416+
if [ -f test-cluster.sh ]; then
417+
./test-cluster.sh
418+
else
419+
echo "No test-cluster.sh found, skipping"
420+
fi
411421
412422
- name: Format
413423
# clippy/format produces different result with stable and nightly. Only with nightly.

Cargo.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,9 @@ exclude = [
7777
"examples/raft-kv-memstore-opendal-snapshot-data",
7878
"examples/raft-kv-rocksdb",
7979

80+
"examples/multi-raft-kv",
81+
"examples/multi-raft-sharding",
82+
8083
"rt-monoio",
8184
"rt-compio"
8285
]

Makefile

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ test:
2525
cargo test --features serde
2626
# only crate `tests` has single-term-leader feature
2727
cargo test --features single-term-leader -p tests
28+
# multi-raft feature tests
29+
cargo test --features multi-raft -p openraft
2830
$(MAKE) test-examples
2931

3032
check-parallel:
@@ -40,6 +42,8 @@ test-examples:
4042
cargo test --manifest-path examples/raft-kv-memstore-singlethreaded/Cargo.toml
4143
cargo test --manifest-path examples/raft-kv-rocksdb/Cargo.toml
4244
cargo test --manifest-path examples/rocksstore/Cargo.toml
45+
cargo test --manifest-path examples/multi-raft-kv/Cargo.toml
46+
cargo test --manifest-path examples/multi-raft-sharding/Cargo.toml
4347

4448
bench:
4549
cargo bench --features bench
@@ -82,12 +86,14 @@ lint:
8286
cargo fmt --manifest-path examples/raft-kv-memstore/Cargo.toml
8387
cargo fmt --manifest-path examples/raft-kv-rocksdb/Cargo.toml
8488
cargo clippy --no-deps --all-targets -- -D warnings
85-
cargo clippy --no-deps --manifest-path examples/mem-log/Cargo.toml --all-targets -- -D warnings
89+
cargo clippy --no-deps --manifest-path examples/mem-log/Cargo.toml --all-targets -- -D warnings
8690
cargo clippy --no-deps --manifest-path examples/raft-kv-memstore-network-v2/Cargo.toml --all-targets -- -D warnings
8791
cargo clippy --no-deps --manifest-path examples/raft-kv-memstore-opendal-snapshot-data/Cargo.toml --all-targets -- -D warnings
8892
cargo clippy --no-deps --manifest-path examples/raft-kv-memstore-singlethreaded/Cargo.toml --all-targets -- -D warnings
8993
cargo clippy --no-deps --manifest-path examples/raft-kv-memstore/Cargo.toml --all-targets -- -D warnings
9094
cargo clippy --no-deps --manifest-path examples/raft-kv-rocksdb/Cargo.toml --all-targets -- -D warnings
95+
cargo clippy --no-deps --manifest-path examples/multi-raft-kv/Cargo.toml --all-targets -- -D warnings
96+
cargo clippy --no-deps --manifest-path examples/multi-raft-sharding/Cargo.toml --all-targets -- -D warnings
9197
# Bug: clippy --all-targets reports false warning about unused dep in
9298
# `[dev-dependencies]`:
9399
# https://github.com/rust-lang/rust/issues/72686#issuecomment-635539688

examples/mem-log/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ license = "MIT OR Apache-2.0"
1515
repository = "https://github.com/databendlabs/openraft"
1616

1717
[dependencies]
18-
openraft = { path = "../../openraft", features = ["type-alias"] }
18+
openraft = { path = "../../openraft", default-features = false, features = ["type-alias", "tokio-rt"] }
1919

2020
tokio = { version = "1.0", default-features = false, features = ["sync"] }
2121

examples/multi-raft-kv/Cargo.toml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
[package]
2+
name = "multi-raft-kv"
3+
version = "0.1.0"
4+
readme = "README.md"
5+
6+
edition = "2021"
7+
authors = [
8+
"AriesDevil <ariesdevil77@gmail.com>",
9+
]
10+
categories = ["algorithms", "asynchronous", "data-structures"]
11+
description = "An example Multi-Raft distributed key-value store with 3 groups built upon `openraft`."
12+
homepage = "https://github.com/databendlabs/openraft"
13+
keywords = ["raft", "consensus", "multi-raft"]
14+
license = "MIT OR Apache-2.0"
15+
repository = "https://github.com/databendlabs/openraft"
16+
17+
[dependencies]
18+
mem-log = { path = "../mem-log", features = [] }
19+
# Disable adapt-network-v1 to use GroupNetworkAdapter as RaftNetworkV2
20+
openraft = { path = "../../openraft", default-features = false, features = ["serde", "type-alias", "multi-raft", "tokio-rt"] }
21+
22+
futures = { version = "0.3" }
23+
serde = { version = "1", features = ["derive"] }
24+
serde_json = { version = "1" }
25+
tokio = { version = "1", default-features = false, features = ["sync"] }
26+
tracing = { version = "0.1" }
27+
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
28+
29+
30+
[features]
31+
32+
[package.metadata.docs.rs]
33+
all-features = true
34+

examples/multi-raft-kv/README.md

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Multi-Raft KV Store Example
2+
3+
This example demonstrates how to use OpenRaft's Multi-Raft support to run multiple independent Raft consensus groups within a single process.
4+
5+
## Overview
6+
7+
The example creates a distributed key-value store with **3 Raft groups**:
8+
- **users** - Stores user data
9+
- **orders** - Stores order data
10+
- **products** - Stores product data
11+
12+
Each group runs its own independent Raft consensus, but they share the same network infrastructure.
13+
14+
## Architecture
15+
16+
```
17+
+-----------------------------------------------------------------------+
18+
| Node 1 |
19+
| +-------------------+ +-------------------+ +-------------------+ |
20+
| | Group "users" | | Group "orders" | | Group "products" | |
21+
| | (Raft Instance) | | (Raft Instance) | | (Raft Instance) | |
22+
| +-------------------+ +-------------------+ +-------------------+ |
23+
| | | | |
24+
| +----------------------+----------------------+ |
25+
| | |
26+
| +--------+--------+ |
27+
| | Router | |
28+
| | (shared network)| |
29+
| +-----------------+ |
30+
+------------------------------------------------------------------------+
31+
|
32+
Network Connection
33+
|
34+
+------------------------------------------------------------------------+
35+
| Node 2 |
36+
| +-------------------+ +-------------------+ +-------------------+ |
37+
| | Group "users" | | Group "orders" | | Group "products" | |
38+
| | (Raft Instance) | | (Raft Instance) | | (Raft Instance) | |
39+
| +-------------------+ +-------------------+ +-------------------+ |
40+
+------------------------------------------------------------------------+
41+
```
42+
43+
## Key Concepts
44+
45+
### GroupId
46+
A string identifier that uniquely identifies each Raft group (e.g., "users", "orders", "products").
47+
48+
### Shared Network
49+
Multiple Raft groups share the same network infrastructure (`Router`), reducing connection overhead. Messages are routed to the correct group using the `group_id`.
50+
51+
### Independent Consensus
52+
Each group runs its own Raft consensus independently:
53+
- Separate log storage
54+
- Separate state machine
55+
- Separate leader election
56+
- Separate membership
57+
58+
## Running the Test
59+
60+
```bash
61+
# Run the integration test
62+
cargo test -p multi-raft-kv test_multi_raft_cluster -- --nocapture
63+
64+
# With debug logging
65+
RUST_LOG=debug cargo test -p multi-raft-kv test_multi_raft_cluster -- --nocapture
66+
```
67+
68+
## Code Structure
69+
70+
```
71+
multi-raft-kv/
72+
├── Cargo.toml
73+
├── README.md
74+
├── src/
75+
│ ├── lib.rs # Type definitions and group constants
76+
│ ├── app.rs # Application handler for each group
77+
│ ├── api.rs # API handlers (read, write, raft operations)
78+
│ ├── network.rs # Network implementation with group routing
79+
│ ├── router.rs # Message router for (node_id, group_id)
80+
│ └── store.rs # State machine storage
81+
└── tests/
82+
└── cluster/
83+
├── main.rs
84+
└── test_cluster.rs # Integration tests
85+
```

examples/multi-raft-kv/src/api.rs

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
use std::collections::BTreeMap;
2+
use std::collections::BTreeSet;
3+
4+
use openraft::BasicNode;
5+
use openraft::ReadPolicy;
6+
7+
use crate::app::App;
8+
use crate::decode;
9+
use crate::encode;
10+
use crate::typ::*;
11+
use crate::NodeId;
12+
13+
/// Write a key-value pair to the group's state machine
14+
pub async fn write(app: &mut App, req: String) -> String {
15+
let res = app.raft.client_write(decode(&req)).await;
16+
encode(res)
17+
}
18+
19+
/// Read a value from the group's state machine using linearizable read
20+
pub async fn read(app: &mut App, req: String) -> String {
21+
let key: String = decode(&req);
22+
23+
let ret = app.raft.get_read_linearizer(ReadPolicy::ReadIndex).await;
24+
25+
let res = match ret {
26+
Ok(linearizer) => {
27+
linearizer.await_ready(&app.raft).await.unwrap();
28+
29+
let state_machine = app.state_machine.state_machine.lock().await;
30+
let value = state_machine.data.get(&key).cloned();
31+
32+
let res: Result<String, RaftError<LinearizableReadError>> = Ok(value.unwrap_or_default());
33+
res
34+
}
35+
Err(e) => Err(e),
36+
};
37+
encode(res)
38+
}
39+
40+
// ============================================================================
41+
// Raft Protocol API
42+
// ============================================================================
43+
44+
/// Handle vote request
45+
pub async fn vote(app: &mut App, req: String) -> String {
46+
let res = app.raft.vote(decode(&req)).await;
47+
encode(res)
48+
}
49+
50+
/// Handle append entries request
51+
pub async fn append(app: &mut App, req: String) -> String {
52+
let res = app.raft.append_entries(decode(&req)).await;
53+
encode(res)
54+
}
55+
56+
/// Receive a snapshot and install it
57+
pub async fn snapshot(app: &mut App, req: String) -> String {
58+
let (vote, snapshot_meta, snapshot_data): (Vote, SnapshotMeta, SnapshotData) = decode(&req);
59+
let snapshot = Snapshot {
60+
meta: snapshot_meta,
61+
snapshot: snapshot_data,
62+
};
63+
let res = app.raft.install_full_snapshot(vote, snapshot).await.map_err(RaftError::<Infallible>::Fatal);
64+
encode(res)
65+
}
66+
67+
// ============================================================================
68+
// Management API
69+
// ============================================================================
70+
71+
/// Add a node as **Learner** to this group.
72+
///
73+
/// This should be done before adding a node as a member into the cluster
74+
/// (by calling `change-membership`)
75+
pub async fn add_learner(app: &mut App, req: String) -> String {
76+
let node_id: NodeId = decode(&req);
77+
let node = BasicNode { addr: "".to_string() };
78+
let res = app.raft.add_learner(node_id, node, true).await;
79+
encode(res)
80+
}
81+
82+
/// Changes specified learners to members, or remove members from this group.
83+
pub async fn change_membership(app: &mut App, req: String) -> String {
84+
let node_ids: BTreeSet<NodeId> = decode(&req);
85+
let res = app.raft.change_membership(node_ids, false).await;
86+
encode(res)
87+
}
88+
89+
/// Initialize a single-node cluster for this group.
90+
pub async fn init(app: &mut App) -> String {
91+
let mut nodes = BTreeMap::new();
92+
nodes.insert(app.node_id, BasicNode { addr: "".to_string() });
93+
let res = app.raft.initialize(nodes).await;
94+
encode(res)
95+
}
96+
97+
/// Get the latest metrics of this Raft group
98+
pub async fn metrics(app: &mut App) -> String {
99+
let metrics = app.raft.metrics().borrow().clone();
100+
101+
let res: Result<RaftMetrics, Infallible> = Ok(metrics);
102+
encode(res)
103+
}

0 commit comments

Comments
 (0)