Skip to content

Commit 0b33c57

Browse files
zhangjinpeng87zhangjinpeng87
authored
tikv-8.5: Fix serde version for CI (#393)
* serde version Signed-off-by: zhangjinpeng87 <zzzhangjinpeng@gmail.com>
1 parent 2f9f688 commit 0b33c57

File tree

27 files changed

+291
-201
lines changed

27 files changed

+291
-201
lines changed

‎.github/workflows/rust.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,16 +71,22 @@ jobs:
7171
- uses: Swatinem/rust-cache@v1
7272
with:
7373
sharedKey: ${{ matrix.os }}-stable
74+
- name: Install gRPC dependencies
75+
run: |
76+
sudo apt-get update
77+
sudo apt-get install -y pkg-config libgrpc-dev libgrpc++-dev protobuf-compiler-grpc
7478
- name: Clippy
7579
run: make clippy
7680
env:
7781
WITH_STABLE_TOOLCHAIN: 'force'
82+
GRPCIO_SYS_USE_PKG_CONFIG: '1'
7883
- name: Run tests
7984
run: make test
8085
env:
8186
RUST_BACKTRACE: 1
8287
EXTRA_CARGO_ARGS: '--verbose'
8388
WITH_STABLE_TOOLCHAIN: 'force'
89+
GRPCIO_SYS_USE_PKG_CONFIG: '1'
8490
coverage:
8591
runs-on: ubuntu-latest
8692
needs: nightly
@@ -98,6 +104,10 @@ jobs:
98104
- uses: Swatinem/rust-cache@v1
99105
with:
100106
sharedKey: ubuntu-latest
107+
- name: Install gRPC dependencies
108+
run: |
109+
sudo apt-get update
110+
sudo apt-get install -y pkg-config libgrpc-dev libgrpc++-dev protobuf-compiler-grpc
101111
- name: Install grcov
102112
run: if [[ ! -e ~/.cargo/bin/grcov ]]; then cargo install --locked grcov; fi
103113
- name: Run tests
@@ -106,6 +116,7 @@ jobs:
106116
RUSTFLAGS: '-Cinstrument-coverage'
107117
LLVM_PROFILE_FILE: '%p-%m.profraw'
108118
EXTRA_CARGO_ARGS: '--verbose'
119+
GRPCIO_SYS_USE_PKG_CONFIG: '1'
109120
- name: Run grcov
110121
run: grcov `find . \( -name "*.profraw" \) -print` --binary-path target/debug/deps/ -s . -t lcov --branch --ignore-not-existing --ignore '../**' --ignore '/*' -o coverage.lcov
111122
- name: Upload

‎Cargo.toml‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
name = "raft-engine"
33
version = "0.4.2"
44
authors = ["The TiKV Project Developers"]
5-
edition = "2018"
6-
rust-version = "1.75.0"
5+
edition = "2024"
6+
rust-version = "1.85.0"
77
description = "A persistent storage engine for Multi-Raft logs"
88
readme = "README.md"
99
repository = "https://github.com/tikv/raft-engine"
@@ -58,7 +58,7 @@ protobuf = "2"
5858
rayon = "1.5"
5959
rhai = { version = "1.7", features = ["sync"], optional = true }
6060
scopeguard = "1.1"
61-
serde = { version = "=1.0.194", features = ["derive"] }
61+
serde = { version = "1.0", features = ["derive"] }
6262
serde_repr = "0.1"
6363
strum = { version = "0.26.2", features = ["derive"] }
6464
thiserror = "1.0"
@@ -67,14 +67,18 @@ thiserror = "1.0"
6767
criterion = "0.4"
6868
ctor = "0.2"
6969
env_logger = "0.10"
70-
kvproto = { git = "https://github.com/pingcap/kvproto.git", default-features = false, features = [
70+
grpcio = { version = "=0.10.2", default-features = false, features = [
71+
"protobuf-codec",
72+
] }
73+
kvproto = { git = "https://github.com/pingcap/kvproto.git", branch = "release-8.5", default-features = false, features = [
7174
"protobuf-codec",
7275
] }
7376
raft = { git = "https://github.com/tikv/raft-rs", branch = "master", default-features = false, features = [
7477
"protobuf-codec",
7578
] }
7679
rand = "0.8"
7780
rand_distr = "0.4"
81+
serde_json = "=1.0.146"
7882
tempfile = "3.6"
7983
toml = "0.8"
8084

‎Makefile‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -42,22 +42,31 @@ format:
4242
cargo ${TOOLCHAIN_ARGS} fmt --all
4343

4444
CLIPPY_WHITELIST += -A clippy::bool_assert_comparison
45+
CMAKE_COMPAT := CMAKE="$(CURDIR)/scripts/cmake-wrapper.sh"
4546
## Run clippy.
4647
clippy:
48+
# Fresh lockfile resolution can pull grpcio/grpcio-sys versions that fail on
49+
# macOS arm64 CI images. Force compatible selections before linting.
50+
@if cargo ${TOOLCHAIN_ARGS} tree --all-features -i 'grpcio@0.13.0' >/dev/null 2>&1; then \
51+
cargo ${TOOLCHAIN_ARGS} update -p grpcio@0.13.0 --precise 0.10.2; \
52+
fi
53+
@if cargo ${TOOLCHAIN_ARGS} tree --all-features -i 'grpcio-sys@0.10.1+1.44.0' >/dev/null 2>&1; then \
54+
cargo ${TOOLCHAIN_ARGS} update -p grpcio-sys@0.10.1+1.44.0 --precise 0.10.3+1.44.0-patched; \
55+
fi
4756
ifdef WITH_NIGHTLY_FEATURES
48-
cargo ${TOOLCHAIN_ARGS} clippy --all --features nightly_group,failpoints --all-targets -- -D clippy::all ${CLIPPY_WHITELIST}
57+
${CMAKE_COMPAT} cargo ${TOOLCHAIN_ARGS} clippy --all --features nightly_group,failpoints --all-targets -- -D clippy::all ${CLIPPY_WHITELIST}
4958
else
50-
cargo ${TOOLCHAIN_ARGS} clippy --all --features failpoints --all-targets -- -D clippy::all ${CLIPPY_WHITELIST}
59+
${CMAKE_COMPAT} cargo ${TOOLCHAIN_ARGS} clippy --all --features failpoints --all-targets -- -D clippy::all ${CLIPPY_WHITELIST}
5160
endif
5261

5362
## Run tests.
5463
test:
5564
ifdef WITH_NIGHTLY_FEATURES
56-
cargo ${TOOLCHAIN_ARGS} test --all --features nightly_group ${EXTRA_CARGO_ARGS} -- --nocapture
57-
cargo ${TOOLCHAIN_ARGS} test --test failpoints --features nightly_group,failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
65+
${CMAKE_COMPAT} cargo ${TOOLCHAIN_ARGS} test --all --features nightly_group ${EXTRA_CARGO_ARGS} -- --nocapture
66+
${CMAKE_COMPAT} cargo ${TOOLCHAIN_ARGS} test --test failpoints --features nightly_group,failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
5867
else
59-
cargo ${TOOLCHAIN_ARGS} test --all ${EXTRA_CARGO_ARGS} -- --nocapture
60-
cargo ${TOOLCHAIN_ARGS} test --test failpoints --features failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
68+
${CMAKE_COMPAT} cargo ${TOOLCHAIN_ARGS} test --all ${EXTRA_CARGO_ARGS} -- --nocapture
69+
${CMAKE_COMPAT} cargo ${TOOLCHAIN_ARGS} test --test failpoints --features failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
6170
endif
6271

6372
## Run tests with various features for maximum code coverage.
@@ -66,10 +75,10 @@ test_matrix:
6675
$(error Must run test matrix with nightly features. Please reset WITH_STABLE_TOOLCHAIN.)
6776
else
6877
test_matrix: test
69-
cargo ${TOOLCHAIN_ARGS} test --all ${EXTRA_CARGO_ARGS} -- --nocapture
70-
cargo ${TOOLCHAIN_ARGS} test --test failpoints --features failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
71-
cargo ${TOOLCHAIN_ARGS} test --all --features nightly_group,std_fs ${EXTRA_CARGO_ARGS} -- --nocapture
72-
cargo ${TOOLCHAIN_ARGS} test --test failpoints --features nightly_group,std_fs,failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
78+
${CMAKE_COMPAT} cargo ${TOOLCHAIN_ARGS} test --all ${EXTRA_CARGO_ARGS} -- --nocapture
79+
${CMAKE_COMPAT} cargo ${TOOLCHAIN_ARGS} test --test failpoints --features failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
80+
${CMAKE_COMPAT} cargo ${TOOLCHAIN_ARGS} test --all --features nightly_group,std_fs ${EXTRA_CARGO_ARGS} -- --nocapture
81+
${CMAKE_COMPAT} cargo ${TOOLCHAIN_ARGS} test --test failpoints --features nightly_group,std_fs,failpoints ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture
7382
endif
7483

7584
## Build raft-engine-ctl.

‎examples/fork.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
use std::path::Path;
22
use std::sync::Arc;
33

4-
use raft_engine::env::DefaultFileSystem;
54
use raft_engine::Config;
65
use raft_engine::Engine;
6+
use raft_engine::env::DefaultFileSystem;
77

88
fn main() {
99
let mut args = std::env::args();

‎scripts/cmake-wrapper.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
if [[ $# -gt 0 ]]; then
5+
case "$1" in
6+
--build|--install|--open|--help|--version)
7+
exec cmake "$@"
8+
;;
9+
esac
10+
fi
11+
12+
exec cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 "$@"

‎src/config.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use log::{info, warn};
44
use serde::{Deserialize, Serialize};
55

66
use crate::pipe_log::Version;
7-
use crate::{util::ReadableSize, Result};
7+
use crate::{Result, util::ReadableSize};
88

99
const MIN_RECOVERY_READ_BLOCK_SIZE: usize = 512;
1010
const MIN_RECOVERY_THREADS: usize = 1;
@@ -343,9 +343,11 @@ mod tests {
343343
let mut load: Config = toml::from_str(old).unwrap();
344344
load.sanitize().unwrap();
345345
// Downgrade to older version.
346-
assert!(toml::to_string(&load)
347-
.unwrap()
348-
.contains("tolerate-corrupted-tail-records"));
346+
assert!(
347+
toml::to_string(&load)
348+
.unwrap()
349+
.contains("tolerate-corrupted-tail-records")
350+
);
349351
}
350352

351353
#[test]

‎src/consistency.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22

33
use hashbrown::HashMap;
44

5+
use crate::Result;
56
use crate::file_pipe_log::ReplayMachine;
67
use crate::log_batch::{LogItemBatch, LogItemContent};
78
use crate::pipe_log::{FileId, LogQueue};
8-
use crate::Result;
99

1010
/// A `ConsistencyChecker` scans for log entry holes in a log queue. It will
1111
/// return a list of corrupted raft groups along with their last valid log

‎src/engine.rs‎

Lines changed: 40 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
use std::cell::{Cell, RefCell};
44
use std::marker::PhantomData;
55
use std::path::Path;
6-
use std::sync::{mpsc, Arc, Mutex};
6+
use std::sync::{Arc, Mutex, mpsc};
77
use std::thread::{Builder as ThreadBuilder, JoinHandle};
88
use std::time::{Duration, Instant};
99

1010
use log::{error, info};
11-
use protobuf::{parse_from_bytes, Message};
11+
use protobuf::{Message, parse_from_bytes};
1212

1313
use crate::config::{Config, RecoveryMode};
1414
use crate::consistency::ConsistencyChecker;
@@ -22,7 +22,7 @@ use crate::metrics::*;
2222
use crate::pipe_log::{FileBlockHandle, FileId, LogQueue, PipeLog};
2323
use crate::purge::{PurgeHook, PurgeManager};
2424
use crate::write_barrier::{WriteBarrier, Writer};
25-
use crate::{perf_context, Error, GlobalStats, Result};
25+
use crate::{Error, GlobalStats, Result, perf_context};
2626

2727
const METRICS_FLUSH_INTERVAL: Duration = Duration::from_secs(30);
2828
/// Max times for `write`.
@@ -106,11 +106,13 @@ where
106106
let memtables_clone = memtables.clone();
107107
let metrics_flusher = ThreadBuilder::new()
108108
.name("re-metrics".into())
109-
.spawn(move || loop {
110-
stats_clone.flush_metrics();
111-
memtables_clone.flush_metrics();
112-
if rx.recv_timeout(METRICS_FLUSH_INTERVAL).is_ok() {
113-
break;
109+
.spawn(move || {
110+
loop {
111+
stats_clone.flush_metrics();
112+
memtables_clone.flush_metrics();
113+
if rx.recv_timeout(METRICS_FLUSH_INTERVAL).is_ok() {
114+
break;
115+
}
114116
}
115117
})?;
116118

@@ -648,14 +650,14 @@ where
648650
pub(crate) mod tests {
649651
use super::*;
650652
use crate::env::{ObfuscatedFileSystem, Permission};
651-
use crate::file_pipe_log::{parse_reserved_file_name, FileNameExt};
653+
use crate::file_pipe_log::{FileNameExt, parse_reserved_file_name};
652654
use crate::log_batch::AtomicGroupBuilder;
653655
use crate::pipe_log::Version;
654-
use crate::test_util::{generate_entries, PanicGuard};
656+
use crate::test_util::{PanicGuard, generate_entries};
655657
use crate::util::ReadableSize;
656658
use kvproto::raft_serverpb::RaftLocalState;
657659
use raft::eraftpb::Entry;
658-
use rand::{thread_rng, Rng};
660+
use rand::{Rng, thread_rng};
659661
use std::collections::{BTreeSet, HashSet};
660662
use std::fs::OpenOptions;
661663
use std::path::PathBuf;
@@ -1231,9 +1233,11 @@ pub(crate) mod tests {
12311233
// GC all log entries. Won't trigger purge because total size is not enough.
12321234
let count = engine.compact_to(1, 100);
12331235
assert_eq!(count, 100);
1234-
assert!(!engine
1235-
.purge_manager
1236-
.needs_rewrite_log_files(LogQueue::Append));
1236+
assert!(
1237+
!engine
1238+
.purge_manager
1239+
.needs_rewrite_log_files(LogQueue::Append)
1240+
);
12371241

12381242
// Append more logs to make total size greater than `purge_threshold`.
12391243
for index in 100..250 {
@@ -1243,9 +1247,11 @@ pub(crate) mod tests {
12431247
// GC first 101 log entries.
12441248
assert_eq!(engine.compact_to(1, 101), 1);
12451249
// Needs to purge because the total size is greater than `purge_threshold`.
1246-
assert!(engine
1247-
.purge_manager
1248-
.needs_rewrite_log_files(LogQueue::Append));
1250+
assert!(
1251+
engine
1252+
.purge_manager
1253+
.needs_rewrite_log_files(LogQueue::Append)
1254+
);
12491255

12501256
let old_min_file_seq = engine.file_span(LogQueue::Append).0;
12511257
let will_force_compact = engine.purge_expired_files().unwrap();
@@ -1259,9 +1265,11 @@ pub(crate) mod tests {
12591265

12601266
assert_eq!(engine.compact_to(1, 102), 1);
12611267
// Needs to purge because the total size is greater than `purge_threshold`.
1262-
assert!(engine
1263-
.purge_manager
1264-
.needs_rewrite_log_files(LogQueue::Append));
1268+
assert!(
1269+
engine
1270+
.purge_manager
1271+
.needs_rewrite_log_files(LogQueue::Append)
1272+
);
12651273
let will_force_compact = engine.purge_expired_files().unwrap();
12661274
// The region needs to be force compacted because the threshold is reached.
12671275
assert!(!will_force_compact.is_empty());
@@ -1350,9 +1358,11 @@ pub(crate) mod tests {
13501358
engine.append(11, 1, 11, Some(&data));
13511359

13521360
// The engine needs purge, and all old entries should be rewritten.
1353-
assert!(engine
1354-
.purge_manager
1355-
.needs_rewrite_log_files(LogQueue::Append));
1361+
assert!(
1362+
engine
1363+
.purge_manager
1364+
.needs_rewrite_log_files(LogQueue::Append)
1365+
);
13561366
assert!(engine.purge_expired_files().unwrap().is_empty());
13571367
assert!(engine.file_span(LogQueue::Append).0 > 1);
13581368

@@ -1386,9 +1396,11 @@ pub(crate) mod tests {
13861396
}
13871397
}
13881398

1389-
assert!(engine
1390-
.purge_manager
1391-
.needs_rewrite_log_files(LogQueue::Append));
1399+
assert!(
1400+
engine
1401+
.purge_manager
1402+
.needs_rewrite_log_files(LogQueue::Append)
1403+
);
13921404
assert!(engine.purge_expired_files().unwrap().is_empty());
13931405
}
13941406

@@ -1410,7 +1422,7 @@ pub(crate) mod tests {
14101422
let empty_entry = Entry::new();
14111423
assert_eq!(empty_entry.compute_size(), 0);
14121424
log_batch
1413-
.add_entries::<Entry>(0, &[empty_entry.clone()])
1425+
.add_entries::<Entry>(0, std::slice::from_ref(&empty_entry))
14141426
.unwrap();
14151427
engine.write(&mut log_batch, false).unwrap();
14161428
let empty_state = RaftLocalState::new();
@@ -1420,7 +1432,7 @@ pub(crate) mod tests {
14201432
.unwrap();
14211433
engine.write(&mut log_batch, false).unwrap();
14221434
log_batch
1423-
.add_entries::<Entry>(2, &[empty_entry.clone()])
1435+
.add_entries::<Entry>(2, std::slice::from_ref(&empty_entry))
14241436
.unwrap();
14251437
log_batch
14261438
.put_message(2, b"key".to_vec(), &empty_state)

‎src/env/log_fd/unix.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,12 @@ use log::error;
88
use std::io::Result as IoResult;
99
use std::os::unix::io::RawFd;
1010

11+
use nix::NixPath;
1112
use nix::errno::Errno;
1213
use nix::fcntl::{self, OFlag};
1314
use nix::sys::stat::Mode;
1415
use nix::sys::uio::{pread, pwrite};
15-
use nix::unistd::{close, ftruncate, lseek, Whence};
16-
use nix::NixPath;
16+
use nix::unistd::{Whence, close, ftruncate, lseek};
1717

1818
fn from_nix_error(e: nix::Error, custom: &'static str) -> std::io::Error {
1919
let kind = std::io::Error::from(e).kind();

‎src/env/obfuscated.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
use std::io::{Read, Result as IoResult, Seek, SeekFrom, Write};
44
use std::path::Path;
5-
use std::sync::atomic::{AtomicUsize, Ordering};
65
use std::sync::Arc;
6+
use std::sync::atomic::{AtomicUsize, Ordering};
77

88
use crate::env::{DefaultFileSystem, FileSystem, Permission, WriteExt};
99

0 commit comments

Comments
 (0)
⚡