diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8c54d93637..dec0742148 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,22 +14,31 @@ jobs: complete: if: always() - needs: [fmt, cargo-deny, rust-check-git-rev-deps, build] - runs-on: ubuntu-22.04 + needs: [fmt, cargo-deny, rust-check-git-rev-deps, build-linux, build-mac] + runs-on: + - namespace-profile-jammy-1-stellar-core + - nscloud-cache-exp-do-not-commit steps: - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') run: exit 1 fmt: - runs-on: ubuntu-22.04 + runs-on: + - namespace-profile-jammy-1-stellar-core + - nscloud-cache-exp-do-not-commit steps: - - uses: actions/checkout@v4 - - run: rustup component add rustfmt - - run: rustup update - - run: cargo fmt --all --check + - uses: namespacelabs/nscloud-checkout-action@v7 + with: + fetch-depth: 1 + submodules: recursive + - run: rustup component add rustfmt + - run: rustup update + - run: cargo fmt --all --check cargo-deny: - runs-on: ubuntu-22.04 + runs-on: + - namespace-profile-jammy-1-stellar-core + - nscloud-cache-exp-do-not-commit strategy: matrix: checks: @@ -38,33 +47,36 @@ jobs: # Prevent sudden announcement of a new advisory from failing ci: continue-on-error: ${{ matrix.checks == 'advisories' }} steps: - - uses: actions/checkout@v4 - - uses: EmbarkStudios/cargo-deny-action@8d73959fce1cdc8989f23fdf03bec6ae6a6576ef - with: - command: check ${{ matrix.checks }} - # leave arguments empty so we don't test --all-features - # which will see conflicting env versions - arguments: + - uses: namespacelabs/nscloud-checkout-action@v7 + with: + fetch-depth: 1 + submodules: recursive + - uses: EmbarkStudios/cargo-deny-action@8d73959fce1cdc8989f23fdf03bec6ae6a6576ef + with: + command: check ${{ matrix.checks }} + # leave arguments empty so we don't test --all-features + # which will see conflicting env versions + arguments: rust-check-git-rev-deps: - runs-on: ubuntu-22.04 + runs-on: + - namespace-profile-jammy-1-stellar-core + - nscloud-cache-exp-do-not-commit steps: - - uses: actions/checkout@v4 - - uses: stellar/actions/rust-check-git-rev-deps@main - - build: - runs-on: ubuntu-jammy-16-cores-amd64 - env: - CACHED_PATHS: | - ~/.ccache - ~/.cargo - target + - uses: namespacelabs/nscloud-checkout-action@v7 + with: + fetch-depth: 1 + submodules: recursive + - uses: stellar/actions/rust-check-git-rev-deps@main + + build-linux: + runs-on: + - namespace-profile-jammy-32-stellar-core;overrides.cache-tag=config-${{ matrix.toolchain }}-${{ matrix.protocol }} strategy: fail-fast: false matrix: toolchain: [ "gcc", "clang"] protocol: ["current", "next"] - scenario: ["", "--check-test-tx-meta"] steps: - name: Fix kernel mmap rnd bits # Asan in llvm provided in ubuntu 22.04 is incompatible with @@ -72,66 +84,32 @@ jobs: # using leading to random crashes: https://reviews.llvm.org/D148280 run: sudo sysctl vm.mmap_rnd_bits=28 - # We start with as cheap as possible a cache probe to see if we have an exact hit on this - # git commit ID (for a given OS/toolchain/protocol). If so we can skip all subsequent - # steps: we already tested this exact SHA. - # - # Unfortunately due to the way github actions control flow works, we have to repeat the - # step 'if' condition in each step, and cannot actually "exit early" and skip the job. - # There are a lot of duplicate bug reports filed on this aspect of github actions, don't - # bother filing another. - - name: Probe Cache - id: cache - uses: actions/cache/restore@v4 + - name: Checkout with Namespace Git mirrors cache + uses: namespacelabs/nscloud-checkout-action@v7 with: - path: ${{ env.CACHED_PATHS }} - key: ${{ runner.os }}-${{ matrix.toolchain }}-${{ matrix.protocol }}-${{ github.sha }} - lookup-only: true - - # When we have a cache miss on the exact commit SHA, we restore from the prefix without it. - # This will restore the most-recently-written cache (github does this date ordering itself). - - name: Restore Cache - if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/restore@v4 - with: - path: ${{ env.CACHED_PATHS }} - key: ${{ steps.cache.outputs.cache-primary-key }} - restore-keys: | - ${{ runner.os }}-${{ matrix.toolchain }}-${{ matrix.protocol }} + fetch-depth: 1 + submodules: recursive - - uses: actions/checkout@v4 - if: steps.cache.outputs.cache-hit != 'true' + - name: Configure Namespace cache volume + uses: namespacelabs/nscloud-cache-action@v1 with: - fetch-depth: 200 - submodules: true - - name: install core packages - if: steps.cache.outputs.cache-hit != 'true' - run: | - sudo apt-get update - sudo apt-get -y install --no-install-recommends apt-utils dialog git iproute2 procps lsb-release - - name: install tool chain - if: steps.cache.outputs.cache-hit != 'true' - run: | - sudo apt-get -y install libstdc++-10-dev clang-format-12 ccache lldb - if test "${{ matrix.toolchain }}" = "gcc" ; then - sudo apt-get -y install cpp-10 gcc-10 g++-10 - else - sudo apt-get -y install clang-12 llvm-12 - fi + path: | + ~/.cargo + build-${{matrix.toolchain}}-${{matrix.protocol}} + + - name: install rustup + run: ./install-rust.sh + - name: install rustup components - if: steps.cache.outputs.cache-hit != 'true' run: rustup component add rustfmt + - name: install cargo-cache - if: steps.cache.outputs.cache-hit != 'true' run: cargo install --locked cargo-cache --version 0.8.3 + - name: install cargo-sweep - if: steps.cache.outputs.cache-hit != 'true' run: cargo install --locked cargo-sweep --version 0.7.0 - - name: install dependencies - if: steps.cache.outputs.cache-hit != 'true' - run: sudo apt-get -y install postgresql git build-essential pkg-config autoconf automake libtool bison flex libpq-dev parallel libunwind-dev sed perl + - name: Build - if: steps.cache.outputs.cache-hit != 'true' run: | if test "${{ matrix.toolchain }}" = "gcc" ; then export CC='gcc' @@ -141,11 +119,52 @@ jobs: export CXX='clang++' fi echo Build with $CC and $CXX - ./ci-build.sh --use-temp-db --protocol ${{ matrix.protocol }} ${{ matrix.scenario }} + ./ci-build.sh --use-temp-db --protocol ${{ matrix.protocol }} + + build-mac: + runs-on: + - namespace-profile-macos-sequoia;overrides.cache-tag=config-macos-sequoia + steps: + + - name: Checkout with Namespace Git mirrors cache + uses: namespacelabs/nscloud-checkout-action@v7 + with: + fetch-depth: 1 + submodules: recursive - # We only _save_ to the cache when we had a cache miss, are running on master, and are the non-txmeta scenario. - - uses: actions/cache/save@v4 - if: ${{ steps.cache.outputs.cache-hit != 'true' && github.ref_name == 'master' && matrix.scenario == ''}} + - name: Configure Namespace cache volume + uses: namespacelabs/nscloud-cache-action@v1 with: - path: ${{ env.CACHED_PATHS }} - key: ${{ steps.cache.outputs.cache-primary-key }} \ No newline at end of file + path: | + ~/.cargo + build-clang-current + + - name: install prerequisites and uninstall brew rust, add rustup + run: | + brew install libsodium libtool autoconf automake pkg-config libpq openssl parallel ccache bison gnu-sed perl coreutils + brew uninstall rust rustup + brew install rustup + + - name: install rustup components + run: | + rustup-init -y + rustup component add rustfmt rustc cargo clippy rust-src rust-std + + - name: install cargo-cache + run: | + cargo install --locked cargo-cache --version 0.8.3 + + - name: install cargo-sweep + run: | + cargo install --locked cargo-sweep --version 0.7.0 + + - name: Build + run: | + export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$(brew --prefix)/opt/libpq/lib/pkgconfig" + export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$(brew --prefix)/opt/openssl@3/lib/pkgconfig" + export PATH="$(brew --prefix bison)/bin:$PATH" + export CC='clang' + export CXX='clang++' + export CLANG_VERSION=none + export SKIP_FORMAT_CHECK=1 + ./ci-build.sh --disable-postgres --protocol current diff --git a/ci-build.sh b/ci-build.sh index 1122dfecf5..8f21dbdbce 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -26,14 +26,6 @@ while [[ -n "$1" ]]; do export TEMP_POSTGRES=1 echo Using temp database ;; - "--check-test-tx-meta") - if [[ -z "${PROTOCOL}" ]]; then - echo 'must specify --protocol before --check-test-tx-meta' - exit 1 - fi - export TEST_SPEC='[tx]' - export STELLAR_CORE_TEST_PARAMS="--ll fatal -r simple --all-versions --rng-seed 12345 --check-test-tx-meta ${PWD}/test-tx-meta-baseline-${PROTOCOL}" - ;; "--protocol") PROTOCOL="$1" shift @@ -61,15 +53,16 @@ while [[ -n "$1" ]]; do done -echo $TRAVIS_PULL_REQUEST - NPROCS=$(getconf _NPROCESSORS_ONLN) echo "Found $NPROCS processors" date +mkdir -p "build-${CC}-${PROTOCOL}" +cd "build-${CC}-${PROTOCOL}" + # Try to ensure we're using the real g++ and clang++ versions we want -mkdir bin +mkdir -p bin export PATH=`pwd`/bin:$PATH echo "PATH is $PATH" @@ -80,19 +73,19 @@ if test $CXX = 'clang++'; then # Use CLANG_VERSION environment variable if set, otherwise default to 12 CLANG_VER=${CLANG_VERSION:-12} which clang-${CLANG_VER} - ln -s `which clang-${CLANG_VER}` bin/clang + ln -sf `which clang-${CLANG_VER}` bin/clang which clang++-${CLANG_VER} - ln -s `which clang++-${CLANG_VER}` bin/clang++ + ln -sf `which clang++-${CLANG_VER}` bin/clang++ which llvm-symbolizer-${CLANG_VER} - ln -s `which llvm-symbolizer-${CLANG_VER}` bin/llvm-symbolizer + ln -sf `which llvm-symbolizer-${CLANG_VER}` bin/llvm-symbolizer clang -v llvm-symbolizer --version || true elif test $CXX = 'g++'; then RUN_PARTITIONS=$(seq $NPROCS $((2*NPROCS-1))) which gcc-10 - ln -s `which gcc-10` bin/gcc + ln -sf `which gcc-10` bin/gcc which g++-10 - ln -s `which g++-10` bin/g++ + ln -sf `which g++-10` bin/g++ which g++ g++ -v fi @@ -111,11 +104,11 @@ export ASAN_OPTIONS="quarantine_size_mb=100:malloc_context_size=4:detect_leaks=0 echo "config_flags = $config_flags" #### ccache config -export CCACHE_DIR=$HOME/.ccache +export CCACHE_DIR=$(pwd)/.ccache export CCACHE_COMPRESS=true export CCACHE_COMPRESSLEVEL=9 # cache size should be large enough for a full build -export CCACHE_MAXSIZE=500M +export CCACHE_MAXSIZE=800M export CCACHE_CPP2=true # periodically check to see if caches are old and purge them if so @@ -127,11 +120,10 @@ if [ -d "$CCACHE_DIR" ] ; then fi ccache -p - ccache -s date -time ./autogen.sh -time ./configure $config_flags +time (cd .. && ./autogen.sh) +time ../configure $config_flags if [ -z "${SKIP_FORMAT_CHECK}" ]; then make format d=`git diff | wc -l` @@ -155,9 +147,8 @@ date time make -j$(($NPROCS - 1)) ccache -s -### incrementally purge old content from cargo source cache and target directory -cargo cache trim --limit 100M -cargo sweep --maxsize 500MB +### incrementally purge old content from target directory +(cd .. && CARGO_TARGET_DIR="build-${CC}-${PROTOCOL}/target" cargo sweep --maxsize 800MB) if [ $WITH_TESTS -eq 0 ] ; then echo "Build done, skipping tests" @@ -182,7 +173,12 @@ export NUM_PARTITIONS=$((NPROCS*2)) export RUN_PARTITIONS export RND_SEED=$(($(date +%s) / 86400)) # Convert to days since epoch echo "Using RND_SEED: $RND_SEED" -ulimit -n 256 +ulimit -n 4096 +time make check + +echo Running fixed check-test-tx-meta tests +export TEST_SPEC='[tx]' +export STELLAR_CORE_TEST_PARAMS="--ll fatal -r simple --all-versions --rng-seed 12345 --check-test-tx-meta ${PWD}/../test-tx-meta-baseline-${PROTOCOL}" time make check echo All done diff --git a/configure.ac b/configure.ac index 09201b44ba..25cf001bca 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # under the Apache License, Version 2.0. See the COPYING file at the root # of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 -AC_PREREQ([2.68]) +AC_PREREQ([2.71]) AC_INIT([stellar-core],[0.1],[],[],[http://www.stellar.org]) # tar-ustar is required for long file names when libsodium is bundled AM_INIT_AUTOMAKE([-Wall -Wextra -Wconversion subdir-objects tar-ustar silent-rules]) @@ -230,6 +230,29 @@ AS_IF([test "x$enable_ccache" = "xyes"], [ esac ]) +AC_ARG_ENABLE([sccache], + AS_HELP_STRING([--enable-ccache], [build with sccache])) +AS_IF([test "x$enable_sccache" = "xyes"], [ + AC_CHECK_PROGS([SCCACHE], [sccache]) + AS_IF([test -z "$SCCACHE"], [ + AC_MSG_ERROR([sccache enabled but not found]) + ]) + case "$CC" in + *sccache\ *) + ;; + *) + CC="sccache ${CC}" + ;; + esac + case "$CXX" in + *sccache\ *) + ;; + *) + CXX="sccache ${CXX}" + ;; + esac +]) + # Permit user to enable AFL instrumentation AC_ARG_ENABLE([afl], AS_HELP_STRING([--enable-afl], @@ -261,30 +284,7 @@ AS_IF([test "x$enable_afl" = "xyes"], [ ]) AM_CONDITIONAL([USE_AFL_FUZZ], [test "x$enable_afl" == "xyes"]) -# check to see if we need to append -lstdc++fs or -lc++fs to access -# functionality from (for some reason this was thought -# a good idea in gcc 8 and clang 8) -AC_MSG_CHECKING([to see if works without any extra libs]) -AC_RUN_IFELSE([AC_LANG_PROGRAM([[#include ]], - [[return std::filesystem::exists(std::filesystem::path("hello"));]])], - [AC_MSG_RESULT([yes]); FS_WORKS=yes], - [AC_MSG_RESULT([no]); FS_WORKS=no]) -for testlib in -lstdc++fs -lc++fs; do - if test "$FS_WORKS" = "no"; then - fs_save_LIBS="$LIBS" - LIBS="$testlib $LIBS" - AC_MSG_CHECKING([to see if works with $testlib]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([[#include ]], - [[return std::filesystem::exists(std::filesystem::path("hello"));]])], - [AC_MSG_RESULT([yes]); FS_WORKS=yes], - [AC_MSG_RESULT([no]); FS_WORKS=no; LIBS="$fs_save_LIBS"]) - fi -done -if test "$FS_WORKS" = "no"; then - AC_MSG_ERROR([C++17 does not work with any known linker flags]) -fi - -# prefer 10 as it's the one we use +# prefer 12 as it's the one we use AC_CHECK_PROGS(CLANG_FORMAT, [clang-format-12 clang-format]) AM_CONDITIONAL([USE_CLANG_FORMAT], [test "x$CLANG_FORMAT" != "x"]) diff --git a/docker/setup b/docker/setup index b2c50639bf..c4c1341b7f 100755 --- a/docker/setup +++ b/docker/setup @@ -7,9 +7,26 @@ export DEBIAN_FRONTEND=noninteractive apt-get update # We need apt-transport-https for adding apt.stellar.org in the Dockerfile -apt-get install -y wget curl python-six python3-colorama python3-pip postgresql-client sqlite3 apt-transport-https gnupg2 +apt-get install -y wget curl python3-six python3-colorama python3-pip postgresql-client sqlite3 apt-transport-https gnupg2 + +if grep "VERSION_CODENAME=jammy" /etc/os-release; then + echo "Setting up LLVM repo in the 22.04 Ubuntu base" + wget -O /etc/apt/trusted.gpg.d/apt.llvm.org.asc https://apt.llvm.org/llvm-snapshot.gpg.key + + cat > /etc/apt/sources.list.d/llvm.list </dev/null 2>&1 && exit 0 + # Fail on any error set -e diff --git a/src/Makefile.am b/src/Makefile.am index dd7944cdce..e85e4db885 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -65,11 +65,11 @@ endif # !BUILD_TESTS if ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION CARGO_FEATURE_NEXT = --features next main/XDRFilesSha256.cpp: $(SRC_X_FILES) Makefile $(top_srcdir)/hash-xdrs.sh - $(top_srcdir)/hash-xdrs.sh protocol-next >$@ + $(top_srcdir)/hash-xdrs.sh $(top_srcdir)/src/protocol-next >$@ else CARGO_FEATURE_NEXT = main/XDRFilesSha256.cpp: $(SRC_X_FILES) Makefile $(top_srcdir)/hash-xdrs.sh - $(top_srcdir)/hash-xdrs.sh protocol-curr >$@ + $(top_srcdir)/hash-xdrs.sh $(top_srcdir)/src/protocol-curr >$@ endif stellar_core_LDADD = $(soci_LIBS) $(libmedida_LIBS) \ @@ -89,6 +89,7 @@ BUILT_SOURCES = $(SRC_X_FILES:.x=.h) main/StellarCoreVersion.cpp main/XDRFilesSh $(SRC_X_FILES:.x=.h): $(XDRC) SUFFIXES = .x .h .rs .x.h: + mkdir -p $(@D) $(XDRC) -hh -pedantic -o $@ $< BISON=bison @@ -124,9 +125,11 @@ RUST_TOOLCHAIN_CHANNEL=$(shell sed -n 's/channel *= *"\([^"]*\)"/\1/p' $(RUST_TO endif CARGO=cargo +$(RUST_TOOLCHAIN_CHANNEL) -# we pass RUST_TOOLCHAIN_CHANNEL by environment variable +# we pass some configuration by environment variable # to tests since they can't take command-line arguments. export RUST_TOOLCHAIN_CHANNEL +export top_srcdir +export top_builddir RUST_BUILD_DIR=$(top_builddir)/src/rust RUST_BIN_DIR=$(RUST_BUILD_DIR)/bin @@ -233,11 +236,12 @@ rust/RustBridge.cpp: rust/src/bridge.rs $(SRC_RUST_FILES) Makefile $(RUST_CXXBRI $(RUST_DEP_TREE_STAMP): $(wildcard rust/soroban/*/Cargo.*) Makefile $(RUST_TOOLCHAIN_FILE) rm -f $@ + mkdir -p $(RUST_BUILD_DIR)/src/dep-trees for proto in $(ALL_SOROBAN_PROTOCOLS); \ do \ - $(CARGO) tree --manifest-path rust/soroban/$${proto}/Cargo.toml --locked --package soroban-env-host --edges no-dev --target all \ + $(CARGO) tree --manifest-path $(top_srcdir)/src/rust/soroban/$${proto}/Cargo.toml --locked --package soroban-env-host --edges no-dev --target all \ | sed -e "s@$(abspath $(top_srcdir))/@@g" > $(RUST_BUILD_DIR)/src/dep-trees/$${proto}-actual.txt ; \ - if ! diff -u rust/src/dep-trees/$${proto}-expect.txt $(RUST_BUILD_DIR)/src/dep-trees/$${proto}-actual.txt; \ + if ! diff -u $(top_srcdir)/src/rust/src/dep-trees/$${proto}-expect.txt $(RUST_BUILD_DIR)/src/dep-trees/$${proto}-actual.txt; \ then \ echo "dep trees differ, please update $${proto}-expect.txt or roll back submodule"; \ exit 1; \ @@ -272,6 +276,7 @@ CARGOFLAGS_BUILDSTD := $(if $(findstring sanitizer,$(RUSTFLAGS_SANI)),-Zbuild-st RUSTFLAGS_CFGS := $(if $(findstring sanitizer,$(RUSTFLAGS_SANI)),--cfg curve25519_dalek_backend=\"serial\",) $(LIBRUST_STELLAR_CORE): $(RUST_HOST_DEPFILES) $(SRC_RUST_FILES) Makefile $(RUST_TOOLCHAIN_FILE) + cd $(abspath $(top_srcdir))/src/rust && \ CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" CXXSTDLIB="$(CXXSTDLIB)" LDFLAGS="$(LDFLAGS)" \ RUSTFLAGS="$(RUSTFLAGS_SANI) $(RUSTFLAGS_CFGS)" \ CARGO_NET_GIT_FETCH_WITH_CLI=true \ @@ -346,9 +351,11 @@ $(SOROBAN_LIBS_STAMP): $(wildcard rust/soroban/p*/Cargo.lock) Makefile $(RUST_DE FEATURE_FLAGS="$(CARGO_FEATURE_TRACY)" \ ;; \ esac ; \ - cd $(abspath $(RUST_BUILD_DIR))/soroban/$$proto && \ + mkdir -p $(SOROBAN_BUILD_DIR)/$$proto/target && \ + cd $(abspath $(top_srcdir))/src/rust/soroban/$$proto && \ CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" CXXSTDLIB="$(CXXSTDLIB)" LDFLAGS="$(LDFLAGS)" \ RUSTFLAGS="-Cmetadata=$$proto" \ + CARGO_TARGET_DIR=$(SOROBAN_BUILD_DIR)/$$proto/target \ CARGO_NET_GIT_FETCH_WITH_CLI=true \ $(CARGO) build \ --package soroban-env-host \ @@ -376,6 +383,7 @@ $(SOROBAN_LIBS_STAMP): $(wildcard rust/soroban/p*/Cargo.lock) Makefile $(RUST_DE LIBRUST_STELLAR_CORE=$(RUST_TARGET_DIR)/$(RUST_PROFILE_DIR)/librust_stellar_core.a $(LIBRUST_STELLAR_CORE): $(RUST_HOST_DEPFILES) $(SRC_RUST_FILES) Makefile $(SOROBAN_LIBS_STAMP) $(RUST_TOOLCHAIN_FILE) + cd $(abspath $(top_srcdir))/src/rust && \ CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" CXXSTDLIB="$(CXXSTDLIB)" LDFLAGS="$(LDFLAGS)" \ CARGO_NET_GIT_FETCH_WITH_CLI=true \ $(CARGO) rustc \ diff --git a/src/crypto/SecretKey.cpp b/src/crypto/SecretKey.cpp index f3f7abc79b..c2723b616c 100644 --- a/src/crypto/SecretKey.cpp +++ b/src/crypto/SecretKey.cpp @@ -42,8 +42,7 @@ namespace stellar // has no effect on correctness. static std::mutex gVerifySigCacheMutex; -static RandomEvictionCache gVerifySigCache(0xffff, - /* separatePRNG */ true); +static RandomEvictionCache gVerifySigCache(0xffff); static uint64_t gVerifyCacheHit = 0; static uint64_t gVerifyCacheMiss = 0; @@ -322,10 +321,10 @@ PubKeyUtils::clearVerifySigCache() } void -PubKeyUtils::maybeSeedVerifySigCache(unsigned int seed) +PubKeyUtils::seedVerifySigCache(unsigned int seed) { std::lock_guard guard(gVerifySigCacheMutex); - gVerifySigCache.maybeSeed(seed); + gVerifySigCache.seed(seed); } void diff --git a/src/crypto/SecretKey.h b/src/crypto/SecretKey.h index 7ac86454b7..4934c1d84c 100644 --- a/src/crypto/SecretKey.h +++ b/src/crypto/SecretKey.h @@ -140,7 +140,7 @@ bool verifySig(PublicKey const& key, Signature const& signature, ByteSlice const& bin); void clearVerifySigCache(); -void maybeSeedVerifySigCache(unsigned int seed); +void seedVerifySigCache(unsigned int seed); void flushVerifySigCacheCounts(uint64_t& hits, uint64_t& misses); PublicKey random(); diff --git a/src/herder/QuorumIntersectionCheckerImpl.cpp b/src/herder/QuorumIntersectionCheckerImpl.cpp index 180c3fc53f..ee8b3226dd 100644 --- a/src/herder/QuorumIntersectionCheckerImpl.cpp +++ b/src/herder/QuorumIntersectionCheckerImpl.cpp @@ -277,7 +277,7 @@ QuorumIntersectionCheckerImpl::QuorumIntersectionCheckerImpl( , mQuiet(quiet) , mTSC() , mInterruptFlag(interruptFlag) - , mCachedQuorums(MAX_CACHED_QUORUMS_SIZE, /*separatePRNG=*/true) + , mCachedQuorums(MAX_CACHED_QUORUMS_SIZE) , mRand(seed) { buildGraph(qmap); diff --git a/src/herder/test/TxSetTests.cpp b/src/herder/test/TxSetTests.cpp index eefc7b242c..8f3fd9efe3 100644 --- a/src/herder/test/TxSetTests.cpp +++ b/src/herder/test/TxSetTests.cpp @@ -2260,11 +2260,10 @@ TEST_CASE("txset nomination", "[txset]") { runIteration(); } - namespace fs = std::filesystem; - fs::path resultsPath = "testdata/txset/" + testName; - fs::create_directories(resultsPath.parent_path()); + auto resultsPath = getSrcTestDataPath("txset/" + testName); if (getenv("GENERATE_TEST_TXSETS")) { + std::filesystem::create_directories(resultsPath.parent_path()); std::ofstream resultsStream(resultsPath); for (auto const& result : testResults) { diff --git a/src/history/test/SerializeTests.cpp b/src/history/test/SerializeTests.cpp index acdae3c0e6..2ad2e694fe 100644 --- a/src/history/test/SerializeTests.cpp +++ b/src/history/test/SerializeTests.cpp @@ -4,6 +4,7 @@ #include "history/HistoryArchive.h" #include "test/Catch2.h" +#include "test/test.h" #include #include @@ -19,9 +20,8 @@ TEST_CASE("Serialization round trip", "[history]") "stellar-history.testnet.6714239.networkPassphrase.v2.json"}; for (size_t i = 0; i < testFiles.size(); i++) { - std::string fnPath = "testdata/"; - std::string testFilePath = fnPath + testFiles[i]; - SECTION("Serialize " + testFilePath) + auto testFilePath = getBuildTestDataPath(testFiles[i]); + SECTION("Serialize " + testFilePath.string()) { std::ifstream in(testFilePath); REQUIRE(in); diff --git a/src/ledger/test/LedgerCloseMetaStreamTests.cpp b/src/ledger/test/LedgerCloseMetaStreamTests.cpp index 15cce51582..759a839706 100644 --- a/src/ledger/test/LedgerCloseMetaStreamTests.cpp +++ b/src/ledger/test/LedgerCloseMetaStreamTests.cpp @@ -515,13 +515,11 @@ TEST_CASE_VERSIONS("meta stream contains reasonable meta", "[ledgerclosemeta]") if (ledgerSeq == targetSeq) { - std::string refJsonPath; - refJsonPath = fmt::format( - FMT_STRING( - "testdata/ledger-close-meta{}-v{}-protocol-{}{}.json"), + auto refJsonPath = getSrcTestDataPath(fmt::format( + FMT_STRING("ledger-close-meta{}-v{}-protocol-{}{}.json"), enableClassicEvents ? "-enable-classic-events" : "", lcm.v(), cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION, - isSoroban ? "-soroban" : ""); + isSoroban ? "-soroban" : "")); normalizeMeta(lcm); std::string have = xdrToCerealString(lcm, "LedgerCloseMeta"); if (getenv("GENERATE_TEST_LEDGER_CLOSE_META")) diff --git a/src/main/CommandLine.cpp b/src/main/CommandLine.cpp index ce2d0c4612..f7420e7803 100644 --- a/src/main/CommandLine.cpp +++ b/src/main/CommandLine.cpp @@ -1930,14 +1930,10 @@ runApplyLoad(CommandLineArgs const& args) config.IGNORE_MESSAGE_LIMITS_FOR_TESTING = true; } - if (mode != ApplyLoadMode::MAX_SAC_TPS) - { - TmpDirManager tdm(std::string("soroban-storage-meta-")); - TmpDir td = tdm.tmpDir("soroban-meta-ok"); - std::string metaPath = td.getName() + "/stream.xdr"; - - config.METADATA_OUTPUT_STREAM = metaPath; - } + TmpDirManager tdm(std::string("soroban-storage-meta-")); + TmpDir td = tdm.tmpDir("soroban-meta-ok"); + std::string metaPath = td.getName() + "/stream.xdr"; + config.METADATA_OUTPUT_STREAM = metaPath; VirtualClock clock(VirtualClock::REAL_TIME); auto appPtr = Application::create(clock, config); diff --git a/src/main/test/ApplicationUtilsTests.cpp b/src/main/test/ApplicationUtilsTests.cpp index ec40ad8a1e..19a686a8b0 100644 --- a/src/main/test/ApplicationUtilsTests.cpp +++ b/src/main/test/ApplicationUtilsTests.cpp @@ -173,18 +173,18 @@ TEST_CASE("application major version numbers", "[applicationutils]") TEST_CASE("standalone quorum intersection check", "[applicationutils]") { Config cfg = getTestConfig(); - const std::string JSON_ROOT = "testdata/check-quorum-intersection-json/"; + auto JSON_ROOT = getSrcTestDataPath("check-quorum-intersection-json"); SECTION("enjoys quorum intersection") { REQUIRE(checkQuorumIntersectionFromJson( - JSON_ROOT + "enjoys-intersection.json", cfg)); + JSON_ROOT / "enjoys-intersection.json", cfg)); } SECTION("does not enjoy quorum intersection") { REQUIRE(!checkQuorumIntersectionFromJson( - JSON_ROOT + "no-intersection.json", cfg)); + JSON_ROOT / "no-intersection.json", cfg)); } SECTION("malformed JSON") @@ -193,17 +193,17 @@ TEST_CASE("standalone quorum intersection check", "[applicationutils]") // Malformed key REQUIRE_THROWS_AS( - checkQuorumIntersectionFromJson(JSON_ROOT + "bad-key.json", cfg), + checkQuorumIntersectionFromJson(JSON_ROOT / "bad-key.json", cfg), KeyUtils::InvalidStrKey); // Wrong datatype REQUIRE_THROWS_AS(checkQuorumIntersectionFromJson( - JSON_ROOT + "bad-threshold-type.json", cfg), + JSON_ROOT / "bad-threshold-type.json", cfg), std::runtime_error); // No such file REQUIRE_THROWS_AS( - checkQuorumIntersectionFromJson(JSON_ROOT + "no-file.json", cfg), + checkQuorumIntersectionFromJson(JSON_ROOT / "no-file.json", cfg), std::runtime_error); } -} \ No newline at end of file +} diff --git a/src/main/test/ConfigTests.cpp b/src/main/test/ConfigTests.cpp index 05bdbd483a..808241f342 100644 --- a/src/main/test/ConfigTests.cpp +++ b/src/main/test/ConfigTests.cpp @@ -147,7 +147,7 @@ TEST_CASE("resolve node id", "[config]") TEST_CASE("load validators config", "[config]") { Config c; - c.load("testdata/stellar-core_example_validators.cfg"); + c.load(getBuildTestDataPath("stellar-core_example_validators.cfg")); auto actualS = c.toString(c.QUORUM_SET); std::string expected = R"({ "t" : 4, @@ -456,9 +456,8 @@ TEST_CASE("load example configs", "[config]") "stellar-core_testnet_validator.cfg"}; for (auto const& fn : testFiles) { - std::string fnPath = "testdata/"; - fnPath += fn; - SECTION("load config " + fnPath) + auto fnPath = getBuildTestDataPath(fn); + SECTION("load config " + fnPath.string()) { c.load(fnPath); } @@ -628,4 +627,4 @@ PUBLIC_KEY="GBVZFVEARURUJTN5ABZPKW36FHKVJK2GHXEVY2SZCCNU5I3CQMTZ3OES" )"; std::stringstream ss(configStr); c.load(ss); -} \ No newline at end of file +} diff --git a/src/overlay/OverlayManagerImpl.cpp b/src/overlay/OverlayManagerImpl.cpp index 7d593df8d7..8762cbabd2 100644 --- a/src/overlay/OverlayManagerImpl.cpp +++ b/src/overlay/OverlayManagerImpl.cpp @@ -321,7 +321,7 @@ OverlayManagerImpl::OverlayManagerImpl(Application& app) mApp.getConfig().TARGET_PEER_CONNECTIONS, mSurveyManager) , mResolvingPeersWithBackoff(true) , mResolvingPeersRetryCount(0) - , mScheduledMessages(100000, true) + , mScheduledMessages(100000) { mPeerSources[PeerType::INBOUND] = std::make_unique( mPeerManager, RandomPeerSource::nextAttemptCutoff(PeerType::INBOUND)); diff --git a/src/simulation/ApplyLoad.cpp b/src/simulation/ApplyLoad.cpp index 5e3724cce2..d750934a11 100644 --- a/src/simulation/ApplyLoad.cpp +++ b/src/simulation/ApplyLoad.cpp @@ -652,7 +652,6 @@ ApplyLoad::setupBucketList() archivedEntries, {}); } } - lh.ledgerSeq++; mDataEntryCount = currentLiveKey; releaseAssertOrThrow(mTotalHotArchiveEntries <= currentHotArchiveKey); diff --git a/src/test/check-sorobans b/src/test/check-sorobans index 3bde728460..74e5eaf920 100755 --- a/src/test/check-sorobans +++ b/src/test/check-sorobans @@ -28,9 +28,10 @@ SKIP_TESTS="host::declared_size::test::test_expected_size" set -e set -x -cd rust/soroban +cd "${top_srcdir}/src/rust/soroban" for i in p??; do cd $i + CARGO_TARGET_DIR="${top_builddir}/src/rust/soroban/${i}/target" \ RUSTFLAGS="-Cmetadata=${i}" cargo +${RUST_TOOLCHAIN_CHANNEL} \ test \ --locked \ diff --git a/src/test/selftest-nopg b/src/test/selftest-nopg index 18021b9ed2..ef297b0cdd 100755 --- a/src/test/selftest-nopg +++ b/src/test/selftest-nopg @@ -4,4 +4,4 @@ # under the Apache License, Version 2.0. See the COPYING file at the root # of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 -exec ./test/selftest-parallel ./test/run-selftest-nopg +exec "${top_srcdir}/src/test/selftest-parallel" "${top_srcdir}/src/test/run-selftest-nopg" diff --git a/src/test/selftest-parallel b/src/test/selftest-parallel index 8f37ec532d..6195171d86 100755 --- a/src/test/selftest-parallel +++ b/src/test/selftest-parallel @@ -89,5 +89,5 @@ for i in $RUN_PARTITIONS; do cat $TEST_PARTITIONS_DIR/test-partition-$i.txt >> $TESTS done -cat $TESTS | parallel -j $nparts -n $BATCHSIZE --line-buffer --tag-string '{%}' --progress --halt-on-error 2 runpart '{%}' "{}" +cat $TESTS | parallel -j $nparts -n $BATCHSIZE --line-buffer --tag-string '{%}' --halt-on-error 2 runpart '{%}' "{}" diff --git a/src/test/selftest-pg b/src/test/selftest-pg index b856805799..c99a178cb4 100755 --- a/src/test/selftest-pg +++ b/src/test/selftest-pg @@ -4,4 +4,4 @@ # under the Apache License, Version 2.0. See the COPYING file at the root # of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 -exec ./test/selftest-parallel ./test/run-selftest-pg +exec "${top_srcdir}/src/test/selftest-parallel" "${top_srcdir}/src/test/run-selftest-pg" diff --git a/src/test/test.cpp b/src/test/test.cpp index f0a49ce396..56e1b90d2b 100644 --- a/src/test/test.cpp +++ b/src/test/test.cpp @@ -337,6 +337,32 @@ getTestConfig(int instanceNumber, Config::TestDbMode mode) return *cfgs[instanceNumber]; } +std::filesystem::path +getSrcTestDataPath(std::filesystem::path rel) +{ + namespace fs = std::filesystem; + fs::path testdata("testdata"); + char* srcdir = getenv("top_srcdir"); + if (srcdir) + { + testdata = fs::path(srcdir) / "src" / testdata; + } + return testdata / rel; +} + +std::filesystem::path +getBuildTestDataPath(std::filesystem::path rel) +{ + namespace fs = std::filesystem; + fs::path testdata("testdata"); + char* builddir = getenv("top_builddir"); + if (builddir) + { + testdata = fs::path(builddir) / "src" / testdata; + } + return testdata / rel; +} + int runTest(CommandLineArgs const& args) { diff --git a/src/test/test.h b/src/test/test.h index a026102a77..27c9fac306 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -19,6 +19,9 @@ struct TransactionMeta; Config const& getTestConfig(int instanceNumber = 0, Config::TestDbMode mode = Config::TESTDB_DEFAULT); +std::filesystem::path getSrcTestDataPath(std::filesystem::path rel); +std::filesystem::path getBuildTestDataPath(std::filesystem::path rel); + void cleanupTmpDirs(); // Records or checks a TxMetadata value against a persistent record diff --git a/src/util/Math.cpp b/src/util/Math.cpp index adf11997be..957e7a8733 100644 --- a/src/util/Math.cpp +++ b/src/util/Math.cpp @@ -6,6 +6,7 @@ #include "crypto/SecretKey.h" #include "crypto/ShortHash.h" #include "util/GlobalChecks.h" +#include "util/RandomEvictionCache.h" #include "util/UnorderedMap.h" #include #include @@ -184,10 +185,11 @@ reinitializeAllGlobalStateWithSeedInternal(unsigned int seed) { lastGlobalSeed = seed; PubKeyUtils::clearVerifySigCache(); - PubKeyUtils::maybeSeedVerifySigCache(seed); + PubKeyUtils::seedVerifySigCache(seed); srand(seed); getGlobalRandomEngine().seed(seed); randHash::initialize(); + randomEvictionCacheSeed = seed; } void diff --git a/src/util/RandomEvictionCache.h b/src/util/RandomEvictionCache.h index dd129481a4..04da3558cb 100644 --- a/src/util/RandomEvictionCache.h +++ b/src/util/RandomEvictionCache.h @@ -13,6 +13,14 @@ namespace stellar { +// The global prng generator can only be accessed from the main thread, but +// there's no reason RandomEvictionCache should be main-thread only. To get +// around this, each RandomEvictionCache will have a local prng generator by +// default with a seed specific to the cache. While technically there is a race +// on this seed since it's managed via main thread's `initializeAllGlobalState` +// functions, we only actually reset the seed in tests. +static unsigned int randomEvictionCacheSeed{0}; + // Implements a simple fixed-size cache that does // least-recent-out-of-2-random-choices eviction. Degrades more-gracefully // across pathological load patterns than LRU, and also takes somewhat less @@ -57,8 +65,6 @@ class RandomEvictionCache : public NonMovableOrCopyable // Each cache keeps some counters just to monitor its performance. Counters mCounters; - // Use a dedicated random engine, with an option to disable - bool const mSeparatePRNG{true}; stellar_default_random_engine mRandEngine; // Randomly pick two elements and evict the less-recently-used one. @@ -71,11 +77,7 @@ class RandomEvictionCache : public NonMovableOrCopyable return; } auto getRandIndex = [&]() { - if (mSeparatePRNG) - { - return rand_uniform(0, sz - 1, mRandEngine); - } - return rand_uniform(0, sz - 1); + return rand_uniform(0, sz - 1, mRandEngine); }; MapValueType*& vp1 = mValuePtrs.at(getRandIndex()); MapValueType*& vp2 = mValuePtrs.at(getRandIndex()); @@ -89,26 +91,16 @@ class RandomEvictionCache : public NonMovableOrCopyable public: explicit RandomEvictionCache(size_t maxSize) - : mMaxSize(maxSize), mSeparatePRNG(true) - { - mValueMap.reserve(maxSize + 1); - mValuePtrs.reserve(maxSize + 1); - } - - RandomEvictionCache(size_t maxSize, bool separatePRNG) - : mMaxSize(maxSize), mSeparatePRNG(separatePRNG) + : mMaxSize(maxSize), mRandEngine(randomEvictionCacheSeed) { mValueMap.reserve(maxSize + 1); mValuePtrs.reserve(maxSize + 1); } void - maybeSeed(unsigned int seed) + seed(unsigned int seed) { - if (mSeparatePRNG) - { - mRandEngine.seed(seed); - } + mRandEngine.seed(seed); } size_t