From 5d4caea6f9240590a59d56f646aa37e962ac9f84 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Mon, 22 Sep 2025 16:24:37 -0700 Subject: [PATCH 01/15] Small fixes to apply load --- src/main/CommandLine.cpp | 12 ++++-------- src/simulation/ApplyLoad.cpp | 1 - 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/main/CommandLine.cpp b/src/main/CommandLine.cpp index ce2d0c4612..f7420e7803 100644 --- a/src/main/CommandLine.cpp +++ b/src/main/CommandLine.cpp @@ -1930,14 +1930,10 @@ runApplyLoad(CommandLineArgs const& args) config.IGNORE_MESSAGE_LIMITS_FOR_TESTING = true; } - if (mode != ApplyLoadMode::MAX_SAC_TPS) - { - TmpDirManager tdm(std::string("soroban-storage-meta-")); - TmpDir td = tdm.tmpDir("soroban-meta-ok"); - std::string metaPath = td.getName() + "/stream.xdr"; - - config.METADATA_OUTPUT_STREAM = metaPath; - } + TmpDirManager tdm(std::string("soroban-storage-meta-")); + TmpDir td = tdm.tmpDir("soroban-meta-ok"); + std::string metaPath = td.getName() + "/stream.xdr"; + config.METADATA_OUTPUT_STREAM = metaPath; VirtualClock clock(VirtualClock::REAL_TIME); auto appPtr = Application::create(clock, config); diff --git a/src/simulation/ApplyLoad.cpp b/src/simulation/ApplyLoad.cpp index 5e3724cce2..d750934a11 100644 --- a/src/simulation/ApplyLoad.cpp +++ b/src/simulation/ApplyLoad.cpp @@ -652,7 +652,6 @@ ApplyLoad::setupBucketList() archivedEntries, {}); } } - lh.ledgerSeq++; mDataEntryCount = currentLiveKey; releaseAssertOrThrow(mTotalHotArchiveEntries <= currentHotArchiveKey); From 0ec0683791cf2e43a50eebe4e3fbc0b36b5cd382 Mon Sep 17 00:00:00 2001 From: Garand Tyson Date: Mon, 22 Sep 2025 16:59:03 -0700 Subject: [PATCH 02/15] properly initialize seed for RandomEviction cache --- src/util/Math.cpp | 2 ++ src/util/RandomEvictionCache.h | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/util/Math.cpp b/src/util/Math.cpp index adf11997be..0db91b6f06 100644 --- a/src/util/Math.cpp +++ b/src/util/Math.cpp @@ -6,6 +6,7 @@ #include "crypto/SecretKey.h" #include "crypto/ShortHash.h" #include "util/GlobalChecks.h" +#include "util/RandomEvictionCache.h" #include "util/UnorderedMap.h" #include #include @@ -188,6 +189,7 @@ reinitializeAllGlobalStateWithSeedInternal(unsigned int seed) srand(seed); getGlobalRandomEngine().seed(seed); randHash::initialize(); + randomEvictionCacheSeed = seed; } void diff --git a/src/util/RandomEvictionCache.h b/src/util/RandomEvictionCache.h index dd129481a4..636a3e5a83 100644 --- a/src/util/RandomEvictionCache.h +++ b/src/util/RandomEvictionCache.h @@ -13,6 +13,14 @@ namespace stellar { +// The global prng generator can only be accessed from the main thread, but +// there's no reason RandomEvictionCache should be main-thread only. To get +// around this, each RandomEvictionCache will have a local prng generator by +// default with a seed specific to the cache. While technically there is a race +// on this seed since it's managed via main thread's `initializeAllGlobalState` +// functions, we only actually reset the seed in tests. +static unsigned int randomEvictionCacheSeed{0}; + // Implements a simple fixed-size cache that does // least-recent-out-of-2-random-choices eviction. Degrades more-gracefully // across pathological load patterns than LRU, and also takes somewhat less @@ -89,7 +97,9 @@ class RandomEvictionCache : public NonMovableOrCopyable public: explicit RandomEvictionCache(size_t maxSize) - : mMaxSize(maxSize), mSeparatePRNG(true) + : mMaxSize(maxSize) + , mSeparatePRNG(true) + , mRandEngine(randomEvictionCacheSeed) { mValueMap.reserve(maxSize + 1); mValuePtrs.reserve(maxSize + 1); From 01199535aa5304e5d8930b4106b513f173a404a2 Mon Sep 17 00:00:00 2001 From: Siddharth Suresh Date: Mon, 22 Sep 2025 16:59:43 -0700 Subject: [PATCH 03/15] Remove option to use global prng in random evictino cache --- src/crypto/SecretKey.cpp | 7 +++--- src/crypto/SecretKey.h | 2 +- src/herder/QuorumIntersectionCheckerImpl.cpp | 2 +- src/overlay/OverlayManagerImpl.cpp | 2 +- src/util/Math.cpp | 2 +- src/util/RandomEvictionCache.h | 26 +++----------------- 6 files changed, 11 insertions(+), 30 deletions(-) diff --git a/src/crypto/SecretKey.cpp b/src/crypto/SecretKey.cpp index f3f7abc79b..c2723b616c 100644 --- a/src/crypto/SecretKey.cpp +++ b/src/crypto/SecretKey.cpp @@ -42,8 +42,7 @@ namespace stellar // has no effect on correctness. static std::mutex gVerifySigCacheMutex; -static RandomEvictionCache gVerifySigCache(0xffff, - /* separatePRNG */ true); +static RandomEvictionCache gVerifySigCache(0xffff); static uint64_t gVerifyCacheHit = 0; static uint64_t gVerifyCacheMiss = 0; @@ -322,10 +321,10 @@ PubKeyUtils::clearVerifySigCache() } void -PubKeyUtils::maybeSeedVerifySigCache(unsigned int seed) +PubKeyUtils::seedVerifySigCache(unsigned int seed) { std::lock_guard guard(gVerifySigCacheMutex); - gVerifySigCache.maybeSeed(seed); + gVerifySigCache.seed(seed); } void diff --git a/src/crypto/SecretKey.h b/src/crypto/SecretKey.h index 7ac86454b7..4934c1d84c 100644 --- a/src/crypto/SecretKey.h +++ b/src/crypto/SecretKey.h @@ -140,7 +140,7 @@ bool verifySig(PublicKey const& key, Signature const& signature, ByteSlice const& bin); void clearVerifySigCache(); -void maybeSeedVerifySigCache(unsigned int seed); +void seedVerifySigCache(unsigned int seed); void flushVerifySigCacheCounts(uint64_t& hits, uint64_t& misses); PublicKey random(); diff --git a/src/herder/QuorumIntersectionCheckerImpl.cpp b/src/herder/QuorumIntersectionCheckerImpl.cpp index 180c3fc53f..ee8b3226dd 100644 --- a/src/herder/QuorumIntersectionCheckerImpl.cpp +++ b/src/herder/QuorumIntersectionCheckerImpl.cpp @@ -277,7 +277,7 @@ QuorumIntersectionCheckerImpl::QuorumIntersectionCheckerImpl( , mQuiet(quiet) , mTSC() , mInterruptFlag(interruptFlag) - , mCachedQuorums(MAX_CACHED_QUORUMS_SIZE, /*separatePRNG=*/true) + , mCachedQuorums(MAX_CACHED_QUORUMS_SIZE) , mRand(seed) { buildGraph(qmap); diff --git a/src/overlay/OverlayManagerImpl.cpp b/src/overlay/OverlayManagerImpl.cpp index 7d593df8d7..8762cbabd2 100644 --- a/src/overlay/OverlayManagerImpl.cpp +++ b/src/overlay/OverlayManagerImpl.cpp @@ -321,7 +321,7 @@ OverlayManagerImpl::OverlayManagerImpl(Application& app) mApp.getConfig().TARGET_PEER_CONNECTIONS, mSurveyManager) , mResolvingPeersWithBackoff(true) , mResolvingPeersRetryCount(0) - , mScheduledMessages(100000, true) + , mScheduledMessages(100000) { mPeerSources[PeerType::INBOUND] = std::make_unique( mPeerManager, RandomPeerSource::nextAttemptCutoff(PeerType::INBOUND)); diff --git a/src/util/Math.cpp b/src/util/Math.cpp index 0db91b6f06..957e7a8733 100644 --- a/src/util/Math.cpp +++ b/src/util/Math.cpp @@ -185,7 +185,7 @@ reinitializeAllGlobalStateWithSeedInternal(unsigned int seed) { lastGlobalSeed = seed; PubKeyUtils::clearVerifySigCache(); - PubKeyUtils::maybeSeedVerifySigCache(seed); + PubKeyUtils::seedVerifySigCache(seed); srand(seed); getGlobalRandomEngine().seed(seed); randHash::initialize(); diff --git a/src/util/RandomEvictionCache.h b/src/util/RandomEvictionCache.h index 636a3e5a83..04da3558cb 100644 --- a/src/util/RandomEvictionCache.h +++ b/src/util/RandomEvictionCache.h @@ -65,8 +65,6 @@ class RandomEvictionCache : public NonMovableOrCopyable // Each cache keeps some counters just to monitor its performance. Counters mCounters; - // Use a dedicated random engine, with an option to disable - bool const mSeparatePRNG{true}; stellar_default_random_engine mRandEngine; // Randomly pick two elements and evict the less-recently-used one. @@ -79,11 +77,7 @@ class RandomEvictionCache : public NonMovableOrCopyable return; } auto getRandIndex = [&]() { - if (mSeparatePRNG) - { - return rand_uniform(0, sz - 1, mRandEngine); - } - return rand_uniform(0, sz - 1); + return rand_uniform(0, sz - 1, mRandEngine); }; MapValueType*& vp1 = mValuePtrs.at(getRandIndex()); MapValueType*& vp2 = mValuePtrs.at(getRandIndex()); @@ -97,28 +91,16 @@ class RandomEvictionCache : public NonMovableOrCopyable public: explicit RandomEvictionCache(size_t maxSize) - : mMaxSize(maxSize) - , mSeparatePRNG(true) - , mRandEngine(randomEvictionCacheSeed) - { - mValueMap.reserve(maxSize + 1); - mValuePtrs.reserve(maxSize + 1); - } - - RandomEvictionCache(size_t maxSize, bool separatePRNG) - : mMaxSize(maxSize), mSeparatePRNG(separatePRNG) + : mMaxSize(maxSize), mRandEngine(randomEvictionCacheSeed) { mValueMap.reserve(maxSize + 1); mValuePtrs.reserve(maxSize + 1); } void - maybeSeed(unsigned int seed) + seed(unsigned int seed) { - if (mSeparatePRNG) - { - mRandEngine.seed(seed); - } + mRandEngine.seed(seed); } size_t From a755b8ae378c7e6599937c17fa4f235bb4236fee Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Tue, 23 Sep 2025 23:26:10 -0700 Subject: [PATCH 04/15] Remove obsolete test (that happens to do AC_RUN and breaks some environments) This test is obsolete in the first place (we don't build on clang 8 at all anymore) but also, incidentally, since it runs AC_RUN_IFELSE it winds up executing a binary it built, and this happens after asan is enabled, and so in a few specific environments where asan doesn't work (eg. old versions of docker when running under qemu-user; also seemingly pbuilder) this causes configuration to fail entirely. There's no real way to make asan work in these environments, we have to just avoid running conftests after asan is turned on. We could move this test above the asan enabling part of configure, but again, it's an obsolete test; simpler just to remove it entirely. --- configure.ac | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/configure.ac b/configure.ac index 09201b44ba..c69efd0cce 100644 --- a/configure.ac +++ b/configure.ac @@ -261,30 +261,7 @@ AS_IF([test "x$enable_afl" = "xyes"], [ ]) AM_CONDITIONAL([USE_AFL_FUZZ], [test "x$enable_afl" == "xyes"]) -# check to see if we need to append -lstdc++fs or -lc++fs to access -# functionality from (for some reason this was thought -# a good idea in gcc 8 and clang 8) -AC_MSG_CHECKING([to see if works without any extra libs]) -AC_RUN_IFELSE([AC_LANG_PROGRAM([[#include ]], - [[return std::filesystem::exists(std::filesystem::path("hello"));]])], - [AC_MSG_RESULT([yes]); FS_WORKS=yes], - [AC_MSG_RESULT([no]); FS_WORKS=no]) -for testlib in -lstdc++fs -lc++fs; do - if test "$FS_WORKS" = "no"; then - fs_save_LIBS="$LIBS" - LIBS="$testlib $LIBS" - AC_MSG_CHECKING([to see if works with $testlib]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([[#include ]], - [[return std::filesystem::exists(std::filesystem::path("hello"));]])], - [AC_MSG_RESULT([yes]); FS_WORKS=yes], - [AC_MSG_RESULT([no]); FS_WORKS=no; LIBS="$fs_save_LIBS"]) - fi -done -if test "$FS_WORKS" = "no"; then - AC_MSG_ERROR([C++17 does not work with any known linker flags]) -fi - -# prefer 10 as it's the one we use +# prefer 12 as it's the one we use AC_CHECK_PROGS(CLANG_FORMAT, [clang-format-12 clang-format]) AM_CONDITIONAL([USE_CLANG_FORMAT], [test "x$CLANG_FORMAT" != "x"]) From 4f103e5d8459b625c396a236e6e0596c91d5277b Mon Sep 17 00:00:00 2001 From: Jacek N Date: Wed, 24 Sep 2025 18:11:11 +0100 Subject: [PATCH 05/15] Changes to support Ubuntu 24.04 builds ### What Update docker building to support Ubuntu 24.04 ### Why We need to start publishing 24.04 packages --- docker/setup | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docker/setup b/docker/setup index b2c50639bf..c4c1341b7f 100755 --- a/docker/setup +++ b/docker/setup @@ -7,9 +7,26 @@ export DEBIAN_FRONTEND=noninteractive apt-get update # We need apt-transport-https for adding apt.stellar.org in the Dockerfile -apt-get install -y wget curl python-six python3-colorama python3-pip postgresql-client sqlite3 apt-transport-https gnupg2 +apt-get install -y wget curl python3-six python3-colorama python3-pip postgresql-client sqlite3 apt-transport-https gnupg2 + +if grep "VERSION_CODENAME=jammy" /etc/os-release; then + echo "Setting up LLVM repo in the 22.04 Ubuntu base" + wget -O /etc/apt/trusted.gpg.d/apt.llvm.org.asc https://apt.llvm.org/llvm-snapshot.gpg.key + + cat > /etc/apt/sources.list.d/llvm.list < Date: Tue, 16 Sep 2025 21:54:25 -0700 Subject: [PATCH 06/15] Update autoconf to new version, to silence noise-warnings --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index c69efd0cce..1a38031dc5 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ # under the Apache License, Version 2.0. See the COPYING file at the root # of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 -AC_PREREQ([2.68]) +AC_PREREQ([2.71]) AC_INIT([stellar-core],[0.1],[],[],[http://www.stellar.org]) # tar-ustar is required for long file names when libsodium is bundled AM_INIT_AUTOMAKE([-Wall -Wextra -Wconversion subdir-objects tar-ustar silent-rules]) From da057e8fa2505f82418781afc8e41c3e6f98e657 Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Fri, 26 Sep 2025 17:26:17 -0700 Subject: [PATCH 07/15] Add support for sccache to configure.ac --- configure.ac | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/configure.ac b/configure.ac index 1a38031dc5..25cf001bca 100644 --- a/configure.ac +++ b/configure.ac @@ -230,6 +230,29 @@ AS_IF([test "x$enable_ccache" = "xyes"], [ esac ]) +AC_ARG_ENABLE([sccache], + AS_HELP_STRING([--enable-ccache], [build with sccache])) +AS_IF([test "x$enable_sccache" = "xyes"], [ + AC_CHECK_PROGS([SCCACHE], [sccache]) + AS_IF([test -z "$SCCACHE"], [ + AC_MSG_ERROR([sccache enabled but not found]) + ]) + case "$CC" in + *sccache\ *) + ;; + *) + CC="sccache ${CC}" + ;; + esac + case "$CXX" in + *sccache\ *) + ;; + *) + CXX="sccache ${CXX}" + ;; + esac +]) + # Permit user to enable AFL instrumentation AC_ARG_ENABLE([afl], AS_HELP_STRING([--enable-afl], From ec1822f147f988bfea7dd466f6f6b5fab2517362 Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Fri, 26 Sep 2025 17:28:19 -0700 Subject: [PATCH 08/15] Make build steps a little more idempotent --- ci-build.sh | 12 ++++++------ install-rust.sh | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 1122dfecf5..48649c5733 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -69,7 +69,7 @@ echo "Found $NPROCS processors" date # Try to ensure we're using the real g++ and clang++ versions we want -mkdir bin +mkdir -p bin export PATH=`pwd`/bin:$PATH echo "PATH is $PATH" @@ -80,19 +80,19 @@ if test $CXX = 'clang++'; then # Use CLANG_VERSION environment variable if set, otherwise default to 12 CLANG_VER=${CLANG_VERSION:-12} which clang-${CLANG_VER} - ln -s `which clang-${CLANG_VER}` bin/clang + ln -sf `which clang-${CLANG_VER}` bin/clang which clang++-${CLANG_VER} - ln -s `which clang++-${CLANG_VER}` bin/clang++ + ln -sf `which clang++-${CLANG_VER}` bin/clang++ which llvm-symbolizer-${CLANG_VER} - ln -s `which llvm-symbolizer-${CLANG_VER}` bin/llvm-symbolizer + ln -sf `which llvm-symbolizer-${CLANG_VER}` bin/llvm-symbolizer clang -v llvm-symbolizer --version || true elif test $CXX = 'g++'; then RUN_PARTITIONS=$(seq $NPROCS $((2*NPROCS-1))) which gcc-10 - ln -s `which gcc-10` bin/gcc + ln -sf `which gcc-10` bin/gcc which g++-10 - ln -s `which g++-10` bin/g++ + ln -sf `which g++-10` bin/g++ which g++ g++ -v fi diff --git a/install-rust.sh b/install-rust.sh index 7c63afb5f0..d66cbe0198 100755 --- a/install-rust.sh +++ b/install-rust.sh @@ -7,6 +7,8 @@ # is version-controlled and run in CI to make the dependency explicit (and to # use a version newer than the debian packages). +rustup --version >/dev/null 2>&1 && exit 0 + # Fail on any error set -e From 2113f2cbaf500d20d52b8e23d0f3e75e23a65d62 Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Fri, 26 Sep 2025 17:30:02 -0700 Subject: [PATCH 09/15] Move --check-test-tx-meta into mandatory 2nd pass of ci-build.sh --- ci-build.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 48649c5733..7eb4268571 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -26,14 +26,6 @@ while [[ -n "$1" ]]; do export TEMP_POSTGRES=1 echo Using temp database ;; - "--check-test-tx-meta") - if [[ -z "${PROTOCOL}" ]]; then - echo 'must specify --protocol before --check-test-tx-meta' - exit 1 - fi - export TEST_SPEC='[tx]' - export STELLAR_CORE_TEST_PARAMS="--ll fatal -r simple --all-versions --rng-seed 12345 --check-test-tx-meta ${PWD}/test-tx-meta-baseline-${PROTOCOL}" - ;; "--protocol") PROTOCOL="$1" shift @@ -185,6 +177,11 @@ echo "Using RND_SEED: $RND_SEED" ulimit -n 256 time make check +echo Running fixed check-test-tx-meta tests +export TEST_SPEC='[tx]' +export STELLAR_CORE_TEST_PARAMS="--ll fatal -r simple --all-versions --rng-seed 12345 --check-test-tx-meta ${PWD}/../test-tx-meta-baseline-${PROTOCOL}" +time make check + echo All done date exit 0 From 4996d91ac6761369c09e6bbf465b7019285dc2ef Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Fri, 26 Sep 2025 17:33:53 -0700 Subject: [PATCH 10/15] bump ulimit for more concurrent tests --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 7eb4268571..4815b85a8b 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -174,7 +174,7 @@ export NUM_PARTITIONS=$((NPROCS*2)) export RUN_PARTITIONS export RND_SEED=$(($(date +%s) / 86400)) # Convert to days since epoch echo "Using RND_SEED: $RND_SEED" -ulimit -n 256 +ulimit -n 4096 time make check echo Running fixed check-test-tx-meta tests From 6325c7f4aa0242ab4341e1d65f9bdb46e20e77c1 Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Fri, 26 Sep 2025 17:34:16 -0700 Subject: [PATCH 11/15] bump ccache size from 500M => 800M --- ci-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-build.sh b/ci-build.sh index 4815b85a8b..b61341b270 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -107,7 +107,7 @@ export CCACHE_DIR=$HOME/.ccache export CCACHE_COMPRESS=true export CCACHE_COMPRESSLEVEL=9 # cache size should be large enough for a full build -export CCACHE_MAXSIZE=500M +export CCACHE_MAXSIZE=800M export CCACHE_CPP2=true # periodically check to see if caches are old and purge them if so From ef4639c5c3441346787f53a0835e3f0188fbab4c Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Fri, 26 Sep 2025 17:34:30 -0700 Subject: [PATCH 12/15] Remove old TRAVIS variable --- ci-build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index b61341b270..93ff7d1546 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -53,8 +53,6 @@ while [[ -n "$1" ]]; do done -echo $TRAVIS_PULL_REQUEST - NPROCS=$(getconf _NPROCESSORS_ONLN) echo "Found $NPROCS processors" From 8bd38341d525203014f5421e976458136fdb14f9 Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Fri, 26 Sep 2025 17:47:13 -0700 Subject: [PATCH 13/15] Support out-of-tree builds --- ci-build.sh | 15 ++++++----- src/Makefile.am | 18 ++++++++----- src/herder/test/TxSetTests.cpp | 5 ++-- src/history/test/SerializeTests.cpp | 6 ++--- .../test/LedgerCloseMetaStreamTests.cpp | 8 +++--- src/main/test/ApplicationUtilsTests.cpp | 14 +++++----- src/main/test/ConfigTests.cpp | 9 +++---- src/test/check-sorobans | 3 ++- src/test/selftest-nopg | 2 +- src/test/selftest-parallel | 2 +- src/test/selftest-pg | 2 +- src/test/test.cpp | 26 +++++++++++++++++++ src/test/test.h | 3 +++ 13 files changed, 73 insertions(+), 40 deletions(-) diff --git a/ci-build.sh b/ci-build.sh index 93ff7d1546..8f21dbdbce 100755 --- a/ci-build.sh +++ b/ci-build.sh @@ -58,6 +58,9 @@ NPROCS=$(getconf _NPROCESSORS_ONLN) echo "Found $NPROCS processors" date +mkdir -p "build-${CC}-${PROTOCOL}" +cd "build-${CC}-${PROTOCOL}" + # Try to ensure we're using the real g++ and clang++ versions we want mkdir -p bin @@ -101,7 +104,7 @@ export ASAN_OPTIONS="quarantine_size_mb=100:malloc_context_size=4:detect_leaks=0 echo "config_flags = $config_flags" #### ccache config -export CCACHE_DIR=$HOME/.ccache +export CCACHE_DIR=$(pwd)/.ccache export CCACHE_COMPRESS=true export CCACHE_COMPRESSLEVEL=9 # cache size should be large enough for a full build @@ -117,11 +120,10 @@ if [ -d "$CCACHE_DIR" ] ; then fi ccache -p - ccache -s date -time ./autogen.sh -time ./configure $config_flags +time (cd .. && ./autogen.sh) +time ../configure $config_flags if [ -z "${SKIP_FORMAT_CHECK}" ]; then make format d=`git diff | wc -l` @@ -145,9 +147,8 @@ date time make -j$(($NPROCS - 1)) ccache -s -### incrementally purge old content from cargo source cache and target directory -cargo cache trim --limit 100M -cargo sweep --maxsize 500MB +### incrementally purge old content from target directory +(cd .. && CARGO_TARGET_DIR="build-${CC}-${PROTOCOL}/target" cargo sweep --maxsize 800MB) if [ $WITH_TESTS -eq 0 ] ; then echo "Build done, skipping tests" diff --git a/src/Makefile.am b/src/Makefile.am index dd7944cdce..f9ab9cd864 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -65,11 +65,11 @@ endif # !BUILD_TESTS if ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION CARGO_FEATURE_NEXT = --features next main/XDRFilesSha256.cpp: $(SRC_X_FILES) Makefile $(top_srcdir)/hash-xdrs.sh - $(top_srcdir)/hash-xdrs.sh protocol-next >$@ + $(top_srcdir)/hash-xdrs.sh $(top_srcdir)/src/protocol-next >$@ else CARGO_FEATURE_NEXT = main/XDRFilesSha256.cpp: $(SRC_X_FILES) Makefile $(top_srcdir)/hash-xdrs.sh - $(top_srcdir)/hash-xdrs.sh protocol-curr >$@ + $(top_srcdir)/hash-xdrs.sh $(top_srcdir)/src/protocol-curr >$@ endif stellar_core_LDADD = $(soci_LIBS) $(libmedida_LIBS) \ @@ -89,6 +89,7 @@ BUILT_SOURCES = $(SRC_X_FILES:.x=.h) main/StellarCoreVersion.cpp main/XDRFilesSh $(SRC_X_FILES:.x=.h): $(XDRC) SUFFIXES = .x .h .rs .x.h: + mkdir -p $(@D) $(XDRC) -hh -pedantic -o $@ $< BISON=bison @@ -124,9 +125,11 @@ RUST_TOOLCHAIN_CHANNEL=$(shell sed -n 's/channel *= *"\([^"]*\)"/\1/p' $(RUST_TO endif CARGO=cargo +$(RUST_TOOLCHAIN_CHANNEL) -# we pass RUST_TOOLCHAIN_CHANNEL by environment variable +# we pass some configuration by environment variable # to tests since they can't take command-line arguments. export RUST_TOOLCHAIN_CHANNEL +export top_srcdir +export top_builddir RUST_BUILD_DIR=$(top_builddir)/src/rust RUST_BIN_DIR=$(RUST_BUILD_DIR)/bin @@ -233,11 +236,12 @@ rust/RustBridge.cpp: rust/src/bridge.rs $(SRC_RUST_FILES) Makefile $(RUST_CXXBRI $(RUST_DEP_TREE_STAMP): $(wildcard rust/soroban/*/Cargo.*) Makefile $(RUST_TOOLCHAIN_FILE) rm -f $@ + mkdir -p $(RUST_BUILD_DIR)/src/dep-trees for proto in $(ALL_SOROBAN_PROTOCOLS); \ do \ - $(CARGO) tree --manifest-path rust/soroban/$${proto}/Cargo.toml --locked --package soroban-env-host --edges no-dev --target all \ + $(CARGO) tree --manifest-path $(top_srcdir)/src/rust/soroban/$${proto}/Cargo.toml --locked --package soroban-env-host --edges no-dev --target all \ | sed -e "s@$(abspath $(top_srcdir))/@@g" > $(RUST_BUILD_DIR)/src/dep-trees/$${proto}-actual.txt ; \ - if ! diff -u rust/src/dep-trees/$${proto}-expect.txt $(RUST_BUILD_DIR)/src/dep-trees/$${proto}-actual.txt; \ + if ! diff -u $(top_srcdir)/src/rust/src/dep-trees/$${proto}-expect.txt $(RUST_BUILD_DIR)/src/dep-trees/$${proto}-actual.txt; \ then \ echo "dep trees differ, please update $${proto}-expect.txt or roll back submodule"; \ exit 1; \ @@ -346,9 +350,11 @@ $(SOROBAN_LIBS_STAMP): $(wildcard rust/soroban/p*/Cargo.lock) Makefile $(RUST_DE FEATURE_FLAGS="$(CARGO_FEATURE_TRACY)" \ ;; \ esac ; \ - cd $(abspath $(RUST_BUILD_DIR))/soroban/$$proto && \ + mkdir -p $(SOROBAN_BUILD_DIR)/$$proto/target && \ + cd $(abspath $(top_srcdir))/src/rust/soroban/$$proto && \ CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" CXXSTDLIB="$(CXXSTDLIB)" LDFLAGS="$(LDFLAGS)" \ RUSTFLAGS="-Cmetadata=$$proto" \ + CARGO_TARGET_DIR=$(SOROBAN_BUILD_DIR)/$$proto/target \ CARGO_NET_GIT_FETCH_WITH_CLI=true \ $(CARGO) build \ --package soroban-env-host \ diff --git a/src/herder/test/TxSetTests.cpp b/src/herder/test/TxSetTests.cpp index eefc7b242c..8f3fd9efe3 100644 --- a/src/herder/test/TxSetTests.cpp +++ b/src/herder/test/TxSetTests.cpp @@ -2260,11 +2260,10 @@ TEST_CASE("txset nomination", "[txset]") { runIteration(); } - namespace fs = std::filesystem; - fs::path resultsPath = "testdata/txset/" + testName; - fs::create_directories(resultsPath.parent_path()); + auto resultsPath = getSrcTestDataPath("txset/" + testName); if (getenv("GENERATE_TEST_TXSETS")) { + std::filesystem::create_directories(resultsPath.parent_path()); std::ofstream resultsStream(resultsPath); for (auto const& result : testResults) { diff --git a/src/history/test/SerializeTests.cpp b/src/history/test/SerializeTests.cpp index acdae3c0e6..2ad2e694fe 100644 --- a/src/history/test/SerializeTests.cpp +++ b/src/history/test/SerializeTests.cpp @@ -4,6 +4,7 @@ #include "history/HistoryArchive.h" #include "test/Catch2.h" +#include "test/test.h" #include #include @@ -19,9 +20,8 @@ TEST_CASE("Serialization round trip", "[history]") "stellar-history.testnet.6714239.networkPassphrase.v2.json"}; for (size_t i = 0; i < testFiles.size(); i++) { - std::string fnPath = "testdata/"; - std::string testFilePath = fnPath + testFiles[i]; - SECTION("Serialize " + testFilePath) + auto testFilePath = getBuildTestDataPath(testFiles[i]); + SECTION("Serialize " + testFilePath.string()) { std::ifstream in(testFilePath); REQUIRE(in); diff --git a/src/ledger/test/LedgerCloseMetaStreamTests.cpp b/src/ledger/test/LedgerCloseMetaStreamTests.cpp index 15cce51582..759a839706 100644 --- a/src/ledger/test/LedgerCloseMetaStreamTests.cpp +++ b/src/ledger/test/LedgerCloseMetaStreamTests.cpp @@ -515,13 +515,11 @@ TEST_CASE_VERSIONS("meta stream contains reasonable meta", "[ledgerclosemeta]") if (ledgerSeq == targetSeq) { - std::string refJsonPath; - refJsonPath = fmt::format( - FMT_STRING( - "testdata/ledger-close-meta{}-v{}-protocol-{}{}.json"), + auto refJsonPath = getSrcTestDataPath(fmt::format( + FMT_STRING("ledger-close-meta{}-v{}-protocol-{}{}.json"), enableClassicEvents ? "-enable-classic-events" : "", lcm.v(), cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION, - isSoroban ? "-soroban" : ""); + isSoroban ? "-soroban" : "")); normalizeMeta(lcm); std::string have = xdrToCerealString(lcm, "LedgerCloseMeta"); if (getenv("GENERATE_TEST_LEDGER_CLOSE_META")) diff --git a/src/main/test/ApplicationUtilsTests.cpp b/src/main/test/ApplicationUtilsTests.cpp index ec40ad8a1e..19a686a8b0 100644 --- a/src/main/test/ApplicationUtilsTests.cpp +++ b/src/main/test/ApplicationUtilsTests.cpp @@ -173,18 +173,18 @@ TEST_CASE("application major version numbers", "[applicationutils]") TEST_CASE("standalone quorum intersection check", "[applicationutils]") { Config cfg = getTestConfig(); - const std::string JSON_ROOT = "testdata/check-quorum-intersection-json/"; + auto JSON_ROOT = getSrcTestDataPath("check-quorum-intersection-json"); SECTION("enjoys quorum intersection") { REQUIRE(checkQuorumIntersectionFromJson( - JSON_ROOT + "enjoys-intersection.json", cfg)); + JSON_ROOT / "enjoys-intersection.json", cfg)); } SECTION("does not enjoy quorum intersection") { REQUIRE(!checkQuorumIntersectionFromJson( - JSON_ROOT + "no-intersection.json", cfg)); + JSON_ROOT / "no-intersection.json", cfg)); } SECTION("malformed JSON") @@ -193,17 +193,17 @@ TEST_CASE("standalone quorum intersection check", "[applicationutils]") // Malformed key REQUIRE_THROWS_AS( - checkQuorumIntersectionFromJson(JSON_ROOT + "bad-key.json", cfg), + checkQuorumIntersectionFromJson(JSON_ROOT / "bad-key.json", cfg), KeyUtils::InvalidStrKey); // Wrong datatype REQUIRE_THROWS_AS(checkQuorumIntersectionFromJson( - JSON_ROOT + "bad-threshold-type.json", cfg), + JSON_ROOT / "bad-threshold-type.json", cfg), std::runtime_error); // No such file REQUIRE_THROWS_AS( - checkQuorumIntersectionFromJson(JSON_ROOT + "no-file.json", cfg), + checkQuorumIntersectionFromJson(JSON_ROOT / "no-file.json", cfg), std::runtime_error); } -} \ No newline at end of file +} diff --git a/src/main/test/ConfigTests.cpp b/src/main/test/ConfigTests.cpp index 05bdbd483a..808241f342 100644 --- a/src/main/test/ConfigTests.cpp +++ b/src/main/test/ConfigTests.cpp @@ -147,7 +147,7 @@ TEST_CASE("resolve node id", "[config]") TEST_CASE("load validators config", "[config]") { Config c; - c.load("testdata/stellar-core_example_validators.cfg"); + c.load(getBuildTestDataPath("stellar-core_example_validators.cfg")); auto actualS = c.toString(c.QUORUM_SET); std::string expected = R"({ "t" : 4, @@ -456,9 +456,8 @@ TEST_CASE("load example configs", "[config]") "stellar-core_testnet_validator.cfg"}; for (auto const& fn : testFiles) { - std::string fnPath = "testdata/"; - fnPath += fn; - SECTION("load config " + fnPath) + auto fnPath = getBuildTestDataPath(fn); + SECTION("load config " + fnPath.string()) { c.load(fnPath); } @@ -628,4 +627,4 @@ PUBLIC_KEY="GBVZFVEARURUJTN5ABZPKW36FHKVJK2GHXEVY2SZCCNU5I3CQMTZ3OES" )"; std::stringstream ss(configStr); c.load(ss); -} \ No newline at end of file +} diff --git a/src/test/check-sorobans b/src/test/check-sorobans index 3bde728460..74e5eaf920 100755 --- a/src/test/check-sorobans +++ b/src/test/check-sorobans @@ -28,9 +28,10 @@ SKIP_TESTS="host::declared_size::test::test_expected_size" set -e set -x -cd rust/soroban +cd "${top_srcdir}/src/rust/soroban" for i in p??; do cd $i + CARGO_TARGET_DIR="${top_builddir}/src/rust/soroban/${i}/target" \ RUSTFLAGS="-Cmetadata=${i}" cargo +${RUST_TOOLCHAIN_CHANNEL} \ test \ --locked \ diff --git a/src/test/selftest-nopg b/src/test/selftest-nopg index 18021b9ed2..ef297b0cdd 100755 --- a/src/test/selftest-nopg +++ b/src/test/selftest-nopg @@ -4,4 +4,4 @@ # under the Apache License, Version 2.0. See the COPYING file at the root # of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 -exec ./test/selftest-parallel ./test/run-selftest-nopg +exec "${top_srcdir}/src/test/selftest-parallel" "${top_srcdir}/src/test/run-selftest-nopg" diff --git a/src/test/selftest-parallel b/src/test/selftest-parallel index 8f37ec532d..6195171d86 100755 --- a/src/test/selftest-parallel +++ b/src/test/selftest-parallel @@ -89,5 +89,5 @@ for i in $RUN_PARTITIONS; do cat $TEST_PARTITIONS_DIR/test-partition-$i.txt >> $TESTS done -cat $TESTS | parallel -j $nparts -n $BATCHSIZE --line-buffer --tag-string '{%}' --progress --halt-on-error 2 runpart '{%}' "{}" +cat $TESTS | parallel -j $nparts -n $BATCHSIZE --line-buffer --tag-string '{%}' --halt-on-error 2 runpart '{%}' "{}" diff --git a/src/test/selftest-pg b/src/test/selftest-pg index b856805799..c99a178cb4 100755 --- a/src/test/selftest-pg +++ b/src/test/selftest-pg @@ -4,4 +4,4 @@ # under the Apache License, Version 2.0. See the COPYING file at the root # of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 -exec ./test/selftest-parallel ./test/run-selftest-pg +exec "${top_srcdir}/src/test/selftest-parallel" "${top_srcdir}/src/test/run-selftest-pg" diff --git a/src/test/test.cpp b/src/test/test.cpp index f0a49ce396..56e1b90d2b 100644 --- a/src/test/test.cpp +++ b/src/test/test.cpp @@ -337,6 +337,32 @@ getTestConfig(int instanceNumber, Config::TestDbMode mode) return *cfgs[instanceNumber]; } +std::filesystem::path +getSrcTestDataPath(std::filesystem::path rel) +{ + namespace fs = std::filesystem; + fs::path testdata("testdata"); + char* srcdir = getenv("top_srcdir"); + if (srcdir) + { + testdata = fs::path(srcdir) / "src" / testdata; + } + return testdata / rel; +} + +std::filesystem::path +getBuildTestDataPath(std::filesystem::path rel) +{ + namespace fs = std::filesystem; + fs::path testdata("testdata"); + char* builddir = getenv("top_builddir"); + if (builddir) + { + testdata = fs::path(builddir) / "src" / testdata; + } + return testdata / rel; +} + int runTest(CommandLineArgs const& args) { diff --git a/src/test/test.h b/src/test/test.h index a026102a77..27c9fac306 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -19,6 +19,9 @@ struct TransactionMeta; Config const& getTestConfig(int instanceNumber = 0, Config::TestDbMode mode = Config::TESTDB_DEFAULT); +std::filesystem::path getSrcTestDataPath(std::filesystem::path rel); +std::filesystem::path getBuildTestDataPath(std::filesystem::path rel); + void cleanupTmpDirs(); // Records or checks a TxMetadata value against a persistent record From 4b392f471305e50b8e649d189c5f3d8cce6fc1ff Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Fri, 26 Sep 2025 17:47:38 -0700 Subject: [PATCH 14/15] Switch build.yml to namespace runners --- .github/workflows/build.yml | 133 ++++++++++++++---------------------- 1 file changed, 52 insertions(+), 81 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8c54d93637..8bb57fae7b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,21 +15,30 @@ jobs: complete: if: always() needs: [fmt, cargo-deny, rust-check-git-rev-deps, build] - runs-on: ubuntu-22.04 + runs-on: + - namespace-profile-jammy-1-stellar-core + - nscloud-cache-exp-do-not-commit steps: - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') run: exit 1 fmt: - runs-on: ubuntu-22.04 + runs-on: + - namespace-profile-jammy-1-stellar-core + - nscloud-cache-exp-do-not-commit steps: - - uses: actions/checkout@v4 - - run: rustup component add rustfmt - - run: rustup update - - run: cargo fmt --all --check + - uses: namespacelabs/nscloud-checkout-action@v7 + with: + fetch-depth: 1 + submodules: recursive + - run: rustup component add rustfmt + - run: rustup update + - run: cargo fmt --all --check cargo-deny: - runs-on: ubuntu-22.04 + runs-on: + - namespace-profile-jammy-1-stellar-core + - nscloud-cache-exp-do-not-commit strategy: matrix: checks: @@ -38,33 +47,36 @@ jobs: # Prevent sudden announcement of a new advisory from failing ci: continue-on-error: ${{ matrix.checks == 'advisories' }} steps: - - uses: actions/checkout@v4 - - uses: EmbarkStudios/cargo-deny-action@8d73959fce1cdc8989f23fdf03bec6ae6a6576ef - with: - command: check ${{ matrix.checks }} - # leave arguments empty so we don't test --all-features - # which will see conflicting env versions - arguments: + - uses: namespacelabs/nscloud-checkout-action@v7 + with: + fetch-depth: 1 + submodules: recursive + - uses: EmbarkStudios/cargo-deny-action@8d73959fce1cdc8989f23fdf03bec6ae6a6576ef + with: + command: check ${{ matrix.checks }} + # leave arguments empty so we don't test --all-features + # which will see conflicting env versions + arguments: rust-check-git-rev-deps: - runs-on: ubuntu-22.04 + runs-on: + - namespace-profile-jammy-1-stellar-core + - nscloud-cache-exp-do-not-commit steps: - - uses: actions/checkout@v4 - - uses: stellar/actions/rust-check-git-rev-deps@main + - uses: namespacelabs/nscloud-checkout-action@v7 + with: + fetch-depth: 1 + submodules: recursive + - uses: stellar/actions/rust-check-git-rev-deps@main build: - runs-on: ubuntu-jammy-16-cores-amd64 - env: - CACHED_PATHS: | - ~/.ccache - ~/.cargo - target + runs-on: + - namespace-profile-jammy-32-stellar-core;overrides.cache-tag=config-${{ matrix.toolchain }}-${{ matrix.protocol }} strategy: fail-fast: false matrix: toolchain: [ "gcc", "clang"] protocol: ["current", "next"] - scenario: ["", "--check-test-tx-meta"] steps: - name: Fix kernel mmap rnd bits # Asan in llvm provided in ubuntu 22.04 is incompatible with @@ -72,66 +84,32 @@ jobs: # using leading to random crashes: https://reviews.llvm.org/D148280 run: sudo sysctl vm.mmap_rnd_bits=28 - # We start with as cheap as possible a cache probe to see if we have an exact hit on this - # git commit ID (for a given OS/toolchain/protocol). If so we can skip all subsequent - # steps: we already tested this exact SHA. - # - # Unfortunately due to the way github actions control flow works, we have to repeat the - # step 'if' condition in each step, and cannot actually "exit early" and skip the job. - # There are a lot of duplicate bug reports filed on this aspect of github actions, don't - # bother filing another. - - name: Probe Cache - id: cache - uses: actions/cache/restore@v4 + - name: Checkout with Namespace Git mirrors cache + uses: namespacelabs/nscloud-checkout-action@v7 with: - path: ${{ env.CACHED_PATHS }} - key: ${{ runner.os }}-${{ matrix.toolchain }}-${{ matrix.protocol }}-${{ github.sha }} - lookup-only: true + fetch-depth: 1 + submodules: recursive - # When we have a cache miss on the exact commit SHA, we restore from the prefix without it. - # This will restore the most-recently-written cache (github does this date ordering itself). - - name: Restore Cache - if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/restore@v4 + - name: Configure Namespace cache volume + uses: namespacelabs/nscloud-cache-action@v1 with: - path: ${{ env.CACHED_PATHS }} - key: ${{ steps.cache.outputs.cache-primary-key }} - restore-keys: | - ${{ runner.os }}-${{ matrix.toolchain }}-${{ matrix.protocol }} + path: | + ~/.cargo + build-${{matrix.toolchain}}-${{matrix.protocol}} + + - name: install rustup + run: ./install-rust.sh - - uses: actions/checkout@v4 - if: steps.cache.outputs.cache-hit != 'true' - with: - fetch-depth: 200 - submodules: true - - name: install core packages - if: steps.cache.outputs.cache-hit != 'true' - run: | - sudo apt-get update - sudo apt-get -y install --no-install-recommends apt-utils dialog git iproute2 procps lsb-release - - name: install tool chain - if: steps.cache.outputs.cache-hit != 'true' - run: | - sudo apt-get -y install libstdc++-10-dev clang-format-12 ccache lldb - if test "${{ matrix.toolchain }}" = "gcc" ; then - sudo apt-get -y install cpp-10 gcc-10 g++-10 - else - sudo apt-get -y install clang-12 llvm-12 - fi - name: install rustup components - if: steps.cache.outputs.cache-hit != 'true' run: rustup component add rustfmt + - name: install cargo-cache - if: steps.cache.outputs.cache-hit != 'true' run: cargo install --locked cargo-cache --version 0.8.3 + - name: install cargo-sweep - if: steps.cache.outputs.cache-hit != 'true' run: cargo install --locked cargo-sweep --version 0.7.0 - - name: install dependencies - if: steps.cache.outputs.cache-hit != 'true' - run: sudo apt-get -y install postgresql git build-essential pkg-config autoconf automake libtool bison flex libpq-dev parallel libunwind-dev sed perl + - name: Build - if: steps.cache.outputs.cache-hit != 'true' run: | if test "${{ matrix.toolchain }}" = "gcc" ; then export CC='gcc' @@ -141,11 +119,4 @@ jobs: export CXX='clang++' fi echo Build with $CC and $CXX - ./ci-build.sh --use-temp-db --protocol ${{ matrix.protocol }} ${{ matrix.scenario }} - - # We only _save_ to the cache when we had a cache miss, are running on master, and are the non-txmeta scenario. - - uses: actions/cache/save@v4 - if: ${{ steps.cache.outputs.cache-hit != 'true' && github.ref_name == 'master' && matrix.scenario == ''}} - with: - path: ${{ env.CACHED_PATHS }} - key: ${{ steps.cache.outputs.cache-primary-key }} \ No newline at end of file + ./ci-build.sh --use-temp-db --protocol ${{ matrix.protocol }} From a2547be7f33b3345c398338f7ef0d1973191a995 Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Wed, 29 Oct 2025 14:37:54 -0700 Subject: [PATCH 15/15] Add build-mac job --- .github/workflows/build.yml | 52 +++++++++++++++++++++++++++++++++++-- src/Makefile.am | 2 ++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8bb57fae7b..dec0742148 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,7 @@ jobs: complete: if: always() - needs: [fmt, cargo-deny, rust-check-git-rev-deps, build] + needs: [fmt, cargo-deny, rust-check-git-rev-deps, build-linux, build-mac] runs-on: - namespace-profile-jammy-1-stellar-core - nscloud-cache-exp-do-not-commit @@ -69,7 +69,7 @@ jobs: submodules: recursive - uses: stellar/actions/rust-check-git-rev-deps@main - build: + build-linux: runs-on: - namespace-profile-jammy-32-stellar-core;overrides.cache-tag=config-${{ matrix.toolchain }}-${{ matrix.protocol }} strategy: @@ -120,3 +120,51 @@ jobs: fi echo Build with $CC and $CXX ./ci-build.sh --use-temp-db --protocol ${{ matrix.protocol }} + + build-mac: + runs-on: + - namespace-profile-macos-sequoia;overrides.cache-tag=config-macos-sequoia + steps: + + - name: Checkout with Namespace Git mirrors cache + uses: namespacelabs/nscloud-checkout-action@v7 + with: + fetch-depth: 1 + submodules: recursive + + - name: Configure Namespace cache volume + uses: namespacelabs/nscloud-cache-action@v1 + with: + path: | + ~/.cargo + build-clang-current + + - name: install prerequisites and uninstall brew rust, add rustup + run: | + brew install libsodium libtool autoconf automake pkg-config libpq openssl parallel ccache bison gnu-sed perl coreutils + brew uninstall rust rustup + brew install rustup + + - name: install rustup components + run: | + rustup-init -y + rustup component add rustfmt rustc cargo clippy rust-src rust-std + + - name: install cargo-cache + run: | + cargo install --locked cargo-cache --version 0.8.3 + + - name: install cargo-sweep + run: | + cargo install --locked cargo-sweep --version 0.7.0 + + - name: Build + run: | + export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$(brew --prefix)/opt/libpq/lib/pkgconfig" + export PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$(brew --prefix)/opt/openssl@3/lib/pkgconfig" + export PATH="$(brew --prefix bison)/bin:$PATH" + export CC='clang' + export CXX='clang++' + export CLANG_VERSION=none + export SKIP_FORMAT_CHECK=1 + ./ci-build.sh --disable-postgres --protocol current diff --git a/src/Makefile.am b/src/Makefile.am index f9ab9cd864..e85e4db885 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -276,6 +276,7 @@ CARGOFLAGS_BUILDSTD := $(if $(findstring sanitizer,$(RUSTFLAGS_SANI)),-Zbuild-st RUSTFLAGS_CFGS := $(if $(findstring sanitizer,$(RUSTFLAGS_SANI)),--cfg curve25519_dalek_backend=\"serial\",) $(LIBRUST_STELLAR_CORE): $(RUST_HOST_DEPFILES) $(SRC_RUST_FILES) Makefile $(RUST_TOOLCHAIN_FILE) + cd $(abspath $(top_srcdir))/src/rust && \ CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" CXXSTDLIB="$(CXXSTDLIB)" LDFLAGS="$(LDFLAGS)" \ RUSTFLAGS="$(RUSTFLAGS_SANI) $(RUSTFLAGS_CFGS)" \ CARGO_NET_GIT_FETCH_WITH_CLI=true \ @@ -382,6 +383,7 @@ $(SOROBAN_LIBS_STAMP): $(wildcard rust/soroban/p*/Cargo.lock) Makefile $(RUST_DE LIBRUST_STELLAR_CORE=$(RUST_TARGET_DIR)/$(RUST_PROFILE_DIR)/librust_stellar_core.a $(LIBRUST_STELLAR_CORE): $(RUST_HOST_DEPFILES) $(SRC_RUST_FILES) Makefile $(SOROBAN_LIBS_STAMP) $(RUST_TOOLCHAIN_FILE) + cd $(abspath $(top_srcdir))/src/rust && \ CC="$(CC)" CXX="$(CXX)" LD="$(LD)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" CXXSTDLIB="$(CXXSTDLIB)" LDFLAGS="$(LDFLAGS)" \ CARGO_NET_GIT_FETCH_WITH_CLI=true \ $(CARGO) rustc \