diff --git a/.clang-format b/.clang-format index 400349f157..96477b5f77 100644 --- a/.clang-format +++ b/.clang-format @@ -4,72 +4,71 @@ BreakBeforeBraces: Stroustrup IndentWidth: 4 TabWidth: 4 AlignAfterOpenBracket: AlwaysBreak -AlignConsecutiveMacros: 'true' -AlignConsecutiveAssignments: 'false' -AlignConsecutiveDeclarations: 'false' +AlignConsecutiveMacros: "true" +AlignConsecutiveAssignments: "false" +AlignConsecutiveDeclarations: "false" AlignEscapedNewlines: Right -AlignOperands: 'true' -AlignTrailingComments: 'true' -AllowAllArgumentsOnNextLine: 'false' -AllowAllConstructorInitializersOnNextLine: 'false' -AllowAllParametersOfDeclarationOnNextLine: 'false' -AllowShortBlocksOnASingleLine: 'true' -AllowShortCaseLabelsOnASingleLine: 'false' +AlignOperands: "true" +AlignTrailingComments: "true" +AllowAllArgumentsOnNextLine: "false" +AllowAllConstructorInitializersOnNextLine: "false" +AllowAllParametersOfDeclarationOnNextLine: "false" +AllowShortBlocksOnASingleLine: "true" +AllowShortCaseLabelsOnASingleLine: "false" AllowShortFunctionsOnASingleLine: None AllowShortIfStatementsOnASingleLine: Never AlwaysBreakAfterReturnType: None -BinPackArguments: 'false' -BinPackParameters: 'false' +BinPackArguments: "true" +BinPackParameters: OnePerLine BreakBeforeBinaryOperators: NonAssignment -BreakBeforeTernaryOperators: 'true' +BreakBeforeTernaryOperators: "true" BreakConstructorInitializers: BeforeComma BreakInheritanceList: BeforeComma -CompactNamespaces: 'false' -ConstructorInitializerAllOnOneLineOrOnePerLine: 'true' -ConstructorInitializerIndentWidth: '4' -ContinuationIndentWidth: '4' -Cpp11BracedListStyle: 'false' -FixNamespaceComments: 'true' +CompactNamespaces: "false" +ConstructorInitializerAllOnOneLineOrOnePerLine: "true" +ConstructorInitializerIndentWidth: "4" +ContinuationIndentWidth: "4" +Cpp11BracedListStyle: "false" +FixNamespaceComments: "true" IncludeBlocks: Regroup -IndentCaseLabels: 'true' +IndentCaseLabels: "true" IndentPPDirectives: None -IndentWrappedFunctionNames: 'false' -KeepEmptyLinesAtTheStartOfBlocks: 'false' -MaxEmptyLinesToKeep: '1' +IndentWrappedFunctionNames: "false" +KeepEmptyLinesAtTheStartOfBlocks: "false" +MaxEmptyLinesToKeep: "1" NamespaceIndentation: None PointerAlignment: Left -ReflowComments: 'true' -SortIncludes: 'true' -SortUsingDeclarations: 'true' -SpaceAfterCStyleCast: 'false' -SpaceAfterLogicalNot: 'true' -SpaceAfterTemplateKeyword: 'true' -SpaceBeforeAssignmentOperators: 'true' -SpaceBeforeCpp11BracedList: 'true' -SpaceBeforeCtorInitializerColon: 'true' -SpaceBeforeInheritanceColon: 'true' +ReflowComments: "true" +SortIncludes: "true" +SortUsingDeclarations: "true" +SpaceAfterCStyleCast: "false" +SpaceAfterLogicalNot: "true" +SpaceAfterTemplateKeyword: "true" +SpaceBeforeAssignmentOperators: "true" +SpaceBeforeCpp11BracedList: "true" +SpaceBeforeCtorInitializerColon: "true" +SpaceBeforeInheritanceColon: "true" SpaceBeforeParens: ControlStatements -SpaceBeforeRangeBasedForLoopColon: 'true' -SpaceInEmptyParentheses: 'false' -SpacesBeforeTrailingComments: '3' -SpacesInAngles: 'false' -SpacesInCStyleCastParentheses: 'false' -SpacesInContainerLiterals: 'true' -SpacesInParentheses: 'false' -SpacesInSquareBrackets: 'false' -UseTab: 'Always' +SpaceBeforeRangeBasedForLoopColon: "true" +SpaceInEmptyParentheses: "false" +SpacesBeforeTrailingComments: "3" +SpacesInAngles: "false" +SpacesInCStyleCastParentheses: "false" +SpacesInContainerLiterals: "true" +SpacesInParentheses: "false" +SpacesInSquareBrackets: "false" +UseTab: "Always" --- Language: Cpp Standard: Cpp03 -ColumnLimit: '240' +ColumnLimit: "120" --- Language: ObjC -ColumnLimit: '240' +ColumnLimit: "120" --- Language: Java -ColumnLimit: '240' +ColumnLimit: "120" --- Language: CSharp -ColumnLimit: '240' -... +ColumnLimit: "120" diff --git a/.github/workflows/central-controller.yaml b/.github/workflows/central-controller.yaml new file mode 100644 index 0000000000..209cc0522f --- /dev/null +++ b/.github/workflows/central-controller.yaml @@ -0,0 +1,82 @@ +# on: +# workflow_dispatch: +on: + push: + workflow_dispatch: + +jobs: + central_controller: + name: Central Controller Build + strategy: + matrix: + runner: [gha-runner-x64, gha-runner-arm64] + runs-on: ${{ matrix.runner }} + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: GCP Auth + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.DOCKER_REGISTRY_WRITER}} + + - name: Set up GCloud CLI + uses: google-github-actions/setup-gcloud@v2 + + - name: Docker Auth + run: gcloud auth configure-docker us-central1-docker.pkg.dev --quiet + + - name: Get branch name and sanitize + id: branch + run: | + BRANCH_NAME="${GITHUB_REF##*/}" + SANITIZED_BRANCH="${BRANCH_NAME//\//-}" + echo "branch_name=$SANITIZED_BRANCH" >> $GITHUB_OUTPUT + + - name: Get short git commit SHA + id: sha + run: | + calculatedSha=$(git rev-parse --short ${{ github.sha }}) + echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV + + - name: Build & Push Docker Image + run: | + docker build -t us-central1-docker.pkg.dev/zerotier-421eb9/docker-images/ztcentral-controller:${{ env.COMMIT_SHORT_SHA }}-${{ steps.branch.outputs.branch_name }}-${{ runner.arch }} -f ext/central-controller-docker/Dockerfile.new --provenance false . --push + + multi-arch-docker: + runs-on: gha-runner-x64 + needs: central_controller + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: GCP Auth + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.DOCKER_REGISTRY_WRITER}} + + - name: Set up GCloud CLI + uses: google-github-actions/setup-gcloud@v2 + + - name: Docker Auth + run: gcloud auth configure-docker us-central1-docker.pkg.dev --quiet + + - name: Get branch name and sanitize + id: branch + run: | + BRANCH_NAME="${GITHUB_REF##*/}" + SANITIZED_BRANCH="${BRANCH_NAME//\//-}" + echo "branch_name=$SANITIZED_BRANCH" >> $GITHUB_OUTPUT + + - name: Get short git commit SHA + id: sha + run: | + calculatedSha=$(git rev-parse --short ${{ github.sha }}) + echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV + + - name: Create and push multi-arch manifest + run: | + docker manifest create us-central1-docker.pkg.dev/zerotier-421eb9/docker-images/ztcentral-controller:${{ env.COMMIT_SHORT_SHA }}-${{ steps.branch.outputs.branch_name }} \ + --amend us-central1-docker.pkg.dev/zerotier-421eb9/docker-images/ztcentral-controller:${{ env.COMMIT_SHORT_SHA }}-${{ steps.branch.outputs.branch_name }}-X64 \ + --amend us-central1-docker.pkg.dev/zerotier-421eb9/docker-images/ztcentral-controller:${{ env.COMMIT_SHORT_SHA }}-${{ steps.branch.outputs.branch_name }}-ARM64 + docker manifest push us-central1-docker.pkg.dev/zerotier-421eb9/docker-images/ztcentral-controller:${{ env.COMMIT_SHORT_SHA }}-${{ steps.branch.outputs.branch_name }} diff --git a/.gitignore b/.gitignore index f5befd99f3..d52abee96a 100755 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,4 @@ __pycache__ *.pyc *_source.tar.bz2 snap/.snapcraft +build/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index fff7808e10..c13af6dcd7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,12 +1,167 @@ # CMake build script for libzerotiercore.a -cmake_minimum_required (VERSION 2.8) -project (zerotiercore) +cmake_minimum_required (VERSION 3.13) +project (zerotier-one LANGUAGES CXX C) +option(ZT1_CENTRAL_CONTROLLER "Build with ZeroTier Central Controller support" OFF) +option(ADDRESS_SANITIZER "Build with Address Sanitizer enabled (only for x86_64/arm64)" OFF +) set (PROJ_DIR ${PROJECT_SOURCE_DIR}) -set (ZT_DEFS -std=c++11) -file(GLOB core_src_glob ${PROJ_DIR}/node/*.cpp) -add_library(zerotiercore STATIC ${core_src_glob}) +execute_process(COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE CPU_ARCHITECTURE) +set(CPU_ARCHITECTURE ${CPU_ARCHITECTURE} CACHE STRING "Target CPU architecture (detected automatically)") +message(STATUS "Detected CPU architecture: ${CPU_ARCHITECTURE}") -target_compile_options(zerotiercore PRIVATE ${ZT_DEFS}) +if(CPU_ARCHITECTURE STREQUAL "x86_64") + add_definitions( + -DZT_ARCHITECTURE=2 + -DZT_USE_X64_ASM_SALSA=1 + -DZT_USE_X64_ASM_ED25519=1 + -DZT_SSO_SUPPORTED=1 + -DZT_USE_X64_ASM_SALSA2012=1 + -DZT_USE_FAST_X64_ED25519=1 + ) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -z noexecstack") + + if(ADDRESS_SANITIZER) + add_compile_options(-fsanitize=address) + add_link_options(-fsanitize=address) + endif() +elseif(CPU_ARCHITECTURE STREQUAL "aarch64" OR CPU_ARCHITECTURE STREQUAL "arm64") + add_definitions(-DZT_ARCHITECTURE=4 -DZT_ARCH_ARM_HAS_NEON=1 -DZT_SSO_SUPPORTED=1 -DZT_AES_NEON=1) + add_compile_options(-march=armv8-a+crypto -mtune=generic -mstrict-align) + + if(ADDRESS_SANITIZER) + add_compile_options(-fsanitize=address) + add_link_options(-fsanitize=address) + endif() +endif() + +include(FetchContent) +include(ExternalProject) + +add_definitions(-DCMAKE_BUILD) + +find_package(OpenSSL REQUIRED) +find_package(nlohmann_json REQUIRED) +find_package(inja REQUIRED) + +set(CMAKE_THREDAD_PREFER_PTHREAD TRUE CACHE INTERNAL "Use pthreads" FORCE) +set(THREADS_PREFER_PTHREAD_FLAG TRUE CACHE INTERNAL "Use pthreads" FORCE) +find_package(Threads REQUIRED) +if(CMAKE_USE_PTHREADS_INIT) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pthread") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") +endif() + +if(ZT1_CENTRAL_CONTROLLER) + find_package(PostgreSQL REQUIRED) + find_package(opentelemetry-cpp REQUIRED COMPONENTS api sdk exporters_otlp_grpc exporters_otlp_http exporters_prometheus ) + find_package(google_cloud_cpp_bigtable REQUIRED) + find_package(google_cloud_cpp_pubsub REQUIRED) +else() + find_package(opentelemetry-cpp REQUIRED COMPONENTS api) +endif() + +if(ZT1_CENTRAL_CONTROLLER) + set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to conform to" FORCE) + set(CMAKE_CXX_STANDARD_REQUIRED True CACHE BOOL "C++ standard required" FORCE) + set(CMAKE_CXX_EXTENSIONS ON CACHE BOOL "Enable compiler-specific extensions" FORCE) + add_definitions(-DZT_NONFREE_CONTROLLER=1 -DZT_CONTROLLER_USE_LIBPQ=1 -DZT1_CENTRAL_CONTROLLER=1 -DZT_OPENTELEMETRY_ENABLED=1) + set(CONTROLLER_RUST_FEATURES ztcontroller) + set(RUST_BUILD_COMMAND cargo build --release -F ${CONTROLLER_RUST_FEATURES}) +else() + set(CMAKE_CXX_STANDARD 11 CACHE STRING "C++ standard to conform to") + set(CMAKE_CXX_STANDARD_REQUIRED True CACHE BOOL "C++ standard required") + set(CMAKE_CXX_EXTENSIONS ON CACHE BOOL "Enable compiler-specific extensions") + set(RUST_BUILD_COMMAND "cargo build --release") +endif() + +# build rustybits +ExternalProject_Add( + rustybits_build + DOWNLOAD_COMMAND "" + CONFIGURE_COMMAND "" + # PREFIX ${PROJ_DIR}/rustybits + BUILD_COMMAND cd ${PROJ_DIR}/rustybits && ${RUST_BUILD_COMMAND} + INSTALL_COMMAND "" + BUILD_BYPRODUCTS ${PROJ_DIR}/rustybits/target/release/librustybits.a ${PROJ_DIR}/rustybits/target/rustybits.h + ) +set(RUSTYBITS_LIB ${PROJ_DIR}/rustybits/target/release/librustybits.a) +set(RUSTYBITS_INCLUDE_DIR ${PROJ_DIR}/rustybits/target) +add_library(rustybits STATIC IMPORTED GLOBAL) +set_property(TARGET rustybits PROPERTY IMPORTED_LOCATION ${RUSTYBITS_LIB}) +add_dependencies(rustybits rustybits_build) + + + +# Get & build dependencies not in conda +include (cmake/cpp-httplib.cmake) +include (cmake/redis-plus-plus.cmake) +include (cmake/miniupnpc.cmake) + +add_subdirectory(ext) +add_subdirectory(node) +add_subdirectory(osdep) +add_subdirectory(service) +add_subdirectory(nonfree) + +set(LINKED_LIBRARIES + prometheus-cpp-lite + zerotier-service + zerotier-osdep + zerotier-core + zerotier-controller + Threads::Threads + nlohmann_json::nlohmann_json + opentelemetry-cpp::api + rustybits + OpenSSL::Crypto + OpenSSL::SSL + Threads::Threads +) + +if(ZT1_CENTRAL_CONTROLLER) + list(APPEND LINKED_LIBRARIES + opentelemetry-cpp::sdk + opentelemetry-cpp::trace + opentelemetry-cpp::proto_grpc + opentelemetry-cpp::otlp_grpc_client + opentelemetry-cpp::otlp_grpc_exporter + opentelemetry-cpp::otlp_grpc_log_record_exporter + opentelemetry-cpp::otlp_grpc_metrics_exporter + opentelemetry-cpp::otlp_http_exporter + opentelemetry-cpp::otlp_http_log_record_exporter + opentelemetry-cpp::otlp_http_metric_exporter + opentelemetry-cpp::prometheus_exporter + ) +endif() + +if (APPLE) + find_library(COREFOUNDATION_LIBRARY CoreFoundation) + find_library(SECURITY_LIBRARY Security) + find_library(SYSTEM_CONFIGURATION_LIBRARY SystemConfiguration) + find_library(CARBON Carbon) + find_library(CORESERVICES_LIBRARY CoreServices) + list(APPEND LINKED_LIBRARIES ${COREFOUNDATION_LIBRARY} ${SECURITY_LIBRARY} ${CARBON_LIBRARY} ${SYSTEM_CONFIGURATION_LIBRARY} ${CORESERVICES_LIBRARY}) +endif() + +add_executable(zerotier-one + one.cpp + ext/http-parser/http_parser.c) + +target_link_libraries(zerotier-one + ${LINKED_LIBRARIES} +) + +add_executable(zerotier-selftest + selftest.cpp +) + +target_link_libraries(zerotier-selftest + zerotier-core + zerotier-osdep + Threads::Threads + nlohmann_json::nlohmann_json + prometheus-cpp-lite + Threads::Threads) diff --git a/build_central_controller.sh b/build_central_controller.sh new file mode 100755 index 0000000000..5c6665a0cb --- /dev/null +++ b/build_central_controller.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -e + +cmake -DCMAKE_BUILD_TYPE=Release -DZT1_CENTRAL_CONTROLLER=1 -DCMAKE_INSTALL_PREFIX=$PWD -S . -B build/ -DCMAKE_INSTALL_PREFIX=$(shell pwd)/build-out +cmake --build build/ --target all -j4 --verbose + +ARCH=$(uname -m) +if [ "$ARCH" = "x86_64" ]; then + ARCH="amd64" +elif [ "$ARCH" = "aarch64" ]; then + ARCH="arm64" +fi + +if [ -z "$TARGET_DOCKER_REPO" ]; then + TARGET_DOCKER_REPO="us-central1-docker.pkg.dev/zerotier-421eb9/docker-images" +fi + +docker build -platform linux/$ARCH -t $TARGET_DOCKER_REPO/ztcentral-controller:$(git rev-parse --short HEAD)-$ARCH -f ext/central-controller/docker/Dockerfile.central-controller . --load --push \ No newline at end of file diff --git a/cmake/cpp-httplib.cmake b/cmake/cpp-httplib.cmake new file mode 100644 index 0000000000..d1834c4081 --- /dev/null +++ b/cmake/cpp-httplib.cmake @@ -0,0 +1,21 @@ + +set(FETCHCONTENT_QUIET OFF) + +FetchContent_Declare( + cpp-httplib + GIT_REPOSITORY https://github.com/yhirose/cpp-httplib.git + GIT_TAG v0.25.0 + GIT_SHALLOW ON +) +set(BUILD_SHARED_LIBS OFF CACHE INTERNAL "") +set(HTTPLIB_COMPILE OFF CACHE INTERNAL "") +set(HTTPLIB_USE_ZLIB_IF_AVAILABLE ON CACHE INTERNAL "Use zlib if available") +set(HTTPLIB_USE_BROTLI_IF_AVAILABLE ON CACHE INTERNAL "Use brotli if available") +set(HTTPLIB_USE_OPENSSL_IF_AVAILABLE ON CACHE INTERNAL "Use OpenSSL if available") +set(HTTPLIB_USE_ZSTD_IF_AVAILABLE ON CACHE INTERNAL "Use zstd if available") +FetchContent_MakeAvailable(cpp-httplib) + +if (NOT TARGET httplib::httplib) + message(FATAL_ERROR "A required cpp-httplib target (cpp-httplib) was not imported") +endif() + diff --git a/cmake/miniupnpc.cmake b/cmake/miniupnpc.cmake new file mode 100644 index 0000000000..44565a58ad --- /dev/null +++ b/cmake/miniupnpc.cmake @@ -0,0 +1,32 @@ +set(FETCHCONTENT_QUIET OFF) + +FetchContent_Declare( + miniupnpc + URL https://github.com/miniupnp/miniupnp/releases/download/miniupnpc_2_3_3/miniupnpc-2.3.3.tar.gz + DOWNLOAD_EXTRACT_TIMESTAMP TRUE +) +set(UPNPC_BUILD_TESTS FALSE CACHE INTERNAL "Build tests" FORCE) +set(UPNPC_BUILD_SHARED FALSE CACHE INTERNAL "Build shared library" FORCE) +set(UPNPC_BUILD_STATIC TRUE CACHE INTERNAL "Build static library" FORCE) +set(UPNPC_BUILD_SAMPLE FALSE CACHE INTERNAL "Build sample" FORCE) + +FetchContent_MakeAvailable(miniupnpc) + +if (NOT TARGET miniupnpc::miniupnpc) + message(FATAL_ERROR "A required miniupnpc target (miniupnpc::miniupnpc) was not imported") +endif() +add_definitions(-DZT_USE_MINIUPNPC) + +FetchContent_Declare( + libnatpmp + GIT_REPOSITORY https://github.com/miniupnp/libnatpmp.git + GIT_TAG master + GIT_SHALLOW ON + PATCH_COMMAND git apply ${CMAKE_SOURCE_DIR}/ext/cmake-patches/libnatpmp.patch + UPDATE_DISCONNECTED TRUE +) +FetchContent_MakeAvailable(libnatpmp) + +if (NOT TARGET natpmp) + message(FATAL_ERROR "A required libnatpmp target (natpmp) was not imported") +endif() diff --git a/cmake/redis-plus-plus.cmake b/cmake/redis-plus-plus.cmake new file mode 100644 index 0000000000..7fa99ea733 --- /dev/null +++ b/cmake/redis-plus-plus.cmake @@ -0,0 +1,19 @@ +set(FETCHCONTENT_QUIET OFF) + +FetchContent_Declare( + redis-plus-plus + GIT_REPOSITORY https://github.com/sewenew/redis-plus-plus.git + GIT_TAG 1.3.15 + GIT_SHALLOW ON +) +set(REDIS_PLUS_PLUS_BUILD_STATIC ON CACHE INTERNAL "Build static library" FORCE) +set(REDIS_PLUS_PLUS_BUILD_SHARED ON CACHE INTERNAL "Build shared library" FORCE) +set(REDIS_PLUS_PLUS_BUILD_TEST OFF CACHE INTERNAL "Build tests" FORCE) +set(REDIS_PLUS_PLUS_BUILD_STATIC_WITH_PIC ON CACHE INTERNAL "Build static library with PIC" FORCE) +set(BUILD_SHARED_LIBS OFF CACHE INTERNAL "Build shared libraries" FORCE) + +FetchContent_MakeAvailable(redis-plus-plus) +if(NOT TARGET redis++::redis++_static) + message(FATAL_ERROR "A required redis-plus-plus target (redis++::redis++) was not imported") +endif() +message(STATUS "redis-plus-plus imported") diff --git a/conda_env_build.yml b/conda_env_build.yml new file mode 100644 index 0000000000..9a51cb7b7e --- /dev/null +++ b/conda_env_build.yml @@ -0,0 +1,20 @@ +name: central_controller +channels: + - conda-forge +dependencies: + - conda-forge::cmake=3.27.4 + - conda-forge::git + - conda-forge::cxx-compiler + - conda-forge::c-compiler + - conda-forge::make + - conda-forge::pkg-config + - conda-forge::libpqxx=7.10.1 + - conda-forge::libopentelemetry-cpp=1.21.0 + - conda-forge::libopentelemetry-cpp-headers=1.21.0 + - conda-forge::google-cloud-cpp=2.38.0 + - conda-forge::libgoogle-cloud=2.38.0 + - conda-forge::rust=1.89.0 + - conda-forge::inja=3.3.0 + - conda-forge::libhiredis=1.3.0 + - conda-forge::nlohmann_json=3.12.0 + - conda-forge::jemalloc=5.3.0 diff --git a/conda_env_run.yml b/conda_env_run.yml new file mode 100644 index 0000000000..3ad8b606f2 --- /dev/null +++ b/conda_env_run.yml @@ -0,0 +1,14 @@ +name: central_controller +channels: + - conda-forge +dependencies: + - conda-forge::pkg-config + - conda-forge::libpqxx=7.10.1 + - conda-forge::libopentelemetry-cpp=1.21.0 + - conda-forge::google-cloud-cpp=2.38.0 + - conda-forge::libgoogle-cloud=2.38.0 + - conda-forge::inja=3.3.0 + - conda-forge::libhiredis=1.3.0 + - conda-forge::nlohmann_json=3.12.0 + - conda-forge::jemalloc=5.3.0 + - conda-forge::postgresql=17.6 diff --git a/ext/CMakeLists.txt b/ext/CMakeLists.txt new file mode 100644 index 0000000000..650174241c --- /dev/null +++ b/ext/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 3.13) +project(ext) + +file(GLOB_RECURSE PROM_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/prometheus-cpp-lite-1.0/core/include/*.h ) +set(PROMETHEUS_CPP_LITE_HEADERS ${PROM_CORE_HEADERS} ${CMAKE_CURRENT_SOURCE_DIR}/prometheus-cpp-lite-1.0/simpleapi/include) + + +add_library(prometheus-cpp-lite INTERFACE) +target_sources(prometheus-cpp-lite INTERFACE ${PROMETHEUS_CPP_LITE_HEADERS}) +target_include_directories(prometheus-cpp-lite INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/prometheus-cpp-lite-1.0/simpleapi/include ${CMAKE_CURRENT_SOURCE_DIR}/prometheus-cpp-lite-1.0/core/include) +add_custom_target(prometheus-cpp-lite-ide SOURCES ${PROMETHEUS_CPP_LITE_HEADERS}) +set(prometheus-cpp-lite_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/prometheus-cpp-lite-1.0/simpleapi/include ${CMAKE_CURRENT_SOURCE_DIR}/prometheus-cpp-lite-1.0/core/include PARENT_SCOPE) diff --git a/ext/central-controller-docker/Dockerfile.conda b/ext/central-controller-docker/Dockerfile.conda new file mode 100644 index 0000000000..7d1bbc64cc --- /dev/null +++ b/ext/central-controller-docker/Dockerfile.conda @@ -0,0 +1,25 @@ +FROM golang:bookworm AS go_base +RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest + + +FROM continuumio/miniconda3:25.3.1-1 + +LABEL maintainer="ZeroTier Inc." + +ADD environment.yml /environment.yml +RUN conda env create -f /environment.yml && \ + conda clean -a -y +SHELL ["conda", "run", "--no-capture-output", "-n", "central_controller", "/bin/bash", "-c"] + +COPY --from=go_base /go/bin/migrate /usr/local/bin/migrate +COPY ext/central-controller-docker/migrations /migrations +ADD build/zerotier-one /usr/local/bin/zerotier-one +RUN chmod a+x /usr/local/bin/zerotier-one +RUN echo "/opt/conda/envs/central_controller/lib" > /etc/ld.so.conf.d/conda-central-controller.conf && \ + echo "/opt/conda/envs/central_controller/`uname -m`-conda-linux-gnu/lib" > /etc/ld.so.conf.d/conda-central-controller-x64.conf && \ + ldconfig + +ADD ext/central-controller-docker/main-new.sh /main.sh +RUN chmod a+x /main.sh + +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "central_controller", "/bin/bash", "-c", "/main.sh"] diff --git a/ext/central-controller-docker/Dockerfile.conda_builder b/ext/central-controller-docker/Dockerfile.conda_builder new file mode 100644 index 0000000000..42be3ab8bc --- /dev/null +++ b/ext/central-controller-docker/Dockerfile.conda_builder @@ -0,0 +1,10 @@ +FROM continuumio/miniconda3:25.3.1-1 + +LABEL maintainer="ZeroTier Inc." + +ADD conda_env_build.yml /environment.yml +RUN conda env create -f /environment.yml && \ + conda clean -a -y + +SHELL ["conda", "run", "--no-capture-output", "-n", "central_controller", "/bin/bash", "-c"] +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "central_controller", "/bin/bash"] \ No newline at end of file diff --git a/ext/central-controller-docker/Dockerfile.new b/ext/central-controller-docker/Dockerfile.new new file mode 100644 index 0000000000..18245494cc --- /dev/null +++ b/ext/central-controller-docker/Dockerfile.new @@ -0,0 +1,25 @@ +FROM us-central1-docker.pkg.dev/zerotier-421eb9/docker-images/controller_conda_builder:latest AS controller_conda_builder +ADD . /ZeroTierOne +WORKDIR /ZeroTierOne +SHELL ["conda", "run", "--no-capture-output", "-n", "central_controller", "/bin/bash", "-c"] +RUN cmake -B build -S . -DCMAKE_BUILD_TYPE=Release -DZT1_CENTRAL_CONTROLLER=1 && cmake --build build/ --target all -j4 --verbose + +FROM golang:bookworm AS go_base +RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest + +FROM continuumio/miniconda3:25.3.1-1 +LABEL maintainer="ZeroTier Inc." +ADD conda_env_run.yml /environment.yml +RUN conda env create -f /environment.yml && \ + conda clean -a -y +SHELL ["conda", "run", "--no-capture-output", "-n", "central_controller", "/bin/bash", "-c"] +COPY --from=go_base /go/bin/migrate /usr/local/bin/migrate +COPY --from=controller_conda_builder /ZeroTierOne/build/zerotier-one /usr/local/bin/zerotier-one +COPY ext/central-controller-docker/migrations /migrations +RUN chmod a+x /usr/local/bin/zerotier-one +# RUN echo "/opt/conda/envs/central_controller/lib" > /etc/ld.so.conf.d/conda-central-controller.conf && \ +# echo "/opt/conda/envs/central_controller/`uname -m`-conda-linux-gnu/lib" > /etc/ld.so.conf.d/conda-central-controller-x64.conf && \ +# ldconfig +ADD ext/central-controller-docker/main-new.sh /main.sh +RUN chmod a+x /main.sh +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "central_controller", "/bin/bash", "-c", "/main.sh"] \ No newline at end of file diff --git a/ext/central-controller-docker/main-new.sh b/ext/central-controller-docker/main-new.sh new file mode 100755 index 0000000000..b7085997a4 --- /dev/null +++ b/ext/central-controller-docker/main-new.sh @@ -0,0 +1,180 @@ +#!/bin/bash + +# conda init +# conda activate central_controller + +if [ -z "$ZT_DB_HOST" ]; then + echo '*** FAILED: ZT_DB_HOST environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_PORT" ]; then + echo '*** FAILED: ZT_DB_PORT environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_NAME" ]; then + echo '*** FAILED: ZT_DB_NAME environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_USER" ]; then + echo '*** FAILED: ZT_DB_USER environment variable not defined' + exit 1 +fi +if [ -z "$ZT_DB_PASSWORD" ]; then + echo '*** FAILED: ZT_DB_PASSWORD environment variable not defined' + exit 1 +fi + +REDIS="" +if [ "$ZT_USE_REDIS" == "true" ]; then + if [ -z "$ZT_REDIS_HOST" ]; then + echo '*** FAILED: ZT_REDIS_HOST environment variable not defined' + exit 1 + fi + + if [ -z "$ZT_REDIS_PORT" ]; then + echo '*** FAILED: ZT_REDIS_PORT enivronment variable not defined' + exit 1 + fi + + if [ -z "$ZT_REDIS_CLUSTER_MODE" ]; then + echo '*** FAILED: ZT_REDIS_CLUSTER_MODE environment variable not defined' + exit 1 + fi + + REDIS=", \"redis\": { + \"hostname\": \"${ZT_REDIS_HOST}\", + \"port\": ${ZT_REDIS_PORT}, + \"clusterMode\": ${ZT_REDIS_CLUSTER_MODE}, + \"password\": \"${ZT_REDIS_PASSWORD}\" + } + " +else + REDIS=", \"redis\": null" +fi + +mkdir -p /var/lib/zerotier-one + +pushd /var/lib/zerotier-one +if [ -d "$ZT_IDENTITY_PATH" ]; then + echo '*** Using existing ZT identity from path $ZT_IDENTITY_PATH' + + ln -s $ZT_IDENTITY_PATH/identity.public identity.public + ln -s $ZT_IDENTITY_PATH/identity.secret identity.secret + if [ -L "$ZT_IDENTITY_PATH/authtoken.secret" ] && [ -e "$ZT_IDENTITY_PATH/authtoken.secret" ]; then + ln -s $ZT_IDENTITY_PATH/authtoken.secret authtoken.secret + ln -s $ZT_IDENTITY_PATH/authtoken.secret metricstoken.secret + fi +fi +popd + +DEFAULT_PORT=9993 +DEFAULT_LB_MODE=false + +APP_NAME="controller-$(cat /var/lib/zerotier-one/identity.public | cut -d ':' -f 1)" + +BIGTABLE_CONF="" +if [ "$ZT_USE_BIGTABLE" == "true" ]; then + if [ -z "$ZT_BIGTABLE_PROJECT" ] || [ -z "$ZT_BIGTABLE_INSTANCE" ] || [ -z "$ZT_BIGTABLE_TABLE" ]; then + echo '*** FAILED: ZT_BIGTABLE_PROJECT, ZT_BIGTABLE_INSTANCE, and ZT_BIGTABLE_TABLE environment variables must all be defined to use Bigtable as a controller backend' + exit 1 + fi + + BIGTABLE_CONF=", \"bigtable\": { + \"project_id\": \"${ZT_BIGTABLE_PROJECT}\", + \"instance_id\": \"${ZT_BIGTABLE_INSTANCE}\", + \"table_id\": \"${ZT_BIGTABLE_TABLE}\" + } + " +fi + + +PUBSUB_CONF="" +if [ "$ZT_USE_PUBSUB" == "true" ]; then + if [ -z "$ZT_PUBSUB_PROJECT" ]; then + echo '*** FAILED: ZT_PUBSUB_PROJECT environment variable must be defined to use PubSub as a controller backend' + exit 1 + fi + + if [ -z "$ZT_PUBSUB_MEMBER_CHANGE_RECV_TOPIC" ] || [ -z "$ZT_PUBSUB_MEMBER_CHANGE_SEND_TOPIC" ] || [ -z "$ZT_PUBSUB_NETWORK_CHANGE_RECV_TOPIC" ] || [ -z "$ZT_PUBSUB_NETWORK_CHANGE_SEND_TOPIC" ]; then + echo '*** FAILED: ZT_PUBSUB_MEMBER_CHANGE_RECV_TOPIC, ZT_PUBSUB_MEMBER_CHANGE_SEND_TOPIC, ZT_PUBSUB_NETWORK_CHANGE_RECV_TOPIC, and ZT_PUBSUB_NETWORK_CHANGE_SEND_TOPIC environment variables must all be defined to use PubSub as a controller backend' + exit 1 + fi + + PUBSUB_CONF=", \"pubsub\": { + \"project_id\": \"${ZT_PUBSUB_PROJECT}\", + \"member_change_recv_topic\": \"${ZT_PUBSUB_MEMBER_CHANGE_RECV_TOPIC}\", + \"member_change_send_topic\": \"${ZT_PUBSUB_MEMBER_CHANGE_SEND_TOPIC}\", + \"network_change_recv_topic\": \"${ZT_PUBSUB_NETWORK_CHANGE_RECV_TOPIC}\", + \"network_change_send_topic\": \"${ZT_PUBSUB_NETWORK_CHANGE_SEND_TOPIC}\" + } +" +fi + +echo "{ + \"settings\": { + \"controllerDbPath\": \"postgres:host=${ZT_DB_HOST} port=${ZT_DB_PORT} dbname=${ZT_DB_NAME} user=${ZT_DB_USER} password=${ZT_DB_PASSWORD} application_name=${APP_NAME} sslmode=prefer sslcert=${DB_CLIENT_CERT} sslkey=${DB_CLIENT_KEY} sslrootcert=${DB_SERVER_CA}\", + \"portMappingEnabled\": true, + \"softwareUpdate\": \"disable\", + \"interfacePrefixBlacklist\": [ + \"inot\", + \"nat64\" + ], + \"lowBandwidthMode\": ${ZT_LB_MODE:-$DEFAULT_LB_MODE}, + \"ssoRedirectURL\": \"${ZT_SSO_REDIRECT_URL}\", + \"allowManagementFrom\": [\"127.0.0.1\", \"::1\", \"10.0.0.0/8\"], + \"otel\": { + \"exporterEndpoint\": \"${ZT_EXPORTER_ENDPOINT}\", + \"exporterSampleRate\": ${ZT_EXPORTER_SAMPLE_RATE:-0} + } + ${REDIS} + }, + \"controller\": { + \"listenMode\": \"${ZT_LISTEN_MODE:-pgsql}\", + \"statusMode\": \"${ZT_STATUS_MODE:-pgsql}\" + ${REDIS} + ${BIGTABLE_CONF} + ${PUBSUB_CONF} + } +} +" > /var/lib/zerotier-one/local.conf + +if [ -n "$DB_SERVER_CA" ]; then + echo "secret list" + chmod 600 /secrets/db/*.pem + ls -l /secrets/db/ + until pg_isready -h ${ZT_DB_HOST} -p ${ZT_DB_PORT} -d "sslmode=prefer sslcert=${DB_CLIENT_CERT} sslkey=${DB_CLIENT_KEY} sslrootcert=${DB_SERVER_CA}"; do + echo "Waiting for PostgreSQL..."; + sleep 2; + done +else + until pg_isready -h ${ZT_DB_HOST} -p ${ZT_DB_PORT}; do + echo "Waiting for PostgreSQL..."; + sleep 2; + done +fi + + +echo "Migrating database (if needed)..." +if [ -n "$DB_SERVER_CA" ]; then + /usr/local/bin/migrate -source file:///migrations -database "postgres://$ZT_DB_USER:$ZT_DB_PASSWORD@$ZT_DB_HOST:$ZT_DB_PORT/$ZT_DB_NAME?x-migrations-table=controller_migrations&sslmode=verify-full&sslrootcert=$DB_SERVER_CA&sslcert=$DB_CLIENT_CERT&sslkey=$DB_CLIENT_KEY" up +else + /usr/local/bin/migrate -source file:///migrations -database "postgres://$ZT_DB_USER:$ZT_DB_PASSWORD@$ZT_DB_HOST:$ZT_DB_PORT/$ZT_DB_NAME?x-migrations-table=controller_migrations&sslmode=disable" up +fi + +if [ -n "$ZT_TEMPORAL_HOST" ] && [ -n "$ZT_TEMPORAL_PORT" ]; then + echo "waiting for temporal..." + while ! nc -z ${ZT_TEMPORAL_HOST} ${ZT_TEMPORAL_PORT}; do + echo "waiting..."; + sleep 1; + done + echo "Temporal is up" +fi + +cat /var/lib/zerotier-one/local.conf + +export GOOGLE_CLOUD_CPP_ENABLE_CLOG=yes +export LIBC_FATAL_STDERR_=1 +export GLIBCXX_FORCE_NEW=1 +export GLIBCPP_FORCE_NEW=1 +export LD_PRELOAD="/opt/conda/envs/central_controller/lib/libjemalloc.so.2" +exec /usr/local/bin/zerotier-one -p${ZT_CONTROLLER_PORT:-$DEFAULT_PORT} /var/lib/zerotier-one diff --git a/ext/central-controller-docker/migrations/0003_frontend.down.sql b/ext/central-controller-docker/migrations/0003_frontend.down.sql new file mode 100644 index 0000000000..870eeaa42a --- /dev/null +++ b/ext/central-controller-docker/migrations/0003_frontend.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE network_memberships_ctl + DROP COLUMN frontend; + +ALTER TABLE networks_ctl + DROP COLUMN frontend; \ No newline at end of file diff --git a/ext/central-controller-docker/migrations/0003_frontend.up.sql b/ext/central-controller-docker/migrations/0003_frontend.up.sql new file mode 100644 index 0000000000..cb9824496e --- /dev/null +++ b/ext/central-controller-docker/migrations/0003_frontend.up.sql @@ -0,0 +1,5 @@ +ALTER TABLE networks_ctl + ADD COLUMN frontend TEXT NOT NULL DEFAULT 'cv2'; + +ALTER TABLE network_memberships_ctl + ADD COLUMN frontend TEXT NOT NULL DEFAULT 'cv2'; \ No newline at end of file diff --git a/ext/central-controller-docker/migrations/0004_use_controller_flag.down.sql b/ext/central-controller-docker/migrations/0004_use_controller_flag.down.sql new file mode 100644 index 0000000000..afcd9c3f76 --- /dev/null +++ b/ext/central-controller-docker/migrations/0004_use_controller_flag.down.sql @@ -0,0 +1 @@ +ALTER TABLE controllers_ctl DROP COLUMN IF EXISTS use_for_new_networks; \ No newline at end of file diff --git a/ext/central-controller-docker/migrations/0004_use_controller_flag.up.sql b/ext/central-controller-docker/migrations/0004_use_controller_flag.up.sql new file mode 100644 index 0000000000..b723b0172c --- /dev/null +++ b/ext/central-controller-docker/migrations/0004_use_controller_flag.up.sql @@ -0,0 +1 @@ +ALTER TABLE controllers_ctl ADD COLUMN IF NOT EXISTS use_for_new_networks boolean DEFAULT true NOT NULL; \ No newline at end of file diff --git a/ext/central-controller-docker/migrations/0005_controller_log.down.sql b/ext/central-controller-docker/migrations/0005_controller_log.down.sql new file mode 100644 index 0000000000..fecab6b5b3 --- /dev/null +++ b/ext/central-controller-docker/migrations/0005_controller_log.down.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS ctl_check_time_ix; +DROP INDEX IF EXISTS ctl_id_ix; +DROP TABLE IF EXISTS controller_log; diff --git a/ext/central-controller-docker/migrations/0005_controller_log.up.sql b/ext/central-controller-docker/migrations/0005_controller_log.up.sql new file mode 100644 index 0000000000..eafb9edec8 --- /dev/null +++ b/ext/central-controller-docker/migrations/0005_controller_log.up.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS controller_log ( + controller_id CHAR(10) NOT NULL REFERENCES controllers_ctl(id) ON DELETE CASCADE, + check_time TIMESTAMP WITH TIME ZONE NOT NULL, + load_factor REAL, + PRIMARY KEY (controller_id, check_time) +); +CREATE INDEX IF NOT EXISTS ctl_check_time_ix ON public.controller_log USING btree (check_time); +CREATE INDEX IF NOT EXISTS ctl_id_ix ON public.controller_log USING btree (controller_id); + +CREATE TABLE IF NOT EXISTS sso_expiry ( + nonce TEXT PRIMARY KEY, + nonce_expiration TIMESTAMP WITH TIME ZONE NOT NULL, + network_id CHARACTER(16) NOT NULL, + member_id CHARACTER(10) NOT NULL, + creation_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT (current_timestamp AT TIME ZONE 'UTC'), + email TEXT, + authentication_expiry_time TIMESTAMP WITH TIME ZONE, + FOREIGN KEY (network_id, member_id) REFERENCES network_memberships_ctl(network_id, device_id) ON DELETE CASCADE +); diff --git a/ext/cmake-patches/libnatpmp.patch b/ext/cmake-patches/libnatpmp.patch new file mode 100644 index 0000000000..5b14957076 --- /dev/null +++ b/ext/cmake-patches/libnatpmp.patch @@ -0,0 +1,19 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 16af4aa..1eb89f7 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -1,6 +1,6 @@ + cmake_minimum_required(VERSION 3.5) + +-file(STRINGS VERSION PVER) ++set(PVER "20120821") + + project(natpmp + LANGUAGES C +diff --git a/VERSION b/VERSION +deleted file mode 100644 +index 2ed7994..0000000 +--- a/VERSION ++++ /dev/null +@@ -1 +0,0 @@ +-20120821 diff --git a/ext/prometheus-cpp-lite-1.0/3rdparty/http-client-lite/CMakeLists.txt b/ext/prometheus-cpp-lite-1.0/3rdparty/http-client-lite/CMakeLists.txt index 177cdb974c..e7412e7f2e 100644 --- a/ext/prometheus-cpp-lite-1.0/3rdparty/http-client-lite/CMakeLists.txt +++ b/ext/prometheus-cpp-lite-1.0/3rdparty/http-client-lite/CMakeLists.txt @@ -5,7 +5,7 @@ # Distributed under the MIT License. # (See accompanying file LICENSE or copy at https://mit-license.org/) -cmake_minimum_required(VERSION 3.3 FATAL_ERROR) +cmake_minimum_required(VERSION 3.13 FATAL_ERROR) project(http-client-lite) ## Set output binary diff --git a/ext/prometheus-cpp-lite-1.0/core/CMakeLists.txt b/ext/prometheus-cpp-lite-1.0/core/CMakeLists.txt index 70c932218f..38ff920ba8 100644 --- a/ext/prometheus-cpp-lite-1.0/core/CMakeLists.txt +++ b/ext/prometheus-cpp-lite-1.0/core/CMakeLists.txt @@ -1,5 +1,5 @@ project(prometheus-cpp-lite-core) -cmake_minimum_required(VERSION 3.2) +cmake_minimum_required(VERSION 3.13) file(GLOB_RECURSE PROMETHEUS_CPP_LITE_HEADERS *.h) diff --git a/ext/prometheus-cpp-lite-1.0/simpleapi/CMakeLists.txt b/ext/prometheus-cpp-lite-1.0/simpleapi/CMakeLists.txt index 19dc9e1f3a..fce2ef7228 100644 --- a/ext/prometheus-cpp-lite-1.0/simpleapi/CMakeLists.txt +++ b/ext/prometheus-cpp-lite-1.0/simpleapi/CMakeLists.txt @@ -1,5 +1,5 @@ project(prometheus-cpp-simpleapi) -cmake_minimum_required(VERSION 3.2) +cmake_minimum_required(VERSION 3.13) add_library (${PROJECT_NAME} STATIC "./src/simpleapi.cpp" ) target_sources (${PROJECT_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include/prometheus/simpleapi.h") diff --git a/make-linux.mk b/make-linux.mk index 77d3e0b1a4..9814819dc7 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -14,6 +14,7 @@ DEFS?= LDLIBS?= DESTDIR?= EXTRA_DEPS?= +ZT_CARGO_FLAGS?= include objects.mk @@ -77,7 +78,6 @@ ifeq ($(ZT_DEBUG),1) override CFLAGS+=-Wall -Wno-deprecated -g -O -pthread $(INCLUDES) $(DEFS) override CXXFLAGS+=-Wall -Wno-deprecated -g -O -std=c++17 -pthread $(INCLUDES) $(DEFS) ZT_TRACE=1 - ZT_CARGO_FLAGS= # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in -O0 even on a 3ghz box! node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CXXFLAGS=-Wall -O2 -g -pthread $(INCLUDES) $(DEFS) @@ -87,7 +87,7 @@ else CXXFLAGS?=-O3 -fstack-protector override CXXFLAGS+=-Wall -Wno-deprecated -std=c++17 -pthread $(INCLUDES) -DNDEBUG $(DEFS) LDFLAGS?=-pie -Wl,-z,relro,-z,now - ZT_CARGO_FLAGS=--release + ZT_CARGO_FLAGS+=--release endif ifeq ($(ZT_QNAP), 1) @@ -311,9 +311,9 @@ ifeq ($(ZT_SSO_SUPPORTED), 1) ifeq ($(ZT_EMBEDDED),) override DEFS+=-DZT_SSO_SUPPORTED=1 ifeq ($(ZT_DEBUG),1) - LDLIBS+=rustybits/target/debug/libzeroidc.a -ldl -lssl -lcrypto + LDLIBS+=rustybits/target/debug/librustybits.a -ldl -lssl -lcrypto else - LDLIBS+=rustybits/target/release/libzeroidc.a -ldl -lssl -lcrypto + LDLIBS+=rustybits/target/release/librustybits.a -ldl -lssl -lcrypto endif endif endif @@ -350,11 +350,12 @@ ifeq ($(ZT_CONTROLLER),1) override CXXFLAGS+=-Wall -Wno-deprecated -std=c++17 -pthread $(INCLUDES) -DNDEBUG $(DEFS) override LDLIBS+=-Lext/libpqxx-7.7.3/install/ubuntu22.04/$(EXT_ARCH)/lib -lpqxx -lpq ext/hiredis-1.0.2/lib/ubuntu22.04/$(EXT_ARCH)/libhiredis.a ext/redis-plus-plus-1.3.3/install/ubuntu22.04/$(EXT_ARCH)/lib/libredis++.a -lssl -lcrypto override DEFS+=-DZT_CONTROLLER_USE_LIBPQ -DZT_NO_PEER_METRICS -DZT_OPENTELEMETRY_ENABLED - override INCLUDES+=-I/usr/include/postgresql -Iext/libpqxx-7.7.3/install/ubuntu22.04/$(EXT_ARCH)/include -Iext/hiredis-1.0.2/include/ -Iext/redis-plus-plus-1.3.3/install/ubuntu22.04/$(EXT_ARCH)/include/sw/ + override INCLUDES+=-I/usr/include/postgresql -Iext/libpqxx-7.7.3/install/ubuntu22.04/$(EXT_ARCH)/include -Iext/hiredis-1.0.2/include/ -Iext/redis-plus-plus-1.3.3/install/ubuntu22.04/$(EXT_ARCH)/include + override ZT_CARGO_FLAGS+=-F ztcontroller ifeq ($(ZT_DEBUG),1) - override LDLIBS+=rustybits/target/debug/libsmeeclient.a + override LDLIBS+=rustybits/target/debug/librustybits.a else - override LDLIBS+=rustybits/target/release/libsmeeclient.a + override LDLIBS+=rustybits/target/release/librustybits.a endif endif @@ -415,7 +416,7 @@ zerotier-idtool: zerotier-one zerotier-cli: zerotier-one ln -sf zerotier-one zerotier-cli -$(ONE_OBJS): zeroidc smeeclient +$(ONE_OBJS): zeroidc rustybits libzerotiercore.a: FORCE make CFLAGS="-O3 -fstack-protector -fPIC" CXXFLAGS="-O3 -std=c++17 -fstack-protector -fPIC" $(CORE_OBJS) @@ -483,19 +484,12 @@ debug: FORCE ifeq ($(ZT_SSO_SUPPORTED), 1) ifeq ($(ZT_EMBEDDED),) zeroidc: FORCE - export PATH=/${HOME}/.cargo/bin:$$PATH; cd rustybits && cargo build $(ZT_CARGO_FLAGS) -p zeroidc + export PATH=/${HOME}/.cargo/bin:$$PATH; cd rustybits && cargo build $(ZT_CARGO_FLAGS) endif else zeroidc: endif -ifeq ($(ZT_CONTROLLER), 1) -smeeclient: FORCE - export PATH=/${HOME}/.cargo/bin:$$PATH; cd rustybits && cargo build $(ZT_CARGO_FLAGS) -p smeeclient -else -smeeclient: -endif - # Note: keep the symlinks in /var/lib/zerotier-one to the binaries since these # provide backward compatibility with old releases where the binaries actually # lived here. Folks got scripts. diff --git a/make-mac.mk b/make-mac.mk index e63fb7f88b..28b3a605fa 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -31,14 +31,7 @@ include objects.mk ONE_OBJS+=osdep/MacEthernetTap.o osdep/MacKextEthernetTap.o osdep/MacDNSHelper.o ext/http-parser/http_parser.o LIBS+=-framework CoreServices -framework SystemConfiguration -framework CoreFoundation -framework Security -ifeq ($(ZT_CONTROLLER),1) - ZT_NONFREE=1 -endif -ifeq ($(ZT_NONFREE),1) - include objects-nonfree.mk - ONE_OBJS+=$(CONTROLLER_OBJS) - override DEFS += -DZT_NONFREE_CONTROLLER -endif +EXTRA_CARGO_FLAGS?= ifeq ($(ZT_OFFICIAL_RELEASE),1) ZT_USE_MINIUPNPC=1 @@ -62,9 +55,10 @@ ONE_OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connec ifeq ($(ZT_CONTROLLER),1) MACOS_VERSION_MIN=10.15 override CXXFLAGS=$(CFLAGS) -std=c++17 -stdlib=libc++ - LIBS+=-L/opt/homebrew/lib -L/usr/local/opt/libpqxx/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/openssl/lib/ -lpqxx -lpq -lssl -lcrypto -lgssapi_krb5 ext/redis-plus-plus-1.1.1/install/macos/lib/libredis++.a ext/hiredis-0.14.1/lib/macos/libhiredis.a rustybits/target/libsmeeclient.a + LIBS+=-L/opt/homebrew/lib -L/usr/local/opt/libpqxx/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/openssl/lib/ -lpqxx -lpq -lssl -lcrypto -lgssapi_krb5 ext/redis-plus-plus-1.1.1/install/macos/lib/libredis++.a ext/hiredis-0.14.1/lib/macos/libhiredis.a rustybits/target/librustybits.a override DEFS+=-DZT_CONTROLLER_USE_LIBPQ -DZT_CONTROLLER_USE_REDIS -DZT_CONTROLLER INCLUDES+=-I/opt/homebrew/include -I/opt/homebrew/opt/libpq/include -I/usr/local/opt/libpq/include -I/usr/local/opt/libpqxx/include -Iext/hiredis-0.14.1/include/ -Iext/redis-plus-plus-1.1.1/install/macos/include/sw/ -Irustybits/target/ + EXTRA_CARGO_FLAGS+=-F ztcontroller else MACOS_VERSION_MIN=10.13 endif @@ -82,7 +76,6 @@ ifeq ($(ZT_DEBUG),1) ARCH_FLAGS= CFLAGS+=-Wall -g $(INCLUDES) $(DEFS) $(ARCH_FLAGS) STRIP=echo - EXTRA_CARGO_FLAGS= RUST_VARIANT=debug # The following line enables optimization for the crypto code, since # C25519 in particular is almost UNUSABLE in heavy testing without it. @@ -91,7 +84,7 @@ else CFLAGS?=-O3 -fstack-protector-strong CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -mmacosx-version-min=$(MACOS_VERSION_MIN) -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) STRIP=strip - EXTRA_CARGO_FLAGS=--release + EXTRA_CARGO_FLAGS+=--release RUST_VARIANT=release endif @@ -132,11 +125,11 @@ osdep/MacDNSHelper.o: osdep/MacDNSHelper.mm $(CXX) $(CXXFLAGS) -c osdep/MacDNSHelper.mm -o osdep/MacDNSHelper.o ifeq ($(ZT_CONTROLLER),1) -one: otel zeroidc smeeclient $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent +one: otel rustybits $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent else -one: otel zeroidc $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent +one: otel rustybits $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent endif - $(CXX) $(CXXFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LIBS) rustybits/target/libzeroidc.a + $(CXX) $(CXXFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LIBS) rustybits/target/librustybits.a # $(STRIP) zerotier-one ln -sf zerotier-one zerotier-idtool ln -sf zerotier-one zerotier-cli @@ -144,21 +137,12 @@ endif zerotier-one: one -zeroidc: rustybits/target/libzeroidc.a - -ifeq ($(ZT_CONTROLLER),1) -smeeclient: rustybits/target/libsmeeclient.a - -rustybits/target/libsmeeclient.a: FORCE - cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p smeeclient --target=x86_64-apple-darwin $(EXTRA_CARGO_FLAGS) - cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p smeeclient --target=aarch64-apple-darwin $(EXTRA_CARGO_FLAGS) - cd rustybits && lipo -create target/x86_64-apple-darwin/$(RUST_VARIANT)/libsmeeclient.a target/aarch64-apple-darwin/$(RUST_VARIANT)/libsmeeclient.a -output target/libsmeeclient.a -endif +rustybits: rustybits/target/rustybits.a -rustybits/target/libzeroidc.a: FORCE - cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p zeroidc --target=x86_64-apple-darwin $(EXTRA_CARGO_FLAGS) - cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p zeroidc --target=aarch64-apple-darwin $(EXTRA_CARGO_FLAGS) - cd rustybits && lipo -create target/x86_64-apple-darwin/$(RUST_VARIANT)/libzeroidc.a target/aarch64-apple-darwin/$(RUST_VARIANT)/libzeroidc.a -output target/libzeroidc.a +rustybits/target/rustybits.a: FORCE + cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build --target=x86_64-apple-darwin $(EXTRA_CARGO_FLAGS) + cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build --target=aarch64-apple-darwin $(EXTRA_CARGO_FLAGS) + cd rustybits && lipo -create target/x86_64-apple-darwin/$(RUST_VARIANT)/librustybits.a target/aarch64-apple-darwin/$(RUST_VARIANT)/librustybits.a -output target/librustybits.a central-controller: make ARCH_FLAGS="-arch x86_64" ZT_CONTROLLER=1 one @@ -167,7 +151,7 @@ zerotier-idtool: one zerotier-cli: one -$(ONE_OBJS): zeroidc +$(ONE_OBJS): rustybits libzerotiercore.a: $(CORE_OBJS) ar rcs libzerotiercore.a $(CORE_OBJS) @@ -176,7 +160,7 @@ libzerotiercore.a: $(CORE_OBJS) core: libzerotiercore.a selftest: $(CORE_OBJS) $(ONE_OBJS) selftest.o - $(CXX) $(CXXFLAGS) -o zerotier-selftest selftest.o $(CORE_OBJS) $(ONE_OBJS) $(LIBS) rustybits/target/libzeroidc.a + $(CXX) $(CXXFLAGS) -o zerotier-selftest selftest.o $(CORE_OBJS) $(ONE_OBJS) $(LIBS) rustybits/target/librustybits.a $(STRIP) zerotier-selftest zerotier-selftest: selftest diff --git a/node/CMakeLists.txt b/node/CMakeLists.txt new file mode 100644 index 0000000000..b74935e1cd --- /dev/null +++ b/node/CMakeLists.txt @@ -0,0 +1,94 @@ +cmake_minimum_required(VERSION 3.13) +project(zerotier-core LANGUAGES CXX ASM) + + +file (GLOB core_src_glob ${PROJ_DIR}/node/*.cpp) +file (GLOB core_hdr_glob ${PROJ_DIR}/node/*.hpp) + +if(${CPU_ARCHITECTURE} STREQUAL "x86_64") + set(CMAKE_ASM_FLAGS "${CFLAGS} -x assembler-with-cpp -z noexecstack") + set(ASM_SALSA_DIR ${CMAKE_SOURCE_DIR}/ext/x64-salsa2012-asm) + set(ASM_ED25519_DIR ${CMAKE_SOURCE_DIR}/ext/ed25519-amd64-asm) + list(APPEND core_src_glob + ${ASM_SALSA_DIR}/salsa2012.s + #${ASM_ED25519_DIR}/batch.c + ${ASM_ED25519_DIR}/choose_t.s + ${ASM_ED25519_DIR}/consts.s + ${ASM_ED25519_DIR}/fe25519_add.s + ${ASM_ED25519_DIR}/fe25519_freeze.s + ${ASM_ED25519_DIR}/fe25519_getparity.c + ${ASM_ED25519_DIR}/fe25519_invert.c + ${ASM_ED25519_DIR}/fe25519_iseq.c + ${ASM_ED25519_DIR}/fe25519_iszero.c + ${ASM_ED25519_DIR}/fe25519_mul.s + ${ASM_ED25519_DIR}/fe25519_neg.c + ${ASM_ED25519_DIR}/fe25519_pack.c + ${ASM_ED25519_DIR}/fe25519_pow2523.c + ${ASM_ED25519_DIR}/fe25519_setint.c + ${ASM_ED25519_DIR}/fe25519_square.s + ${ASM_ED25519_DIR}/fe25519_sub.s + ${ASM_ED25519_DIR}/fe25519_unpack.c + ${ASM_ED25519_DIR}/ge25519_add_p1p1.s + ${ASM_ED25519_DIR}/ge25519_add.c + ${ASM_ED25519_DIR}/ge25519_base.c + ${ASM_ED25519_DIR}/ge25519_dbl_p1p1.s + ${ASM_ED25519_DIR}/ge25519_double_scalarmult.c + ${ASM_ED25519_DIR}/ge25519_double.c + ${ASM_ED25519_DIR}/ge25519_isneutral.c + ${ASM_ED25519_DIR}/ge25519_multi_scalarmult.c + ${ASM_ED25519_DIR}/ge25519_nielsadd_p1p1.s + ${ASM_ED25519_DIR}/ge25519_nielsadd2.s + ${ASM_ED25519_DIR}/ge25519_p1p1_to_p2.s + ${ASM_ED25519_DIR}/ge25519_p1p1_to_p3.s + ${ASM_ED25519_DIR}/ge25519_pack.c + ${ASM_ED25519_DIR}/ge25519_pnielsadd_p1p1.s + ${ASM_ED25519_DIR}/ge25519_scalarmult_base.c + ${ASM_ED25519_DIR}/ge25519_unpackneg.c + ${ASM_ED25519_DIR}/heap_rootreplaced_1limb.s + ${ASM_ED25519_DIR}/heap_rootreplaced_2limbs.s + ${ASM_ED25519_DIR}/heap_rootreplaced_3limbs.s + ${ASM_ED25519_DIR}/heap_rootreplaced.s + ${ASM_ED25519_DIR}/hram.c + ${ASM_ED25519_DIR}/index_heap.c + #${ASM_ED25519_DIR}/keypair.c + #${ASM_ED25519_DIR}/open.c + ${ASM_ED25519_DIR}/sc25519_add.s + ${ASM_ED25519_DIR}/sc25519_barrett.s + ${ASM_ED25519_DIR}/sc25519_from_shortsc.c + ${ASM_ED25519_DIR}/sc25519_from32bytes.c + ${ASM_ED25519_DIR}/sc25519_from64bytes.c + ${ASM_ED25519_DIR}/sc25519_iszero.c + ${ASM_ED25519_DIR}/sc25519_lt.s + ${ASM_ED25519_DIR}/sc25519_mul_shortsc.c + ${ASM_ED25519_DIR}/sc25519_mul.c + ${ASM_ED25519_DIR}/sc25519_slide.c + ${ASM_ED25519_DIR}/sc25519_sub_nored.s + ${ASM_ED25519_DIR}/sc25519_to32bytes.c + ${ASM_ED25519_DIR}/sc25519_window4.c + ${ASM_ED25519_DIR}/sign.c + ${ASM_ED25519_DIR}/ull4_mul.s + ) + list(APPEND core_hdr_glob + ${ASM_SALSA_DIR}/salsa2012.h + ${ASM_ED25519_DIR}/fe25519.h + ${ASM_ED25519_DIR}/ge25519.h + ${ASM_ED25519_DIR}/hram.h + ${ASM_ED25519_DIR}/index_heap.h + ${ASM_ED25519_DIR}/sc25519.h) + set_property(SOURCE ${ASM_ED25519_DIR}/fe25519_freeze.s PROPERTY COMPILE_FLAGS "-z noexecstack") +elseif(${CPU_ARCHITECTURE} STREQUAL "aarch64") + +endif() + +add_library(zerotier-core STATIC ${core_src_glob} ${core_hdr_glob}) + +target_include_directories(zerotier-core + PRIVATE + ${prometheus-cpp-lite_INCLUDE} +) +target_link_libraries(zerotier-core + PRIVATE + nlohmann_json::nlohmann_json + Threads::Threads + prometheus-cpp-lite + Threads::Threads) diff --git a/nonfree/CMakeLists.txt b/nonfree/CMakeLists.txt new file mode 100644 index 0000000000..82d587d7cb --- /dev/null +++ b/nonfree/CMakeLists.txt @@ -0,0 +1,3 @@ +cmake_minimum_required(VERSION 3.13) + +add_subdirectory(controller) \ No newline at end of file diff --git a/nonfree/controller/BigTableStatusWriter.cpp b/nonfree/controller/BigTableStatusWriter.cpp new file mode 100644 index 0000000000..44180b65a3 --- /dev/null +++ b/nonfree/controller/BigTableStatusWriter.cpp @@ -0,0 +1,156 @@ +#include "BigTableStatusWriter.hpp" + +#include "ControllerConfig.hpp" +#include "PubSubWriter.hpp" + +#include +#include +#include +#include + +namespace cbt = google::cloud::bigtable; + +namespace ZeroTier { + +const std::string nodeInfoColumnFamily = "node_info"; +const std::string checkInColumnFamily = "check_in"; + +const std::string osColumn = "os"; +const std::string archColumn = "arch"; +const std::string versionColumn = "version"; +const std::string ipv4Column = "ipv4"; +const std::string ipv6Column = "ipv6"; +const std::string lastSeenColumn = "last_seen"; + +BigTableStatusWriter::BigTableStatusWriter( + const std::string& project_id, + const std::string& instance_id, + const std::string& table_id) + : _project_id(project_id) + , _instance_id(instance_id) + , _table_id(table_id) + , _table(nullptr) +{ + _table = new cbt::Table(cbt::MakeDataConnection(), cbt::TableResource(_project_id, _instance_id, _table_id)); + fprintf( + stderr, "BigTableStatusWriter for project %s instance %s table %s\n", project_id.c_str(), instance_id.c_str(), + table_id.c_str()); +} + +BigTableStatusWriter::~BigTableStatusWriter() +{ + writePending(); + + if (_table != nullptr) { + delete _table; + _table = nullptr; + } +} + +void BigTableStatusWriter::updateNodeStatus( + const std::string& network_id, + const std::string& node_id, + const std::string& os, + const std::string& arch, + const std::string& version, + const InetAddress& address, + int64_t last_seen, + const std::string& frontend) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("BigTableStatusWriter"); + auto span = tracer->StartSpan("BigTableStatusWriter::updateNodeStatus"); + auto scope = tracer->WithActiveSpan(span); + + std::lock_guard l(_lock); + _pending.push_back({ network_id, node_id, os, arch, version, address, last_seen, frontend }); +} + +size_t BigTableStatusWriter::queueLength() const +{ + std::lock_guard l(_lock); + return _pending.size(); +} + +void BigTableStatusWriter::writePending() +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("BigTableStatusWriter"); + auto span = tracer->StartSpan("BigTableStatusWriter::writePending"); + auto scope = tracer->WithActiveSpan(span); + + std::vector toWrite; + { + std::lock_guard l(_lock); + toWrite.swap(_pending); + } + if (toWrite.empty()) { + return; + } + fprintf(stderr, "Writing %zu pending status entries to BigTable\n", toWrite.size()); + + cbt::BulkMutation bulk; + for (const auto& entry : toWrite) { + std::string row_key = entry.network_id + "#" + entry.node_id; + + // read the latest values from BigTable for this row key + std::map latest_values; + try { + auto row = _table->ReadRow(row_key, cbt::Filter::Latest(1)); + if (row->first) { + for (const auto& cell : row->second.cells()) { + if (cell.family_name() == nodeInfoColumnFamily) { + latest_values[cell.column_qualifier()] = cell.value(); + } + } + } + } + catch (const std::exception& e) { + fprintf(stderr, "Exception reading from BigTable: %s\n", e.what()); + } + + cbt::SingleRowMutation m(row_key); + + // only update if value has changed + if (latest_values[osColumn] != entry.os) { + m.emplace_back(cbt::SetCell(nodeInfoColumnFamily, osColumn, entry.os)); + } + if (latest_values[archColumn] != entry.arch) { + m.emplace_back(cbt::SetCell(nodeInfoColumnFamily, archColumn, entry.arch)); + } + if (latest_values[versionColumn] != entry.version) { + m.emplace_back(cbt::SetCell(nodeInfoColumnFamily, versionColumn, entry.version)); + } + + char buf[64] = { 0 }; + std::string addressStr = entry.address.toString(buf); + if (entry.address.ss_family == AF_INET) { + m.emplace_back(cbt::SetCell(checkInColumnFamily, ipv4Column, std::move(addressStr))); + } + else if (entry.address.ss_family == AF_INET6) { + m.emplace_back(cbt::SetCell(checkInColumnFamily, ipv6Column, std::move(addressStr))); + } + int64_t ts = entry.last_seen; + m.emplace_back(cbt::SetCell(checkInColumnFamily, lastSeenColumn, std::move(ts))); + bulk.emplace_back(m); + } + + fprintf(stderr, "Applying %zu mutations to BigTable\n", bulk.size()); + + try { + std::vector failures = _table->BulkApply(std::move(bulk)); + fprintf(stderr, "BigTable write completed with %zu failures\n", failures.size()); + for (auto const& r : failures) { + // Handle error (log it, retry, etc.) + std::cerr << "Error writing to BigTable: " << r.status() << "\n"; + } + } + catch (const std::exception& e) { + fprintf(stderr, "Exception writing to BigTable: %s\n", e.what()); + span->SetAttribute("error", e.what()); + span->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + return; + } +} + +} // namespace ZeroTier \ No newline at end of file diff --git a/nonfree/controller/BigTableStatusWriter.hpp b/nonfree/controller/BigTableStatusWriter.hpp new file mode 100644 index 0000000000..ce687121f9 --- /dev/null +++ b/nonfree/controller/BigTableStatusWriter.hpp @@ -0,0 +1,44 @@ +#ifndef BIGTABLESTATUSWRITER_HPP +#define BIGTABLESTATUSWRITER_HPP + +#include "StatusWriter.hpp" + +#include +#include +#include +#include + +namespace ZeroTier { + +class PubSubWriter; + +class BigTableStatusWriter : public StatusWriter { + public: + BigTableStatusWriter(const std::string& project_id, const std::string& instance_id, const std::string& table_id); + virtual ~BigTableStatusWriter(); + + virtual void updateNodeStatus( + const std::string& network_id, + const std::string& node_id, + const std::string& os, + const std::string& arch, + const std::string& version, + const InetAddress& address, + int64_t last_seen, + const std::string& frontend) override; + virtual size_t queueLength() const override; + virtual void writePending() override; + + private: + const std::string _project_id; + const std::string _instance_id; + const std::string _table_id; + + mutable std::mutex _lock; + std::vector _pending; + google::cloud::bigtable::Table* _table; +}; + +} // namespace ZeroTier + +#endif \ No newline at end of file diff --git a/nonfree/controller/CMakeLists.txt b/nonfree/controller/CMakeLists.txt new file mode 100644 index 0000000000..9a50709777 --- /dev/null +++ b/nonfree/controller/CMakeLists.txt @@ -0,0 +1,102 @@ +cmake_minimum_required(VERSION 3.13) +project(zerotier-controller) + + +set(SRC_FILES + DB.cpp + DB.hpp + DBMirrorSet.cpp + DBMirrorSet.hpp + EmbeddedNetworkController.cpp + EmbeddedNetworkController.hpp + FileDB.cpp + FileDB.hpp + CtlUtil.cpp + CtlUtil.hpp +) + +set(INCLUDE_DIRS + "${httplib_SOURCE_DIR}" + ${RUSTYBITS_INCLUDE_DIR} +) + +set(LINK_LIBS + zerotier-osdep + nlohmann_json::nlohmann_json + opentelemetry-cpp::api + rustybits + Threads::Threads + prometheus-cpp-lite + Threads::Threads +) + +if (ZT1_CENTRAL_CONTROLLER) + find_package(PostgreSQL REQUIRED) + find_package(protobuf REQUIRED) + + list(APPEND SRC_FILES + CV1.cpp + CV1.hpp + CV2.cpp + CV2.hpp + CentralDB.cpp + CentralDB.hpp + ControllerConfig.hpp + ControllerChangeNotifier.cpp + ControllerChangeNotifier.hpp + NotificationListener.hpp + PostgreSQL.cpp + PostgreSQL.hpp + PubSubListener.cpp + PubSubListener.hpp + PubSubWriter.cpp + PubSubWriter.hpp + Redis.hpp + RedisListener.cpp + RedisListener.hpp + StatusWriter.cpp + StatusWriter.hpp + BigTableStatusWriter.cpp + BigTableStatusWriter.hpp + PostgresStatusWriter.cpp + PostgresStatusWriter.hpp + Redis.hpp + RedisStatusWriter.cpp + RedisStatusWriter.hpp + ) + + list(APPEND INCLUDE_DIRS + ${PostgreSQL_INCLUDE_DIRS} + "${redis++_BUILD_DIR}/src" + ${pqxx_INCLUDE_DIRS} + "${CMAKE_CURRENT_BINARY_DIR}" + ) + + list(APPEND LINK_LIBS + redis++::redis++_static + pqxx + ${PostgreSQL_LIBRARIES} + google-cloud-cpp::bigtable + google-cloud-cpp::pubsub + ) +endif() + +add_library(zerotier-controller STATIC ${SRC_FILES}) + +if (ZT1_CENTRAL_CONTROLLER) + file(GLOB PROTO_FILES "${CMAKE_CURRENT_SOURCE_DIR}/protobuf/*.proto") + protobuf_generate( + TARGET zerotier-controller + LANGUAGE cpp + PROTOS ${PROTO_FILES} + APPEND_PATH + ) +endif() + +target_include_directories(zerotier-controller PRIVATE ${INCLUDE_DIRS}) + +add_dependencies(zerotier-controller redis++::redis++) + +target_link_libraries(zerotier-controller + ${LINK_LIBS} +) diff --git a/nonfree/controller/CV1.cpp b/nonfree/controller/CV1.cpp index cf4496ad1f..dd45360cb5 100644 --- a/nonfree/controller/CV1.cpp +++ b/nonfree/controller/CV1.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include // #define REDIS_TRACE 1 @@ -58,11 +58,14 @@ CV1::CV1(const Identity& myId, const char* path, int listenPort, RedisConfig* rc auto span = tracer->StartSpan("cv1::CV1"); auto scope = tracer->WithActiveSpan(span); + rustybits::init_async_runtime(); + char myAddress[64]; _myAddressStr = myId.address().toString(myAddress); _connString = std::string(path); auto f = std::make_shared(_connString); - _pool = std::make_shared >(15, 5, std::static_pointer_cast(f)); + _pool = + std::make_shared >(15, 5, std::static_pointer_cast(f)); memset(_ssoPsk, 0, sizeof(_ssoPsk)); char* const ssoPskHex = getenv("ZT_SSO_PSK"); @@ -89,7 +92,11 @@ CV1::CV1(const Identity& myId, const char* path, int listenPort, RedisConfig* rc txn.commit(); if (dbVersion < DB_MINIMUM_VERSION) { - fprintf(stderr, "Central database schema version too low. This controller version requires a minimum schema version of %d. Please upgrade your Central instance", DB_MINIMUM_VERSION); + fprintf( + stderr, + "Central database schema version too low. This controller version requires a minimum schema version of " + "%d. Please upgrade your Central instance", + DB_MINIMUM_VERSION); exit(1); } _pool->unborrow(c); @@ -124,7 +131,9 @@ CV1::CV1(const Identity& myId, const char* path, int listenPort, RedisConfig* rc _readyLock.lock(); - fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL waiting for initial data download..." ZT_EOL_S, ::_timestr(), (unsigned long long)_myAddress.toInt()); + fprintf( + stderr, "[%s] NOTICE: %.10llx controller PostgreSQL waiting for initial data download..." ZT_EOL_S, + ::_timestr(), (unsigned long long)_myAddress.toInt()); _waitNoticePrinted = true; initializeNetworks(); @@ -144,10 +153,12 @@ CV1::CV1(const Identity& myId, const char* path, int listenPort, RedisConfig* rc CV1::~CV1() { if (_smee != NULL) { - smeeclient::smee_client_delete(_smee); + rustybits::smee_client_delete(_smee); _smee = NULL; } + rustybits::shutdown_async_runtime(); + _run = 0; std::this_thread::sleep_for(std::chrono::milliseconds(100)); @@ -185,8 +196,9 @@ void CV1::configureSmee() if (scheme != NULL && host != NULL && port != NULL && ns != NULL && task_queue != NULL) { fprintf(stderr, "creating smee client\n"); - std::string hostPort = std::string(scheme) + std::string("://") + std::string(host) + std::string(":") + std::string(port); - this->_smee = smeeclient::smee_client_new(hostPort.c_str(), ns, task_queue); + std::string hostPort = + std::string(scheme) + std::string("://") + std::string(host) + std::string(":") + std::string(port); + this->_smee = rustybits::smee_client_new(hostPort.c_str(), ns, task_queue); } else { fprintf(stderr, "Smee client not configured\n"); @@ -315,7 +327,11 @@ void CV1::eraseMember(const uint64_t networkId, const uint64_t memberId) _memberChanged(tmp.first, nullJson, true); } -void CV1::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch) +void CV1::nodeIsOnline( + const uint64_t networkId, + const uint64_t memberId, + const InetAddress& physicalAddress, + const char* osArch) { auto provider = opentelemetry::trace::Provider::GetTracerProvider(); auto tracer = provider->GetTracer("cv1"); @@ -380,15 +396,17 @@ AuthInfo CV1::getSSOAuthInfo(const nlohmann::json& member, const std::string& re std::string nonce = ""; // check if the member exists first. - pqxx::row count = w.exec_params1("SELECT count(id) FROM ztc_member WHERE id = $1 AND network_id = $2 AND deleted = false", memberId, networkId); + pqxx::row count = w.exec_params1( + "SELECT count(id) FROM ztc_member WHERE id = $1 AND network_id = $2 AND deleted = false", memberId, + networkId); if (count[0].as() == 1) { // get active nonce, if exists. pqxx::result r = w.exec_params( "SELECT nonce FROM ztc_sso_expiry " "WHERE network_id = $1 AND member_id = $2 " - "AND ((NOW() AT TIME ZONE 'UTC') <= authentication_expiry_time) AND ((NOW() AT TIME ZONE 'UTC') <= nonce_expiration)", - networkId, - memberId); + "AND ((NOW() AT TIME ZONE 'UTC') <= authentication_expiry_time) AND ((NOW() AT TIME ZONE 'UTC') <= " + "nonce_expiration)", + networkId, memberId); if (r.size() == 0) { // no active nonce. @@ -397,8 +415,7 @@ AuthInfo CV1::getSSOAuthInfo(const nlohmann::json& member, const std::string& re "SELECT nonce FROM ztc_sso_expiry " "WHERE network_id = $1 AND member_id = $2 " "AND authentication_expiry_time IS NULL AND ((NOW() AT TIME ZONE 'UTC') <= nonce_expiration)", - networkId, - memberId); + networkId, memberId); if (r.size() == 1) { // we have an existing nonce. Use it @@ -416,10 +433,7 @@ AuthInfo CV1::getSSOAuthInfo(const nlohmann::json& member, const std::string& re "INSERT INTO ztc_sso_expiry " "(nonce, nonce_expiration, network_id, member_id) VALUES " "($1, TO_TIMESTAMP($2::double precision/1000), $3, $4)", - nonce, - OSUtils::now() + 300000, - networkId, - memberId); + nonce, OSUtils::now() + 300000, networkId, memberId); w.commit(); } @@ -465,7 +479,9 @@ AuthInfo CV1::getSSOAuthInfo(const nlohmann::json& member, const std::string& re sso_version = r.at(0)[4].as >().value_or(1); } else if (r.size() > 1) { - fprintf(stderr, "ERROR: More than one auth endpoint for an organization?!?!? NetworkID: %s\n", networkId.c_str()); + fprintf( + stderr, "ERROR: More than one auth endpoint for an organization?!?!? NetworkID: %s\n", + networkId.c_str()); } else { fprintf(stderr, "No client or auth endpoint?!?\n"); @@ -483,13 +499,10 @@ AuthInfo CV1::getSSOAuthInfo(const nlohmann::json& member, const std::string& re if (info.version == 0) { char url[2048] = { 0 }; OSUtils::ztsnprintf( - url, - sizeof(authenticationURL), - "%s?response_type=id_token&response_mode=form_post&scope=openid+email+profile&redirect_uri=%s&nonce=%s&state=%s&client_id=%s", - authorization_endpoint.c_str(), - url_encode(redirectURL).c_str(), - nonce.c_str(), - state_hex, + url, sizeof(authenticationURL), + "%s?response_type=id_token&response_mode=form_post&scope=openid+email+profile&redirect_uri=%s&" + "nonce=%s&state=%s&client_id=%s", + authorization_endpoint.c_str(), url_encode(redirectURL).c_str(), nonce.c_str(), state_hex, client_id.c_str()); info.authenticationURL = std::string(url); } @@ -503,18 +516,17 @@ AuthInfo CV1::getSSOAuthInfo(const nlohmann::json& member, const std::string& re #ifdef ZT_DEBUG fprintf( stderr, - "ssoClientID: %s\nissuerURL: %s\nssoNonce: %s\nssoState: %s\ncentralAuthURL: %s\nprovider: %s\n", - info.ssoClientID.c_str(), - info.issuerURL.c_str(), - info.ssoNonce.c_str(), - info.ssoState.c_str(), - info.centralAuthURL.c_str(), - provider.c_str()); + "ssoClientID: %s\nissuerURL: %s\nssoNonce: %s\nssoState: %s\ncentralAuthURL: %s\nprovider: " + "%s\n", + info.ssoClientID.c_str(), info.issuerURL.c_str(), info.ssoNonce.c_str(), info.ssoState.c_str(), + info.centralAuthURL.c_str(), provider.c_str()); #endif } } else { - fprintf(stderr, "client_id: %s\nauthorization_endpoint: %s\n", client_id.c_str(), authorization_endpoint.c_str()); + fprintf( + stderr, "client_id: %s\nauthorization_endpoint: %s\n", client_id.c_str(), + authorization_endpoint.c_str()); } } @@ -560,13 +572,18 @@ void CV1::initializeNetworks() char qbuf[2048] = { 0 }; sprintf( qbuf, - "SELECT n.id, (EXTRACT(EPOCH FROM n.creation_time AT TIME ZONE 'UTC')*1000)::bigint as creation_time, n.capabilities, " - "n.enable_broadcast, (EXTRACT(EPOCH FROM n.last_modified AT TIME ZONE 'UTC')*1000)::bigint AS last_modified, n.mtu, n.multicast_limit, n.name, n.private, n.remote_trace_level, " - "n.remote_trace_target, n.revision, n.rules, n.tags, n.v4_assign_mode, n.v6_assign_mode, n.sso_enabled, (CASE WHEN n.sso_enabled THEN noc.client_id ELSE NULL END) as client_id, " + "SELECT n.id, (EXTRACT(EPOCH FROM n.creation_time AT TIME ZONE 'UTC')*1000)::bigint as creation_time, " + "n.capabilities, " + "n.enable_broadcast, (EXTRACT(EPOCH FROM n.last_modified AT TIME ZONE 'UTC')*1000)::bigint AS " + "last_modified, n.mtu, n.multicast_limit, n.name, n.private, n.remote_trace_level, " + "n.remote_trace_target, n.revision, n.rules, n.tags, n.v4_assign_mode, n.v6_assign_mode, n.sso_enabled, " + "(CASE WHEN n.sso_enabled THEN noc.client_id ELSE NULL END) as client_id, " "(CASE WHEN n.sso_enabled THEN oc.authorization_endpoint ELSE NULL END) as authorization_endpoint, " "(CASE WHEN n.sso_enabled THEN oc.provider ELSE NULL END) as provider, d.domain, d.servers, " - "ARRAY(SELECT CONCAT(host(ip_range_start),'|', host(ip_range_end)) FROM ztc_network_assignment_pool WHERE network_id = n.id) AS assignment_pool, " - "ARRAY(SELECT CONCAT(host(address),'/',bits::text,'|',COALESCE(host(via), 'NULL'))FROM ztc_network_route WHERE network_id = n.id) AS routes " + "ARRAY(SELECT CONCAT(host(ip_range_start),'|', host(ip_range_end)) FROM ztc_network_assignment_pool WHERE " + "network_id = n.id) AS assignment_pool, " + "ARRAY(SELECT CONCAT(host(address),'/',bits::text,'|',COALESCE(host(via), 'NULL'))FROM ztc_network_route " + "WHERE network_id = n.id) AS routes " "FROM ztc_network n " "LEFT OUTER JOIN ztc_org o " " ON o.owner_id = n.owner_id " @@ -800,7 +817,9 @@ void CV1::initializeNetworks() if (++this->_ready == 2) { if (_waitNoticePrinted) { - fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), (unsigned long long)_myAddress.toInt()); + fprintf( + stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), + (unsigned long long)_myAddress.toInt()); } _readyLock.unlock(); } @@ -886,12 +905,14 @@ void CV1::initializeMembers() " FROM ztc_sso_expiry e " " INNER JOIN ztc_network n1 " " ON n1.id = e.network_id AND n1.deleted = TRUE " - " WHERE e.network_id = m.network_id AND e.member_id = m.id AND n.sso_enabled = TRUE AND e.authentication_expiry_time IS NOT NULL " + " WHERE e.network_id = m.network_id AND e.member_id = m.id AND n.sso_enabled = TRUE AND " + "e.authentication_expiry_time IS NOT NULL " " ORDER BY e.authentication_expiry_time DESC LIMIT 1 " " ) " " ELSE NULL " " END) AS authentication_expiry_time, " - "ARRAY(SELECT DISTINCT address FROM ztc_member_ip_assignment WHERE member_id = m.id AND network_id = m.network_id) AS assigned_addresses " + "ARRAY(SELECT DISTINCT address FROM ztc_member_ip_assignment WHERE member_id = m.id AND network_id = " + "m.network_id) AS assigned_addresses " "FROM ztc_member m " "INNER JOIN ztc_network n " " ON n.id = m.network_id " @@ -1076,7 +1097,9 @@ void CV1::initializeMembers() if (++this->_ready == 2) { if (_waitNoticePrinted) { - fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), (unsigned long long)_myAddress.toInt()); + fprintf( + stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), + (unsigned long long)_myAddress.toInt()); } _readyLock.unlock(); } @@ -1137,13 +1160,17 @@ void CV1::heartbeat() pqxx::work w { *c->c }; pqxx::result res = w.exec0( - "INSERT INTO ztc_controller (id, cluster_host, last_alive, public_identity, v_major, v_minor, v_rev, v_build, host_port, use_redis, redis_member_status) " + "INSERT INTO ztc_controller (id, cluster_host, last_alive, public_identity, v_major, v_minor, " + "v_rev, v_build, host_port, use_redis, redis_member_status) " "VALUES (" - + w.quote(controllerId) + ", " + w.quote(hostname) + ", TO_TIMESTAMP(" + now + "::double precision/1000), " + w.quote(publicIdentity) + ", " + major + ", " + minor + ", " + rev + ", " + build + ", " + host_port + ", " - + use_redis + ", " + redis_mem_status + + w.quote(controllerId) + ", " + w.quote(hostname) + ", TO_TIMESTAMP(" + now + + "::double precision/1000), " + w.quote(publicIdentity) + ", " + major + ", " + minor + ", " + rev + + ", " + build + ", " + host_port + ", " + use_redis + ", " + redis_mem_status + ") " - "ON CONFLICT (id) DO UPDATE SET cluster_host = EXCLUDED.cluster_host, last_alive = EXCLUDED.last_alive, " - "public_identity = EXCLUDED.public_identity, v_major = EXCLUDED.v_major, v_minor = EXCLUDED.v_minor, " + "ON CONFLICT (id) DO UPDATE SET cluster_host = EXCLUDED.cluster_host, last_alive = " + "EXCLUDED.last_alive, " + "public_identity = EXCLUDED.public_identity, v_major = EXCLUDED.v_major, v_minor = " + "EXCLUDED.v_minor, " "v_rev = EXCLUDED.v_rev, v_build = EXCLUDED.v_rev, host_port = EXCLUDED.host_port, " "use_redis = EXCLUDED.use_redis, redis_member_status = EXCLUDED.redis_member_status"); w.commit(); @@ -1187,7 +1214,8 @@ void CV1::membersDbWatcher() } if (_run == 1) { - fprintf(stderr, "ERROR: %s membersDbWatcher should still be running! Exiting Controller.\n", _myAddressStr.c_str()); + fprintf( + stderr, "ERROR: %s membersDbWatcher should still be running! Exiting Controller.\n", _myAddressStr.c_str()); exit(9); } fprintf(stderr, "Exited membersDbWatcher\n"); @@ -1293,7 +1321,9 @@ void CV1::networksDbWatcher() } if (_run == 1) { - fprintf(stderr, "ERROR: %s networksDbWatcher should still be running! Exiting Controller.\n", _myAddressStr.c_str()); + fprintf( + stderr, "ERROR: %s networksDbWatcher should still be running! Exiting Controller.\n", + _myAddressStr.c_str()); exit(8); } fprintf(stderr, "Exited networksDbWatcher\n"); @@ -1370,7 +1400,9 @@ void CV1::_networksWatcher_Redis() } } catch (std::exception& e) { - fprintf(stderr, "json parse error in networkWatcher_Redis: what: %s json: %s\n", e.what(), a.second.c_str()); + fprintf( + stderr, "json parse error in networkWatcher_Redis: what: %s json: %s\n", e.what(), + a.second.c_str()); } } if (_rc->clusterMode) { @@ -1455,7 +1487,8 @@ void CV1::commitThread() continue; } - pqxx::row mrow = w.exec_params1("SELECT COUNT(id) FROM ztc_member WHERE id = $1 AND network_id = $2", memberId, networkId); + pqxx::row mrow = w.exec_params1( + "SELECT COUNT(id) FROM ztc_member WHERE id = $1 AND network_id = $2", memberId, networkId); int membercount = mrow[0].as(); bool isNewMember = false; @@ -1465,27 +1498,17 @@ void CV1::commitThread() pqxx::result res = w.exec_params0( "INSERT INTO ztc_member (id, network_id, active_bridge, authorized, capabilities, " "identity, last_authorized_time, last_deauthorized_time, no_auto_assign_ips, " - "remote_trace_level, remote_trace_target, revision, tags, v_major, v_minor, v_rev, v_proto) " + "remote_trace_level, remote_trace_target, revision, tags, v_major, v_minor, v_rev, " + "v_proto) " "VALUES ($1, $2, $3, $4, $5, $6, " "TO_TIMESTAMP($7::double precision/1000), TO_TIMESTAMP($8::double precision/1000), " "$9, $10, $11, $12, $13, $14, $15, $16, $17)", - memberId, - networkId, - (bool)config["activeBridge"], - (bool)config["authorized"], - OSUtils::jsonDump(config["capabilities"], -1), - OSUtils::jsonString(config["identity"], ""), - (uint64_t)config["lastAuthorizedTime"], - (uint64_t)config["lastDeauthorizedTime"], - (bool)config["noAutoAssignIps"], - (int)config["remoteTraceLevel"], - target, - (uint64_t)config["revision"], - OSUtils::jsonDump(config["tags"], -1), - (int)config["vMajor"], - (int)config["vMinor"], - (int)config["vRev"], - (int)config["vProto"]); + memberId, networkId, (bool)config["activeBridge"], (bool)config["authorized"], + OSUtils::jsonDump(config["capabilities"], -1), OSUtils::jsonString(config["identity"], ""), + (uint64_t)config["lastAuthorizedTime"], (uint64_t)config["lastDeauthorizedTime"], + (bool)config["noAutoAssignIps"], (int)config["remoteTraceLevel"], target, + (uint64_t)config["revision"], OSUtils::jsonDump(config["tags"], -1), (int)config["vMajor"], + (int)config["vMinor"], (int)config["vRev"], (int)config["vProto"]); } else { // existing member @@ -1497,27 +1520,18 @@ void CV1::commitThread() "no_auto_assign_ips = $9, remote_trace_level = $10, remote_trace_target= $11, " "revision = $12, tags = $13, v_major = $14, v_minor = $15, v_rev = $16, v_proto = $17 " "WHERE id = $1 AND network_id = $2", - memberId, - networkId, - (bool)config["activeBridge"], - (bool)config["authorized"], - OSUtils::jsonDump(config["capabilities"], -1), - OSUtils::jsonString(config["identity"], ""), - (uint64_t)config["lastAuthorizedTime"], - (uint64_t)config["lastDeauthorizedTime"], - (bool)config["noAutoAssignIps"], - (int)config["remoteTraceLevel"], - target, - (uint64_t)config["revision"], - OSUtils::jsonDump(config["tags"], -1), - (int)config["vMajor"], - (int)config["vMinor"], - (int)config["vRev"], - (int)config["vProto"]); + memberId, networkId, (bool)config["activeBridge"], (bool)config["authorized"], + OSUtils::jsonDump(config["capabilities"], -1), OSUtils::jsonString(config["identity"], ""), + (uint64_t)config["lastAuthorizedTime"], (uint64_t)config["lastDeauthorizedTime"], + (bool)config["noAutoAssignIps"], (int)config["remoteTraceLevel"], target, + (uint64_t)config["revision"], OSUtils::jsonDump(config["tags"], -1), (int)config["vMajor"], + (int)config["vMinor"], (int)config["vRev"], (int)config["vProto"]); } if (! isNewMember) { - pqxx::result res = w.exec_params0("DELETE FROM ztc_member_ip_assignment WHERE member_id = $1 AND network_id = $2", memberId, networkId); + pqxx::result res = w.exec_params0( + "DELETE FROM ztc_member_ip_assignment WHERE member_id = $1 AND network_id = $2", memberId, + networkId); } std::vector assignments; @@ -1529,7 +1543,10 @@ void CV1::commitThread() continue; } - pqxx::result res = w.exec_params0("INSERT INTO ztc_member_ip_assignment (member_id, network_id, address) VALUES ($1, $2, $3) ON CONFLICT (network_id, member_id, address) DO NOTHING", memberId, networkId, addr); + pqxx::result res = w.exec_params0( + "INSERT INTO ztc_member_ip_assignment (member_id, network_id, address) VALUES ($1, $2, $3) " + "ON CONFLICT (network_id, member_id, address) DO NOTHING", + memberId, networkId, addr); assignments.push_back(addr); } @@ -1573,11 +1590,15 @@ void CV1::commitThread() _memberChanged(memOrig, memNew, qitem.second); } else { - fprintf(stderr, "%s: Can't notify of change. Error parsing nwid or memberid: %llu-%llu\n", _myAddressStr.c_str(), (unsigned long long)nwidInt, (unsigned long long)memberidInt); + fprintf( + stderr, "%s: Can't notify of change. Error parsing nwid or memberid: %llu-%llu\n", + _myAddressStr.c_str(), (unsigned long long)nwidInt, (unsigned long long)memberidInt); } } catch (std::exception& e) { - fprintf(stderr, "%s ERROR: Error updating member %s-%s: %s\n", _myAddressStr.c_str(), networkId.c_str(), memberId.c_str(), e.what()); + fprintf( + stderr, "%s ERROR: Error updating member %s-%s: %s\n", _myAddressStr.c_str(), networkId.c_str(), + memberId.c_str(), e.what()); mspan->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); } } @@ -1606,12 +1627,14 @@ void CV1::commitThread() // did not previously exist. If the record already exists owner_id is left // unchanged, so owner_id should be left out of the update clause. pqxx::result res = w.exec_params0( - "INSERT INTO ztc_network (id, creation_time, owner_id, controller_id, capabilities, enable_broadcast, " + "INSERT INTO ztc_network (id, creation_time, owner_id, controller_id, capabilities, " + "enable_broadcast, " "last_modified, mtu, multicast_limit, name, private, " "remote_trace_level, remote_trace_target, rules, rules_source, " "tags, v4_assign_mode, v6_assign_mode, sso_enabled) VALUES (" "$1, TO_TIMESTAMP($5::double precision/1000), " - "(SELECT user_id AS owner_id FROM ztc_global_permissions WHERE authorize = true AND del = true AND modify = true AND read = true LIMIT 1)," + "(SELECT user_id AS owner_id FROM ztc_global_permissions WHERE authorize = true AND del = true " + "AND modify = true AND read = true LIMIT 1)," "$2, $3, $4, TO_TIMESTAMP($5::double precision/1000), " "$6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) " "ON CONFLICT (id) DO UPDATE set controller_id = EXCLUDED.controller_id, " @@ -1623,25 +1646,15 @@ void CV1::commitThread() "rules_source = EXCLUDED.rules_source, tags = EXCLUDED.tags, " "v4_assign_mode = EXCLUDED.v4_assign_mode, v6_assign_mode = EXCLUDED.v6_assign_mode, " "sso_enabled = EXCLUDED.sso_enabled", - id, - _myAddressStr, - OSUtils::jsonDump(config["capabilities"], -1), - (bool)config["enableBroadcast"], - OSUtils::now(), - (int)config["mtu"], - (int)config["multicastLimit"], - OSUtils::jsonString(config["name"], ""), - (bool)config["private"], - (int)config["remoteTraceLevel"], - remoteTraceTarget, - OSUtils::jsonDump(config["rules"], -1), - rulesSource, - OSUtils::jsonDump(config["tags"], -1), - OSUtils::jsonDump(config["v4AssignMode"], -1), - OSUtils::jsonDump(config["v6AssignMode"], -1), + id, _myAddressStr, OSUtils::jsonDump(config["capabilities"], -1), + (bool)config["enableBroadcast"], OSUtils::now(), (int)config["mtu"], + (int)config["multicastLimit"], OSUtils::jsonString(config["name"], ""), (bool)config["private"], + (int)config["remoteTraceLevel"], remoteTraceTarget, OSUtils::jsonDump(config["rules"], -1), + rulesSource, OSUtils::jsonDump(config["tags"], -1), + OSUtils::jsonDump(config["v4AssignMode"], -1), OSUtils::jsonDump(config["v6AssignMode"], -1), OSUtils::jsonBool(config["ssoEnabled"], false)); - res = w.exec_params0("DELETE FROM ztc_network_assignment_pool WHERE network_id = $1", 0); + res = w.exec_params0("DELETE FROM ztc_network_assignment_pool WHERE network_id = $1", id); auto pool = config["ipAssignmentPools"]; bool err = false; @@ -1652,9 +1665,7 @@ void CV1::commitThread() res = w.exec_params0( "INSERT INTO ztc_network_assignment_pool (network_id, ip_range_start, ip_range_end) " "VALUES ($1, $2, $3)", - id, - start, - end); + id, start, end); } res = w.exec_params0("DELETE FROM ztc_network_route WHERE network_id = $1", id); @@ -1679,7 +1690,9 @@ void CV1::commitThread() via = (*i)["via"]; } - res = w.exec_params0("INSERT INTO ztc_network_route (network_id, address, bits, via) VALUES ($1, $2, $3, $4)", id, targetAddr, targetBits, (via == "NULL" ? NULL : via.c_str())); + res = w.exec_params0( + "INSERT INTO ztc_network_route (network_id, address, bits, via) VALUES ($1, $2, $3, $4)", + id, targetAddr, targetBits, (via == "NULL" ? NULL : via.c_str())); } if (err) { fprintf(stderr, "%s: route add error\n", _myAddressStr.c_str()); @@ -1702,7 +1715,10 @@ void CV1::commitThread() std::string s = servers.str(); - res = w.exec_params0("INSERT INTO ztc_network_dns (network_id, domain, servers) VALUES ($1, $2, $3) ON CONFLICT (network_id) DO UPDATE SET domain = EXCLUDED.domain, servers = EXCLUDED.servers", id, domain, s); + res = w.exec_params0( + "INSERT INTO ztc_network_dns (network_id, domain, servers) VALUES ($1, $2, $3) ON CONFLICT " + "(network_id) DO UPDATE SET domain = EXCLUDED.domain, servers = EXCLUDED.servers", + id, domain, s); w.commit(); @@ -1716,7 +1732,9 @@ void CV1::commitThread() _networkChanged(nwOrig, nwNew, qitem.second); } else { - fprintf(stderr, "%s: Can't notify network changed: %llu\n", _myAddressStr.c_str(), (unsigned long long)nwidInt); + fprintf( + stderr, "%s: Can't notify network changed: %llu\n", _myAddressStr.c_str(), + (unsigned long long)nwidInt); } } catch (std::exception& e) { @@ -1790,7 +1808,9 @@ void CV1::commitThread() std::string memberId = config["id"]; std::string networkId = config["nwid"]; - pqxx::result res = w.exec_params0("UPDATE ztc_member SET hidden = true, deleted = true WHERE id = $1 AND network_id = $2", memberId, networkId); + pqxx::result res = w.exec_params0( + "UPDATE ztc_member SET hidden = true, deleted = true WHERE id = $1 AND network_id = $2", + memberId, networkId); w.commit(); } @@ -1841,7 +1861,7 @@ void CV1::notifyNewMember(const std::string& networkID, const std::string& membe auto span = tracer->StartSpan("cv1::notifyNewMember"); auto scope = tracer->WithActiveSpan(span); - smeeclient::smee_client_notify_network_joined(_smee, networkID.c_str(), memberID.c_str()); + rustybits::smee_client_notify_network_joined(_smee, networkID.c_str(), memberID.c_str()); } void CV1::onlineNotificationThread() @@ -1914,7 +1934,8 @@ void CV1::onlineNotification_Postgres() std::string memberId(memTmp); try { - pqxx::row r = w2.exec_params1("SELECT id, network_id FROM ztc_member WHERE network_id = $1 AND id = $2", networkId, memberId); + pqxx::row r = w2.exec_params1( + "SELECT id, network_id FROM ztc_member WHERE network_id = $1 AND id = $2", networkId, memberId); } catch (pqxx::unexpected_rows& e) { continue; @@ -1933,8 +1954,9 @@ void CV1::onlineNotification_Postgres() } std::stringstream memberUpdate; - memberUpdate << "INSERT INTO ztc_member_status (network_id, member_id, address, last_updated, os, arch) VALUES " - << "('" << networkId << "', '" << memberId << "', "; + memberUpdate + << "INSERT INTO ztc_member_status (network_id, member_id, address, last_updated, os, arch) VALUES " + << "('" << networkId << "', '" << memberId << "', "; if (ipAddr.empty()) { memberUpdate << "NULL, "; } @@ -1945,7 +1967,8 @@ void CV1::onlineNotification_Postgres() << "'" << os << "', " << "'" << arch << "'" << ") " - << " ON CONFLICT (network_id, member_id) DO UPDATE SET address = EXCLUDED.address, last_updated = EXCLUDED.last_updated, " + << " ON CONFLICT (network_id, member_id) DO UPDATE SET address = EXCLUDED.address, " + "last_updated = EXCLUDED.last_updated, " << "os = EXCLUDED.os, arch = EXCLUDED.arch"; pipe.insert(memberUpdate.str()); @@ -1967,7 +1990,9 @@ void CV1::onlineNotification_Postgres() _pool->unborrow(c); ConnectionPoolStats stats = _pool->get_stats(); - fprintf(stderr, "%s pool stats: in use size: %llu, available size: %llu, total: %llu\n", _myAddressStr.c_str(), stats.borrowed_size, stats.pool_size, (stats.borrowed_size + stats.pool_size)); + fprintf( + stderr, "%s pool stats: in use size: %llu, available size: %llu, total: %llu\n", _myAddressStr.c_str(), + stats.borrowed_size, stats.pool_size, (stats.borrowed_size + stats.pool_size)); span->End(); @@ -1975,7 +2000,9 @@ void CV1::onlineNotification_Postgres() } fprintf(stderr, "%s: Fell out of run loop in onlineNotificationThread\n", _myAddressStr.c_str()); if (_run == 1) { - fprintf(stderr, "ERROR: %s onlineNotificationThread should still be running! Exiting Controller.\n", _myAddressStr.c_str()); + fprintf( + stderr, "ERROR: %s onlineNotificationThread should still be running! Exiting Controller.\n", + _myAddressStr.c_str()); exit(6); } } @@ -2029,7 +2056,10 @@ void CV1::onlineNotification_Redis() } } -uint64_t CV1::_doRedisUpdate(sw::redis::Transaction& tx, std::string& controllerId, std::unordered_map, NodeOnlineRecord, _PairHasher>& lastOnline) +uint64_t CV1::_doRedisUpdate( + sw::redis::Transaction& tx, + std::string& controllerId, + std::unordered_map, NodeOnlineRecord, _PairHasher>& lastOnline) { auto provider = opentelemetry::trace::Provider::GetTracerProvider(); auto tracer = provider->GetTracer("cv1"); @@ -2066,7 +2096,11 @@ uint64_t CV1::_doRedisUpdate(sw::redis::Transaction& tx, std::string& controller arch = osArchSplit[1]; } - std::unordered_map record = { { "id", memberId }, { "address", ipAddr }, { "last_updated", std::to_string(ts) }, { "os", os }, { "arch", arch } }; + std::unordered_map record = { { "id", memberId }, + { "address", ipAddr }, + { "last_updated", std::to_string(ts) }, + { "os", os }, + { "arch", arch } }; tx.zadd("nodes-online:{" + controllerId + "}", memberId, ts) .zadd("nodes-online2:{" + controllerId + "}", networkId + "-" + memberId, ts) .zadd("network-nodes-online:{" + controllerId + "}:" + networkId, memberId, ts) @@ -2080,16 +2114,24 @@ uint64_t CV1::_doRedisUpdate(sw::redis::Transaction& tx, std::string& controller // expire records from all-nodes and network-nodes member list uint64_t expireOld = OSUtils::now() - 300000; - tx.zremrangebyscore("nodes-online:{" + controllerId + "}", sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); - tx.zremrangebyscore("nodes-online2:{" + controllerId + "}", sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); - tx.zremrangebyscore("active-networks:{" + controllerId + "}", sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); + tx.zremrangebyscore( + "nodes-online:{" + controllerId + "}", + sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); + tx.zremrangebyscore( + "nodes-online2:{" + controllerId + "}", + sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); + tx.zremrangebyscore( + "active-networks:{" + controllerId + "}", + sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); { std::shared_lock l(_networks_l); for (const auto& it : _networks) { uint64_t nwid_i = it.first; char nwidTmp[64]; OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i); - tx.zremrangebyscore("network-nodes-online:{" + controllerId + "}:" + nwidTmp, sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); + tx.zremrangebyscore( + "network-nodes-online:{" + controllerId + "}:" + nwidTmp, + sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); } } tx.exec(); diff --git a/nonfree/controller/CV1.hpp b/nonfree/controller/CV1.hpp index 4819cde378..2552be3588 100644 --- a/nonfree/controller/CV1.hpp +++ b/nonfree/controller/CV1.hpp @@ -11,15 +11,15 @@ #define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4 -#include "../node/Metrics.hpp" +#include "../../node/Metrics.hpp" #include "ConnectionPool.hpp" #include "PostgreSQL.hpp" #include #include -#include +#include -namespace smeeclient { +namespace rustybits { struct SmeeClient; } @@ -47,7 +47,11 @@ class CV1 : public DB { virtual void eraseNetwork(const uint64_t networkId); virtual void eraseMember(const uint64_t networkId, const uint64_t memberId); virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress); - virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch); + virtual void nodeIsOnline( + const uint64_t networkId, + const uint64_t memberId, + const InetAddress& physicalAddress, + const char* osArch); virtual AuthInfo getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL); virtual bool ready() @@ -55,13 +59,6 @@ class CV1 : public DB { return _ready == 2; } - protected: - struct _PairHasher { - inline std::size_t operator()(const std::pair& p) const - { - return (std::size_t)(p.first ^ p.second); - } - }; virtual void _memberChanged(nlohmann::json& old, nlohmann::json& memberConfig, bool notifyListeners) { DB::_memberChanged(old, memberConfig, notifyListeners); @@ -72,6 +69,14 @@ class CV1 : public DB { DB::_networkChanged(old, networkConfig, notifyListeners); } + protected: + struct _PairHasher { + inline std::size_t operator()(const std::pair& p) const + { + return (std::size_t)(p.first ^ p.second); + } + }; + private: void initializeNetworks(); void initializeMembers(); @@ -88,7 +93,10 @@ class CV1 : public DB { void onlineNotificationThread(); void onlineNotification_Postgres(); void onlineNotification_Redis(); - uint64_t _doRedisUpdate(sw::redis::Transaction& tx, std::string& controllerId, std::unordered_map, NodeOnlineRecord, _PairHasher>& lastOnline); + uint64_t _doRedisUpdate( + sw::redis::Transaction& tx, + std::string& controllerId, + std::unordered_map, NodeOnlineRecord, _PairHasher>& lastOnline); void configureSmee(); void notifyNewMember(const std::string& networkID, const std::string& memberID); @@ -125,7 +133,7 @@ class CV1 : public DB { std::shared_ptr _cluster; bool _redisMemberStatus; - smeeclient::SmeeClient* _smee; + rustybits::SmeeClient* _smee; }; } // namespace ZeroTier diff --git a/nonfree/controller/CV2.cpp b/nonfree/controller/CV2.cpp index 8b83707721..8e2d6cdf97 100644 --- a/nonfree/controller/CV2.cpp +++ b/nonfree/controller/CV2.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include using json = nlohmann::json; @@ -27,13 +28,24 @@ namespace { using namespace ZeroTier; -CV2::CV2(const Identity& myId, const char* path, int listenPort) : DB(), _pool(), _myId(myId), _myAddress(myId.address()), _ready(0), _connected(1), _run(1), _waitNoticePrinted(false), _listenPort(listenPort) +CV2::CV2(const Identity& myId, const char* path, int listenPort) + : DB() + , _pool() + , _myId(myId) + , _myAddress(myId.address()) + , _ready(0) + , _connected(1) + , _run(1) + , _waitNoticePrinted(false) + , _listenPort(listenPort) { auto provider = opentelemetry::trace::Provider::GetTracerProvider(); auto tracer = provider->GetTracer("cv2"); auto span = tracer->StartSpan("cv2::CV2"); auto scope = tracer->WithActiveSpan(span); + rustybits::init_async_runtime(); + fprintf(stderr, "CV2::CV2\n"); char myAddress[64]; _myAddressStr = myId.address().toString(myAddress); @@ -41,7 +53,8 @@ CV2::CV2(const Identity& myId, const char* path, int listenPort) : DB(), _pool() _connString = std::string(path); auto f = std::make_shared(_connString); - _pool = std::make_shared >(15, 5, std::static_pointer_cast(f)); + _pool = + std::make_shared >(15, 5, std::static_pointer_cast(f)); memset(_ssoPsk, 0, sizeof(_ssoPsk)); char* const ssoPskHex = getenv("ZT_SSO_PSK"); @@ -57,7 +70,9 @@ CV2::CV2(const Identity& myId, const char* path, int listenPort) : DB(), _pool() _readyLock.lock(); - fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL waiting for initial data download..." ZT_EOL_S, ::_timestr(), (unsigned long long)_myAddress.toInt()); + fprintf( + stderr, "[%s] NOTICE: %.10llx controller PostgreSQL waiting for initial data download..." ZT_EOL_S, + ::_timestr(), (unsigned long long)_myAddress.toInt()); _waitNoticePrinted = true; initializeNetworks(); @@ -74,6 +89,8 @@ CV2::CV2(const Identity& myId, const char* path, int listenPort) : DB(), _pool() CV2::~CV2() { + rustybits::shutdown_async_runtime(); + _run = 0; std::this_thread::sleep_for(std::chrono::milliseconds(100)); @@ -224,7 +241,11 @@ void CV2::eraseMember(const uint64_t networkId, const uint64_t memberId) //_memberChanged(tmp.first, nullJson, isReady()); } -void CV2::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch) +void CV2::nodeIsOnline( + const uint64_t networkId, + const uint64_t memberId, + const InetAddress& physicalAddress, + const char* osArch) { auto provider = opentelemetry::trace::Provider::GetTracerProvider(); auto tracer = provider->GetTracer("cv2"); @@ -290,21 +311,21 @@ AuthInfo CV2::getSSOAuthInfo(const nlohmann::json& member, const std::string& re // std::string nonce = ""; // // check if the member exists first. - // pqxx::row count = w.exec_params1("SELECT count(id) FROM ztc_member WHERE id = $1 AND network_id = $2 AND deleted = false", memberId, networkId); - // if (count[0].as() == 1) { + // pqxx::row count = w.exec_params1("SELECT count(id) FROM ztc_member WHERE id = $1 AND network_id = $2 AND + // deleted = false", memberId, networkId); if (count[0].as() == 1) { // // get active nonce, if exists. // pqxx::result r = w.exec_params("SELECT nonce FROM ztc_sso_expiry " // "WHERE network_id = $1 AND member_id = $2 " - // "AND ((NOW() AT TIME ZONE 'UTC') <= authentication_expiry_time) AND ((NOW() AT TIME ZONE 'UTC') <= nonce_expiration)", - // networkId, memberId); + // "AND ((NOW() AT TIME ZONE 'UTC') <= authentication_expiry_time) AND ((NOW() AT TIME ZONE 'UTC') + // <= nonce_expiration)", networkId, memberId); // if (r.size() == 0) { // // no active nonce. // // find an unused nonce, if one exists. // pqxx::result r = w.exec_params("SELECT nonce FROM ztc_sso_expiry " // "WHERE network_id = $1 AND member_id = $2 " - // "AND authentication_expiry_time IS NULL AND ((NOW() AT TIME ZONE 'UTC') <= nonce_expiration)", - // networkId, memberId); + // "AND authentication_expiry_time IS NULL AND ((NOW() AT TIME ZONE 'UTC') <= + // nonce_expiration)", networkId, memberId); // if (r.size() == 1) { // // we have an existing nonce. Use it @@ -361,15 +382,14 @@ AuthInfo CV2::getSSOAuthInfo(const nlohmann::json& member, const std::string& re // provider = r.at(0)[3].as>().value_or(""); // sso_version = r.at(0)[4].as>().value_or(1); // } else if (r.size() > 1) { - // fprintf(stderr, "ERROR: More than one auth endpoint for an organization?!?!? NetworkID: %s\n", networkId.c_str()); - // } else { - // fprintf(stderr, "No client or auth endpoint?!?\n"); + // fprintf(stderr, "ERROR: More than one auth endpoint for an organization?!?!? NetworkID: %s\n", + // networkId.c_str()); } else { fprintf(stderr, "No client or auth endpoint?!?\n"); // } // info.version = sso_version; - // // no catch all else because we don't actually care if no records exist here. just continue as normal. - // if ((!client_id.empty())&&(!authorization_endpoint.empty())) { + // // no catch all else because we don't actually care if no records exist here. just continue as + // normal. if ((!client_id.empty())&&(!authorization_endpoint.empty())) { // uint8_t state[48]; // HMACSHA384(_ssoPsk, nonceBytes, sizeof(nonceBytes), state); @@ -396,17 +416,16 @@ AuthInfo CV2::getSSOAuthInfo(const nlohmann::json& member, const std::string& re // #ifdef ZT_DEBUG // fprintf( // stderr, - // "ssoClientID: %s\nissuerURL: %s\nssoNonce: %s\nssoState: %s\ncentralAuthURL: %s\nprovider: %s\n", - // info.ssoClientID.c_str(), - // info.issuerURL.c_str(), - // info.ssoNonce.c_str(), + // "ssoClientID: %s\nissuerURL: %s\nssoNonce: %s\nssoState: %s\ncentralAuthURL: + // %s\nprovider: %s\n", info.ssoClientID.c_str(), info.issuerURL.c_str(), info.ssoNonce.c_str(), // info.ssoState.c_str(), // info.centralAuthURL.c_str(), // provider.c_str()); // #endif // } // } else { - // fprintf(stderr, "client_id: %s\nauthorization_endpoint: %s\n", client_id.c_str(), authorization_endpoint.c_str()); + // fprintf(stderr, "client_id: %s\nauthorization_endpoint: %s\n", client_id.c_str(), + // authorization_endpoint.c_str()); // } // } @@ -480,12 +499,16 @@ void CV2::initializeNetworks() config["lastModified"] = last_modified.value_or(0); config["revision"] = revision.value_or(0); config["capabilities"] = cfgtmp["capabilities"].is_array() ? cfgtmp["capabilities"] : json::array(); - config["enableBroadcast"] = cfgtmp["enableBroadcast"].is_boolean() ? cfgtmp["enableBroadcast"].get() : false; + config["enableBroadcast"] = + cfgtmp["enableBroadcast"].is_boolean() ? cfgtmp["enableBroadcast"].get() : false; config["mtu"] = cfgtmp["mtu"].is_number() ? cfgtmp["mtu"].get() : 2800; - config["multicastLimit"] = cfgtmp["multicastLimit"].is_number() ? cfgtmp["multicastLimit"].get() : 64; + config["multicastLimit"] = + cfgtmp["multicastLimit"].is_number() ? cfgtmp["multicastLimit"].get() : 64; config["private"] = cfgtmp["private"].is_boolean() ? cfgtmp["private"].get() : true; - config["remoteTraceLevel"] = cfgtmp["remoteTraceLevel"].is_number() ? cfgtmp["remoteTraceLevel"].get() : 0; - config["remoteTraceTarget"] = cfgtmp["remoteTraceTarget"].is_string() ? cfgtmp["remoteTraceTarget"].get() : ""; + config["remoteTraceLevel"] = + cfgtmp["remoteTraceLevel"].is_number() ? cfgtmp["remoteTraceLevel"].get() : 0; + config["remoteTraceTarget"] = + cfgtmp["remoteTraceTarget"].is_string() ? cfgtmp["remoteTraceTarget"].get() : ""; config["revision"] = revision.value_or(0); config["rules"] = cfgtmp["rules"].is_array() ? cfgtmp["rules"] : json::array(); config["tags"] = cfgtmp["tags"].is_array() ? cfgtmp["tags"] : json::array(); @@ -509,7 +532,9 @@ void CV2::initializeNetworks() config["objtype"] = "network"; config["routes"] = cfgtmp["routes"].is_array() ? cfgtmp["routes"] : json::array(); config["clientId"] = cfgtmp["clientId"].is_string() ? cfgtmp["clientId"].get() : ""; - config["authorizationEndpoint"] = cfgtmp["authorizationEndpoint"].is_string() ? cfgtmp["authorizationEndpoint"].get() : nullptr; + config["authorizationEndpoint"] = cfgtmp["authorizationEndpoint"].is_string() + ? cfgtmp["authorizationEndpoint"].get() + : nullptr; config["provider"] = cfgtmp["ssoProvider"].is_string() ? cfgtmp["ssoProvider"].get() : ""; if (! cfgtmp["dns"].is_object()) { cfgtmp["dns"] = json::object(); @@ -519,7 +544,8 @@ void CV2::initializeNetworks() else { config["dns"] = cfgtmp["dns"]; } - config["ipAssignmentPools"] = cfgtmp["ipAssignmentPools"].is_array() ? cfgtmp["ipAssignmentPools"] : json::array(); + config["ipAssignmentPools"] = + cfgtmp["ipAssignmentPools"].is_array() ? cfgtmp["ipAssignmentPools"] : json::array(); Metrics::network_count++; @@ -541,7 +567,9 @@ void CV2::initializeNetworks() if (++this->_ready == 2) { if (_waitNoticePrinted) { - fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), (unsigned long long)_myAddress.toInt()); + fprintf( + stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), + (unsigned long long)_myAddress.toInt()); } _readyLock.unlock(); } @@ -568,7 +596,8 @@ void CV2::initializeMembers() char qbuf[2048]; sprintf( qbuf, - "SELECT nm.device_id, nm.network_id, nm.authorized, nm.active_bridge, nm.ip_assignments, nm.no_auto_assign_ips, " + "SELECT nm.device_id, nm.network_id, nm.authorized, nm.active_bridge, nm.ip_assignments, " + "nm.no_auto_assign_ips, " "nm.sso_exempt, (EXTRACT(EPOCH FROM nm.authentication_expiry_time AT TIME ZONE 'UTC')*1000)::bigint, " "(EXTRACT(EPOCH FROM nm.creation_time AT TIME ZONE 'UTC')*1000)::bigint, nm.identity, " "(EXTRACT(EPOCH FROM nm.last_authorized_time AT TIME ZONE 'UTC')*1000)::bigint, " @@ -700,7 +729,9 @@ void CV2::initializeMembers() if (++this->_ready == 2) { if (_waitNoticePrinted) { - fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), (unsigned long long)_myAddress.toInt()); + fprintf( + stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), + (unsigned long long)_myAddress.toInt()); } _readyLock.unlock(); } @@ -754,13 +785,10 @@ void CV2::heartbeat() w.exec_params0( "INSERT INTO controllers_ctl (id, hostname, last_heartbeat, public_identity, version) VALUES " "($1, $2, TO_TIMESTAMP($3::double precision/1000), $4, $5) " - "ON CONFLICT (id) DO UPDATE SET hostname = EXCLUDED.hostname, last_heartbeat = EXCLUDED.last_heartbeat, " + "ON CONFLICT (id) DO UPDATE SET hostname = EXCLUDED.hostname, last_heartbeat = " + "EXCLUDED.last_heartbeat, " "public_identity = EXCLUDED.public_identity, version = EXCLUDED.version", - controllerId, - hostname, - ts, - publicIdentity, - versionStr); + controllerId, hostname, ts, publicIdentity, versionStr); w.commit(); } catch (std::exception& e) { @@ -871,7 +899,7 @@ void CV2::commitThread() target = config["remoteTraceTarget"]; } - pqxx::row nwrow = w.exec_params1("SELECT COUNT(id) FROM networks WHERE id = $1", networkId); + pqxx::row nwrow = w.exec_params1("SELECT COUNT(id) FROM networks_ctl WHERE id = $1", networkId); int nwcount = nwrow[0].as(); if (nwcount != 1) { @@ -882,12 +910,13 @@ void CV2::commitThread() } // only needed for hooks, and no hooks for now - // pqxx::row mrow = w.exec_params1("SELECT COUNT(id) FROM device_networks WHERE device_id = $1 AND network_id = $2", memberId, networkId); - // int membercount = mrow[0].as(); - // bool isNewMember = (membercount == 0); + // pqxx::row mrow = w.exec_params1("SELECT COUNT(id) FROM device_networks WHERE device_id = $1 AND + // network_id = $2", memberId, networkId); int membercount = mrow[0].as(); bool isNewMember = + // (membercount == 0); pqxx::result res = w.exec_params0( - "INSERT INTO network_memberships_ctl (device_id, network_id, authorized, active_bridge, ip_assignments, " + "INSERT INTO network_memberships_ctl (device_id, network_id, authorized, active_bridge, " + "ip_assignments, " "no_auto_assign_ips, sso_exempt, authentication_expiry_time, capabilities, creation_time, " "identity, last_authorized_time, last_deauthorized_time, " "remote_trace_level, remote_trace_target, revision, tags, version_major, version_minor, " @@ -898,35 +927,24 @@ void CV2::commitThread() "ON CONFLICT (device_id, network_id) DO UPDATE SET " "authorized = EXCLUDED.authorized, active_bridge = EXCLUDED.active_bridge, " "ip_assignments = EXCLUDED.ip_assignments, no_auto_assign_ips = EXCLUDED.no_auto_assign_ips, " - "sso_exempt = EXCLUDED.sso_exempt, authentication_expiry_time = EXCLUDED.authentication_expiry_time, " + "sso_exempt = EXCLUDED.sso_exempt, authentication_expiry_time = " + "EXCLUDED.authentication_expiry_time, " "capabilities = EXCLUDED.capabilities, creation_time = EXCLUDED.creation_time, " "identity = EXCLUDED.identity, last_authorized_time = EXCLUDED.last_authorized_time, " "last_deauthorized_time = EXCLUDED.last_deauthorized_time, " - "remote_trace_level = EXCLUDED.remote_trace_level, remote_trace_target = EXCLUDED.remote_trace_target, " + "remote_trace_level = EXCLUDED.remote_trace_level, remote_trace_target = " + "EXCLUDED.remote_trace_target, " "revision = EXCLUDED.revision, tags = EXCLUDED.tags, version_major = EXCLUDED.version_major, " "version_minor = EXCLUDED.version_minor, version_revision = EXCLUDED.version_revision, " "version_protocol = EXCLUDED.version_protocol", - memberId, - networkId, - (bool)config["authorized"], - (bool)config["activeBridge"], - config["ipAssignments"].get >(), - (bool)config["noAutoAssignIps"], - (bool)config["ssoExempt"], - (uint64_t)config["authenticationExpiryTime"], - OSUtils::jsonDump(config["capabilities"], -1), - (uint64_t)config["creationTime"], - OSUtils::jsonString(config["identity"], ""), - (uint64_t)config["lastAuthorizedTime"], - (uint64_t)config["lastDeauthorizedTime"], - (int)config["remoteTraceLevel"], - target, - (uint64_t)config["revision"], - OSUtils::jsonDump(config["tags"], -1), - (int)config["vMajor"], - (int)config["vMinor"], - (int)config["vRev"], - (int)config["vProto"]); + memberId, networkId, (bool)config["authorized"], (bool)config["activeBridge"], + config["ipAssignments"].get >(), (bool)config["noAutoAssignIps"], + (bool)config["ssoExempt"], (uint64_t)config["authenticationExpiryTime"], + OSUtils::jsonDump(config["capabilities"], -1), (uint64_t)config["creationTime"], + OSUtils::jsonString(config["identity"], ""), (uint64_t)config["lastAuthorizedTime"], + (uint64_t)config["lastDeauthorizedTime"], (int)config["remoteTraceLevel"], target, + (uint64_t)config["revision"], OSUtils::jsonDump(config["tags"], -1), (int)config["vMajor"], + (int)config["vMinor"], (int)config["vRev"], (int)config["vProto"]); w.commit(); @@ -962,7 +980,9 @@ void CV2::commitThread() _memberChanged(memOrig, memNew, qitem.second); } else { - fprintf(stderr, "%s: Can't notify of change. Error parsing nwid or memberid: %llu-%llu\n", _myAddressStr.c_str(), (unsigned long long)nwidInt, (unsigned long long)memberidInt); + fprintf( + stderr, "%s: Can't notify of change. Error parsing nwid or memberid: %llu-%llu\n", + _myAddressStr.c_str(), (unsigned long long)nwidInt, (unsigned long long)memberidInt); } } catch (pqxx::data_exception& e) { @@ -980,7 +1000,9 @@ void CV2::commitThread() } catch (std::exception& e) { std::string cfgDump = OSUtils::jsonDump(config, 2); - fprintf(stderr, "%s ERROR: Error updating member %s-%s: %s\njsonDump: %s\n", _myAddressStr.c_str(), networkId.c_str(), memberId.c_str(), e.what(), cfgDump.c_str()); + fprintf( + stderr, "%s ERROR: Error updating member %s-%s: %s\njsonDump: %s\n", _myAddressStr.c_str(), + networkId.c_str(), memberId.c_str(), e.what(), cfgDump.c_str()); mspan->SetStatus(opentelemetry::trace::StatusCode::kError, "std::exception"); mspan->SetAttribute("error", e.what()); mspan->SetAttribute("config", cfgDump); @@ -1002,10 +1024,7 @@ void CV2::commitThread() "VALUES ($1, $2, $3, $4, $5) " "ON CONFLICT (id) DO UPDATE SET " "name = EXCLUDED.name, configuration = EXCLUDED.configuration, revision = EXCLUDED.revision+1", - id, - OSUtils::jsonString(config["name"], ""), - OSUtils::jsonDump(config, -1), - _myAddressStr, + id, OSUtils::jsonString(config["name"], ""), OSUtils::jsonDump(config, -1), _myAddressStr, ((uint64_t)config["revision"])); w.commit(); @@ -1020,7 +1039,9 @@ void CV2::commitThread() _networkChanged(nwOrig, nwNew, qitem.second); } else { - fprintf(stderr, "%s: Can't notify network changed: %llu\n", _myAddressStr.c_str(), (unsigned long long)nwidInt); + fprintf( + stderr, "%s: Can't notify network changed: %llu\n", _myAddressStr.c_str(), + (unsigned long long)nwidInt); } } catch (pqxx::data_exception& e) { @@ -1078,7 +1099,9 @@ void CV2::commitThread() std::string memberId = config["id"]; std::string networkId = config["nwid"]; - pqxx::result res = w.exec_params0("DELETE FROM network_memberships_ctl WHERE device_id = $1 AND network_id = $2", memberId, networkId); + pqxx::result res = w.exec_params0( + "DELETE FROM network_memberships_ctl WHERE device_id = $1 AND network_id = $2", memberId, + networkId); w.commit(); @@ -1168,7 +1191,10 @@ void CV2::onlineNotificationThread() std::string memberId(memTmp); try { - pqxx::row r = w2.exec_params1("SELECT device_id, network_id FROM network_memberships_ctl WHERE network_id = $1 AND device_id = $2", networkId, memberId); + pqxx::row r = w2.exec_params1( + "SELECT device_id, network_id FROM network_memberships_ctl WHERE network_id = $1 AND device_id " + "= $2", + networkId, memberId); } catch (pqxx::unexpected_rows& e) { continue; @@ -1190,15 +1216,16 @@ void CV2::onlineNotificationThread() { ipAddr, ts }, }; - std::string device_network_insert = "INSERT INTO network_memberships_ctl (device_id, network_id, last_seen, os, arch) " - "VALUES ('" - + w2.esc(memberId) + "', '" + w2.esc(networkId) + "', '" + w2.esc(record.dump()) - + "'::JSONB, " - "'" - + w2.esc(os) + "', '" + w2.esc(arch) - + "') " - "ON CONFLICT (device_id, network_id) DO UPDATE SET os = EXCLUDED.os, arch = EXCLUDED.arch, " - "last_seen = network_memberships_ctl.last_seen || EXCLUDED.last_seen"; + std::string device_network_insert = + "INSERT INTO network_memberships_ctl (device_id, network_id, last_seen, os, arch) " + "VALUES ('" + + w2.esc(memberId) + "', '" + w2.esc(networkId) + "', '" + w2.esc(record.dump()) + + "'::JSONB, " + "'" + + w2.esc(os) + "', '" + w2.esc(arch) + + "') " + "ON CONFLICT (device_id, network_id) DO UPDATE SET os = EXCLUDED.os, arch = EXCLUDED.arch, " + "last_seen = network_memberships_ctl.last_seen || EXCLUDED.last_seen"; pipe.insert(device_network_insert); Metrics::pgsql_node_checkin++; @@ -1228,7 +1255,9 @@ void CV2::onlineNotificationThread() fprintf(stderr, "%s: Fell out of run loop in onlineNotificationThread\n", _myAddressStr.c_str()); if (_run == 1) { - fprintf(stderr, "ERROR: %s onlineNotificationThread should still be running! Exiting Controller.\n", _myAddressStr.c_str()); + fprintf( + stderr, "ERROR: %s onlineNotificationThread should still be running! Exiting Controller.\n", + _myAddressStr.c_str()); exit(6); } } diff --git a/nonfree/controller/CV2.hpp b/nonfree/controller/CV2.hpp index 4e30501429..5257ed0372 100644 --- a/nonfree/controller/CV2.hpp +++ b/nonfree/controller/CV2.hpp @@ -17,7 +17,7 @@ #include #include -#include +#include namespace ZeroTier { class CV2 : public DB { @@ -42,6 +42,10 @@ class CV2 : public DB { return _ready == 2; } + virtual void _memberChanged(nlohmann::json& old, nlohmann::json& memberConfig, bool notifyListeners); + + virtual void _networkChanged(nlohmann::json& old, nlohmann::json& networkConfig, bool notifyListeners); + protected: struct _PairHasher { inline std::size_t operator()(const std::pair& p) const @@ -49,9 +53,6 @@ class CV2 : public DB { return (std::size_t)(p.first ^ p.second); } }; - virtual void _memberChanged(nlohmann::json& old, nlohmann::json& memberConfig, bool notifyListeners); - - virtual void _networkChanged(nlohmann::json& old, nlohmann::json& networkConfig, bool notifyListeners); private: void initializeNetworks(); diff --git a/nonfree/controller/CentralDB.cpp b/nonfree/controller/CentralDB.cpp new file mode 100644 index 0000000000..5646f8c778 --- /dev/null +++ b/nonfree/controller/CentralDB.cpp @@ -0,0 +1,1897 @@ +/* + * Copyright (c)2019 ZeroTier, Inc. + * + * Use of this software is governed by the Business Source License included + * in the LICENSE.TXT file in the project's root directory. + * + * Change Date: 2026-01-01 + * + * On the date above, in accordance with the Business Source License, use + * of this software will be governed by version 2.0 of the Apache License. + */ +/****/ + +#include "CentralDB.hpp" + +#ifdef ZT_CONTROLLER_USE_LIBPQ + +#include "../../node/Constants.hpp" +#include "../../node/SHA512.hpp" +#include "../../version.h" +#include "BigTableStatusWriter.hpp" +#include "ControllerChangeNotifier.hpp" +#include "ControllerConfig.hpp" +#include "CtlUtil.hpp" +#include "EmbeddedNetworkController.hpp" +#include "OtelCarrier.hpp" +#include "PostgresStatusWriter.hpp" +#include "PubSubListener.hpp" +#include "PubSubWriter.hpp" +#include "Redis.hpp" +#include "RedisListener.hpp" +#include "RedisStatusWriter.hpp" +#include "opentelemetry/context/propagation/global_propagator.h" +#include "opentelemetry/trace/provider.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +// #define REDIS_TRACE 1 + +using json = nlohmann::json; + +using namespace ZeroTier; + +using Attrs = std::vector >; +using Item = std::pair; +using ItemStream = std::vector; + +CentralDB::CentralDB( + const Identity& myId, + const char* connString, + int listenPort, + CentralDB::ListenerMode listenMode, + CentralDB::StatusWriterMode statusMode, + const ControllerConfig* cc) + : DB() + , _listenerMode(listenMode) + , _statusWriterMode(statusMode) + , _cc(cc) + , _pool() + , _myId(myId) + , _myAddress(myId.address()) + , _ready(0) + , _connected(1) + , _run(1) + , _waitNoticePrinted(false) + , _listenPort(listenPort) + , _redis(NULL) + , _cluster(NULL) + , _redisMemberStatus(false) + , _smee(NULL) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::CentralDB"); + auto scope = tracer->WithActiveSpan(span); + + rustybits::init_async_runtime(); + + char myAddress[64]; + _myAddressStr = myId.address().toString(myAddress); + _connString = std::string(connString); + + auto f = std::make_shared(_connString); + _pool = + std::make_shared >(15, 5, std::static_pointer_cast(f)); + + memset(_ssoPsk, 0, sizeof(_ssoPsk)); + char* const ssoPskHex = getenv("ZT_SSO_PSK"); +#ifdef ZT_TRACE + fprintf(stderr, "ZT_SSO_PSK: %s\n", ssoPskHex); +#endif + if (ssoPskHex) { + // SECURITY: note that ssoPskHex will always be null-terminated if libc actually + // returns something non-NULL. If the hex encodes something shorter than 48 bytes, + // it will be padded at the end with zeroes. If longer, it'll be truncated. + Utils::unhex(ssoPskHex, _ssoPsk, sizeof(_ssoPsk)); + } + const char* redisMemberStatus = getenv("ZT_REDIS_MEMBER_STATUS"); + if (redisMemberStatus && (strcmp(redisMemberStatus, "true") == 0)) { + _redisMemberStatus = true; + fprintf(stderr, "Using redis for member status\n"); + } + + if ((listenMode == LISTENER_MODE_REDIS || statusMode == STATUS_WRITER_MODE_REDIS) && _cc->redisConfig != NULL) { + auto innerspan = tracer->StartSpan("CentralDB::CentralDB::configureRedis"); + auto innerscope = tracer->WithActiveSpan(innerspan); + + sw::redis::ConnectionOptions opts; + sw::redis::ConnectionPoolOptions poolOpts; + opts.host = _cc->redisConfig->hostname; + opts.port = _cc->redisConfig->port; + opts.password = _cc->redisConfig->password; + opts.db = 0; + opts.keep_alive = true; + opts.connect_timeout = std::chrono::seconds(3); + poolOpts.size = 25; + poolOpts.wait_timeout = std::chrono::seconds(5); + poolOpts.connection_lifetime = std::chrono::minutes(3); + poolOpts.connection_idle_time = std::chrono::minutes(1); + if (_cc->redisConfig->clusterMode) { + innerspan->SetAttribute("cluster_mode", "true"); + fprintf(stderr, "Using Redis in Cluster Mode\n"); + _cluster = std::make_shared(opts, poolOpts); + } + else { + innerspan->SetAttribute("cluster_mode", "false"); + fprintf(stderr, "Using Redis in Standalone Mode\n"); + _redis = std::make_shared(opts, poolOpts); + } + } + + _readyLock.lock(); + + fprintf( + stderr, "[%s] NOTICE: %.10llx controller PostgreSQL waiting for initial data download..." ZT_EOL_S, + ::_timestr(), (unsigned long long)_myAddress.toInt()); + _waitNoticePrinted = true; + + initializeNetworks(); + initializeMembers(); + + _heartbeatThread = std::thread(&CentralDB::heartbeat, this); + + switch (listenMode) { + case LISTENER_MODE_REDIS: + fprintf(stderr, "Using Redis for change listeners\n"); + if (_cc->redisConfig != NULL) { + if (_cc->redisConfig->clusterMode) { + _membersDbWatcher = std::make_shared(_myAddressStr, _cluster, this); + _networksDbWatcher = std::make_shared(_myAddressStr, _cluster, this); + } + else { + _membersDbWatcher = std::make_shared(_myAddressStr, _redis, this); + _networksDbWatcher = std::make_shared(_myAddressStr, _redis, this); + } + } + else { + throw std::runtime_error("CentralDB: Redis listener mode selected but no Redis configuration provided"); + } + case LISTENER_MODE_PUBSUB: + fprintf(stderr, "Using PubSub for change listeners\n"); + if (cc->pubSubConfig != NULL) { + _membersDbWatcher = std::make_shared( + _myAddressStr, cc->pubSubConfig->project_id, cc->pubSubConfig->member_change_recv_topic, this); + _networksDbWatcher = std::make_shared( + _myAddressStr, cc->pubSubConfig->project_id, cc->pubSubConfig->network_change_recv_topic, this); + _changeNotifier = std::make_shared( + _myAddressStr, cc->pubSubConfig->project_id, cc->pubSubConfig->member_change_send_topic, + cc->pubSubConfig->network_change_send_topic); + } + else { + throw std::runtime_error( + "CentralDB: PubSub listener mode selected but no PubSub configuration provided"); + } + break; + case LISTENER_MODE_PGSQL: + default: + fprintf(stderr, "Using PostgreSQL for change listeners\n"); + _membersDbWatcher = std::make_shared(this, _pool, "member_" + _myAddressStr, 5); + _networksDbWatcher = std::make_shared(this, _pool, "network_" + _myAddressStr, 5); + break; + } + + std::shared_ptr pubsubWriter; + switch (statusMode) { + case STATUS_WRITER_MODE_REDIS: + fprintf(stderr, "Using Redis for status writer\n"); + if (_cc->redisConfig != NULL) { + if (_cc->redisConfig->clusterMode) { + _statusWriter = std::make_shared(_cluster, _myAddressStr); + } + else { + _statusWriter = std::make_shared(_redis, _myAddressStr); + } + } + else { + throw std::runtime_error("CentralDB: Redis status mode selected but no Redis configuration provided"); + } + break; + case STATUS_WRITER_MODE_BIGTABLE: + fprintf(stderr, "Using BigTable for status writer\n"); + if (cc->bigTableConfig == NULL) { + throw std::runtime_error( + "CentralDB: BigTable status mode selected but no BigTable configuration provided"); + } + if (cc->pubSubConfig == NULL) { + throw std::runtime_error( + "CentralDB: BigTable status mode selected but no PubSub configuration provided"); + } + + _statusWriter = std::make_shared( + cc->bigTableConfig->project_id, cc->bigTableConfig->instance_id, cc->bigTableConfig->table_id); + break; + case STATUS_WRITER_MODE_PGSQL: + default: + fprintf(stderr, "Using PostgreSQL for status writer\n"); + _statusWriter = std::make_shared(_pool); + break; + } + + // start background threads + for (int i = 0; i < ZT_CENTRAL_CONTROLLER_COMMIT_THREADS; ++i) { + _commitThread[i] = std::thread(&CentralDB::commitThread, this); + } + _onlineNotificationThread = std::thread(&CentralDB::onlineNotificationThread, this); + + configureSmee(); +} + +CentralDB::~CentralDB() +{ + if (_smee != NULL) { + rustybits::smee_client_delete(_smee); + _smee = NULL; + } + + rustybits::shutdown_async_runtime(); + + _run = 0; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + _heartbeatThread.join(); + _commitQueue.stop(); + for (int i = 0; i < ZT_CENTRAL_CONTROLLER_COMMIT_THREADS; ++i) { + _commitThread[i].join(); + } + _onlineNotificationThread.join(); +} + +void CentralDB::configureSmee() +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::configureSmee"); + auto scope = tracer->WithActiveSpan(span); + + const char* TEMPORAL_SCHEME = "ZT_TEMPORAL_SCHEME"; + const char* TEMPORAL_HOST = "ZT_TEMPORAL_HOST"; + const char* TEMPORAL_PORT = "ZT_TEMPORAL_PORT"; + const char* TEMPORAL_NAMESPACE = "ZT_TEMPORAL_NAMESPACE"; + const char* SMEE_TASK_QUEUE = "ZT_SMEE_TASK_QUEUE"; + + const char* scheme = getenv(TEMPORAL_SCHEME); + if (scheme == NULL) { + scheme = "http"; + } + const char* host = getenv(TEMPORAL_HOST); + const char* port = getenv(TEMPORAL_PORT); + const char* ns = getenv(TEMPORAL_NAMESPACE); + const char* task_queue = getenv(SMEE_TASK_QUEUE); + + if (scheme != NULL && host != NULL && port != NULL && ns != NULL && task_queue != NULL) { + fprintf(stderr, "creating smee client\n"); + std::string hostPort = + std::string(scheme) + std::string("://") + std::string(host) + std::string(":") + std::string(port); + this->_smee = rustybits::smee_client_new(hostPort.c_str(), ns, task_queue); + } + else { + fprintf(stderr, "Smee client not configured\n"); + } +} + +bool CentralDB::waitForReady() +{ + while (_ready < 2) { + _readyLock.lock(); + _readyLock.unlock(); + } + return true; +} + +bool CentralDB::isReady() +{ + return ((_ready == 2) && (_connected)); +} + +bool CentralDB::save(nlohmann::json& record, bool notifyListeners) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::save"); + auto scope = tracer->WithActiveSpan(span); + + bool modified = false; + try { + if (! record.is_object()) { + fprintf(stderr, "record is not an object?!?\n"); + return false; + } + const std::string objtype = record["objtype"]; + if (objtype == "network") { + auto span = tracer->StartSpan("CentralDB::save::network"); + auto scope = tracer->WithActiveSpan(span); + + fprintf(stderr, "CentralDB network save %s\n", record["id"].get().c_str()); + const uint64_t nwid = OSUtils::jsonIntHex(record["id"], 0ULL); + if (nwid) { + nlohmann::json old; + get(nwid, old); + if ((! old.is_object()) || (! _compareRecords(old, record))) { + fprintf(stderr, "posting network change to commit queue\n"); + record["revision"] = OSUtils::jsonInt(record["revision"], 0ULL) + 1ULL; + _queueItem qi; + qi.jsonData = record; + qi.notifyListeners = notifyListeners; + OtelCarrier > carrier(qi.traceContext); + auto current_ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + auto propagator = + opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + propagator->Inject(carrier, current_ctx); + _commitQueue.post(qi); + modified = true; + } + } + } + else if (objtype == "member") { + auto span = tracer->StartSpan("CentralDB::save::member"); + auto scope = tracer->WithActiveSpan(span); + + std::string networkId = record["nwid"]; + std::string memberId = record["id"]; + const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"], 0ULL); + const uint64_t id = OSUtils::jsonIntHex(record["id"], 0ULL); + fprintf(stderr, "member save %s-%s\n", networkId.c_str(), memberId.c_str()); + if ((id) && (nwid)) { + nlohmann::json network, old; + get(nwid, network, id, old); + if ((! old.is_object()) || (! _compareRecords(old, record))) { + fprintf(stderr, "posting member change to commit queue\n"); + record["revision"] = OSUtils::jsonInt(record["revision"], 0ULL) + 1ULL; + + _queueItem qi; + qi.jsonData = record; + qi.notifyListeners = notifyListeners; + OtelCarrier > carrier(qi.traceContext); + auto current_ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + auto propagator = + opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + propagator->Inject(carrier, current_ctx); + _commitQueue.post(qi); + + modified = true; + } + else { + // fprintf(stderr, "no change\n"); + } + } + } + else { + fprintf(stderr, "uhh waaat\n"); + } + } + catch (std::exception& e) { + fprintf(stderr, "Error on CentralDB::save: %s\n", e.what()); + } + catch (...) { + fprintf(stderr, "Unknown error on CentralDB::save\n"); + } + return modified; +} + +void CentralDB::eraseNetwork(const uint64_t networkId) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::eraseNetwork"); + auto scope = tracer->WithActiveSpan(span); + char networkIdStr[17]; + span->SetAttribute("network_id", Utils::hex(networkId, networkIdStr)); + + fprintf(stderr, "CentralDB::eraseNetwork\n"); + char tmp2[24]; + waitForReady(); + Utils::hex(networkId, tmp2); + + _queueItem qi; + qi.jsonData["id"] = tmp2; + qi.jsonData["objtype"] = "_delete_network"; + qi.notifyListeners = true; + OtelCarrier > carrier(qi.traceContext); + auto current_ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + propagator->Inject(carrier, current_ctx); + _commitQueue.post(qi); + + nlohmann::json nullJson; + _networkChanged(qi.jsonData, nullJson, true); +} + +void CentralDB::eraseMember(const uint64_t networkId, const uint64_t memberId) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::eraseMember"); + auto scope = tracer->WithActiveSpan(span); + char networkIdStr[17]; + char memberIdStr[11]; + span->SetAttribute("network_id", Utils::hex(networkId, networkIdStr)); + span->SetAttribute("member_id", Utils::hex10(memberId, memberIdStr)); + + fprintf(stderr, "CentralDB::eraseMember\n"); + char tmp2[24]; + waitForReady(); + + _queueItem qi; + qi.jsonData["nwid"] = tmp2; + qi.jsonData["id"] = tmp2; + qi.jsonData["objtype"] = "_delete_member"; + qi.notifyListeners = true; + OtelCarrier > carrier(qi.traceContext); + auto current_ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + propagator->Inject(carrier, current_ctx); + _commitQueue.post(qi); + + nlohmann::json nullJson; + _memberChanged(qi.jsonData, nullJson, true); +} + +void CentralDB::nodeIsOnline( + const uint64_t networkId, + const uint64_t memberId, + const InetAddress& physicalAddress, + const char* osArch) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::nodeIsOnline"); + auto scope = tracer->WithActiveSpan(span); + char networkIdStr[17]; + char memberIdStr[11]; + char ipStr[INET6_ADDRSTRLEN]; + span->SetAttribute("network_id", Utils::hex(networkId, networkIdStr)); + span->SetAttribute("member_id", Utils::hex10(memberId, memberIdStr)); + span->SetAttribute("physical_address", physicalAddress.toString(ipStr)); + span->SetAttribute("os_arch", osArch); + + std::lock_guard l(_lastOnline_l); + NodeOnlineRecord& i = _lastOnline[std::pair(networkId, memberId)]; + i.lastSeen = OSUtils::now(); + if (physicalAddress) { + i.physicalAddress = physicalAddress; + } + i.osArch = std::string(osArch); +} + +void CentralDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress) +{ + this->nodeIsOnline(networkId, memberId, physicalAddress, "unknown/unknown"); +} + +AuthInfo CentralDB::getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL) +{ + if (_cc->ssoEnabled) { + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::getSSOAuthInfo"); + auto scope = tracer->WithActiveSpan(span); + + Metrics::db_get_sso_info++; + // NONCE is just a random character string. no semantic meaning + // state = HMAC SHA384 of Nonce based on shared sso key + // + // need nonce timeout in database? make sure it's used within X time + // X is 5 minutes for now. Make configurable later? + // + // how do we tell when a nonce is used? if auth_expiration_time is set + std::string networkId = member["nwid"]; + std::string memberId = member["id"]; + + char authenticationURL[4096] = { 0 }; + AuthInfo info; + info.enabled = true; + + // if (memberId == "a10dccea52" && networkId == "8056c2e21c24673d") { + // fprintf(stderr, "invalid authinfo for grant's machine\n"); + // info.version=1; + // return info; + // } + // fprintf(stderr, "CentralDB::updateMemberOnLoad: %s-%s\n", networkId.c_str(), memberId.c_str()); + std::shared_ptr c; + try { + c = _pool->borrow(); + pqxx::work w(*c->c); + + char nonceBytes[16] = { 0 }; + std::string nonce = ""; + + // check if the member exists first. + pqxx::row count = + w.exec( + "SELECT count(id) FROM ztc_member WHERE id = $1 AND network_id = $2 AND deleted = false", + pqxx::params { memberId, networkId }) + .one_row(); + if (count[0].as() == 1) { + // get active nonce, if exists. + pqxx::result r = w.exec( + "SELECT nonce FROM ztc_sso_expiry " + "WHERE network_id = $1 AND member_id = $2 " + "AND ((NOW() AT TIME ZONE 'UTC') <= authentication_expiry_time) AND ((NOW() AT TIME ZONE 'UTC') <= " + "nonce_expiration)", + pqxx::params { networkId, memberId }); + + if (r.size() == 0) { + // no active nonce. + // find an unused nonce, if one exists. + pqxx::result r = w.exec( + "SELECT nonce FROM ztc_sso_expiry " + "WHERE network_id = $1 AND member_id = $2 " + "AND authentication_expiry_time IS NULL AND ((NOW() AT TIME ZONE 'UTC') <= nonce_expiration)", + pqxx::params { networkId, memberId }); + + if (r.size() == 1) { + // we have an existing nonce. Use it + nonce = r.at(0)[0].as(); + Utils::unhex(nonce.c_str(), nonceBytes, sizeof(nonceBytes)); + } + else if (r.empty()) { + // create a nonce + Utils::getSecureRandom(nonceBytes, 16); + char nonceBuf[64] = { 0 }; + Utils::hex(nonceBytes, sizeof(nonceBytes), nonceBuf); + nonce = std::string(nonceBuf); + + pqxx::result ir = w.exec( + "INSERT INTO ztc_sso_expiry " + "(nonce, nonce_expiration, network_id, member_id) VALUES " + "($1, TO_TIMESTAMP($2::double precision/1000), $3, $4)", + pqxx::params { nonce, OSUtils::now() + 300000, networkId, memberId }); + + w.commit(); + } + else { + // > 1 ?!? Thats an error! + fprintf(stderr, "> 1 unused nonce!\n"); + exit(6); + } + } + else if (r.size() == 1) { + nonce = r.at(0)[0].as(); + Utils::unhex(nonce.c_str(), nonceBytes, sizeof(nonceBytes)); + } + else { + // more than 1 nonce in use? Uhhh... + fprintf(stderr, "> 1 nonce in use for network member?!?\n"); + exit(7); + } + + r = w.exec( + "SELECT oc.client_id, oc.authorization_endpoint, oc.issuer, oc.provider, oc.sso_impl_version " + "FROM ztc_network AS n " + "INNER JOIN ztc_org o " + " ON o.owner_id = n.owner_id " + "LEFT OUTER JOIN ztc_network_oidc_config noc " + " ON noc.network_id = n.id " + "LEFT OUTER JOIN ztc_oidc_config oc " + " ON noc.client_id = oc.client_id AND oc.org_id = o.org_id " + "WHERE n.id = $1 AND n.sso_enabled = true", + pqxx::params { networkId }); + + std::string client_id = ""; + std::string authorization_endpoint = ""; + std::string issuer = ""; + std::string provider = ""; + uint64_t sso_version = 0; + + if (r.size() == 1) { + client_id = r.at(0)[0].as >().value_or(""); + authorization_endpoint = r.at(0)[1].as >().value_or(""); + issuer = r.at(0)[2].as >().value_or(""); + provider = r.at(0)[3].as >().value_or(""); + sso_version = r.at(0)[4].as >().value_or(1); + } + else if (r.size() > 1) { + fprintf( + stderr, "ERROR: More than one auth endpoint for an organization?!?!? NetworkID: %s\n", + networkId.c_str()); + } + else { + fprintf(stderr, "No client or auth endpoint?!?\n"); + } + + info.version = sso_version; + + // no catch all else because we don't actually care if no records exist here. just continue as normal. + if ((! client_id.empty()) && (! authorization_endpoint.empty())) { + uint8_t state[48]; + HMACSHA384(_ssoPsk, nonceBytes, sizeof(nonceBytes), state); + char state_hex[256]; + Utils::hex(state, 48, state_hex); + + if (info.version == 0) { + char url[2048] = { 0 }; + OSUtils::ztsnprintf( + url, sizeof(authenticationURL), + "%s?response_type=id_token&response_mode=form_post&scope=openid+email+profile&redirect_uri=" + "%s&nonce=%s&state=%s&client_id=%s", + authorization_endpoint.c_str(), url_encode(redirectURL).c_str(), nonce.c_str(), state_hex, + client_id.c_str()); + info.authenticationURL = std::string(url); + } + else if (info.version == 1) { + info.ssoClientID = client_id; + info.issuerURL = issuer; + info.ssoProvider = provider; + info.ssoNonce = nonce; + info.ssoState = std::string(state_hex) + "_" + networkId; + info.centralAuthURL = redirectURL; +#ifdef ZT_DEBUG + fprintf( + stderr, + "ssoClientID: %s\nissuerURL: %s\nssoNonce: %s\nssoState: %s\ncentralAuthURL: %s\nprovider: " + "%s\n", + info.ssoClientID.c_str(), info.issuerURL.c_str(), info.ssoNonce.c_str(), + info.ssoState.c_str(), info.centralAuthURL.c_str(), provider.c_str()); +#endif + } + } + else { + fprintf( + stderr, "client_id: %s\nauthorization_endpoint: %s\n", client_id.c_str(), + authorization_endpoint.c_str()); + } + } + + _pool->unborrow(c); + } + catch (std::exception& e) { + span->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "ERROR: Error updating member on load for network %s: %s\n", networkId.c_str(), e.what()); + } + + return info; // std::string(authenticationURL); + } + return AuthInfo(); +} + +void CentralDB::initializeNetworks() +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::initializeNetworks"); + auto scope = tracer->WithActiveSpan(span); + + fprintf(stderr, "Initializing networks...\n"); + + try { + char qbuf[2048]; + sprintf( + qbuf, + "SELECT id, name, configuration , (EXTRACT(EPOCH FROM creation_time AT TIME ZONE 'UTC')*1000)::bigint, " + "(EXTRACT(EPOCH FROM last_modified AT TIME ZONE 'UTC')*1000)::bigint, revision, frontend " + "FROM networks_ctl WHERE controller_id = '%s'", + _myAddressStr.c_str()); + + auto c = _pool->borrow(); + pqxx::work w(*c->c); + + fprintf(stderr, "Load networks from psql...\n"); + auto stream = pqxx::stream_from::query(w, qbuf); + std::tuple< + std::string // network ID + , + std::optional // name + , + std::string // configuration + , + std::optional // creation_time + , + std::optional // last_modified + , + std::optional // revision + , + std::string // frontend + > + row; + uint64_t count = 0; + uint64_t total = 0; + while (stream >> row) { + auto start = std::chrono::high_resolution_clock::now(); + + json empty; + json config; + + initNetwork(config); + + std::string nwid = std::get<0>(row); + std::string name = std::get<1>(row).value_or(""); + json cfgtmp = json::parse(std::get<2>(row)); + std::optional created_at = std::get<3>(row); + std::optional last_modified = std::get<4>(row); + std::optional revision = std::get<5>(row); + std::string frontend = std::get<6>(row); + + config["id"] = nwid; + config["name"] = name; + config["creationTime"] = created_at.value_or(0); + config["lastModified"] = last_modified.value_or(0); + config["revision"] = revision.value_or(0); + config["capabilities"] = cfgtmp["capabilities"].is_array() ? cfgtmp["capabilities"] : json::array(); + config["enableBroadcast"] = + cfgtmp["enableBroadcast"].is_boolean() ? cfgtmp["enableBroadcast"].get() : false; + config["mtu"] = cfgtmp["mtu"].is_number() ? cfgtmp["mtu"].get() : 2800; + config["multicastLimit"] = + cfgtmp["multicastLimit"].is_number() ? cfgtmp["multicastLimit"].get() : 64; + config["private"] = cfgtmp["private"].is_boolean() ? cfgtmp["private"].get() : true; + config["remoteTraceLevel"] = + cfgtmp["remoteTraceLevel"].is_number() ? cfgtmp["remoteTraceLevel"].get() : 0; + config["remoteTraceTarget"] = + cfgtmp["remoteTraceTarget"].is_string() ? cfgtmp["remoteTraceTarget"].get() : ""; + config["revision"] = revision.value_or(0); + config["rules"] = cfgtmp["rules"].is_array() ? cfgtmp["rules"] : json::array(); + config["tags"] = cfgtmp["tags"].is_array() ? cfgtmp["tags"] : json::array(); + if (cfgtmp["v4AssignMode"].is_object()) { + config["v4AssignMode"] = cfgtmp["v4AssignMode"]; + } + else { + config["v4AssignMode"] = json::object(); + config["v4AssignMode"]["zt"] = true; + } + if (cfgtmp["v6AssignMode"].is_object()) { + config["v6AssignMode"] = cfgtmp["v6AssignMode"]; + } + else { + config["v6AssignMode"] = json::object(); + config["v6AssignMode"]["zt"] = true; + config["v6AssignMode"]["6plane"] = true; + config["v6AssignMode"]["rfc4193"] = false; + } + config["ssoEnabled"] = cfgtmp["ssoEnabled"].is_boolean() ? cfgtmp["ssoEnabled"].get() : false; + if (config["ssoConfig"].is_object()) { + config["ssoConfig"] = cfgtmp["ssoConfig"]; + } + else { + config["ssoConfig"] = empty; + } + config["objtype"] = "network"; + config["routes"] = cfgtmp["routes"].is_array() ? cfgtmp["routes"] : json::array(); + config["clientId"] = cfgtmp["clientId"].is_string() ? cfgtmp["clientId"].get() : ""; + config["authorizationEndpoint"] = + cfgtmp["authorizationEndpoint"].is_string() ? cfgtmp["authorizationEndpoint"].get() : ""; + config["provider"] = cfgtmp["ssoProvider"].is_string() ? cfgtmp["ssoProvider"].get() : ""; + if (! cfgtmp["dns"].is_object()) { + cfgtmp["dns"] = json::object(); + cfgtmp["dns"]["domain"] = ""; + cfgtmp["dns"]["servers"] = json::array(); + } + else { + config["dns"] = cfgtmp["dns"]; + } + config["ipAssignmentPools"] = + cfgtmp["ipAssignmentPools"].is_array() ? cfgtmp["ipAssignmentPools"] : json::array(); + config["frontend"] = frontend; + + Metrics::network_count++; + + _networkChanged(empty, config, false); + + auto end = std::chrono::high_resolution_clock::now(); + auto dur = std::chrono::duration_cast(end - start); + + total += dur.count(); + ++count; + if (count > 0 && count % 10000 == 0) { + fprintf(stderr, "Averaging %lu us per network\n", (total / count)); + } + } + + w.commit(); + _pool->unborrow(c); + fprintf(stderr, "done.\n"); + + if (++this->_ready == 2) { + if (_waitNoticePrinted) { + fprintf( + stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), + (unsigned long long)_myAddress.toInt()); + } + _readyLock.unlock(); + } + fprintf(stderr, "network init done\n"); + } + catch (std::exception& e) { + fprintf(stderr, "ERROR: Error initializing networks: %s\n", e.what()); + span->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + std::this_thread::sleep_for(std::chrono::milliseconds(5000)); + exit(-1); + } +} + +void CentralDB::initializeMembers() +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::initializeMembers"); + auto scope = tracer->WithActiveSpan(span); + + std::string memberId; + std::string networkId; + try { + std::unordered_map networkMembers; + fprintf(stderr, "Initializing Members...\n"); + + std::string setKeyBase = "network-nodes-all:{" + _myAddressStr + "}:"; + + if (_redisMemberStatus) { + fprintf(stderr, "Initialize Redis for members...\n"); + std::unique_lock l(_networks_l); + std::unordered_set deletes; + for (auto it : _networks) { + uint64_t nwid_i = it.first; + char nwidTmp[64] = { 0 }; + OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i); + std::string nwid(nwidTmp); + std::string key = setKeyBase + nwid; + deletes.insert(key); + } + + if (! deletes.empty()) { + try { + if (_cc->redisConfig->clusterMode) { + auto tx = _cluster->transaction(_myAddressStr, true, false); + for (std::string k : deletes) { + tx.del(k); + } + tx.exec(); + } + else { + auto tx = _redis->transaction(true, false); + for (std::string k : deletes) { + tx.del(k); + } + tx.exec(); + } + } + catch (sw::redis::Error& e) { + // ignore + } + } + } + + char qbuf[2048]; + sprintf( + qbuf, + "SELECT nm.device_id, nm.network_id, nm.authorized, nm.active_bridge, nm.ip_assignments, " + "nm.no_auto_assign_ips, " + "nm.sso_exempt, (EXTRACT(EPOCH FROM nm.authentication_expiry_time AT TIME ZONE 'UTC')*1000)::bigint, " + "(EXTRACT(EPOCH FROM nm.creation_time AT TIME ZONE 'UTC')*1000)::bigint, nm.identity, " + "(EXTRACT(EPOCH FROM nm.last_authorized_time AT TIME ZONE 'UTC')*1000)::bigint, " + "(EXTRACT(EPOCH FROM nm.last_deauthorized_time AT TIME ZONE 'UTC')*1000)::bigint, " + "nm.remote_trace_level, nm.remote_trace_target, nm.revision, nm.capabilities, nm.tags, " + "nm.frontend " + "FROM network_memberships_ctl nm " + "INNER JOIN networks_ctl n " + " ON nm.network_id = n.id " + "WHERE n.controller_id = '%s'", + _myAddressStr.c_str()); + + auto c = _pool->borrow(); + pqxx::work w(*c->c); + fprintf(stderr, "Load members from psql...\n"); + auto stream = pqxx::stream_from::query(w, qbuf); + std::tuple< + std::string // device ID + , + std::string // network ID + , + bool // authorized + , + std::optional // active_bridge + , + std::optional // ip_assignments + , + std::optional // no_auto_assign_ips + , + std::optional // sso_exempt + , + std::optional // authentication_expiry_time + , + std::optional // creation_time + , + std::optional // identity + , + std::optional // last_authorized_time + , + std::optional // last_deauthorized_time + , + std::optional // remote_trace_level + , + std::optional // remote_trace_target + , + std::optional // revision + , + std::optional // capabilities + , + std::optional // tags + , + std::string // frontend + > + row; + + auto tmp = std::chrono::high_resolution_clock::now(); + uint64_t count = 0; + uint64_t total = 0; + while (stream >> row) { + auto start = std::chrono::high_resolution_clock::now(); + json empty; + json config; + + initMember(config); + + memberId = std::get<0>(row); + networkId = std::get<1>(row); + bool authorized = std::get<2>(row); + std::optional active_bridge = std::get<3>(row); + std::string ip_assignments = std::get<4>(row).value_or(""); + std::optional no_auto_assign_ips = std::get<5>(row); + std::optional sso_exempt = std::get<6>(row); + std::optional authentication_expiry_time = std::get<7>(row); + std::optional creation_time = std::get<8>(row); + std::optional identity = std::get<9>(row); + std::optional last_authorized_time = std::get<10>(row); + std::optional last_deauthorized_time = std::get<11>(row); + std::optional remote_trace_level = std::get<12>(row); + std::optional remote_trace_target = std::get<13>(row); + std::optional revision = std::get<14>(row); + std::optional capabilities = std::get<15>(row); + std::optional tags = std::get<16>(row); + + networkMembers.insert(std::pair(setKeyBase + networkId, memberId)); + + config["objtype"] = "member"; + config["id"] = memberId; + config["address"] = identity.value_or(""); + config["nwid"] = networkId; + config["authorized"] = authorized; + config["activeBridge"] = active_bridge.value_or(false); + config["ipAssignments"] = json::array(); + if (ip_assignments != "{}") { + std::string tmp = ip_assignments.substr(1, ip_assignments.length() - 2); + std::vector addrs = split(tmp, ','); + for (auto it = addrs.begin(); it != addrs.end(); ++it) { + config["ipAssignments"].push_back(*it); + } + } + config["capabilities"] = json::parse(capabilities.value_or("[]")); + config["creationTime"] = creation_time.value_or(0); + config["lastAuthorizedTime"] = last_authorized_time.value_or(0); + config["lastDeauthorizedTime"] = last_deauthorized_time.value_or(0); + config["noAutoAssignIPs"] = no_auto_assign_ips.value_or(false); + config["remoteTraceLevel"] = remote_trace_level.value_or(0); + config["remoteTraceTarget"] = remote_trace_target.value_or(nullptr); + config["revision"] = revision.value_or(0); + config["ssoExempt"] = sso_exempt.value_or(false); + config["authenticationExpiryTime"] = authentication_expiry_time.value_or(0); + config["tags"] = json::parse(tags.value_or("[]")); + config["frontend"] = std::get<17>(row); + + Metrics::member_count++; + + _memberChanged(empty, config, false); + + memberId = ""; + networkId = ""; + + auto end = std::chrono::high_resolution_clock::now(); + auto dur = std::chrono::duration_cast(end - start); + total += dur.count(); + ++count; + if (count > 0 && count % 10000 == 0) { + fprintf(stderr, "Averaging %llu us per member\n", (total / count)); + } + } + if (count > 0) { + fprintf(stderr, "Took %llu us per member to load\n", (total / count)); + } + + stream.complete(); + + w.commit(); + _pool->unborrow(c); + fprintf(stderr, "done.\n"); + + if (_listenerMode == LISTENER_MODE_REDIS) + if (! networkMembers.empty()) { + if (_redisMemberStatus) { + fprintf(stderr, "Load member data into redis...\n"); + if (_cc->redisConfig->clusterMode) { + auto tx = _cluster->transaction(_myAddressStr, true, false); + uint64_t count = 0; + for (auto it : networkMembers) { + tx.sadd(it.first, it.second); + if (++count % 30000 == 0) { + tx.exec(); + tx = _cluster->transaction(_myAddressStr, true, false); + } + } + tx.exec(); + } + else { + auto tx = _redis->transaction(true, false); + uint64_t count = 0; + for (auto it : networkMembers) { + tx.sadd(it.first, it.second); + if (++count % 30000 == 0) { + tx.exec(); + tx = _redis->transaction(true, false); + } + } + tx.exec(); + } + fprintf(stderr, "done.\n"); + } + } + + fprintf(stderr, "Done loading members...\n"); + + if (++this->_ready == 2) { + if (_waitNoticePrinted) { + fprintf( + stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), + (unsigned long long)_myAddress.toInt()); + } + _readyLock.unlock(); + } + } + catch (sw::redis::Error& e) { + span->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "ERROR: Error initializing members (redis): %s\n", e.what()); + exit(-1); + } + catch (std::exception& e) { + span->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "ERROR: Error initializing member: %s-%s %s\n", networkId.c_str(), memberId.c_str(), e.what()); + exit(-1); + } +} + +void CentralDB::heartbeat() +{ + char publicId[1024]; + char hostnameTmp[1024]; + _myId.toString(false, publicId); + if (gethostname(hostnameTmp, sizeof(hostnameTmp)) != 0) { + hostnameTmp[0] = (char)0; + } + else { + for (int i = 0; i < (int)sizeof(hostnameTmp); ++i) { + if ((hostnameTmp[i] == '.') || (hostnameTmp[i] == 0)) { + hostnameTmp[i] = (char)0; + break; + } + } + } + const char* controllerId = _myAddressStr.c_str(); + const char* publicIdentity = publicId; + const char* hostname = hostnameTmp; + + while (_run == 1) { + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::heartbeat"); + auto scope = tracer->WithActiveSpan(span); + + // fprintf(stderr, "%s: heartbeat\n", controllerId); + auto c = _pool->borrow(); + int64_t ts = OSUtils::now(); + + if (c->c) { + std::string major = std::to_string(ZEROTIER_ONE_VERSION_MAJOR); + std::string minor = std::to_string(ZEROTIER_ONE_VERSION_MINOR); + std::string rev = std::to_string(ZEROTIER_ONE_VERSION_REVISION); + std::string version = major + "." + minor + "." + rev; + std::string versionStr = "v" + version; + + try { + pqxx::work w { *c->c }; + w.exec( + "INSERT INTO controllers_ctl (id, hostname, last_heartbeat, public_identity, version) VALUES " + "($1, $2, TO_TIMESTAMP($3::double precision/1000), $4, $5) " + "ON CONFLICT (id) DO UPDATE SET hostname = EXCLUDED.hostname, last_heartbeat = " + "EXCLUDED.last_heartbeat, " + "public_identity = EXCLUDED.public_identity, version = EXCLUDED.version", + pqxx::params { controllerId, hostname, ts, publicIdentity, versionStr }) + .no_rows(); + w.commit(); + } + catch (std::exception& e) { + fprintf(stderr, "%s: Heartbeat update failed: %s\n", controllerId, e.what()); + span->End(); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + continue; + } + } + _pool->unborrow(c); + + try { + if (_listenerMode == LISTENER_MODE_REDIS && _redisMemberStatus) { + if (_cc->redisConfig->clusterMode) { + _cluster->zadd("controllers", "controllerId", ts); + } + else { + _redis->zadd("controllers", "controllerId", ts); + } + } + } + catch (sw::redis::Error& e) { + fprintf(stderr, "ERROR: Redis error in heartbeat thread: %s\n", e.what()); + } + + span->End(); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + fprintf(stderr, "Exited heartbeat thread\n"); +} + +void CentralDB::commitThread() +{ + fprintf(stderr, "%s: commitThread start\n", _myAddressStr.c_str()); + _queueItem qitem; + while (_commitQueue.get(qitem) & (_run == 1)) { + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + + fprintf(stderr, "checking qitem trace context\n"); + for (auto const& kv : qitem.traceContext) { + fprintf(stderr, "traceContext: %s: %s\n", kv.first.c_str(), kv.second.c_str()); + } + + auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + OtelCarrier > carrier(qitem.traceContext); + auto current_ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + auto new_context = propagator->Extract(carrier, current_ctx); + auto remote_span = opentelemetry::trace::GetSpan(new_context); + auto remote_scope = tracer->WithActiveSpan(remote_span); + + { + auto span = tracer->StartSpan("CentralDB::commitThread"); + auto scope = tracer->WithActiveSpan(span); + + fprintf(stderr, "commitThread tick\n"); + if (! qitem.jsonData.is_object()) { + fprintf(stderr, "not an object\n"); + continue; + } + + std::shared_ptr c; + try { + c = _pool->borrow(); + } + catch (std::exception& e) { + fprintf(stderr, "ERROR: %s\n", e.what()); + continue; + } + + if (! c) { + fprintf(stderr, "Error getting database connection\n"); + continue; + } + + Metrics::pgsql_commit_ticks++; + try { + nlohmann::json& config = (qitem.jsonData); + const std::string objtype = config["objtype"]; + if (objtype == "member") { + auto mspan = tracer->StartSpan("CentralDB::commitThread::member"); + auto mscope = tracer->WithActiveSpan(mspan); + + // fprintf(stderr, "%s: commitThread: member\n", _myAddressStr.c_str()); + std::string memberId; + std::string networkId; + + try { + pqxx::work w(*c->c); + + memberId = config["id"]; + networkId = config["nwid"]; + fprintf(stderr, "commit member %s-%s\n", networkId.c_str(), memberId.c_str()); + + std::string target = "NULL"; + if (! config["remoteTraceTarget"].is_null()) { + target = config["remoteTraceTarget"]; + } + + // get network and the frontend it is assigned to + // if network does not exist, skip member update + pqxx::row nwrow = + w.exec( + "SELECT COUNT(id), frontend FROM networks_ctl WHERE id = $1 GROUP BY frontend", + pqxx::params { networkId }) + .one_row(); + int nwcount = nwrow[0].as(); + std::string frontend = nwrow[1].as(); + + if (nwcount != 1) { + fprintf(stderr, "network %s does not exist. skipping member upsert\n", networkId.c_str()); + w.abort(); + _pool->unborrow(c); + continue; + } + + pqxx::row mrow = + w.exec( + "SELECT COUNT(device_id) FROM network_memberships_ctl WHERE device_id = $1 " + "AND network_id = $2", + pqxx::params { memberId, networkId }) + .one_row(); + int membercount = mrow[0].as(); + bool isNewMember = (membercount == 0); + + std::string change_source; + if (! config["change_source"].is_null()) { + change_source = config["change_source"]; + } + else { + change_source = "controller"; + } + if (! isNewMember && change_source != "controller" && frontend != change_source) { + fprintf( + stderr, "skipping member %s-%s update. change source: %s, frontend: %s\n", + networkId.c_str(), memberId.c_str(), change_source.c_str(), frontend.c_str()); + // if it is not a new member and the change source is not the controller and doesn't match + // the frontend, don't apply the change. + continue; + } + + std::vector ipAssignments; + fprintf( + stderr, "Saving IP Assignments: \n\tipAssignments: %s\n", + OSUtils::jsonDump(config["ipAssignments"], -1).c_str()); + if (config["ipAssignments"].is_array()) { + for (auto& ip : config["ipAssignments"]) { + if (ip.is_string()) { + ipAssignments.push_back(ip.get()); + } + } + } + + fprintf(stderr, "member json: %s\n", config.dump().c_str()); + + int64_t vMajor = OSUtils::jsonUInt(config["vMajor"], 0); + int64_t vMinor = OSUtils::jsonUInt(config["vMinor"], 0); + int64_t vRev = OSUtils::jsonUInt(config["vRev"], 0); + int64_t vProto = OSUtils::jsonUInt(config["vProto"], 0); + if (vMajor < 0) + vMajor = 0; + if (vMinor < 0) + vMinor = 0; + if (vRev < 0) + vRev = 0; + if (vProto < 0) + vProto = 0; + + pqxx::result res = + w.exec( + "INSERT INTO network_memberships_ctl (device_id, network_id, authorized, " + "active_bridge, " + "ip_assignments, " + "no_auto_assign_ips, sso_exempt, authentication_expiry_time, capabilities, " + "creation_time, " + "identity, last_authorized_time, last_deauthorized_time, " + "remote_trace_level, remote_trace_target, revision, tags, version_major, " + "version_minor, " + "version_revision, version_protocol) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, TO_TIMESTAMP($8::double precision/1000), $9, " + "TO_TIMESTAMP($10::double precision/1000), $11, TO_TIMESTAMP($12::double " + "precision/1000), " + "TO_TIMESTAMP($13::double precision/1000), $14, $15, $16, $17, $18, $19, $20, $21) " + "ON CONFLICT (device_id, network_id) DO UPDATE SET " + "authorized = EXCLUDED.authorized, active_bridge = EXCLUDED.active_bridge, " + "ip_assignments = EXCLUDED.ip_assignments, no_auto_assign_ips = " + "EXCLUDED.no_auto_assign_ips, " + "sso_exempt = EXCLUDED.sso_exempt, authentication_expiry_time = " + "EXCLUDED.authentication_expiry_time, " + "capabilities = EXCLUDED.capabilities, creation_time = EXCLUDED.creation_time, " + "identity = EXCLUDED.identity, last_authorized_time = EXCLUDED.last_authorized_time, " + "last_deauthorized_time = EXCLUDED.last_deauthorized_time, " + "remote_trace_level = EXCLUDED.remote_trace_level, remote_trace_target = " + "EXCLUDED.remote_trace_target, " + "revision = EXCLUDED.revision, tags = EXCLUDED.tags, version_major = " + "EXCLUDED.version_major, " + "version_minor = EXCLUDED.version_minor, version_revision = " + "EXCLUDED.version_revision, " + "version_protocol = EXCLUDED.version_protocol", + pqxx::params { memberId, + networkId, + OSUtils::jsonBool(config["authorized"], false), + OSUtils::jsonBool(config["activeBridge"], false), + ipAssignments, + OSUtils::jsonBool(config["noAutoAssignIps"], false), + OSUtils::jsonBool(config["ssoExempt"], false), + OSUtils::jsonInt(config["authenticationExpiryTime"], 0), + OSUtils::jsonDump(config["capabilities"], -1), + OSUtils::jsonInt(config["creationTime"], OSUtils::now()), + OSUtils::jsonString(config["identity"], ""), + OSUtils::jsonInt(config["lastAuthorizedTime"], 0), + OSUtils::jsonInt(config["lastDeauthorizedTime"], 0), + OSUtils::jsonInt(config["remoteTraceLevel"], 0), + target, + OSUtils::jsonInt(config["revision"], 0), + OSUtils::jsonDump(config["tags"], -1), + vMajor, + vMinor, + vRev, + vProto }) + .no_rows(); + + w.commit(); + + if (_listenerMode == LISTENER_MODE_PUBSUB) { + // Publish change to pubsub stream + + if (config["change_source"].is_null() || config["change_source"] == "controller") { + nlohmann::json oldMember; + nlohmann::json newMember = config; + if (! isNewMember) { + oldMember = _getNetworkMember(w, networkId, memberId); + } + _changeNotifier->notifyMemberChange(oldMember, newMember, frontend); + } + } + + if (_smee != NULL && isNewMember) { + // TODO: Smee Notifications for New Members + // pqxx::row row = w.exec_params1( + // "SELECT " + // " count(h.hook_id) " + // "FROM " + // " ztc_hook h " + // " INNER JOIN ztc_org o ON o.org_id = h.org_id " + // " INNER JOIN ztc_network n ON n.owner_id = o.owner_id " + // " WHERE " + // "n.id = $1 ", + // networkId); + // int64_t hookCount = row[0].as(); + // if (hookCount > 0) { + // notifyNewMember(networkId, memberId); + // } + } + + const uint64_t nwidInt = OSUtils::jsonIntHex(config["nwid"], 0ULL); + const uint64_t memberidInt = OSUtils::jsonIntHex(config["id"], 0ULL); + if (nwidInt && memberidInt) { + nlohmann::json nwOrig; + nlohmann::json memOrig; + + nlohmann::json memNew(config); + + get(nwidInt, nwOrig, memberidInt, memOrig); + + _memberChanged(memOrig, memNew, qitem.notifyListeners); + } + else { + fprintf( + stderr, "%s: Can't notify of change. Error parsing nwid or memberid: %llu-%llu\n", + _myAddressStr.c_str(), (unsigned long long)nwidInt, (unsigned long long)memberidInt); + } + } + catch (std::exception& e) { + fprintf( + stderr, "%s ERROR: Error updating member %s-%s: %s\n", _myAddressStr.c_str(), + networkId.c_str(), memberId.c_str(), e.what()); + mspan->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + } + } + else if (objtype == "network") { + auto nspan = tracer->StartSpan("CentralDB::commitThread::network"); + auto nscope = tracer->WithActiveSpan(nspan); + + try { + // fprintf(stderr, "%s: commitThread: network\n", _myAddressStr.c_str()); + pqxx::work w(*c->c); + + std::string id = config["id"]; + fprintf(stderr, "commit network %s\n", id.c_str()); + + pqxx::row nwrow = + w.exec("SELECT COUNT(id) frontend FROM networks_ctl WHERE id = $1", pqxx::params { id }) + .one_row(); + int nwcount = nwrow[0].as(); + bool isNewNetwork = (nwcount == 0); + std::string frontend = ""; + + if (! isNewNetwork) { + pqxx::row nwrow = + w.exec("SELECT frontend FROM networks_ctl WHERE id = $1", pqxx::params { id }) + .one_row(); + frontend = nwrow[0].as(); + } + + std::string change_source; + if (! config["change_source"].is_null()) { + change_source = config["change_source"]; + } + + if (! isNewNetwork && change_source != "controller" && frontend != change_source) { + // if it is not a new network and the change source is not the controller and doesn't match + // the frontend, don't apply the change. + fprintf( + stderr, + "Skipping network update %s. isNewNetwork: %s, change_source: %s, frontend: %s\n", + id.c_str(), isNewNetwork ? "true" : "false", change_source.c_str(), frontend.c_str()); + continue; + } + + pqxx::result res = w.exec( + "INSERT INTO networks_ctl (id, name, configuration, controller_id, revision, frontend) " + "VALUES ($1, $2, $3, $4, $5, $6) " + "ON CONFLICT (id) DO UPDATE SET " + "name = EXCLUDED.name, configuration = EXCLUDED.configuration, revision = " + "EXCLUDED.revision+1, " + "frontend = EXCLUDED.frontend", + pqxx::params { id, OSUtils::jsonString(config["name"], ""), OSUtils::jsonDump(config, -1), + _myAddressStr, ((uint64_t)config["revision"]), change_source }); + + w.commit(); + + if (_listenerMode == LISTENER_MODE_PUBSUB) { + // Publish change to pubsub stream + if (config["change_source"].is_null() || config["change_source"] == "controller") { + nlohmann::json oldNetwork; + nlohmann::json newNetwork = config; + if (! isNewNetwork) { + oldNetwork = _getNetwork(w, id); + } + _changeNotifier->notifyNetworkChange(oldNetwork, newNetwork, frontend); + } + } + + const uint64_t nwidInt = OSUtils::jsonIntHex(config["id"], 0ULL); + if (nwidInt) { + nlohmann::json nwOrig; + nlohmann::json nwNew(config); + + get(nwidInt, nwOrig); + + _networkChanged(nwOrig, nwNew, qitem.notifyListeners); + } + else { + fprintf( + stderr, "%s: Can't notify network changed: %llu\n", _myAddressStr.c_str(), + (unsigned long long)nwidInt); + } + } + catch (std::exception& e) { + nspan->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "%s ERROR: Error updating network: %s\n", _myAddressStr.c_str(), e.what()); + } + if (_listenerMode == LISTENER_MODE_REDIS && _redisMemberStatus) { + try { + std::string id = config["id"]; + std::string controllerId = _myAddressStr.c_str(); + std::string key = "networks:{" + controllerId + "}"; + if (_cc->redisConfig->clusterMode) { + _cluster->sadd(key, id); + } + else { + _redis->sadd(key, id); + } + } + catch (sw::redis::Error& e) { + nspan->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "ERROR: Error adding network to Redis: %s\n", e.what()); + } + } + } + else if (objtype == "_delete_network") { + auto dspan = tracer->StartSpan("CentralDB::commitThread::_delete_network"); + auto dscope = tracer->WithActiveSpan(dspan); + + // fprintf(stderr, "%s: commitThread: delete network\n", _myAddressStr.c_str()); + try { + pqxx::work w(*c->c); + std::string networkId = config["id"]; + fprintf(stderr, "Deleting network %s\n", networkId.c_str()); + w.exec("DELETE FROM network_memberships_ctl WHERE network_id = $1", pqxx::params { networkId }); + w.exec("DELETE FROM networks_ctl WHERE id = $1", pqxx::params { networkId }); + + w.commit(); + + uint64_t nwidInt = OSUtils::jsonIntHex(config["nwid"], 0ULL); + json oldConfig; + get(nwidInt, oldConfig); + json empty; + _networkChanged(oldConfig, empty, qitem.notifyListeners); + } + catch (std::exception& e) { + dspan->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "%s ERROR: Error deleting network: %s\n", _myAddressStr.c_str(), e.what()); + } + if (_listenerMode == LISTENER_MODE_REDIS && _redisMemberStatus) { + try { + std::string id = config["id"]; + std::string controllerId = _myAddressStr.c_str(); + std::string key = "networks:{" + controllerId + "}"; + if (_cc->redisConfig->clusterMode) { + _cluster->srem(key, id); + _cluster->del("network-nodes-online:{" + controllerId + "}:" + id); + } + else { + _redis->srem(key, id); + _redis->del("network-nodes-online:{" + controllerId + "}:" + id); + } + } + catch (sw::redis::Error& e) { + dspan->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "ERROR: Error adding network to Redis: %s\n", e.what()); + } + } + } + else if (objtype == "_delete_member") { + auto mspan = tracer->StartSpan("CentralDB::commitThread::_delete_member"); + auto mscope = tracer->WithActiveSpan(mspan); + + // fprintf(stderr, "%s commitThread: delete member\n", _myAddressStr.c_str()); + try { + pqxx::work w(*c->c); + + std::string memberId = config["id"]; + std::string networkId = config["nwid"]; + + fprintf(stderr, "Deleting member %s-%s\n", networkId.c_str(), memberId.c_str()); + + pqxx::result res = + w.exec( + "DELETE FROM network_memberships_ctl WHERE device_id = $1 AND network_id = $2", + pqxx::params { memberId, networkId }) + .no_rows(); + + w.commit(); + + uint64_t nwidInt = OSUtils::jsonIntHex(config["nwid"], 0ULL); + uint64_t memberidInt = OSUtils::jsonIntHex(config["id"], 0ULL); + + nlohmann::json networkConfig; + nlohmann::json oldConfig; + + get(nwidInt, networkConfig, memberidInt, oldConfig); + json empty; + _memberChanged(oldConfig, empty, qitem.notifyListeners); + } + catch (std::exception& e) { + mspan->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "%s ERROR: Error deleting member: %s\n", _myAddressStr.c_str(), e.what()); + } + if (_listenerMode == LISTENER_MODE_REDIS && _redisMemberStatus) { + try { + std::string memberId = config["id"]; + std::string networkId = config["nwid"]; + std::string controllerId = _myAddressStr.c_str(); + std::string key = "network-nodes-all:{" + controllerId + "}:" + networkId; + if (_cc->redisConfig->clusterMode) { + _cluster->srem(key, memberId); + _cluster->del("member:{" + controllerId + "}:" + networkId + ":" + memberId); + } + else { + _redis->srem(key, memberId); + _redis->del("member:{" + controllerId + "}:" + networkId + ":" + memberId); + } + } + catch (sw::redis::Error& e) { + mspan->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "ERROR: Error deleting member from Redis: %s\n", e.what()); + } + } + } + else { + fprintf(stderr, "%s ERROR: unknown objtype\n", _myAddressStr.c_str()); + } + } + catch (std::exception& e) { + span->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "%s ERROR: Error getting objtype: %s\n", _myAddressStr.c_str(), e.what()); + } + _pool->unborrow(c); + c.reset(); + } + } + + fprintf(stderr, "%s commitThread finished\n", _myAddressStr.c_str()); +} + +void CentralDB::notifyNewMember(const std::string& networkID, const std::string& memberID) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::notifyNewMember"); + auto scope = tracer->WithActiveSpan(span); + + rustybits::smee_client_notify_network_joined(_smee, networkID.c_str(), memberID.c_str()); +} + +void CentralDB::onlineNotificationThread() +{ + waitForReady(); + while (_run == 1) { + { + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("CentralDB"); + auto span = tracer->StartSpan("CentralDB::onlineNotificationThread"); + auto scope = tracer->WithActiveSpan(span); + + try { + std::unordered_map, NodeOnlineRecord, _PairHasher> lastOnline; + { + std::lock_guard l(_lastOnline_l); + lastOnline.swap(_lastOnline); + } + + uint64_t updateCount = 0; + auto c = _pool->borrow(); + pqxx::work w(*c->c); + for (auto i = lastOnline.begin(); i != lastOnline.end(); ++i) { + updateCount += 1; + uint64_t nwid_i = i->first.first; + char nwidTmp[64]; + char memTmp[64]; + char ipTmp[64]; + OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i); + OSUtils::ztsnprintf(memTmp, sizeof(memTmp), "%.10llx", i->first.second); + nlohmann::json network, member; + + if (! get(nwid_i, network, i->first.second, member)) { + continue; // skip non existent networks/members + } + + std::string networkId(nwidTmp); + std::string memberId(memTmp); + + try { + // check if the member exists first. + // + // exec_params1 will throw pqxx::unexpected_rows if not exactly one row is returned. If that's + // the case, skip this record and move on. + pqxx::row r = + w.exec( + "SELECT device_id, network_id FROM network_memberships_ctl WHERE network_id = " + "$1 AND device_id " + "= $2", + pqxx::params { networkId, memberId }) + .one_row(); + } + catch (pqxx::unexpected_rows& e) { + continue; + } + + int64_t ts = i->second.lastSeen; + std::string ipAddr = i->second.physicalAddress.toIpString(ipTmp); + std::string timestamp = std::to_string(ts); + std::string osArch = i->second.osArch; + std::vector osArchSplit = split(osArch, '/'); + std::string os = "unknown"; + std::string arch = "unknown"; + std::string frontend = OSUtils::jsonString(network["frontend"], ""); + + int vMajor = OSUtils::jsonInt(member["vMajor"], 0); + int vMinor = OSUtils::jsonInt(member["vMinor"], 0); + int vRev = OSUtils::jsonInt(member["vRev"], 0); + std::string version; + if (vMajor <= 0 && vMinor <= 0 && vRev <= 0) { + vMajor = 0; + vMinor = 0; + vRev = 0; + version = "unknown"; + } + else { + version = + "v" + std::to_string(vMajor) + "." + std::to_string(vMinor) + "." + std::to_string(vRev); + } + if (osArchSplit.size() == 2) { + os = osArchSplit[0]; + arch = osArchSplit[1]; + } + + _statusWriter->updateNodeStatus( + networkId, memberId, os, arch, version, i->second.physicalAddress, ts, frontend); + } + _statusWriter->writePending(); + w.commit(); + _pool->unborrow(c); + } + catch (std::exception& e) { + fprintf(stderr, "%s: error in onlinenotification thread: %s\n", _myAddressStr.c_str(), e.what()); + } + } + + std::this_thread::sleep_for(std::chrono::seconds(10)); + } +} + +nlohmann::json CentralDB::_getNetworkMember(pqxx::work& tx, const std::string networkID, const std::string memberID) +{ + nlohmann::json out; + + try { + pqxx::row row = + tx.exec( + "SELECT nm.device_id, nm.network_id, nm.authorized, nm.active_bridge, nm.ip_assignments, " + "nm.no_auto_assign_ips, " + "nm.sso_exempt, (EXTRACT(EPOCH FROM nm.authentication_expiry_time AT TIME ZONE 'UTC')*1000)::bigint, " + "(EXTRACT(EPOCH FROM nm.creation_time AT TIME ZONE 'UTC')*1000)::bigint, nm.identity, " + "(EXTRACT(EPOCH FROM nm.last_authorized_time AT TIME ZONE 'UTC')*1000)::bigint, " + "(EXTRACT(EPOCH FROM nm.last_deauthorized_time AT TIME ZONE 'UTC')*1000)::bigint, " + "nm.remote_trace_level, nm.remote_trace_target, nm.revision, nm.capabilities, nm.tags, " + "nm.frontend " + "FROM network_memberships_ctl nm " + "INNER JOIN networks_ctl n " + " ON nm.network_id = n.id " + "WHERE nm.network_id = $1 AND nm.device_id = $2", + pqxx::params { networkID, memberID }) + .one_row(); + + bool authorized = row[2].as(); + std::optional active_bridge = + row[3].is_null() ? std::optional() : std::optional(row[3].as()); + std::string ip_assignments = row[4].is_null() ? "{}" : row[4].as(); + std::optional no_auto_assign_ips = + row[5].is_null() ? std::optional() : std::optional(row[5].as()); + std::optional sso_exempt = + row[6].is_null() ? std::optional() : std::optional(row[6].as()); + std::optional authentication_expiry_time = + row[7].is_null() ? std::optional() : std::optional(row[7].as()); + std::optional creation_time = + row[8].is_null() ? std::optional() : std::optional(row[8].as()); + std::optional identity = + row[9].is_null() ? std::optional() : std::optional(row[9].as()); + std::optional last_authorized_time = + row[10].is_null() ? std::optional() : std::optional(row[10].as()); + std::optional last_deauthorized_time = + row[11].is_null() ? std::optional() : std::optional(row[11].as()); + std::optional remote_trace_level = + row[12].is_null() ? std::optional() : std::optional(row[12].as()); + std::optional remote_trace_target = + row[13].is_null() ? std::optional() : std::optional(row[13].as()); + std::optional revision = + row[14].is_null() ? std::optional() : std::optional(row[14].as()); + std::optional capabilities = + row[15].is_null() ? std::optional() : std::optional(row[15].as()); + std::optional tags = + row[16].is_null() ? std::optional() : std::optional(row[16].as()); + std::string frontend = row[17].is_null() ? "" : row[17].as(); + + out["objtype"] = "member"; + out["id"] = memberID; + out["nwid"] = networkID; + out["address"] = identity.value_or(""); + out["authorized"] = authorized; + out["activeBridge"] = active_bridge.value_or(false); + out["ipAssignments"] = json::array(); + if (ip_assignments != "{}" && ip_assignments != "[]") { + std::string tmp = ip_assignments.substr(1, ip_assignments.length() - 2); + std::vector addrs = split(tmp, ','); + for (auto it = addrs.begin(); it != addrs.end(); ++it) { + out["ipAssignments"].push_back(*it); + } + } + out["capabilities"] = json::parse(capabilities.value_or("[]")); + out["creationTime"] = creation_time.value_or(0); + out["lastAuthorizedTime"] = last_authorized_time.value_or(0); + out["lastDeauthorizedTime"] = last_deauthorized_time.value_or(0); + out["noAutoAssignIps"] = no_auto_assign_ips.value_or(false); + out["remoteTraceLevel"] = remote_trace_level.value_or(0); + out["remoteTraceTarget"] = remote_trace_target.value_or(nullptr); + out["revision"] = revision.value_or(0); + out["ssoExempt"] = sso_exempt.value_or(false); + out["authenticationExpiryTime"] = authentication_expiry_time.value_or(0); + out["tags"] = json::parse(tags.value_or("[]")); + out["frontend"] = frontend; + } + catch (std::exception& e) { + fprintf( + stderr, "ERROR: Error getting network member %s-%s: %s\n", networkID.c_str(), memberID.c_str(), e.what()); + return nlohmann::json(); + } + + return out; +} + +nlohmann::json CentralDB::_getNetwork(pqxx::work& tx, const std::string networkID) +{ + nlohmann::json out; + + try { + std::optional name; + std::string cfg; + std::optional creation_time; + std::optional last_modified; + std::optional revision; + std::string frontend; + + pqxx::row row = tx.exec( + "SELECT id, name, configuration , (EXTRACT(EPOCH FROM creation_time AT TIME ZONE " + "'UTC')*1000)::bigint, " + "(EXTRACT(EPOCH FROM last_modified AT TIME ZONE 'UTC')*1000)::bigint, revision, frontend " + "FROM networks_ctl WHERE id = $1", + pqxx::params { networkID }) + .one_row(); + + cfg = row[2].as(); + creation_time = row[3].is_null() ? std::optional() : std::optional(row[3].as()); + last_modified = row[4].is_null() ? std::optional() : std::optional(row[4].as()); + revision = row[5].is_null() ? std::optional() : std::optional(row[5].as()); + frontend = row[6].is_null() ? "" : row[6].as(); + + nlohmann::json cfgtmp = nlohmann::json::parse(cfg); + if (! cfgtmp.is_object()) { + fprintf(stderr, "ERROR: Network %s configuration is not a JSON object\n", networkID.c_str()); + return nlohmann::json(); + } + + out["objtype"] = "network"; + out["id"] = row[0].as(); + out["name"] = row[1].is_null() ? "" : row[1].as(); + out["creationTime"] = creation_time.value_or(0); + out["lastModified"] = last_modified.value_or(0); + out["revision"] = revision.value_or(0); + out["capabilities"] = cfgtmp["capabilities"].is_array() ? cfgtmp["capabilities"] : json::array(); + out["enableBroadcast"] = cfgtmp["enableBroadcast"].is_boolean() ? cfgtmp["enableBroadcast"].get() : false; + out["mtu"] = cfgtmp["mtu"].is_number() ? cfgtmp["mtu"].get() : 2800; + out["multicastLimit"] = cfgtmp["multicastLimit"].is_number() ? cfgtmp["multicastLimit"].get() : 64; + out["private"] = cfgtmp["private"].is_boolean() ? cfgtmp["private"].get() : true; + out["remoteTraceLevel"] = + cfgtmp["remoteTraceLevel"].is_number() ? cfgtmp["remoteTraceLevel"].get() : 0; + out["remoteTraceTarget"] = + cfgtmp["remoteTraceTarget"].is_string() ? cfgtmp["remoteTraceTarget"].get() : ""; + out["revision"] = revision.value_or(0); + out["rules"] = cfgtmp["rules"].is_array() ? cfgtmp["rules"] : json::array(); + out["tags"] = cfgtmp["tags"].is_array() ? cfgtmp["tags"] : json::array(); + if (cfgtmp["v4AssignMode"].is_object()) { + out["v4AssignMode"] = cfgtmp["v4AssignMode"]; + } + else { + out["v4AssignMode"] = json::object(); + out["v4AssignMode"]["zt"] = true; + } + if (cfgtmp["v6AssignMode"].is_object()) { + out["v6AssignMode"] = cfgtmp["v6AssignMode"]; + } + else { + out["v6AssignMode"] = json::object(); + out["v6AssignMode"]["zt"] = true; + out["v6AssignMode"]["6plane"] = true; + out["v6AssignMode"]["rfc4193"] = false; + } + out["ssoEnabled"] = cfgtmp["ssoEnabled"].is_boolean() ? cfgtmp["ssoEnabled"].get() : false; + out["objtype"] = "network"; + out["routes"] = cfgtmp["routes"].is_array() ? cfgtmp["routes"] : json::array(); + out["clientId"] = cfgtmp["clientId"].is_string() ? cfgtmp["clientId"].get() : ""; + out["authorizationEndpoint"] = + cfgtmp["authorizationEndpoint"].is_string() ? cfgtmp["authorizationEndpoint"].get() : nullptr; + out["provider"] = cfgtmp["ssoProvider"].is_string() ? cfgtmp["ssoProvider"].get() : ""; + if (! cfgtmp["dns"].is_object()) { + cfgtmp["dns"] = json::object(); + cfgtmp["dns"]["domain"] = ""; + cfgtmp["dns"]["servers"] = json::array(); + } + else { + out["dns"] = cfgtmp["dns"]; + } + out["ipAssignmentPools"] = cfgtmp["ipAssignmentPools"].is_array() ? cfgtmp["ipAssignmentPools"] : json::array(); + out["frontend"] = row[6].as(); + } + catch (std::exception& e) { + fprintf(stderr, "ERROR: Error getting network %s: %s\n", networkID.c_str(), e.what()); + return nlohmann::json(); + } + return out; +} + +#endif // ZT_CONTROLLER_USE_LIBPQ diff --git a/nonfree/controller/CentralDB.hpp b/nonfree/controller/CentralDB.hpp new file mode 100644 index 0000000000..ed4a160609 --- /dev/null +++ b/nonfree/controller/CentralDB.hpp @@ -0,0 +1,158 @@ +#ifdef ZT_CONTROLLER_USE_LIBPQ + +#ifndef ZT_CONTROLLER_CENTRAL_DB_HPP +#define ZT_CONTROLLER_CENTRAL_DB_HPP + +#define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4 + +#include "../../node/Metrics.hpp" +#include "ConnectionPool.hpp" +#include "DB.hpp" +#include "NotificationListener.hpp" +#include "PostgreSQL.hpp" +#include "StatusWriter.hpp" + +#include +#include +#include + +namespace rustybits { +struct SmeeClient; +} + +namespace ZeroTier { +struct RedisConfig; +struct ControllerConfig; +struct ControllerChangeNotifier; + +class CentralDB : public DB { + public: + enum ListenerMode { + LISTENER_MODE_PGSQL = 0, + LISTENER_MODE_REDIS = 1, + LISTENER_MODE_PUBSUB = 2, + }; + + enum StatusWriterMode { + STATUS_WRITER_MODE_PGSQL = 0, + STATUS_WRITER_MODE_REDIS = 1, + STATUS_WRITER_MODE_BIGTABLE = 2, + }; + + CentralDB( + const Identity& myId, + const char* connString, + int listenPort, + CentralDB::ListenerMode mode, + CentralDB::StatusWriterMode statusMode, + const ControllerConfig* cc); + virtual ~CentralDB(); + + virtual bool waitForReady(); + virtual bool isReady(); + virtual bool save(nlohmann::json& record, bool notifyListeners); + virtual void eraseNetwork(const uint64_t networkId); + virtual void eraseMember(const uint64_t networkId, const uint64_t memberId); + virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress); + virtual void nodeIsOnline( + const uint64_t networkId, + const uint64_t memberId, + const InetAddress& physicalAddress, + const char* osArch); + virtual AuthInfo getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL); + + virtual bool ready() + { + return _ready == 2; + } + + virtual void _memberChanged(nlohmann::json& old, nlohmann::json& memberConfig, bool notifyListeners) + { + DB::_memberChanged(old, memberConfig, notifyListeners); + } + + virtual void _networkChanged(nlohmann::json& old, nlohmann::json& networkConfig, bool notifyListeners) + { + DB::_networkChanged(old, networkConfig, notifyListeners); + } + + protected: + struct _PairHasher { + inline std::size_t operator()(const std::pair& p) const + { + return (std::size_t)(p.first ^ p.second); + } + }; + + private: + void initializeNetworks(); + void initializeMembers(); + void heartbeat(); + + void commitThread(); + void onlineNotificationThread(); + + void configureSmee(); + void notifyNewMember(const std::string& networkID, const std::string& memberID); + + nlohmann::json _getNetworkMember(pqxx::work& tx, const std::string networkID, const std::string memberID); + + nlohmann::json _getNetwork(pqxx::work& tx, const std::string networkID); + + private: + enum OverrideMode { ALLOW_PGBOUNCER_OVERRIDE = 0, NO_OVERRIDE = 1 }; + + ListenerMode _listenerMode; + StatusWriterMode _statusWriterMode; + const ControllerConfig* _cc; + std::shared_ptr > _pool; + + const Identity _myId; + const Address _myAddress; + std::string _myAddressStr; + std::string _connString; + + struct _queueItem { + _queueItem() : jsonData(), notifyListeners(false), traceContext() + { + } + + ~_queueItem() + { + } + + nlohmann::json jsonData; + bool notifyListeners; + std::map traceContext; + }; + BlockingQueue<_queueItem> _commitQueue; + + std::thread _heartbeatThread; + std::shared_ptr _membersDbWatcher; + std::shared_ptr _networksDbWatcher; + std::shared_ptr _statusWriter; + std::shared_ptr _changeNotifier; + std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS]; + std::thread _onlineNotificationThread; + + std::unordered_map, NodeOnlineRecord, _PairHasher> _lastOnline; + + mutable std::mutex _lastOnline_l; + mutable std::mutex _readyLock; + std::atomic _ready, _connected, _run; + mutable volatile bool _waitNoticePrinted; + + int _listenPort; + uint8_t _ssoPsk[48]; + + std::shared_ptr _redis; + std::shared_ptr _cluster; + bool _redisMemberStatus; + + rustybits::SmeeClient* _smee; +}; + +} // namespace ZeroTier + +#endif // ZT_CONTROLLER_CENTRAL_DB_HPP +#endif // ZT_CONTROLLER_USE_LIBPQ \ No newline at end of file diff --git a/nonfree/controller/ControllerChangeNotifier.cpp b/nonfree/controller/ControllerChangeNotifier.cpp new file mode 100644 index 0000000000..2f1bf67fb4 --- /dev/null +++ b/nonfree/controller/ControllerChangeNotifier.cpp @@ -0,0 +1,38 @@ +#include "ControllerChangeNotifier.hpp" + +#include "PubSubWriter.hpp" + +namespace ZeroTier { + +PubSubChangeNotifier::PubSubChangeNotifier( + std::string controllerID, + std::string project, + std::string memberChangeTopic, + std::string networkChangeTopic) + : ControllerChangeNotifier() + , _networkChangeWriter(std::make_shared(project, networkChangeTopic, controllerID)) + , _memberChangeWriter(std::make_shared(project, memberChangeTopic, controllerID)) +{ +} + +PubSubChangeNotifier::~PubSubChangeNotifier() +{ +} + +void PubSubChangeNotifier::notifyNetworkChange( + const nlohmann::json& oldNetwork, + const nlohmann::json& newNetwork, + const std::string& frontend) +{ + _networkChangeWriter->publishNetworkChange(oldNetwork, newNetwork, frontend); +} + +void PubSubChangeNotifier::notifyMemberChange( + const nlohmann::json& oldMember, + const nlohmann::json newMember, + const std::string& frontend) +{ + _memberChangeWriter->publishMemberChange(oldMember, newMember, frontend); +} + +} // namespace ZeroTier \ No newline at end of file diff --git a/nonfree/controller/ControllerChangeNotifier.hpp b/nonfree/controller/ControllerChangeNotifier.hpp new file mode 100644 index 0000000000..2a5e000ff4 --- /dev/null +++ b/nonfree/controller/ControllerChangeNotifier.hpp @@ -0,0 +1,53 @@ +#ifndef CONTROLLERCHANGENOTIFIER_HPP +#define CONTROLLERCHANGENOTIFIER_HPP + +#include +#include +#include + +namespace ZeroTier { + +class PubSubWriter; + +class ControllerChangeNotifier { + public: + virtual ~ControllerChangeNotifier() = default; + + virtual void notifyNetworkChange( + const nlohmann::json& oldNetwork, + const nlohmann::json& newNetwork, + const std::string& frontend = "") = 0; + + virtual void notifyMemberChange( + const nlohmann::json& oldMember, + const nlohmann::json newMember, + const std::string& frontend = "") = 0; +}; + +class PubSubChangeNotifier : public ControllerChangeNotifier { + public: + PubSubChangeNotifier( + std::string controllerID, + std::string project, + std::string memberChangeTopic, + std::string networkChangeTopic); + virtual ~PubSubChangeNotifier(); + + virtual void notifyNetworkChange( + const nlohmann::json& oldNetwork, + const nlohmann::json& newNetwork, + const std::string& frontend = "") override; + + virtual void notifyMemberChange( + const nlohmann::json& oldMember, + const nlohmann::json newMember, + const std::string& frontend = "") override; + + private: + std::shared_ptr _networkChangeWriter; + std::shared_ptr _memberChangeWriter; +}; + +} // namespace ZeroTier + +#endif // CONTROLLERCHANGENOTIFIER_HPP \ No newline at end of file diff --git a/nonfree/controller/ControllerConfig.hpp b/nonfree/controller/ControllerConfig.hpp new file mode 100644 index 0000000000..2df1736c7a --- /dev/null +++ b/nonfree/controller/ControllerConfig.hpp @@ -0,0 +1,60 @@ +#ifndef CONTROLLER_CONFIG_HPP +#define CONTROLLER_CONFIG_HPP + +#include "Redis.hpp" + +#include + +namespace ZeroTier { + +struct PubSubConfig { + std::string project_id; + std::string member_change_recv_topic; + std::string member_change_send_topic; + std::string network_change_recv_topic; + std::string network_change_send_topic; +}; + +struct BigTableConfig { + std::string project_id; + std::string instance_id; + std::string table_id; +}; + +struct ControllerConfig { + bool ssoEnabled; + std::string listenMode; + std::string statusMode; + RedisConfig* redisConfig; + PubSubConfig* pubSubConfig; + BigTableConfig* bigTableConfig; + + ControllerConfig() + : ssoEnabled(false) + , listenMode("") + , statusMode("") + , redisConfig(nullptr) + , pubSubConfig(nullptr) + , bigTableConfig(nullptr) + { + } + + ~ControllerConfig() + { + if (redisConfig) { + delete redisConfig; + redisConfig = nullptr; + } + if (pubSubConfig) { + delete pubSubConfig; + pubSubConfig = nullptr; + } + if (bigTableConfig) { + delete bigTableConfig; + bigTableConfig = nullptr; + } + } +}; + +} // namespace ZeroTier +#endif // CONTROLLER_CONFIG_HPP \ No newline at end of file diff --git a/nonfree/controller/CtlUtil.cpp b/nonfree/controller/CtlUtil.cpp index 51a5535cfb..32ce008ba2 100644 --- a/nonfree/controller/CtlUtil.cpp +++ b/nonfree/controller/CtlUtil.cpp @@ -9,6 +9,23 @@ #include #include +#ifdef ZT1_CENTRAL_CONTROLLER +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pubsub = ::google::cloud::pubsub; +namespace pubsub_admin = ::google::cloud::pubsub_admin; +namespace bigtable_admin = ::google::cloud::bigtable_admin; +#endif + namespace ZeroTier { const char* _timestr() @@ -63,6 +80,118 @@ std::string url_encode(const std::string& value) return escaped.str(); } -} // namespace ZeroTier +std::string random_hex_string(std::size_t length) +{ + static const char hex_chars[] = "0123456789abcdef"; + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, 15); + + std::string result; + result.reserve(length); + for (std::size_t i = 0; i < length; ++i) { + result += hex_chars[dis(gen)]; + } + return result; +} + +#ifdef ZT1_CENTRAL_CONTROLLER +void create_gcp_pubsub_topic_if_needed(std::string project_id, std::string topic_id) +{ + // This is a no-op if the topic already exists. + auto topicAdminClient = pubsub_admin::TopicAdminClient(pubsub_admin::MakeTopicAdminConnection()); + auto topicName = pubsub::Topic(project_id, topic_id).FullName(); + auto topicResult = topicAdminClient.GetTopic(topicName); + if (! topicResult.ok()) { + // Only create if not found + if (topicResult.status().code() == google::cloud::StatusCode::kNotFound) { + auto createResult = topicAdminClient.CreateTopic(topicName); + if (! createResult.ok()) { + fprintf(stderr, "Failed to create topic: %s\n", createResult.status().message().c_str()); + throw std::runtime_error("Failed to create topic"); + } + fprintf(stderr, "Created topic: %s\n", topicName.c_str()); + } + else { + fprintf(stderr, "Failed to get topic: %s\n", topicResult.status().message().c_str()); + throw std::runtime_error("Failed to get topic"); + } + } +} +void create_gcp_pubsub_subscription_if_needed( + std::string project_id, + std::string subscription_id, + std::string topic_id, + std::string controller_id) +{ + // This is a no-op if the subscription already exists. + auto subscriptionAdminClient = + pubsub_admin::SubscriptionAdminClient(pubsub_admin::MakeSubscriptionAdminConnection()); + auto topicName = pubsub::Topic(project_id, topic_id).FullName(); + auto subscriptionName = pubsub::Subscription(project_id, subscription_id).FullName(); + + auto sub = subscriptionAdminClient.GetSubscription(subscriptionName); + if (! sub.ok()) { + if (sub.status().code() == google::cloud::StatusCode::kNotFound) { + fprintf(stderr, "Creating subscription %s for topic %s\n", subscriptionName.c_str(), topicName.c_str()); + google::pubsub::v1::Subscription request; + request.set_name(subscriptionName); + request.set_topic(pubsub::Topic(project_id, topic_id).FullName()); + request.set_filter("(attributes.controller_id=\"" + controller_id + "\")"); + request.set_enable_message_ordering(true); + auto createResult = subscriptionAdminClient.CreateSubscription(request); + if (! createResult.ok()) { + fprintf(stderr, "Failed to create subscription: %s\n", createResult.status().message().c_str()); + throw std::runtime_error("Failed to create subscription"); + } + fprintf(stderr, "Created subscription: %s\n", subscriptionName.c_str()); + } + else { + fprintf(stderr, "Failed to get subscription: %s\n", sub.status().message().c_str()); + throw std::runtime_error("Failed to get subscription"); + } + } +} + +// void create_bigtable_table(std::string project_id, std::string instance_id) +// { +// auto bigtableAdminClient = +// bigtable_admin::BigtableTableAdminClient(bigtable_admin::MakeBigtableTableAdminConnection()); + +// std::string table_id = "member_status"; +// std::string table_name = "projects/" + project_id + "/instances/" + instance_id + "/tables/" + table_id; + +// // Check if the table exists +// auto table = bigtableAdminClient.GetTable(table_name); +// if (! table.ok()) { +// if (table.status().code() == google::cloud::StatusCode::kNotFound) { +// google::bigtable::admin::v2::Table table_config; +// table_config.set_name(table_id); +// auto families = table_config.mutable_column_families(); +// // Define column families +// // Column family "node_info" with max 1 version +// // google::bigtable::admin::v2::ColumnFamily* node_info = table_config.add_column_families(); +// // Column family "check_in" with max 1 version + +// auto create_result = bigtableAdminClient.CreateTable( +// "projects/" + project_id + "/instances/" + instance_id, table_id, table_config); + +// if (! create_result.ok()) { +// fprintf( +// stderr, "Failed to create Bigtable table member_status: %s\n", +// create_result.status().message().c_str()); +// throw std::runtime_error("Failed to create Bigtable table"); +// } +// fprintf(stderr, "Created Bigtable table: member_status\n"); +// } +// else { +// fprintf(stderr, "Failed to get Bigtable table member_status: %s\n", table.status().message().c_str()); +// throw std::runtime_error("Failed to get Bigtable table"); +// } +// } +// } +#endif + +} // namespace ZeroTier #endif diff --git a/nonfree/controller/CtlUtil.hpp b/nonfree/controller/CtlUtil.hpp index 5347f9a4d1..c10322012a 100644 --- a/nonfree/controller/CtlUtil.hpp +++ b/nonfree/controller/CtlUtil.hpp @@ -15,6 +15,19 @@ const char* _timestr(); std::vector split(std::string str, char delim); std::string url_encode(const std::string& value); + +std::string random_hex_string(std::size_t length); + +#ifdef ZT1_CENTRAL_CONTROLLER +void create_gcp_pubsub_topic_if_needed(std::string project_id, std::string topic_id); + +void create_gcp_pubsub_subscription_if_needed( + std::string project_id, + std::string subscription_id, + std::string topic_id, + std::string controller_id); +#endif + } // namespace ZeroTier #endif // namespace ZeroTier diff --git a/nonfree/controller/DB.cpp b/nonfree/controller/DB.cpp index e71fe6eeef..c6d7f74484 100644 --- a/nonfree/controller/DB.cpp +++ b/nonfree/controller/DB.cpp @@ -164,7 +164,8 @@ bool DB::get(const uint64_t networkId, nlohmann::json& network) auto tracer = provider->GetTracer("db"); auto span = tracer->StartSpan("db::getNetwork"); auto scope = tracer->WithActiveSpan(span); - char networkIdStr[17]; + char networkIdStr[32]; + memset(networkIdStr, 0, sizeof(networkIdStr)); span->SetAttribute("network_id", Utils::hex(networkId, networkIdStr)); waitForReady(); @@ -190,8 +191,10 @@ bool DB::get(const uint64_t networkId, nlohmann::json& network, const uint64_t m auto tracer = provider->GetTracer("db"); auto span = tracer->StartSpan("db::getNetworkAndMember"); auto scope = tracer->WithActiveSpan(span); - char networkIdStr[17]; - char memberIdStr[11]; + char networkIdStr[32]; + memset(networkIdStr, 0, sizeof(networkIdStr)); + char memberIdStr[32]; + memset(memberIdStr, 0, sizeof(memberIdStr)); span->SetAttribute("network_id", Utils::hex(networkId, networkIdStr)); span->SetAttribute("member_id", Utils::hex(networkId, memberIdStr)); @@ -216,14 +219,21 @@ bool DB::get(const uint64_t networkId, nlohmann::json& network, const uint64_t m return true; } -bool DB::get(const uint64_t networkId, nlohmann::json& network, const uint64_t memberId, nlohmann::json& member, NetworkSummaryInfo& info) +bool DB::get( + const uint64_t networkId, + nlohmann::json& network, + const uint64_t memberId, + nlohmann::json& member, + NetworkSummaryInfo& info) { auto provider = opentelemetry::trace::Provider::GetTracerProvider(); auto tracer = provider->GetTracer("db"); auto span = tracer->StartSpan("db::getNetworkAndMemberAndSummary"); auto scope = tracer->WithActiveSpan(span); - char networkIdStr[17]; - char memberIdStr[11]; + char networkIdStr[32]; + memset(networkIdStr, 0, sizeof(networkIdStr)); + char memberIdStr[32]; + memset(memberIdStr, 0, sizeof(memberIdStr)); span->SetAttribute("network_id", Utils::hex(networkId, networkIdStr)); span->SetAttribute("member_id", Utils::hex(memberId, memberIdStr)); @@ -256,7 +266,8 @@ bool DB::get(const uint64_t networkId, nlohmann::json& network, std::vectorGetTracer("db"); auto span = tracer->StartSpan("db::getNetworkAndMembers"); auto scope = tracer->WithActiveSpan(span); - char networkIdStr[17]; + char networkIdStr[32]; + memset(networkIdStr, 0, sizeof(networkIdStr)); span->SetAttribute("network_id", Utils::hex(networkId, networkIdStr)); waitForReady(); @@ -500,9 +511,14 @@ void DB::_networkChanged(nlohmann::json& old, nlohmann::json& networkConfig, boo this->get(networkId, network, members); for (auto i = members.begin(); i != members.end(); ++i) { const std::string nodeID = (*i)["id"]; + fprintf( + stderr, "Deauthorizing member %s on network %s due to network deletion\n", nodeID.c_str(), + ids.c_str()); const uint64_t memberId = Utils::hexStrToU64(nodeID.c_str()); std::unique_lock ll(_changeListeners_l); for (auto j = _changeListeners.begin(); j != _changeListeners.end(); ++j) { + fprintf( + stderr, "Notifying listener of deauthorization of %s on %s\n", nodeID.c_str(), ids.c_str()); (*j)->onNetworkMemberDeauthorize(this, networkId, memberId); } } diff --git a/nonfree/controller/DB.hpp b/nonfree/controller/DB.hpp index 0d8026ab7e..0eb4b6f2b8 100644 --- a/nonfree/controller/DB.hpp +++ b/nonfree/controller/DB.hpp @@ -136,6 +136,9 @@ class DB { _changeListeners.push_back(listener); } + virtual void _memberChanged(nlohmann::json& old, nlohmann::json& memberConfig, bool notifyListeners); + virtual void _networkChanged(nlohmann::json& old, nlohmann::json& networkConfig, bool notifyListeners); + protected: static inline bool _compareRecords(const nlohmann::json& a, const nlohmann::json& b) { @@ -172,8 +175,6 @@ class DB { std::shared_mutex lock; }; - virtual void _memberChanged(nlohmann::json& old, nlohmann::json& memberConfig, bool notifyListeners); - virtual void _networkChanged(nlohmann::json& old, nlohmann::json& networkConfig, bool notifyListeners); void _fillSummaryInfo(const std::shared_ptr<_Network>& nw, NetworkSummaryInfo& info); std::vector _changeListeners; diff --git a/nonfree/controller/EmbeddedNetworkController.cpp b/nonfree/controller/EmbeddedNetworkController.cpp index 9011f5f5cf..eb8c8c0431 100644 --- a/nonfree/controller/EmbeddedNetworkController.cpp +++ b/nonfree/controller/EmbeddedNetworkController.cpp @@ -26,12 +26,16 @@ #ifdef ZT_CONTROLLER_USE_LIBPQ #include "CV1.hpp" #include "CV2.hpp" +#include "CentralDB.hpp" +#ifdef ZT1_CENTRAL_CONTROLLER +#include "ControllerConfig.hpp" +#endif #endif -#include "../node/CertificateOfMembership.hpp" -#include "../node/Dictionary.hpp" -#include "../node/NetworkConfig.hpp" -#include "../node/Node.hpp" +#include "../../node/CertificateOfMembership.hpp" +#include "../../node/Dictionary.hpp" +#include "../../node/NetworkConfig.hpp" +#include "../../node/Node.hpp" #include "opentelemetry/trace/provider.h" using json = nlohmann::json; @@ -116,29 +120,17 @@ static json _renderRule(ZT_VirtualNetworkRule& rule) case ZT_NETWORK_RULE_MATCH_MAC_SOURCE: r["type"] = "MATCH_MAC_SOURCE"; OSUtils::ztsnprintf( - tmp, - sizeof(tmp), - "%.2x:%.2x:%.2x:%.2x:%.2x:%.2x", - (unsigned int)rule.v.mac[0], - (unsigned int)rule.v.mac[1], - (unsigned int)rule.v.mac[2], - (unsigned int)rule.v.mac[3], - (unsigned int)rule.v.mac[4], - (unsigned int)rule.v.mac[5]); + tmp, sizeof(tmp), "%.2x:%.2x:%.2x:%.2x:%.2x:%.2x", (unsigned int)rule.v.mac[0], + (unsigned int)rule.v.mac[1], (unsigned int)rule.v.mac[2], (unsigned int)rule.v.mac[3], + (unsigned int)rule.v.mac[4], (unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_MAC_DEST: r["type"] = "MATCH_MAC_DEST"; OSUtils::ztsnprintf( - tmp, - sizeof(tmp), - "%.2x:%.2x:%.2x:%.2x:%.2x:%.2x", - (unsigned int)rule.v.mac[0], - (unsigned int)rule.v.mac[1], - (unsigned int)rule.v.mac[2], - (unsigned int)rule.v.mac[3], - (unsigned int)rule.v.mac[4], - (unsigned int)rule.v.mac[5]); + tmp, sizeof(tmp), "%.2x:%.2x:%.2x:%.2x:%.2x:%.2x", (unsigned int)rule.v.mac[0], + (unsigned int)rule.v.mac[1], (unsigned int)rule.v.mac[2], (unsigned int)rule.v.mac[3], + (unsigned int)rule.v.mac[4], (unsigned int)rule.v.mac[5]); r["mac"] = tmp; break; case ZT_NETWORK_RULE_MATCH_IPV4_SOURCE: @@ -518,7 +510,12 @@ static bool _parseRule(json& r, ZT_VirtualNetworkRule& rule) } // anonymous namespace -EmbeddedNetworkController::EmbeddedNetworkController(Node* node, const char* ztPath, const char* dbPath, int listenPort, RedisConfig* rc) +EmbeddedNetworkController::EmbeddedNetworkController( + Node* node, + const char* ztPath, + const char* dbPath, + int listenPort, + RedisConfig* rc) : _startTime(OSUtils::now()) , _listenPort(listenPort) , _node(node) @@ -566,6 +563,61 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node* node, const char* ztP { } +#ifdef ZT1_CENTRAL_CONTROLLER +EmbeddedNetworkController::EmbeddedNetworkController( + Node* node, + const char* ztPath, + const char* dbPath, + int listenPort, + const ControllerConfig* cc) + : _startTime(OSUtils::now()) + , _listenPort(listenPort) + , _node(node) + , _ztPath(ztPath) + , _path(dbPath) + , _signingId() + , _signingIdAddressString() + , _sender((NetworkController::Sender*)0) + , _db(this) + , _queue() + , _threads() + , _threads_l() + , _memberStatus() + , _memberStatus_l() + , _expiringSoon() + , _expiringSoon_l() + , _rc(nullptr) + , _cc(cc) + , _ssoExpiryRunning(true) + , _ssoExpiry(std::thread(&EmbeddedNetworkController::_ssoExpiryThread, this)) +#ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK + , _member_status_lookup { "nc_member_status_lookup", "" } + , _member_status_lookup_count { "nc_member_status_lookup_count", "" } + , _node_is_online { "nc_node_is_online", "" } + , _node_is_online_count { "nc_node_is_online_count", "" } + , _get_and_init_member { "nc_get_and_init_member", "" } + , _get_and_init_member_count { "nc_get_and_init_member_count", "" } + , _have_identity { "nc_have_identity", "" } + , _have_identity_count { "nc_have_identity_count", "" } + , _determine_auth { "nc_determine_auth", "" } + , _determine_auth_count { "nc_determine_auth_count", "" } + , _sso_check { "nc_sso_check", "" } + , _sso_check_count { "nc_sso_check_count", "" } + , _auth_check { "nc_auth_check", "" } + , _auth_check_count { "nc_auth_check_count", "" } + , _json_schlep { "nc_json_schlep", "" } + , _json_schlep_count { "nc_json_schlep_count", "" } + , _issue_certificate { "nc_issue_certificate", "" } + , _issue_certificate_count { "nc_issue_certificate_count", "" } + , _save_member { "nc_save_member", "" } + , _save_member_count { "nc_save_member_count", "" } + , _send_netconf { "nc_send_netconf2", "" } + , _send_netconf_count { "nc_send_netconf2_count", "" } +#endif +{ +} +#endif + EmbeddedNetworkController::~EmbeddedNetworkController() { std::lock_guard l(_threads_l); @@ -594,6 +646,47 @@ void EmbeddedNetworkController::init(const Identity& signingId, Sender* sender) _sender = sender; _signingIdAddressString = signingId.address().toString(tmp); +#ifdef ZT1_CENTRAL_CONTROLLER + if (! _cc) { + throw std::runtime_error("controller config required"); + } + + if (_path.length() > 9 && (_path.substr(0, 9) != "postgres:")) { + throw std::runtime_error("central controller requires postgres db"); + } + + std::string connString = _path.substr(9); + + CentralDB::ListenerMode lm; + if (_cc->listenMode == "pgsql") { + lm = CentralDB::LISTENER_MODE_PGSQL; + } + else if (_cc->listenMode == "redis") { + lm = CentralDB::LISTENER_MODE_REDIS; + } + else if (_cc->listenMode == "pubsub") { + lm = CentralDB::LISTENER_MODE_PUBSUB; + } + else { + throw std::runtime_error("unsupported listen mode"); + } + + CentralDB::StatusWriterMode sm; + if (_cc->statusMode == "pgsql") { + sm = CentralDB::STATUS_WRITER_MODE_PGSQL; + } + else if (_cc->statusMode == "redis") { + sm = CentralDB::STATUS_WRITER_MODE_REDIS; + } + else if (_cc->statusMode == "bigtable") { + sm = CentralDB::STATUS_WRITER_MODE_BIGTABLE; + } + else { + throw std::runtime_error("unsupported status mode"); + } + + _db.addDB(std::shared_ptr(new CentralDB(_signingId, connString.c_str(), _listenPort, lm, sm, _cc))); +#else #ifdef ZT_CONTROLLER_USE_LIBPQ if ((_path.length() > 9) && (_path.substr(0, 9) == "postgres:")) { fprintf(stderr, "CV1\n"); @@ -610,18 +703,25 @@ void EmbeddedNetworkController::init(const Identity& signingId, Sender* sender) #ifdef ZT_CONTROLLER_USE_LIBPQ } #endif +#endif // ZT1_CENTRAL_CONTROLLER _db.waitForReady(); } -void EmbeddedNetworkController::request(uint64_t nwid, const InetAddress& fromAddr, uint64_t requestPacketId, const Identity& identity, const Dictionary& metaData) +void EmbeddedNetworkController::request( + uint64_t nwid, + const InetAddress& fromAddr, + uint64_t requestPacketId, + const Identity& identity, + const Dictionary& metaData) { auto provider = opentelemetry::trace::Provider::GetTracerProvider(); auto tracer = provider->GetTracer("embedded_controller"); auto span = tracer->StartSpan("embedded_controller::request"); auto scope = tracer->WithActiveSpan(span); - if (((! _signingId) || (! _signingId.hasPrivate())) || (_signingId.address().toInt() != (nwid >> 24)) || (! _sender)) + if (((! _signingId) || (! _signingId.hasPrivate())) || (_signingId.address().toInt() != (nwid >> 24)) + || (! _sender)) return; _startThreads(); @@ -670,7 +770,9 @@ std::string EmbeddedNetworkController::networkUpdateFromPostData(uint64_t networ if (b.count("multicastLimit")) network["multicastLimit"] = OSUtils::jsonInt(b["multicastLimit"], 32ULL); if (b.count("mtu")) - network["mtu"] = std::max(std::min((unsigned int)OSUtils::jsonInt(b["mtu"], ZT_DEFAULT_MTU), (unsigned int)ZT_MAX_MTU), (unsigned int)ZT_MIN_MTU); + network["mtu"] = std::max( + std::min((unsigned int)OSUtils::jsonInt(b["mtu"], ZT_DEFAULT_MTU), (unsigned int)ZT_MAX_MTU), + (unsigned int)ZT_MIN_MTU); if (b.count("remoteTraceTarget")) { const std::string rtt(OSUtils::jsonString(b["remoteTraceTarget"], "")); @@ -743,7 +845,9 @@ std::string EmbeddedNetworkController::networkUpdateFromPostData(uint64_t networ for (unsigned long i = 0; i < relays.size(); ++i) { json& relay = relays[i]; if (relay.is_string()) { - nrelays.push_back(Address(Utils::hexStrToU64(OSUtils::jsonString(relay, "0").c_str()) & 0xffffffffffULL).toString(rtmp)); + nrelays.push_back( + Address(Utils::hexStrToU64(OSUtils::jsonString(relay, "0").c_str()) & 0xffffffffffULL) + .toString(rtmp)); } } } @@ -945,7 +1049,10 @@ std::string EmbeddedNetworkController::networkUpdateFromPostData(uint64_t networ return network.dump(); } -void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, httplib::Server& sv6, const std::function setContent) +void EmbeddedNetworkController::configureHTTPControlPlane( + httplib::Server& s, + httplib::Server& sv6, + const std::function setContent) { // Control plane Endpoints std::string controllerPath = "/controller"; @@ -966,12 +1073,9 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht char tmp[4096]; const bool dbOk = _db.isReady(); OSUtils::ztsnprintf( - tmp, - sizeof(tmp), + tmp, sizeof(tmp), "{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu,\n\t\"databaseReady\": %s\n}\n", - ZT_NETCONF_CONTROLLER_API_VERSION, - (unsigned long long)OSUtils::now(), - dbOk ? "true" : "false"); + ZT_NETCONF_CONTROLLER_API_VERSION, (unsigned long long)OSUtils::now(), dbOk ? "true" : "false"); if (! dbOk) { res.status = 503; @@ -1361,7 +1465,8 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht for (unsigned long i = 0; i < tags.size(); ++i) { json& tag = tags[i]; if ((tag.is_array()) && (tag.size() == 2)) - mtags[OSUtils::jsonInt(tag[0], 0ULL) & 0xffffffffULL] = OSUtils::jsonInt(tag[1], 0ULL) & 0xffffffffULL; + mtags[OSUtils::jsonInt(tag[0], 0ULL) & 0xffffffffULL] = + OSUtils::jsonInt(tag[1], 0ULL) & 0xffffffffULL; } json mtagsa = json::array(); for (std::map::iterator t(mtags.begin()); t != mtags.end(); ++t) { @@ -1491,7 +1596,9 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace& rt) } const int64_t now = OSUtils::now(); - OSUtils::ztsnprintf(id, sizeof(id), "%.10llx-%.16llx-%.10llx-%.4x", _signingId.address().toInt(), now, rt.origin, (unsigned int)(idCounter++ & 0xffff)); + OSUtils::ztsnprintf( + id, sizeof(id), "%.10llx-%.16llx-%.10llx-%.4x", _signingId.address().toInt(), now, rt.origin, + (unsigned int)(idCounter++ & 0xffff)); d["id"] = id; d["objtype"] = "trace"; d["ts"] = now; @@ -1519,7 +1626,11 @@ void EmbeddedNetworkController::onNetworkUpdate(const void* db, uint64_t network } } -void EmbeddedNetworkController::onNetworkMemberUpdate(const void* db, uint64_t networkId, uint64_t memberId, const nlohmann::json& member) +void EmbeddedNetworkController::onNetworkMemberUpdate( + const void* db, + uint64_t networkId, + uint64_t memberId, + const nlohmann::json& member) { auto provider = opentelemetry::trace::Provider::GetTracerProvider(); auto tracer = provider->GetTracer("embedded_controller"); @@ -1544,19 +1655,32 @@ void EmbeddedNetworkController::onNetworkMemberDeauthorize(const void* db, uint6 auto span = tracer->StartSpan("embedded_controller::onNetworkMemberDeauthorize"); auto scope = tracer->WithActiveSpan(span); + fprintf(stderr, "Member Deauthorized: nwid=%.16llx, nodeid=%.10llx\n", networkId, memberId); const int64_t now = OSUtils::now(); - Revocation rev((uint32_t)_node->prng(), networkId, 0, now, ZT_REVOCATION_FLAG_FAST_PROPAGATE, Address(memberId), Revocation::CREDENTIAL_TYPE_COM); + Revocation rev( + (uint32_t)_node->prng(), networkId, 0, now, ZT_REVOCATION_FLAG_FAST_PROPAGATE, Address(memberId), + Revocation::CREDENTIAL_TYPE_COM); rev.sign(_signingId); { std::lock_guard l(_memberStatus_l); for (auto i = _memberStatus.begin(); i != _memberStatus.end(); ++i) { - if ((i->first.networkId == networkId) && (i->second.online(now))) + if ((i->first.networkId == networkId) && (i->second.online(now))) { _node->ncSendRevocation(Address(i->first.nodeId), rev); + fprintf(stderr, " Sent revocation to %.10llx\n", i->first.nodeId); + } + else { + fprintf(stderr, " Not sending revocation to %.10llx\n", i->first.nodeId); + } } } } -void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromAddr, uint64_t requestPacketId, const Identity& identity, const Dictionary& metaData) +void EmbeddedNetworkController::_request( + uint64_t nwid, + const InetAddress& fromAddr, + uint64_t requestPacketId, + const Identity& identity, + const Dictionary& metaData) { auto provider = opentelemetry::trace::Provider::GetTracerProvider(); auto tracer = provider->GetTracer("embedded_controller"); @@ -1579,7 +1703,8 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA DB::NetworkSummaryInfo ns; json network, member; - if (((! _signingId) || (! _signingId.hasPrivate())) || (_signingId.address().toInt() != (nwid >> 24)) || (! _sender)) { + if (((! _signingId) || (! _signingId.hasPrivate())) || (_signingId.address().toInt() != (nwid >> 24)) + || (! _sender)) { return; } @@ -1608,7 +1733,8 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA Utils::hex(nwid, nwids); _db.get(nwid, network, identity.address().toInt(), member, ns); if ((! network.is_object()) || (network.empty())) { - _sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_OBJECT_NOT_FOUND, nullptr, 0); + _sender->ncSendError( + nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_OBJECT_NOT_FOUND, nullptr, 0); #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK b3.stop(); #endif @@ -1635,7 +1761,9 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA // known member. try { if (Identity(haveIdStr.c_str()) != identity) { - _sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0); + _sender->ncSendError( + nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, + 0); #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK b4.stop(); #endif @@ -1643,7 +1771,8 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA } } catch (...) { - _sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0); + _sender->ncSendError( + nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0); #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK b4.stop(); #endif @@ -1740,7 +1869,9 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA Dictionary<4096> authInfo; authInfo.add(ZT_AUTHINFO_DICT_KEY_VERSION, (uint64_t)0ULL); authInfo.add(ZT_AUTHINFO_DICT_KEY_AUTHENTICATION_URL, info.authenticationURL.c_str()); - _sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes()); + _sender->ncSendError( + nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, + authInfo.data(), authInfo.sizeBytes()); } else if (info.version == 1) { Dictionary<8192> authInfo; @@ -1751,7 +1882,9 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA authInfo.add(ZT_AUTHINFO_DICT_KEY_STATE, info.ssoState.c_str()); authInfo.add(ZT_AUTHINFO_DICT_KEY_CLIENT_ID, info.ssoClientID.c_str()); authInfo.add(ZT_AUTHINFO_DICT_KEY_SSO_PROVIDER, info.ssoProvider.c_str()); - _sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes()); + _sender->ncSendError( + nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, + authInfo.data(), authInfo.sizeBytes()); } DB::cleanMember(member); _db.save(member, true); @@ -1804,7 +1937,8 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA // If they are not authorized, STOP! DB::cleanMember(member); _db.save(member, true); - _sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0); + _sender->ncSendError( + nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0); #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK b7.stop(); #endif @@ -1830,7 +1964,9 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA if (network.contains("certificateTimeoutWindowSize")) { credentialtmd = (int64_t)network["certificateTimeoutWindowSize"]; } - credentialtmd = std::max(std::min(credentialtmd, ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA), ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA); + credentialtmd = std::max( + std::min(credentialtmd, ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA), + ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MIN_MAX_DELTA); std::unique_ptr nc(new NetworkConfig()); @@ -1843,7 +1979,9 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA if (OSUtils::jsonBool(network["enableBroadcast"], true)) nc->flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST; Utils::scopy(nc->name, sizeof(nc->name), OSUtils::jsonString(network["name"], "").c_str()); - nc->mtu = std::max(std::min((unsigned int)OSUtils::jsonInt(network["mtu"], ZT_DEFAULT_MTU), (unsigned int)ZT_MAX_MTU), (unsigned int)ZT_MIN_MTU); + nc->mtu = std::max( + std::min((unsigned int)OSUtils::jsonInt(network["mtu"], ZT_DEFAULT_MTU), (unsigned int)ZT_MAX_MTU), + (unsigned int)ZT_MIN_MTU); nc->multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"], 32ULL); nc->ssoEnabled = networkSSOEnabled; // OSUtils::jsonBool(network["ssoEnabled"], false); @@ -1912,7 +2050,8 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA json& memberTags = member["tags"]; json& dns = network["dns"]; - // fprintf(stderr, "IP Assignment Pools for Network %s: %s\n", nwids, OSUtils::jsonDump(ipAssignmentPools, 2).c_str()); + // fprintf(stderr, "IP Assignment Pools for Network %s: %s\n", nwids, OSUtils::jsonDump(ipAssignmentPools, + // 2).c_str()); if (metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV, 0) <= 0) { // Old versions with no rules engine support get an allow everything rule. @@ -1985,7 +2124,8 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA for (unsigned long i = 0; i < memberTags.size(); ++i) { json& t = memberTags[i]; if ((t.is_array()) && (t.size() == 2)) - memberTagsById[(uint32_t)(OSUtils::jsonInt(t[0], 0ULL) & 0xffffffffULL)] = (uint32_t)(OSUtils::jsonInt(t[1], 0ULL) & 0xffffffffULL); + memberTagsById[(uint32_t)(OSUtils::jsonInt(t[0], 0ULL) & 0xffffffffULL)] = + (uint32_t)(OSUtils::jsonInt(t[1], 0ULL) & 0xffffffffULL); } } if (tags.is_array()) { // check network tags array for defaults that are not present in member tags @@ -2070,7 +2210,8 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA int routedNetmaskBits = -1; for (unsigned int rk = 0; rk < nc->routeCount; ++rk) { if (reinterpret_cast(&(nc->routes[rk].target))->containsAddress(ip)) { - const int nb = (int)(reinterpret_cast(&(nc->routes[rk].target))->netmaskBits()); + const int nb = + (int)(reinterpret_cast(&(nc->routes[rk].target))->netmaskBits()); if (nb > routedNetmaskBits) routedNetmaskBits = nb; } @@ -2093,7 +2234,8 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA ipAssignments = json::array(); } - if ((ipAssignmentPools.is_array()) && ((v6AssignMode.is_object()) && (OSUtils::jsonBool(v6AssignMode["zt"], false))) && (! haveManagedIpv6AutoAssignment) && (! noAutoAssignIps)) { + if ((ipAssignmentPools.is_array()) && ((v6AssignMode.is_object()) && (OSUtils::jsonBool(v6AssignMode["zt"], false))) + && (! haveManagedIpv6AutoAssignment) && (! noAutoAssignIps)) { for (unsigned long p = 0; ((p < ipAssignmentPools.size()) && (! haveManagedIpv6AutoAssignment)); ++p) { json& pool = ipAssignmentPools[p]; if (pool.is_object()) { @@ -2117,7 +2259,8 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA xx[1] = Utils::hton(x[1] + identity.address().toInt()); } else { - // Otherwise pick random addresses -- this technically doesn't explore the whole range if the lower 64 bit range is >= 1 but that won't matter since that would be huge anyway + // Otherwise pick random addresses -- this technically doesn't explore the whole range if + // the lower 64 bit range is >= 1 but that won't matter since that would be huge anyway Utils::getSecureRandom((void*)xx, 16); if ((e[0] > s[0])) xx[0] %= (e[0] - s[0]); @@ -2136,12 +2279,16 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA // Check if this IP is within a local-to-Ethernet routed network int routedNetmaskBits = 0; for (unsigned int rk = 0; rk < nc->routeCount; ++rk) { - if ((! nc->routes[rk].via.ss_family) && (nc->routes[rk].target.ss_family == AF_INET6) && (reinterpret_cast(&(nc->routes[rk].target))->containsAddress(ip6))) - routedNetmaskBits = reinterpret_cast(&(nc->routes[rk].target))->netmaskBits(); + if ((! nc->routes[rk].via.ss_family) && (nc->routes[rk].target.ss_family == AF_INET6) + && (reinterpret_cast(&(nc->routes[rk].target)) + ->containsAddress(ip6))) + routedNetmaskBits = + reinterpret_cast(&(nc->routes[rk].target))->netmaskBits(); } // If it's routed, then try to claim and assign it and if successful end loop - if ((routedNetmaskBits > 0) && (! std::binary_search(ns.allocatedIps.begin(), ns.allocatedIps.end(), ip6))) { + if ((routedNetmaskBits > 0) + && (! std::binary_search(ns.allocatedIps.begin(), ns.allocatedIps.end(), ip6))) { char tmpip[64]; const std::string ipStr(ip6.toIpString(tmpip)); if (std::find(ipAssignments.begin(), ipAssignments.end(), ipStr) == ipAssignments.end()) { @@ -2160,24 +2307,40 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA } } - if ((ipAssignmentPools.is_array()) && ((v4AssignMode.is_object()) && (OSUtils::jsonBool(v4AssignMode["zt"], false))) && (! haveManagedIpv4AutoAssignment) && (! noAutoAssignIps)) { + fprintf(stderr, "haveManagedIpv4AutoAssignment=%s\n", haveManagedIpv4AutoAssignment ? "true" : "false"); + fprintf(stderr, "ipAssignmentPools.is_array()=%s\n", ipAssignmentPools.is_array() ? "true" : "false"); + fprintf(stderr, "v4AssignMode.is_object()=%s\n", v4AssignMode.is_object() ? "true" : "false"); + fprintf( + stderr, "v4AssignMode[\"zt\"]=%s\n", + (v4AssignMode.is_object() && OSUtils::jsonBool(v4AssignMode["zt"], false)) ? "true" : "false"); + fprintf(stderr, "noAutoAssignIps=%s\n", noAutoAssignIps ? "true" : "false"); + + if ((ipAssignmentPools.is_array()) && ((v4AssignMode.is_object()) && (OSUtils::jsonBool(v4AssignMode["zt"], false))) + && (! haveManagedIpv4AutoAssignment) && (! noAutoAssignIps)) { + fprintf(stderr, "Trying to do managed IPv4 auto-assignment\n"); for (unsigned long p = 0; ((p < ipAssignmentPools.size()) && (! haveManagedIpv4AutoAssignment)); ++p) { json& pool = ipAssignmentPools[p]; + fprintf(stderr, " considering pool %lu: %s\n", p, OSUtils::jsonDump(pool, 2).c_str()); if (pool.is_object()) { InetAddress ipRangeStartIA(OSUtils::jsonString(pool["ipRangeStart"], "").c_str()); InetAddress ipRangeEndIA(OSUtils::jsonString(pool["ipRangeEnd"], "").c_str()); if ((ipRangeStartIA.ss_family == AF_INET) && (ipRangeEndIA.ss_family == AF_INET)) { - uint32_t ipRangeStart = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeStartIA)->sin_addr.s_addr)); - uint32_t ipRangeEnd = Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEndIA)->sin_addr.s_addr)); + uint32_t ipRangeStart = Utils::ntoh( + (uint32_t)(reinterpret_cast(&ipRangeStartIA)->sin_addr.s_addr)); + uint32_t ipRangeEnd = + Utils::ntoh((uint32_t)(reinterpret_cast(&ipRangeEndIA)->sin_addr.s_addr)); - if ((ipRangeEnd < ipRangeStart) || (ipRangeStart == 0)) + if ((ipRangeEnd < ipRangeStart) || (ipRangeStart == 0)) { + fprintf(stderr, " bad ip range range\n"); continue; + } uint32_t ipRangeLen = ipRangeEnd - ipRangeStart; // Start with the LSB of the member's address uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff); - for (uint32_t k = ipRangeStart, trialCount = 0; ((k <= ipRangeEnd) && (trialCount < 1000)); ++k, ++trialCount) { + for (uint32_t k = ipRangeStart, trialCount = 0; ((k <= ipRangeEnd) && (trialCount < 1000)); + ++k, ++trialCount) { uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart; ++ipTrialCounter; if ((ip & 0x000000ff) == 0x000000ff) { @@ -2188,8 +2351,12 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA int routedNetmaskBits = -1; for (unsigned int rk = 0; rk < nc->routeCount; ++rk) { if (nc->routes[rk].target.ss_family == AF_INET) { - uint32_t targetIp = Utils::ntoh((uint32_t)(reinterpret_cast(&(nc->routes[rk].target))->sin_addr.s_addr)); - int targetBits = Utils::ntoh((uint16_t)(reinterpret_cast(&(nc->routes[rk].target))->sin_port)); + uint32_t targetIp = Utils::ntoh( + (uint32_t)(reinterpret_cast(&(nc->routes[rk].target)) + ->sin_addr.s_addr)); + int targetBits = Utils::ntoh( + (uint16_t)(reinterpret_cast(&(nc->routes[rk].target)) + ->sin_port)); if ((ip & (0xffffffff << (32 - targetBits))) == targetIp) { routedNetmaskBits = targetBits; break; @@ -2199,14 +2366,17 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA // If it's routed, then try to claim and assign it and if successful end loop const InetAddress ip4(Utils::hton(ip), 0); - if ((routedNetmaskBits > 0) && (! std::binary_search(ns.allocatedIps.begin(), ns.allocatedIps.end(), ip4))) { + if ((routedNetmaskBits > 0) + && (! std::binary_search(ns.allocatedIps.begin(), ns.allocatedIps.end(), ip4))) { char tmpip[64]; const std::string ipStr(ip4.toIpString(tmpip)); if (std::find(ipAssignments.begin(), ipAssignments.end(), ipStr) == ipAssignments.end()) { + fprintf(stderr, " assigning %s\n", ipStr.c_str()); ipAssignments.push_back(ipStr); member["ipAssignments"] = ipAssignments; if (nc->staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { - struct sockaddr_in* const v4ip = reinterpret_cast(&(nc->staticIps[nc->staticIpCount++])); + struct sockaddr_in* const v4ip = + reinterpret_cast(&(nc->staticIps[nc->staticIpCount++])); v4ip->sin_family = AF_INET; v4ip->sin_port = Utils::hton((uint16_t)routedNetmaskBits); v4ip->sin_addr.s_addr = Utils::hton(ip); @@ -2260,7 +2430,8 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA nc->com = com; } else { - _sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR, nullptr, 0); + _sender->ncSendError( + nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR, nullptr, 0); #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK b9.stop(); #endif @@ -2274,6 +2445,7 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA c10++; b10.start(); #endif + member["change_source"] = "controller"; DB::cleanMember(member); _db.save(member, true); #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK @@ -2284,7 +2456,9 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA c11++; b11.start(); #endif - _sender->ncSendConfig(nwid, requestPacketId, identity.address(), *(nc.get()), metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION, 0) < 6); + _sender->ncSendConfig( + nwid, requestPacketId, identity.address(), *(nc.get()), + metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION, 0) < 6); #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK b11.stop(); #endif @@ -2318,10 +2492,14 @@ void EmbeddedNetworkController::_startThreads() _request(qe->nwid, qe->fromAddr, qe->requestPacketId, qe->identity, qe->metaData); } catch (std::exception& e) { - fprintf(stderr, "ERROR: exception in controller request handling thread: %s" ZT_EOL_S, e.what()); + fprintf( + stderr, "ERROR: exception in controller request handling thread: %s" ZT_EOL_S, + e.what()); } catch (...) { - fprintf(stderr, "ERROR: exception in controller request handling thread: unknown exception" ZT_EOL_S); + fprintf( + stderr, + "ERROR: exception in controller request handling thread: unknown exception" ZT_EOL_S); } delete qe; qe = nullptr; @@ -2354,7 +2532,8 @@ void EmbeddedNetworkController::_ssoExpiryThread() network.clear(); member.clear(); if (_db.get(s->second.networkId, network, s->second.nodeId, member)) { - int64_t authenticationExpiryTime = (int64_t)OSUtils::jsonInt(member["authenticationExpiryTime"], 0); + int64_t authenticationExpiryTime = + (int64_t)OSUtils::jsonInt(member["authenticationExpiryTime"], 0); if (authenticationExpiryTime <= now) { expired.push_back(s->second); } diff --git a/nonfree/controller/EmbeddedNetworkController.hpp b/nonfree/controller/EmbeddedNetworkController.hpp index 81f248ab4f..085ece3c87 100644 --- a/nonfree/controller/EmbeddedNetworkController.hpp +++ b/nonfree/controller/EmbeddedNetworkController.hpp @@ -12,7 +12,11 @@ #include "DB.hpp" #include "DBMirrorSet.hpp" +#ifdef CMAKE_BUILD +#include +#else #include +#endif #include #include #include @@ -26,6 +30,10 @@ namespace ZeroTier { class Node; struct RedisConfig; +#ifdef ZT1_CENTRAL_CONTROLLER +class ControllerConfig; +#endif + class EmbeddedNetworkController : public NetworkController , public DB::ChangeListener { @@ -35,24 +43,46 @@ class EmbeddedNetworkController * @param dbPath Database path (file path or database credentials) */ EmbeddedNetworkController(Node* node, const char* ztPath, const char* dbPath, int listenPort, RedisConfig* rc); +#ifdef ZT1_CENTRAL_CONTROLLER + EmbeddedNetworkController( + Node* node, + const char* ztPath, + const char* dbPath, + int listenPort, + const ControllerConfig* cc); +#endif virtual ~EmbeddedNetworkController(); virtual void init(const Identity& signingId, Sender* sender); void setSSORedirectURL(const std::string& url); - virtual void request(uint64_t nwid, const InetAddress& fromAddr, uint64_t requestPacketId, const Identity& identity, const Dictionary& metaData); + virtual void request( + uint64_t nwid, + const InetAddress& fromAddr, + uint64_t requestPacketId, + const Identity& identity, + const Dictionary& metaData); - void configureHTTPControlPlane(httplib::Server& s, httplib::Server& sV6, const std::function); + void configureHTTPControlPlane( + httplib::Server& s, + httplib::Server& sV6, + const std::function); void handleRemoteTrace(const ZT_RemoteTrace& rt); virtual void onNetworkUpdate(const void* db, uint64_t networkId, const nlohmann::json& network); - virtual void onNetworkMemberUpdate(const void* db, uint64_t networkId, uint64_t memberId, const nlohmann::json& member); + virtual void + onNetworkMemberUpdate(const void* db, uint64_t networkId, uint64_t memberId, const nlohmann::json& member); virtual void onNetworkMemberDeauthorize(const void* db, uint64_t networkId, uint64_t memberId); private: - void _request(uint64_t nwid, const InetAddress& fromAddr, uint64_t requestPacketId, const Identity& identity, const Dictionary& metaData); + void _request( + uint64_t nwid, + const InetAddress& fromAddr, + uint64_t requestPacketId, + const Identity& identity, + const Dictionary& metaData); void _startThreads(); void _ssoExpiryThread(); @@ -128,6 +158,9 @@ class EmbeddedNetworkController std::mutex _expiringSoon_l; RedisConfig* _rc; +#ifdef ZT1_CENTRAL_CONTROLLER + const ControllerConfig* _cc; +#endif std::string _ssoRedirectURL; bool _ssoExpiryRunning; diff --git a/nonfree/controller/FileDB.cpp b/nonfree/controller/FileDB.cpp index 6991959532..906d39706e 100644 --- a/nonfree/controller/FileDB.cpp +++ b/nonfree/controller/FileDB.cpp @@ -4,12 +4,17 @@ #include "FileDB.hpp" -#include "../node/Metrics.hpp" +#include "../../node/Metrics.hpp" #include "opentelemetry/trace/provider.h" namespace ZeroTier { -FileDB::FileDB(const char* path) : DB(), _path(path), _networksPath(_path + ZT_PATH_SEPARATOR_S + "network"), _tracePath(_path + ZT_PATH_SEPARATOR_S + "trace"), _running(true) +FileDB::FileDB(const char* path) + : DB() + , _path(path) + , _networksPath(_path + ZT_PATH_SEPARATOR_S + "network") + , _tracePath(_path + ZT_PATH_SEPARATOR_S + "trace") + , _running(true) { auto provider = opentelemetry::trace::Provider::GetTracerProvider(); auto tracer = provider->GetTracer("filedb"); @@ -37,7 +42,8 @@ FileDB::FileDB(const char* path) : DB(), _path(path), _networksPath(_path + ZT_P std::vector members(OSUtils::listDirectory(membersPath.c_str(), false)); for (auto m = members.begin(); m != members.end(); ++m) { buf.clear(); - if ((m->length() == 15) && (OSUtils::readFile((membersPath + ZT_PATH_SEPARATOR_S + *m).c_str(), buf))) { + if ((m->length() == 15) + && (OSUtils::readFile((membersPath + ZT_PATH_SEPARATOR_S + *m).c_str(), buf))) { try { nlohmann::json member(OSUtils::jsonParse(buf)); const std::string addrs = member["id"]; @@ -101,7 +107,8 @@ bool FileDB::save(nlohmann::json& record, bool notifyListeners) get(nwid, old); if ((! old.is_object()) || (! _compareRecords(old, record))) { record["revision"] = OSUtils::jsonInt(record["revision"], 0ULL) + 1ULL; - OSUtils::ztsnprintf(p1, sizeof(p1), "%s" ZT_PATH_SEPARATOR_S "%.16llx.json", _networksPath.c_str(), nwid); + OSUtils::ztsnprintf( + p1, sizeof(p1), "%s" ZT_PATH_SEPARATOR_S "%.16llx.json", _networksPath.c_str(), nwid); if (! OSUtils::writeFile(p1, OSUtils::jsonDump(record, -1))) { fprintf(stderr, "WARNING: controller unable to write to path: %s" ZT_EOL_S, p1); } @@ -121,10 +128,15 @@ bool FileDB::save(nlohmann::json& record, bool notifyListeners) get(nwid, network, id, old); if ((! old.is_object()) || (! _compareRecords(old, record))) { record["revision"] = OSUtils::jsonInt(record["revision"], 0ULL) + 1ULL; - OSUtils::ztsnprintf(pb, sizeof(pb), "%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member", _networksPath.c_str(), (unsigned long long)nwid); - OSUtils::ztsnprintf(p1, sizeof(p1), "%s" ZT_PATH_SEPARATOR_S "%.10llx.json", pb, (unsigned long long)id); + OSUtils::ztsnprintf( + pb, sizeof(pb), "%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member", + _networksPath.c_str(), (unsigned long long)nwid); + OSUtils::ztsnprintf( + p1, sizeof(p1), "%s" ZT_PATH_SEPARATOR_S "%.10llx.json", pb, (unsigned long long)id); if (! OSUtils::writeFile(p1, OSUtils::jsonDump(record, -1))) { - OSUtils::ztsnprintf(p2, sizeof(p2), "%s" ZT_PATH_SEPARATOR_S "%.16llx", _networksPath.c_str(), (unsigned long long)nwid); + OSUtils::ztsnprintf( + p2, sizeof(p2), "%s" ZT_PATH_SEPARATOR_S "%.16llx", _networksPath.c_str(), + (unsigned long long)nwid); OSUtils::mkdir(p2); OSUtils::mkdir(pb); if (! OSUtils::writeFile(p1, OSUtils::jsonDump(record, -1))) { @@ -154,7 +166,8 @@ void FileDB::eraseNetwork(const uint64_t networkId) char p[16384]; OSUtils::ztsnprintf(p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "%.16llx.json", _networksPath.c_str(), networkId); OSUtils::rm(p); - OSUtils::ztsnprintf(p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "%.16llx", _networksPath.c_str(), (unsigned long long)networkId); + OSUtils::ztsnprintf( + p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "%.16llx", _networksPath.c_str(), (unsigned long long)networkId); OSUtils::rmDashRf(p); _networkChanged(network, nullJson, true); std::lock_guard l(this->_online_l); @@ -171,14 +184,21 @@ void FileDB::eraseMember(const uint64_t networkId, const uint64_t memberId) nlohmann::json network, member, nullJson; get(networkId, network, memberId, member); char p[4096]; - OSUtils::ztsnprintf(p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member" ZT_PATH_SEPARATOR_S "%.10llx.json", _networksPath.c_str(), networkId, memberId); + OSUtils::ztsnprintf( + p, sizeof(p), + "%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member" ZT_PATH_SEPARATOR_S "%.10llx.json", + _networksPath.c_str(), networkId, memberId); OSUtils::rm(p); _memberChanged(member, nullJson, true); std::lock_guard l(this->_online_l); this->_online[networkId].erase(memberId); } -void FileDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch) +void FileDB::nodeIsOnline( + const uint64_t networkId, + const uint64_t memberId, + const InetAddress& physicalAddress, + const char* osArch) { auto provider = opentelemetry::trace::Provider::GetTracerProvider(); auto tracer = provider->GetTracer("filedb"); diff --git a/nonfree/controller/NotificationListener.hpp b/nonfree/controller/NotificationListener.hpp new file mode 100644 index 0000000000..49170228c6 --- /dev/null +++ b/nonfree/controller/NotificationListener.hpp @@ -0,0 +1,32 @@ +#ifndef NOTIFICATION_LISTENER_HPP +#define NOTIFICATION_LISTENER_HPP + +#include + +namespace ZeroTier { + +/** + * Base class for notification listeners + * + * This class is used to receive notifications from various sources such as Redis, PostgreSQL, etc. + */ +class NotificationListener { + public: + NotificationListener() = default; + virtual ~NotificationListener() + { + } + + /** + * Called when a notification is received. + * + * Payload should be parsed and passed to the database handler's save method. + * + * @param payload The payload of the notification. + */ + virtual bool onNotification(const std::string& payload) = 0; +}; + +} // namespace ZeroTier + +#endif // NOTIFICATION_LISTENER_HPP \ No newline at end of file diff --git a/nonfree/controller/OtelCarrier.hpp b/nonfree/controller/OtelCarrier.hpp new file mode 100644 index 0000000000..10a7105305 --- /dev/null +++ b/nonfree/controller/OtelCarrier.hpp @@ -0,0 +1,47 @@ +#ifndef OTEL_CARRIER_HPP +#define OTEL_CARRIER_HPP + +#include "opentelemetry/context/propagation/text_map_propagator.h" +#include "opentelemetry/trace/propagation/http_trace_context.h" + +namespace nostd = opentelemetry::nostd; + +namespace ZeroTier { + +template class OtelCarrier : public opentelemetry::context::propagation::TextMapCarrier { + public: + OtelCarrier(T& headers) : headers_(headers) + { + } + + OtelCarrier() = default; + + virtual nostd::string_view Get(nostd::string_view key) const noexcept override + { + std::string key_to_compare = key.data(); + + if (key == opentelemetry::trace::propagation::kTraceParent) { + key_to_compare = "traceparent"; + } + else if (key == opentelemetry::trace::propagation::kTraceState) { + key_to_compare = "tracestate"; + } + auto it = headers_.find(key_to_compare); + if (it != headers_.end()) { + return it->second; + } + + return ""; + } + + virtual void Set(nostd::string_view key, nostd::string_view value) noexcept override + { + headers_.insert(std::pair(std::string(key), std::string(value))); + } + + T& headers_; +}; + +} // namespace ZeroTier + +#endif // OTEL_CARRIER_HPP \ No newline at end of file diff --git a/nonfree/controller/PostgreSQL.cpp b/nonfree/controller/PostgreSQL.cpp index 57dab6c7e4..e6c96c3c24 100644 --- a/nonfree/controller/PostgreSQL.cpp +++ b/nonfree/controller/PostgreSQL.cpp @@ -6,10 +6,178 @@ #include "PostgreSQL.hpp" +#include "opentelemetry/trace/provider.h" + #include -using namespace nlohmann; +namespace ZeroTier { + +PostgresMemberListener::PostgresMemberListener( + DB* db, + std::shared_ptr > pool, + const std::string& channel, + uint64_t timeout) + : NotificationListener() + , _db(db) + , _pool(pool) + , _notification_timeout(timeout) + , _listenerThread() +{ + _conn = _pool->borrow(); + _receiver = new _notificationReceiver(this, *_conn->c, channel); + _run = true; + _listenerThread = std::thread(&PostgresMemberListener::listen, this); +} + +PostgresMemberListener::~PostgresMemberListener() +{ + _run = false; + if (_listenerThread.joinable()) { + _listenerThread.join(); + } + delete _receiver; + if (_conn) { + _pool->unborrow(_conn); + _conn.reset(); + } +} + +void PostgresMemberListener::listen() +{ + while (_run) { + _conn->c->await_notification(_notification_timeout, 0); + } +} + +bool PostgresMemberListener::onNotification(const std::string& payload) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("PostgresMemberNotificationListener"); + auto span = tracer->StartSpan("PostgresMemberNotificationListener::onNotification"); + auto scope = tracer->WithActiveSpan(span); + span->SetAttribute("payload", payload); + + fprintf(stderr, "Member Notification received: %s\n", payload.c_str()); + Metrics::pgsql_mem_notification++; + nlohmann::json tmp(nlohmann::json::parse(payload)); + nlohmann::json& ov = tmp["old_val"]; + nlohmann::json& nv = tmp["new_val"]; + nlohmann::json oldConfig, newConfig; + if (ov.is_object()) + oldConfig = ov; + if (nv.is_object()) + newConfig = nv; + + if (oldConfig.is_object() && newConfig.is_object()) { + _db->save(newConfig, true); + fprintf(stderr, "payload sent\n"); + } + else if (newConfig.is_object() && ! oldConfig.is_object()) { + // new member + Metrics::member_count++; + _db->save(newConfig, true); + fprintf(stderr, "new member payload sent\n"); + } + else if (! newConfig.is_object() && oldConfig.is_object()) { + // member delete + uint64_t networkId = OSUtils::jsonIntHex(oldConfig["nwid"], 0ULL); + uint64_t memberId = OSUtils::jsonIntHex(oldConfig["id"], 0ULL); + if (memberId && networkId) { + _db->eraseMember(networkId, memberId); + fprintf(stderr, "member delete payload sent\n"); + } + } + return true; +} + +PostgresNetworkListener::PostgresNetworkListener( + DB* db, + std::shared_ptr > pool, + const std::string& channel, + uint64_t timeout) + : NotificationListener() + , _db(db) + , _pool(pool) + , _notification_timeout(timeout) + , _listenerThread() +{ + _conn = _pool->borrow(); + _receiver = new _notificationReceiver(this, *_conn->c, channel); + _run = true; + _listenerThread = std::thread(&PostgresNetworkListener::listen, this); +} + +PostgresNetworkListener::~PostgresNetworkListener() +{ + _run = false; + if (_listenerThread.joinable()) { + _listenerThread.join(); + } + delete _receiver; + if (_conn) { + _pool->unborrow(_conn); + _conn.reset(); + } +} + +void PostgresNetworkListener::listen() +{ + while (_run) { + _conn->c->await_notification(_notification_timeout, 0); + } +} + +bool PostgresNetworkListener::onNotification(const std::string& payload) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("db_network_notification"); + auto span = tracer->StartSpan("db_network_notification::operator()"); + auto scope = tracer->WithActiveSpan(span); + span->SetAttribute("payload", payload); + + fprintf(stderr, "Network Notification received: %s\n", payload.c_str()); + Metrics::pgsql_net_notification++; + nlohmann::json tmp(nlohmann::json::parse(payload)); + + nlohmann::json& ov = tmp["old_val"]; + nlohmann::json& nv = tmp["new_val"]; + nlohmann::json oldConfig, newConfig; + + if (ov.is_object()) + oldConfig = ov; + if (nv.is_object()) + newConfig = nv; + + if (oldConfig.is_object() && newConfig.is_object()) { + std::string nwid = oldConfig["id"]; + span->SetAttribute("action", "network_change"); + span->SetAttribute("network_id", nwid); + _db->save(newConfig, true); + fprintf(stderr, "payload sent\n"); + } + else if (newConfig.is_object() && ! oldConfig.is_object()) { + std::string nwid = newConfig["id"]; + span->SetAttribute("network_id", nwid); + span->SetAttribute("action", "new_network"); + // new network + _db->save(newConfig, true); + fprintf(stderr, "new network payload sent\n"); + } + else if (! newConfig.is_object() && oldConfig.is_object()) { + // network delete + span->SetAttribute("action", "delete_network"); + std::string nwid = oldConfig["id"]; + span->SetAttribute("network_id", nwid); + uint64_t networkId = Utils::hexStrToU64(nwid.c_str()); + span->SetAttribute("network_id_int", networkId); + if (networkId) { + _db->eraseNetwork(networkId); + fprintf(stderr, "network delete payload sent\n"); + } + } + return true; +} -using namespace ZeroTier; +} // namespace ZeroTier #endif diff --git a/nonfree/controller/PostgreSQL.hpp b/nonfree/controller/PostgreSQL.hpp index af40d6467e..5779153cce 100644 --- a/nonfree/controller/PostgreSQL.hpp +++ b/nonfree/controller/PostgreSQL.hpp @@ -9,6 +9,7 @@ #include "ConnectionPool.hpp" #include "DB.hpp" +#include "NotificationListener.hpp" #include "opentelemetry/trace/provider.h" #include @@ -51,7 +52,9 @@ class PostgresConnFactory : public ConnectionFactory { template class MemberNotificationReceiver : public pqxx::notification_receiver { public: - MemberNotificationReceiver(T* p, pqxx::connection& c, const std::string& channel) : pqxx::notification_receiver(c, channel), _psql(p) + MemberNotificationReceiver(T* p, pqxx::connection& c, const std::string& channel) + : pqxx::notification_receiver(c, channel) + , _psql(p) { fprintf(stderr, "initialize MemberNotificationReceiver\n"); } @@ -108,7 +111,9 @@ template class MemberNotificationReceiver : public pqxx::notificati template class NetworkNotificationReceiver : public pqxx::notification_receiver { public: - NetworkNotificationReceiver(T* p, pqxx::connection& c, const std::string& channel) : pqxx::notification_receiver(c, channel), _psql(p) + NetworkNotificationReceiver(T* p, pqxx::connection& c, const std::string& channel) + : pqxx::notification_receiver(c, channel) + , _psql(p) { fprintf(stderr, "initialize NetworkrNotificationReceiver\n"); } @@ -177,6 +182,78 @@ struct NodeOnlineRecord { uint64_t lastSeen; InetAddress physicalAddress; std::string osArch; + std::string version; +}; + +/** + * internal class for listening to PostgreSQL notification channels. + */ +template class _notificationReceiver : public pqxx::notification_receiver { + public: + _notificationReceiver(T* p, pqxx::connection& c, const std::string& channel) + : pqxx::notification_receiver(c, channel) + , _listener(p) + { + fprintf(stderr, "initialize PostgresMemberNotificationListener::_notificationReceiver\n"); + } + + virtual void operator()(const std::string& payload, int backendPid) + { + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("notification_receiver"); + auto span = tracer->StartSpan("notification_receiver::operator()"); + auto scope = tracer->WithActiveSpan(span); + _listener->onNotification(payload); + } + + private: + T* _listener; +}; + +class PostgresMemberListener : public NotificationListener { + public: + PostgresMemberListener( + DB* db, + std::shared_ptr > pool, + const std::string& channel, + uint64_t timeout); + virtual ~PostgresMemberListener(); + + virtual void listen(); + + virtual bool onNotification(const std::string& payload) override; + + private: + bool _run = false; + DB* _db; + std::shared_ptr > _pool; + std::shared_ptr _conn; + uint64_t _notification_timeout; + std::thread _listenerThread; + _notificationReceiver* _receiver; +}; + +class PostgresNetworkListener : public NotificationListener { + public: + PostgresNetworkListener( + DB* db, + std::shared_ptr > pool, + const std::string& channel, + uint64_t timeout); + virtual ~PostgresNetworkListener(); + + virtual void listen(); + + virtual bool onNotification(const std::string& payload) override; + + private: + bool _run = false; + DB* _db; + std::shared_ptr > _pool; + std::shared_ptr _conn; + uint64_t _notification_timeout; + std::thread _listenerThread; + _notificationReceiver* _receiver; }; } // namespace ZeroTier diff --git a/nonfree/controller/PostgresStatusWriter.cpp b/nonfree/controller/PostgresStatusWriter.cpp new file mode 100644 index 0000000000..b44f27bb20 --- /dev/null +++ b/nonfree/controller/PostgresStatusWriter.cpp @@ -0,0 +1,86 @@ +#include "PostgresStatusWriter.hpp" + +#include "../../node/Metrics.hpp" + +#include +#include + +namespace ZeroTier { + +PostgresStatusWriter::PostgresStatusWriter(std::shared_ptr > pool) : _pool(pool) +{ +} + +PostgresStatusWriter::~PostgresStatusWriter() +{ + writePending(); +} + +void PostgresStatusWriter::updateNodeStatus( + const std::string& network_id, + const std::string& node_id, + const std::string& os, + const std::string& arch, + const std::string& version, + const InetAddress& address, + int64_t last_seen, + const std::string& /* frontend unused */) +{ + std::lock_guard l(_lock); + _pending.push_back({ network_id, node_id, os, arch, version, address, last_seen, "" }); +} + +size_t PostgresStatusWriter::queueLength() const +{ + std::lock_guard l(_lock); + return _pending.size(); +} + +void PostgresStatusWriter::writePending() +{ + std::vector toWrite; + { + std::lock_guard l(_lock); + toWrite.swap(_pending); + } + if (toWrite.empty()) { + return; + } + + try { + auto conn = _pool->borrow(); + pqxx::work w(*conn->c); + + pqxx::pipeline pipe(w); + for (const auto& entry : toWrite) { + char iptmp[64] = { 0 }; + nlohmann::json record = { + { entry.address.toIpString(iptmp), entry.last_seen }, + }; + + std::string insert_statement = + "INSERT INTO network_memberships_ctl (device_id, network_id, last_seen, os, arch) " + "VALUES ('" + + w.esc(entry.node_id) + "', '" + w.esc(entry.network_id) + "', '" + w.esc(record.dump()) + + "'::JSONB, " + "'" + + w.esc(entry.os) + "', '" + w.esc(entry.arch) + + "') " + "ON CONFLICT (device_id, network_id) DO UPDATE SET os = EXCLUDED.os, arch = EXCLUDED.arch, " + "last_seen = network_memberships_ctl.last_seen || EXCLUDED.last_seen"; + + pipe.insert(insert_statement); + Metrics::pgsql_node_checkin++; + } + + pipe.complete(); + w.commit(); + _pool->unborrow(conn); + } + catch (const std::exception& e) { + // Log the error + fprintf(stderr, "Error writing to Postgres: %s\n", e.what()); + } +} + +} // namespace ZeroTier \ No newline at end of file diff --git a/nonfree/controller/PostgresStatusWriter.hpp b/nonfree/controller/PostgresStatusWriter.hpp new file mode 100644 index 0000000000..2f4c1f0ea0 --- /dev/null +++ b/nonfree/controller/PostgresStatusWriter.hpp @@ -0,0 +1,40 @@ +#ifndef POSTGRES_STATUS_WRITER_HPP +#define POSTGRES_STATUS_WRITER_HPP + +#include "PostgreSQL.hpp" +#include "StatusWriter.hpp" + +#include +#include +#include +#include + +namespace ZeroTier { + +class PostgresStatusWriter : public StatusWriter { + public: + PostgresStatusWriter(std::shared_ptr > pool); + virtual ~PostgresStatusWriter(); + + virtual void updateNodeStatus( + const std::string& network_id, + const std::string& node_id, + const std::string& os, + const std::string& arch, + const std::string& version, + const InetAddress& address, + int64_t last_seen, + const std::string& /* frontend unused */) override; + virtual size_t queueLength() const override; + virtual void writePending() override; + + private: + std::shared_ptr > _pool; + + mutable std::mutex _lock; + std::vector _pending; +}; + +} // namespace ZeroTier + +#endif // POSTGRES_STATUS_WRITER_HPP \ No newline at end of file diff --git a/nonfree/controller/PubSubListener.cpp b/nonfree/controller/PubSubListener.cpp new file mode 100644 index 0000000000..a155864991 --- /dev/null +++ b/nonfree/controller/PubSubListener.cpp @@ -0,0 +1,614 @@ +#ifdef ZT_CONTROLLER_USE_LIBPQ +#include "PubSubListener.hpp" + +#include "ControllerConfig.hpp" +#include "CtlUtil.hpp" +#include "DB.hpp" +#include "OtelCarrier.hpp" +#include "member.pb.h" +#include "network.pb.h" +#include "opentelemetry/context/propagation/global_propagator.h" +#include "opentelemetry/trace/propagation/http_trace_context.h" +#include "opentelemetry/trace/provider.h" +#include "opentelemetry/trace/tracer.h" +#include "rustybits.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pubsub = ::google::cloud::pubsub; +namespace pubsub_admin = ::google::cloud::pubsub_admin; + +namespace ZeroTier { + +nlohmann::json toJson(const pbmessages::NetworkChange_Network& nc, pbmessages::NetworkChange_ChangeSource source); +nlohmann::json toJson(const pbmessages::MemberChange_Member& mc, pbmessages::MemberChange_ChangeSource source); + +PubSubListener::PubSubListener(std::string controller_id, std::string project, std::string topic) + : _controller_id(controller_id) + , _project(project) + , _topic(topic) + , _subscription_id() + , _run(false) + , _adminClient(pubsub_admin::MakeSubscriptionAdminConnection()) + , _subscription(nullptr) +{ + GOOGLE_PROTOBUF_VERIFY_VERSION; + + _subscription_id = "sub-" + controller_id + "-" + topic; // + "-" + random_hex_string(8); + _subscription = new pubsub::Subscription(_project, _subscription_id); + fprintf( + stderr, "PubSubListener for controller %s project %s topic %s subscription %s\n", controller_id.c_str(), + project.c_str(), topic.c_str(), _subscription_id.c_str()); + + // If PUBSUB_EMULATOR_HOST is set, create the topic if it doesn't exist + const char* emulatorHost = std::getenv("PUBSUB_EMULATOR_HOST"); + if (emulatorHost != nullptr) { + create_gcp_pubsub_topic_if_needed(project, topic); + create_gcp_pubsub_subscription_if_needed(_project, _subscription_id, _topic, _controller_id); + } + + _subscriber = std::make_shared( + pubsub::MakeSubscriberConnection(*_subscription), + google::cloud::Options {}.set(true)); + + _run = true; + _subscriberThread = std::thread(&PubSubListener::subscribe, this); +} + +PubSubListener::~PubSubListener() +{ + _run = false; + if (_subscriberThread.joinable()) { + _subscriberThread.join(); + } + + if (_subscription) { + delete _subscription; + _subscription = nullptr; + } +} + +void PubSubListener::subscribe() +{ + while (_run) { + try { + fprintf(stderr, "Starting new subscription session\n"); + auto session = _subscriber->Subscribe([this](pubsub::Message const& m, pubsub::AckHandler h) { + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("PubSubListener"); + + auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + auto attrs = m.attributes(); + std::map attrs_map; + for (auto const& kv : m.attributes()) { + fprintf(stderr, "Message attribute: %s=%s\n", kv.first.c_str(), kv.second.c_str()); + attrs_map.emplace(kv.first, kv.second); + } + + OtelCarrier > carrier(attrs_map); + + auto current_ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + auto new_context = propagator->Extract(carrier, current_ctx); + auto remote_span = opentelemetry::trace::GetSpan(new_context); + auto remote_scope = tracer->WithActiveSpan(remote_span); + + { + auto span = tracer->StartSpan("PubSubListener::onMessage"); + auto scope = tracer->WithActiveSpan(span); + span->SetAttribute("message_id", m.message_id()); + span->SetAttribute("ordering_key", m.ordering_key()); + + fprintf(stderr, "Received message %s\n", m.message_id().c_str()); + if (onNotification(m.data())) { + std::move(h).ack(); + span->SetStatus(opentelemetry::trace::StatusCode::kOk); + return true; + } + else { + span->SetStatus(opentelemetry::trace::StatusCode::kError, "onNotification failed"); + return false; + } + } + }); + + auto result = session.wait_for(std::chrono::seconds(10)); + if (result == std::future_status::timeout) { + session.cancel(); + continue; + } + + if (! session.valid()) { + fprintf(stderr, "Subscription session no longer valid\n"); + continue; + } + } + catch (google::cloud::Status const& status) { + fprintf(stderr, "Subscription terminated with status: %s\n", status.message().c_str()); + } + } +} + +PubSubNetworkListener::PubSubNetworkListener(std::string controller_id, std::string project, std::string topic, DB* db) + : PubSubListener(controller_id, project, topic) + , _db(db) +{ +} + +PubSubNetworkListener::~PubSubNetworkListener() +{ +} + +bool PubSubNetworkListener::onNotification(const std::string& payload) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("PubSubNetworkListener"); + auto span = tracer->StartSpan("PubSubNetworkListener::onNotification"); + auto scope = tracer->WithActiveSpan(span); + + pbmessages::NetworkChange nc; + if (! nc.ParseFromString(payload)) { + fprintf(stderr, "Failed to parse NetworkChange protobuf message\n"); + span->SetAttribute("error", "Failed to parse NetworkChange protobuf message"); + span->SetStatus(opentelemetry::trace::StatusCode::kError, "Failed to parse protobuf"); + return false; + } + fprintf(stderr, "PubSubNetworkListener: parsed protobuf message. %s\n", nc.DebugString().c_str()); + fprintf(stderr, "Network notification received\n"); + + try { + nlohmann::json oldConfig, newConfig; + + if (nc.has_old()) { + fprintf(stderr, "has old network config\n"); + oldConfig = toJson(nc.old(), nc.change_source()); + } + + if (nc.has_new_()) { + fprintf(stderr, "has new network config\n"); + newConfig = toJson(nc.new_(), nc.change_source()); + } + + if (! nc.has_old() && ! nc.has_new_()) { + fprintf(stderr, "NetworkChange message has no old or new network config\n"); + span->SetAttribute("error", "NetworkChange message has no old or new network config"); + span->SetStatus(opentelemetry::trace::StatusCode::kError, "No old or new config"); + return false; + } + + if (oldConfig.is_object() && newConfig.is_object()) { + // network modification + std::string nwid = oldConfig["id"].get(); + span->SetAttribute("action", "network_change"); + span->SetAttribute("network_id", nwid); + _db->save(newConfig, _db->isReady()); + } + else if (newConfig.is_object() && ! oldConfig.is_object()) { + // new network + std::string nwid = newConfig["id"]; + span->SetAttribute("network_id", nwid); + span->SetAttribute("action", "new_network"); + _db->save(newConfig, _db->isReady()); + } + else if (! newConfig.is_object() && oldConfig.is_object()) { + // network deletion + std::string nwid = oldConfig["id"]; + span->SetAttribute("action", "delete_network"); + span->SetAttribute("network_id", nwid); + + uint64_t networkId = Utils::hexStrToU64(nwid.c_str()); + if (networkId) { + _db->eraseNetwork(networkId); + } + } + } + catch (const nlohmann::json::parse_error& e) { + fprintf(stderr, "PubSubNetworkListener JSON parse error: %s\n", e.what()); + span->SetAttribute("error", e.what()); + span->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "payload: %s\n", payload.c_str()); + return false; + } + catch (const std::exception& e) { + fprintf(stderr, "PubSubNetworkListener Exception in PubSubNetworkListener: %s\n", e.what()); + span->SetAttribute("error", e.what()); + span->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + return false; + } + catch (...) { + fprintf(stderr, "PubSubNetworkListener Unknown exception in PubSubNetworkListener\n"); + span->SetAttribute("error", "Unknown exception in PubSubNetworkListener"); + span->SetStatus(opentelemetry::trace::StatusCode::kError, "Unknown exception"); + return false; + } + fprintf(stderr, "PubSubNetworkListener onNotification complete\n"); + return true; +} + +PubSubMemberListener::PubSubMemberListener(std::string controller_id, std::string project, std::string topic, DB* db) + : PubSubListener(controller_id, project, topic) + , _db(db) +{ +} + +PubSubMemberListener::~PubSubMemberListener() +{ +} + +bool PubSubMemberListener::onNotification(const std::string& payload) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("PubSubMemberListener"); + auto span = tracer->StartSpan("PubSubMemberListener::onNotification"); + auto scope = tracer->WithActiveSpan(span); + + pbmessages::MemberChange mc; + if (! mc.ParseFromString(payload)) { + fprintf(stderr, "Failed to parse MemberChange protobuf message\n"); + span->SetAttribute("error", "Failed to parse MemberChange protobuf message"); + span->SetStatus(opentelemetry::trace::StatusCode::kError, "Failed to parse protobuf"); + return false; + } + fprintf(stderr, "PubSubMemberListener: parsed protobuf message. %s\n", mc.DebugString().c_str()); + fprintf(stderr, "Member notification received"); + + try { + nlohmann::json tmp; + nlohmann::json oldConfig, newConfig; + + if (mc.has_old()) { + fprintf(stderr, "has old member config\n"); + oldConfig = toJson(mc.old(), mc.change_source()); + } + + if (mc.has_new_()) { + fprintf(stderr, "has new member config\n"); + newConfig = toJson(mc.new_(), mc.change_source()); + } + + if (! mc.has_old() && ! mc.has_new_()) { + fprintf(stderr, "MemberChange message has no old or new member config\n"); + span->SetAttribute("error", "MemberChange message has no old or new member config"); + span->SetStatus(opentelemetry::trace::StatusCode::kError, "No old or new config"); + return false; + } + + if (oldConfig.is_object() && newConfig.is_object()) { + // member modification + std::string memberID = oldConfig["id"].get(); + std::string networkID = oldConfig["nwid"].get(); + span->SetAttribute("action", "member_change"); + span->SetAttribute("member_id", memberID); + span->SetAttribute("network_id", networkID); + _db->save(newConfig, _db->isReady()); + } + else if (newConfig.is_object() && ! oldConfig.is_object()) { + // new member + std::string memberID = newConfig["id"].get(); + std::string networkID = newConfig["nwid"].get(); + span->SetAttribute("action", "new_member"); + span->SetAttribute("member_id", memberID); + span->SetAttribute("network_id", networkID); + _db->save(newConfig, _db->isReady()); + } + else if (! newConfig.is_object() && oldConfig.is_object()) { + // member deletion + std::string memberID = oldConfig["id"].get(); + std::string networkID = oldConfig["nwid"].get(); + span->SetAttribute("action", "delete_member"); + span->SetAttribute("member_id", memberID); + span->SetAttribute("network_id", networkID); + + uint64_t networkId = Utils::hexStrToU64(networkID.c_str()); + uint64_t memberId = Utils::hexStrToU64(memberID.c_str()); + if (networkId && memberId) { + _db->eraseMember(networkId, memberId); + } + } + } + catch (const nlohmann::json::parse_error& e) { + fprintf(stderr, "PubSubMemberListener JSON parse error: %s\n", e.what()); + span->SetAttribute("error", e.what()); + span->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + fprintf(stderr, "payload: %s\n", payload.c_str()); + return false; + } + catch (const std::exception& e) { + fprintf(stderr, "PubSubMemberListener Exception in PubSubMemberListener: %s\n", e.what()); + span->SetAttribute("error", e.what()); + span->SetStatus(opentelemetry::trace::StatusCode::kError, e.what()); + return false; + } + return true; +} + +nlohmann::json toJson(const pbmessages::NetworkChange_Network& nc, pbmessages::NetworkChange_ChangeSource source) +{ + nlohmann::json out; + + out["objtype"] = "network"; + out["id"] = nc.network_id(); + out["name"] = nc.name(); + + try { + std::string caps = nc.capabilities(); + if (caps.length() == 0) { + out["capabilities"] = nlohmann::json::array(); + } + else if (caps == "null") { + out["capabilities"] = nlohmann::json::array(); + } + else { + out["capabilities"] = OSUtils::jsonParse(caps); + } + } + catch (const nlohmann::json::parse_error& e) { + fprintf(stderr, "toJson Network capabilities JSON parse error: %s\n", e.what()); + out["capabilities"] = nlohmann::json::array(); + } + + out["mtu"] = nc.mtu(); + out["multicastLimit"] = nc.multicast_limit(); + out["private"] = nc.is_private(); + out["remoteTraceLevel"] = nc.remote_trace_level(); + if (nc.has_remote_trace_target()) { + out["remoteTraceTarget"] = nc.remote_trace_target(); + } + else { + out["remoteTraceTarget"] = ""; + } + + try { + std::string rules = nc.rules(); + if (rules.length() == 0) { + out["rules"] = nlohmann::json::array(); + } + else if (rules == "null") { + out["rules"] = nlohmann::json::array(); + } + else { + out["rules"] = OSUtils::jsonParse(rules); + } + } + catch (const nlohmann::json::parse_error& e) { + fprintf(stderr, "toJson Network rules JSON parse error: %s\n", e.what()); + out["rules"] = nlohmann::json::array(); + } + + out["rulesSource"] = nc.rules_source(); + + try { + std::string tags = nc.tags(); + if (tags.length() == 0) { + out["tags"] = nlohmann::json::array(); + } + else if (tags == "[]") { + out["tags"] = nlohmann::json::array(); + } + else { + out["tags"] = OSUtils::jsonParse(tags); + } + } + catch (const nlohmann::json::parse_error& e) { + fprintf(stderr, "toJson Network tags JSON parse error: %s\n", e.what()); + out["tags"] = nlohmann::json::array(); + } + + if (nc.has_ipv4_assign_mode()) { + nlohmann::json ipv4mode; + ipv4mode["zt"] = nc.ipv4_assign_mode().zt(); + out["v4AssignMode"] = ipv4mode; + } + else { + nlohmann::json ipv4mode = nlohmann::json::object(); + out["zt"] = false; + out["v4AssignMode"] = ipv4mode; + } + + if (nc.has_ipv6_assign_mode()) { + nlohmann::json ipv6mode; + ipv6mode["6plane"] = nc.ipv6_assign_mode().six_plane(); + ipv6mode["rfc4193"] = nc.ipv6_assign_mode().rfc4193(); + ipv6mode["zt"] = nc.ipv6_assign_mode().zt(); + out["v6AssignMode"] = ipv6mode; + } + else { + nlohmann::json ipv6mode = nlohmann::json::object(); + ipv6mode["6plane"] = false; + ipv6mode["rfc4193"] = false; + ipv6mode["zt"] = false; + out["v6AssignMode"] = ipv6mode; + } + + if (nc.assignment_pools_size() > 0) { + nlohmann::json pools = nlohmann::json::array(); + for (const auto& p : nc.assignment_pools()) { + nlohmann::json pool; + pool["ipRangeStart"] = p.start_ip(); + pool["ipRangeEnd"] = p.end_ip(); + pools.push_back(pool); + } + out["ipAssignmentPools"] = pools; + } + else { + out["ipAssignmentPools"] = nlohmann::json::array(); + } + + if (nc.routes_size() > 0) { + nlohmann::json routes = nlohmann::json::array(); + for (const auto& r : nc.routes()) { + nlohmann::json route; + std::string target = r.target(); + if (target.length() > 0) { + route["target"] = r.target(); + if (r.has_via()) { + route["via"] = r.via(); + } + else { + route["via"] = nullptr; + } + routes.push_back(route); + } + } + out["routes"] = routes; + } + + if (nc.has_dns()) { + nlohmann::json dns; + if (nc.dns().nameservers_size() > 0) { + nlohmann::json servers = nlohmann::json::array(); + for (const auto& s : nc.dns().nameservers()) { + servers.push_back(s); + } + dns["servers"] = servers; + } + else { + dns["servers"] = nlohmann::json::array(); + } + dns["domain"] = nc.dns().domain(); + + out["dns"] = dns; + } + + out["ssoEnabled"] = nc.sso_enabled(); + nlohmann::json sso; + if (nc.sso_enabled()) { + sso = nlohmann::json::object(); + if (nc.has_sso_client_id()) { + sso["ssoClientId"] = nc.sso_client_id(); + } + + if (nc.has_sso_authorization_endpoint()) { + sso["ssoAuthorizationEndpoint"] = nc.sso_authorization_endpoint(); + } + + if (nc.has_sso_issuer()) { + sso["ssoIssuer"] = nc.sso_issuer(); + } + + if (nc.has_sso_provider()) { + sso["ssoProvider"] = nc.sso_provider(); + } + } + out["ssoConfig"] = sso; + switch (source) { + case pbmessages::NetworkChange_ChangeSource_CV1: + out["change_source"] = "cv1"; + break; + case pbmessages::NetworkChange_ChangeSource_CV2: + out["change_source"] = "cv2"; + break; + case pbmessages::NetworkChange_ChangeSource_CONTROLLER: + out["change_source"] = "controller"; + break; + default: + out["change_source"] = "unknown"; + break; + } + + return out; +} + +nlohmann::json toJson(const pbmessages::MemberChange_Member& mc, pbmessages::MemberChange_ChangeSource source) +{ + nlohmann::json out; + out["objtype"] = "member"; + out["id"] = mc.device_id(); + out["nwid"] = mc.network_id(); + if (mc.has_remote_trace_target()) { + out["remoteTraceTarget"] = mc.remote_trace_target(); + } + else { + out["remoteTraceTarget"] = ""; + } + out["authorized"] = mc.authorized(); + out["activeBridge"] = mc.active_bridge(); + + auto ipAssignments = mc.ip_assignments(); + if (ipAssignments.size() > 0) { + nlohmann::json assignments = nlohmann::json::array(); + for (const auto& ip : ipAssignments) { + assignments.push_back(ip); + } + out["ipAssignments"] = assignments; + } + + out["noAutoAssignIps"] = mc.no_auto_assign_ips(); + out["ssoExempt"] = mc.sso_exempt(); + out["authenticationExpiryTime"] = mc.auth_expiry_time(); + + try { + std::string caps = mc.capabilities(); + if (caps.length() == 0) { + out["capabilities"] = nlohmann::json::array(); + } + else if (caps == "null") { + out["capabilities"] = nlohmann::json::array(); + } + else { + out["capabilities"] = OSUtils::jsonParse(caps); + } + } + catch (const nlohmann::json::parse_error& e) { + fprintf(stderr, "MemberChange member capabilities JSON parse error: %s\n", e.what()); + fprintf(stderr, "capabilities: %s\n", mc.capabilities().c_str()); + out["capabilities"] = nlohmann::json::array(); + } + + out["creationTime"] = mc.creation_time(); + out["identity"] = mc.identity(); + out["lastAuthorizedTime"] = mc.last_authorized_time(); + out["lastDeauthorizedTime"] = mc.last_deauthorized_time(); + out["remoteTraceLevel"] = mc.remote_trace_level(); + out["revision"] = mc.revision(); + + try { + std::string tags = mc.tags(); + if (tags.length() == 0) { + out["tags"] = nlohmann::json::array(); + } + else if (tags == "null") { + out["tags"] = nlohmann::json::array(); + } + else { + out["tags"] = OSUtils::jsonParse(tags); + } + } + catch (const nlohmann::json::parse_error& e) { + fprintf(stderr, "MemberChange member tags JSON parse error: %s\n", e.what()); + fprintf(stderr, "tags: %s\n", mc.tags().c_str()); + out["tags"] = nlohmann::json::array(); + } + + out["versionMajor"] = mc.version_major(); + out["versionMinor"] = mc.version_minor(); + out["versionRev"] = mc.version_rev(); + out["versionProtocol"] = mc.version_protocol(); + switch (source) { + case pbmessages::MemberChange_ChangeSource_CV1: + out["change_source"] = "cv1"; + break; + case pbmessages::MemberChange_ChangeSource_CV2: + out["change_source"] = "cv2"; + break; + case pbmessages::MemberChange_ChangeSource_CONTROLLER: + out["change_source"] = "controller"; + break; + default: + out["change_source"] = "unknown"; + break; + } + + return out; +} + +} // namespace ZeroTier + +#endif // ZT_CONTROLLER_USE_LIBPQ \ No newline at end of file diff --git a/nonfree/controller/PubSubListener.hpp b/nonfree/controller/PubSubListener.hpp new file mode 100644 index 0000000000..71e65983a2 --- /dev/null +++ b/nonfree/controller/PubSubListener.hpp @@ -0,0 +1,74 @@ +#ifdef ZT_CONTROLLER_USE_LIBPQ + +#ifndef ZT_CONTROLLER_PUBSUBLISTENER_HPP +#define ZT_CONTROLLER_PUBSUBLISTENER_HPP + +#include "NotificationListener.hpp" +#include "rustybits.h" + +#include +#include +#include +#include +#include + +namespace ZeroTier { +class DB; + +/** + * Base class for GCP PubSub listeners + */ +class PubSubListener : public NotificationListener { + public: + PubSubListener(std::string controller_id, std::string project, std::string topic); + virtual ~PubSubListener(); + + virtual bool onNotification(const std::string& payload) = 0; + + protected: + std::string _controller_id; + std::string _project; + std::string _topic; + std::string _subscription_id; + + private: + void subscribe(); + bool _run = false; + google::cloud::pubsub_admin::SubscriptionAdminClient _adminClient; + google::cloud::pubsub::Subscription* _subscription; + std::shared_ptr _subscriber; + std::thread _subscriberThread; +}; + +/** + * Listener for network notifications via GCP PubSub + */ +class PubSubNetworkListener : public PubSubListener { + public: + PubSubNetworkListener(std::string controller_id, std::string project, std::string topic, DB* db); + virtual ~PubSubNetworkListener(); + + virtual bool onNotification(const std::string& payload) override; + + private: + DB* _db; +}; + +/** + * Listener for member notifications via GCP PubSub + */ +class PubSubMemberListener : public PubSubListener { + public: + PubSubMemberListener(std::string controller_id, std::string project, std::string topic, DB* db); + virtual ~PubSubMemberListener(); + + virtual bool onNotification(const std::string& payload) override; + + private: + DB* _db; +}; + +} // namespace ZeroTier + +#endif // ZT_CONTROLLER_PUBSUBLISTENER_HPP +#endif // ZT_CONTROLLER_USE_LIBPQ \ No newline at end of file diff --git a/nonfree/controller/PubSubWriter.cpp b/nonfree/controller/PubSubWriter.cpp new file mode 100644 index 0000000000..c35800abc9 --- /dev/null +++ b/nonfree/controller/PubSubWriter.cpp @@ -0,0 +1,404 @@ +#include "PubSubWriter.hpp" + +#include "../../osdep/OSUtils.hpp" +#include "CtlUtil.hpp" +#include "OtelCarrier.hpp" +#include "member.pb.h" +#include "member_status.pb.h" +#include "network.pb.h" +#include "opentelemetry/context/propagation/global_propagator.h" + +#include +#include +#include +#include +#include +#include + +namespace pubsub = ::google::cloud::pubsub; + +namespace ZeroTier { + +pbmessages::NetworkChange* +networkChangeFromJson(std::string controllerID, const nlohmann::json& oldNetwork, const nlohmann::json& newNetwork); +pbmessages::MemberChange* +memberChangeFromJson(std::string controllerID, const nlohmann::json& oldMember, const nlohmann::json& newMember); + +PubSubWriter::PubSubWriter(std::string project, std::string topic, std::string controller_id) + : _controller_id(controller_id) + , _project(project) + , _topic(topic) +{ + fprintf( + stderr, "PubSubWriter for controller %s project %s topic %s\n", controller_id.c_str(), project.c_str(), + topic.c_str()); + GOOGLE_PROTOBUF_VERIFY_VERSION; + + // If PUBSUB_EMULATOR_HOST is set, create the topic if it doesn't exist + const char* emulatorHost = std::getenv("PUBSUB_EMULATOR_HOST"); + if (emulatorHost != nullptr) { + create_gcp_pubsub_topic_if_needed(project, topic); + } + + auto options = + ::google::cloud::Options {} + .set(pubsub::LimitedTimeRetryPolicy(std::chrono::seconds(5)).clone()) + .set( + pubsub::ExponentialBackoffPolicy(std::chrono::milliseconds(100), std::chrono::seconds(2), 1.3).clone()) + .set(true); + auto publisher = pubsub::MakePublisherConnection(pubsub::Topic(project, topic), std::move(options)); + _publisher = std::make_shared(std::move(publisher)); +} + +PubSubWriter::~PubSubWriter() +{ +} + +bool PubSubWriter::publishMessage( + const std::string& payload, + const std::string& frontend, + const std::string& orderingKey) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("PubSubWriter"); + auto span = tracer->StartSpan("PubSubWriter::publishMessage"); + auto scope = tracer->WithActiveSpan(span); + + fprintf(stderr, "Publishing message to %s\n", _topic.c_str()); + std::vector > attributes; + attributes.emplace_back("controller_id", _controller_id); + + std::map attrs_map; + OtelCarrier > carrier(attrs_map); + auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + auto current_ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + propagator->Inject(carrier, current_ctx); + + for (const auto& kv : attrs_map) { + fprintf(stderr, "Attributes injected: %s=%s\n", kv.first.c_str(), kv.second.c_str()); + attributes.emplace_back(kv.first, kv.second); + } + + if (! frontend.empty()) { + attributes.emplace_back("frontend", frontend); + } + + auto msg_tmp = pubsub::MessageBuilder {}.SetData(payload).SetAttributes(attributes); + if (! orderingKey.empty()) { + msg_tmp.SetOrderingKey(orderingKey); + } + auto msg = std::move(msg_tmp).Build(); + auto message_id = _publisher->Publish(std::move(msg)).get(); + if (! message_id) { + fprintf(stderr, "Failed to publish message: %s\n", std::move(message_id).status().message().c_str()); + return false; + } + + fprintf(stderr, "Published message to %s\n", _topic.c_str()); + return true; +} + +bool PubSubWriter::publishNetworkChange( + const nlohmann::json& oldNetwork, + const nlohmann::json& newNetwork, + const std::string& frontend) +{ + fprintf(stderr, "Publishing network change\n"); + pbmessages::NetworkChange* nc = networkChangeFromJson(_controller_id, oldNetwork, newNetwork); + + std::string networkID; + if (nc->has_new_()) { + networkID = nc->new_().network_id(); + } + else if (nc->has_old()) { + networkID = nc->old().network_id(); + } + + std::string payload; + if (! nc->SerializeToString(&payload)) { + fprintf(stderr, "Failed to serialize NetworkChange protobuf message\n"); + delete nc; + return false; + } + delete nc; + return publishMessage(payload, frontend, networkID); +} + +bool PubSubWriter::publishMemberChange( + const nlohmann::json& oldMember, + const nlohmann::json& newMember, + const std::string& frontend) +{ + fprintf(stderr, "Publishing member change\n"); + pbmessages::MemberChange* mc = memberChangeFromJson(_controller_id, oldMember, newMember); + std::string memberID; + if (mc->has_new_()) { + memberID = mc->new_().network_id() + "-" + mc->new_().device_id(); + } + else if (mc->has_old()) { + memberID = mc->old().network_id() + "-" + mc->old().device_id(); + } + + std::string payload; + if (! mc->SerializeToString(&payload)) { + fprintf(stderr, "Failed to serialize MemberChange protobuf message\n"); + delete mc; + return false; + } + + delete mc; + return publishMessage(payload, frontend, memberID); +} + +bool PubSubWriter::publishStatusChange( + std::string frontend, + std::string network_id, + std::string node_id, + std::string os, + std::string arch, + std::string version, + int64_t last_seen) +{ + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("PubSubWriter"); + auto span = tracer->StartSpan("PubSubWriter::publishStatusChange"); + auto scope = tracer->WithActiveSpan(span); + + pbmessages::MemberStatus_MemberStatusMetadata* metadata = new pbmessages::MemberStatus_MemberStatusMetadata(); + metadata->set_controller_id(_controller_id); + metadata->set_trace_id(""); // TODO: generate a trace ID + + pbmessages::MemberStatus ms; + ms.set_network_id(network_id); + ms.set_member_id(node_id); + ms.set_os(os); + ms.set_arch(arch); + ms.set_version(version); + ms.set_timestamp(last_seen); + ms.set_allocated_metadata(metadata); + + std::string payload; + if (! ms.SerializeToString(&payload)) { + fprintf(stderr, "Failed to serialize StatusChange protobuf message\n"); + return false; + } + + return publishMessage(payload, "", ""); +} + +pbmessages::NetworkChange_Network* networkFromJson(const nlohmann::json& j) +{ + if (! j.is_object()) { + return nullptr; + } + + pbmessages::NetworkChange_Network* n = new pbmessages::NetworkChange_Network(); + try { + n->set_network_id(OSUtils::jsonString(j["id"], "")); + n->set_name(OSUtils::jsonString(j["name"], "")); + n->set_capabilities(OSUtils::jsonDump(j.value("capabilities", "[]"), -1)); + n->set_creation_time(OSUtils::jsonInt(j["creationTime"], 0)); + n->set_enable_broadcast(OSUtils::jsonBool(j["enableBroadcast"], false)); + + for (const auto& p : j["ipAssignmentPools"]) { + if (p.is_object()) { + auto pool = n->add_assignment_pools(); + pool->set_start_ip(OSUtils::jsonString(p["ipRangeStart"], "")); + pool->set_end_ip(OSUtils::jsonString(p["ipRangeEnd"], "")); + } + } + + n->set_mtu(OSUtils::jsonInt(j["mtu"], 2800)); + n->set_multicast_limit(OSUtils::jsonInt(j["multicastLimit"], 32)); + n->set_is_private(OSUtils::jsonBool(j["private"], true)); + n->set_remote_trace_level(OSUtils::jsonInt(j["remoteTraceLevel"], 0)); + n->set_remote_trace_target(OSUtils::jsonString(j["remoteTraceTarget"], "")); + n->set_revision(OSUtils::jsonInt(j["revision"], 0)); + + for (const auto& p : j["routes"]) { + if (p.is_object()) { + auto r = n->add_routes(); + r->set_target(OSUtils::jsonString(p["target"], "")); + r->set_via(OSUtils::jsonString(p["via"], "")); + } + } + std::string rules; + if (j["rules"].is_array()) { + rules = OSUtils::jsonDump(j["rules"], -1); + } + else { + rules = "[]"; + } + n->set_rules(rules); + + std::string tags; + if (j["tags"].is_array()) { + tags = OSUtils::jsonDump(j["tags"], -1); + } + else { + tags = "[]"; + } + n->set_tags(tags); + + pbmessages::NetworkChange_IPV4AssignMode* v4am = new pbmessages::NetworkChange_IPV4AssignMode(); + if (j["v4AssignMode"].is_object()) { + nlohmann::json am = j["v4AssignMode"]; + v4am->set_zt(OSUtils::jsonBool(am["zt"], false)); + } + n->set_allocated_ipv4_assign_mode(v4am); + + pbmessages::NetworkChange_IPV6AssignMode* v6am = new pbmessages::NetworkChange_IPV6AssignMode(); + if (j["v6AssignMode"].is_object()) { + nlohmann::json am = j["v6AssignMode"]; + v6am->set_zt(OSUtils::jsonBool(am["zt"], false)); + v6am->set_six_plane(OSUtils::jsonBool(am["6plane"], false)); + v6am->set_rfc4193(OSUtils::jsonBool(am["rfc4193"], false)); + } + n->set_allocated_ipv6_assign_mode(v6am); + + nlohmann::json jdns = j["dns"]; + if (jdns.is_object()) { + pbmessages::NetworkChange_DNS* dns = new pbmessages::NetworkChange_DNS(); + dns->set_domain(jdns.value("domain", "")); + for (const auto& s : jdns["servers"]) { + if (s.is_string()) { + auto server = dns->add_nameservers(); + *server = s; + } + } + n->set_allocated_dns(dns); + } + + n->set_sso_enabled(OSUtils::jsonBool(j["ssoEnabled"], false)); + nlohmann::json ssocfg = j["ssoConfig"]; + if (ssocfg.is_object()) { + n->set_sso_provider(OSUtils::jsonString(ssocfg["provider"], "")); + n->set_sso_client_id(OSUtils::jsonString(ssocfg["clientId"], "")); + n->set_sso_authorization_endpoint(OSUtils::jsonString(ssocfg["authorizationEndpoint"], "")); + n->set_sso_issuer(OSUtils::jsonString(ssocfg["issuer"], "")); + n->set_sso_provider(OSUtils::jsonString(ssocfg["provider"], "")); + } + + n->set_rules_source(OSUtils::jsonString(j["rulesSource"], "")); + } + catch (const std::exception& e) { + fprintf(stderr, "Exception parsing network JSON: %s\n", e.what()); + delete n; + return nullptr; + } + + return n; +} + +pbmessages::NetworkChange* +networkChangeFromJson(std::string controllerID, const nlohmann::json& oldNetwork, const nlohmann::json& newNetwork) +{ + pbmessages::NetworkChange* nc = new pbmessages::NetworkChange(); + + nc->set_allocated_old(networkFromJson(oldNetwork)); + nc->set_allocated_new_(networkFromJson(newNetwork)); + nc->set_change_source(pbmessages::NetworkChange_ChangeSource::NetworkChange_ChangeSource_CONTROLLER); + + pbmessages::NetworkChange_NetworkChangeMetadata* metadata = new pbmessages::NetworkChange_NetworkChangeMetadata(); + metadata->set_controller_id(controllerID); + metadata->set_trace_id(""); // TODO: generate a trace ID + nc->set_allocated_metadata(metadata); + + return nc; +} + +pbmessages::MemberChange_Member* memberFromJson(const nlohmann::json& j) +{ + if (! j.is_object()) { + fprintf(stderr, "memberFromJson: JSON is not an object\n"); + return nullptr; + } + + fprintf(stderr, "memberFromJSON: %s\n", j.dump().c_str()); + + pbmessages::MemberChange_Member* m = new pbmessages::MemberChange_Member(); + try { + m->set_network_id(OSUtils::jsonString(j["nwid"], "")); + m->set_device_id(OSUtils::jsonString(j["id"], "")); + m->set_identity(OSUtils::jsonString(j["identity"], "")); + m->set_authorized(OSUtils::jsonBool(j["authorized"], false)); + if (j["ipAssignments"].is_array()) { + for (const auto& addr : j["ipAssignments"]) { + if (addr.is_string()) { + auto a = m->add_ip_assignments(); + std::string address = addr.get(); + *a = address; + } + } + } + m->set_active_bridge(OSUtils::jsonBool(j["activeBridge"], false)); + if (j["tags"].is_array()) { + nlohmann::json tags = j["tags"]; + std::string tagsStr = OSUtils::jsonDump(tags, -1); + m->set_tags(tagsStr); + } + else { + nlohmann::json tags = nlohmann::json::array(); + std::string tagsStr = OSUtils::jsonDump(tags, -1); + m->set_tags(tagsStr); + } + if (j["capabilities"].is_array()) { + nlohmann::json caps = j["capabilities"]; + std::string capsStr = OSUtils::jsonDump(caps, -1); + m->set_capabilities(capsStr); + } + else { + nlohmann::json caps = nlohmann::json::array(); + std::string capsStr = OSUtils::jsonDump(caps, -1); + m->set_capabilities(capsStr); + } + m->set_creation_time(OSUtils::jsonInt(j["creationTime"], 0)); + m->set_no_auto_assign_ips(OSUtils::jsonBool(j["noAutoAssignIps"], false)); + m->set_revision(OSUtils::jsonInt(j["revision"], 0)); + m->set_last_authorized_time(OSUtils::jsonInt(j["lastAuthorizedTime"], 0)); + m->set_last_deauthorized_time(OSUtils::jsonInt(j["lastDeauthorizedTime"], 0)); + m->set_last_authorized_credential_type(OSUtils::jsonString(j["lastAuthorizedCredentialType"], "")); + m->set_last_authorized_credential(OSUtils::jsonString(j["lastAuthorizedCredential"], "")); + m->set_version_major(OSUtils::jsonInt(j["versionMajor"], 0)); + m->set_version_minor(OSUtils::jsonInt(j["versionMinor"], 0)); + m->set_version_rev(OSUtils::jsonInt(j["versionRev"], 0)); + m->set_version_protocol(OSUtils::jsonInt(j["versionProtocol"], 0)); + m->set_remote_trace_level(OSUtils::jsonInt(j["remoteTraceLevel"], 0)); + m->set_remote_trace_target(OSUtils::jsonString(j["remoteTraceTarget"], "")); + m->set_sso_exempt(OSUtils::jsonBool(j["ssoExempt"], false)); + m->set_auth_expiry_time(OSUtils::jsonInt(j["authExpiryTime"], 0)); + } + catch (const std::exception& e) { + fprintf(stderr, "Exception parsing member JSON: %s\n", e.what()); + delete m; + return nullptr; + } + fprintf(stderr, "memberFromJSON complete\n"); + return m; +} + +pbmessages::MemberChange* +memberChangeFromJson(std::string controllerID, const nlohmann::json& oldMember, const nlohmann::json& newMember) +{ + fprintf(stderr, "memberrChangeFromJson: old: %s\n", oldMember.dump().c_str()); + fprintf(stderr, "memberrChangeFromJson: new: %s\n", newMember.dump().c_str()); + pbmessages::MemberChange* mc = new pbmessages::MemberChange(); + pbmessages::MemberChange_Member* om = memberFromJson(oldMember); + if (om != nullptr) { + mc->set_allocated_old(om); + } + pbmessages::MemberChange_Member* nm = memberFromJson(newMember); + if (nm != nullptr) { + mc->set_allocated_new_(nm); + } + mc->set_change_source(pbmessages::MemberChange_ChangeSource::MemberChange_ChangeSource_CONTROLLER); + + pbmessages::MemberChange_MemberChangeMetadata* metadata = new pbmessages::MemberChange_MemberChangeMetadata(); + metadata->set_controller_id(controllerID); + metadata->set_trace_id(""); // TODO: generate a trace ID + mc->set_allocated_metadata(metadata); + + return mc; +} + +} // namespace ZeroTier \ No newline at end of file diff --git a/nonfree/controller/PubSubWriter.hpp b/nonfree/controller/PubSubWriter.hpp new file mode 100644 index 0000000000..9db4d0b9dd --- /dev/null +++ b/nonfree/controller/PubSubWriter.hpp @@ -0,0 +1,45 @@ +#ifndef ZT_CONTROLLER_PUBSUBWRITER_HPP +#define ZT_CONTROLLER_PUBSUBWRITER_HPP + +#include +#include +#include +#include + +namespace ZeroTier { + +class PubSubWriter { + public: + PubSubWriter(std::string project, std::string topic, std::string controller_id); + virtual ~PubSubWriter(); + + bool publishNetworkChange( + const nlohmann::json& oldNetwork, + const nlohmann::json& newNetwork, + const std::string& frontend); + + bool + publishMemberChange(const nlohmann::json& oldMember, const nlohmann::json& newMember, const std::string& frontend); + + bool publishStatusChange( + std::string frontend, + std::string network_id, + std::string node_id, + std::string os, + std::string arch, + std::string version, + int64_t last_seen); + + protected: + bool publishMessage(const std::string& payload, const std::string& frontend, const std::string& orderingKey); + + private: + std::string _controller_id; + std::string _project; + std::string _topic; + std::shared_ptr _publisher; +}; + +} // namespace ZeroTier + +#endif // ZT_CONTROLLER_PUBSUBWRITER_HPP \ No newline at end of file diff --git a/nonfree/controller/README_CENTRAL_CONTROLLER.md b/nonfree/controller/README_CENTRAL_CONTROLLER.md new file mode 100644 index 0000000000..1bf2abdf1f --- /dev/null +++ b/nonfree/controller/README_CENTRAL_CONTROLLER.md @@ -0,0 +1,73 @@ +# Central Controller Builds + +NOTE: for ZeroTier, Inc Internal use only. We do not support these builds for external use, nor do we guarantee this will work for anyone but us. + +## Prerequisites + +`cmake` is used for builds and `conda` is used to manage external dependencies. + +First, install `conda`: + +```bash +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh +bash /tmp/miniconda.sh -b -u -p $HOME/miniconda3 +``` + +Initialize conda: + +```bash +source ~/miniconda3/bin/activate +conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main +conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r +conda config --set channel_priority strict +``` + +Install external dependencies: + +```bash +conda env create -f conda_env_build.yaml +conda env activate central_controller +``` + +## Build the Central Controller Binary + + +```bash +cmake -DCMAKE_BUILD_TYPE=Release -DZT1_CENTRAL_CONTROLLER=1 -DCMAKE_INSTALL_PREFIX=$PWD/out -S . -B build/ -DCMAKE_INSTALL_PREFIX=$(shell pwd)/build-out +cmake --build build/ --target all -j8 --verbose +``` + +## Packaging via Docker + +Handled by GitHub Actions + +## Configuration + +Central Controller has new configuration options outside of the normal "settings" block of `local.conf`. + +```json +{ + "settings": { + ...standard zt1 local.conf settings... + }, + "controller": { + "listenMode": (pgsql|redis|pubsub), + "statusMode": (pgsql|redis|bigtable), + "redis": { + "hostname": ..., + "port": 6379, + "clusterMode": true + }, + "pubsub": { + "project_id": + }, + "bigtable": { + "project_id": , + "instance_id": , + "table_id": + } + } +} +``` + +Configuration checks for invalid configurations like `listenMode = "pubsub"`, but without a `"pubsub"` config block. diff --git a/nonfree/controller/RedisListener.cpp b/nonfree/controller/RedisListener.cpp new file mode 100644 index 0000000000..a5100df6cd --- /dev/null +++ b/nonfree/controller/RedisListener.cpp @@ -0,0 +1,238 @@ +#ifdef ZT_CONTROLLER_USE_LIBPQ + +#include "RedisListener.hpp" + +#include "../../node/Metrics.hpp" +#include "nlohmann/json.hpp" +#include "opentelemetry/trace/provider.h" + +#include +#include +#include + +namespace ZeroTier { + +using Attrs = std::vector >; +using Item = std::pair; +using ItemStream = std::vector; + +RedisListener::RedisListener(std::string controller_id, std::shared_ptr redis) + : _controller_id(controller_id) + , _redis(redis) + , _is_cluster(false) + , _run(false) +{ +} + +RedisListener::RedisListener(std::string controller_id, std::shared_ptr cluster) + : _controller_id(controller_id) + , _cluster(cluster) + , _is_cluster(true) + , _run(false) +{ +} + +RedisListener::~RedisListener() +{ + _run = false; + if (_listenThread.joinable()) { + _listenThread.join(); + } +} + +RedisNetworkListener::RedisNetworkListener(std::string controller_id, std::shared_ptr redis, DB* db) + : RedisListener(controller_id, redis) + , _db(db) +{ + // Additional initialization for network listener if needed +} + +RedisNetworkListener::RedisNetworkListener( + std::string controller_id, + std::shared_ptr cluster, + DB* db) + : RedisListener(controller_id, cluster) + , _db(db) +{ + // Additional initialization for network listener if needed +} + +RedisNetworkListener::~RedisNetworkListener() +{ + // Destructor logic if needed +} + +void RedisNetworkListener::listen() +{ + std::string key = "network-stream:{" + _controller_id + "}"; + std::string lastID = "0"; + while (_run) { + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("RedisNetworkListener"); + auto span = tracer->StartSpan("RedisNetworkListener::listen"); + auto scope = tracer->WithActiveSpan(span); + + try { + nlohmann::json tmp; + std::unordered_map result; + if (_is_cluster) { + _cluster->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end())); + } + else { + _redis->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end())); + } + + if (! result.empty()) { + for (auto element : result) { + for (auto rec : element.second) { + std::string id = rec.first; + auto attrs = rec.second; + + for (auto a : attrs) { + try { + tmp = nlohmann::json::parse(a.second); + tmp = nlohmann::json::parse(a.second); + nlohmann::json& ov = tmp["old_val"]; + nlohmann::json& nv = tmp["new_val"]; + nlohmann::json oldConfig, newConfig; + if (ov.is_object()) + oldConfig = ov; + if (nv.is_object()) + newConfig = nv; + if (oldConfig.is_object() || newConfig.is_object()) { + _db->_networkChanged(oldConfig, newConfig, true); + } + } + catch (const nlohmann::json::parse_error& e) { + fprintf(stderr, "JSON parse error: %s\n", e.what()); + } + catch (const std::exception& e) { + fprintf(stderr, "Exception in Redis network listener: %s\n", e.what()); + } + } + if (_is_cluster) { + _cluster->xdel(key, id); + } + else { + _redis->xdel(key, id); + } + lastID = id; + } + Metrics::redis_net_notification++; + } + } + } + catch (sw::redis::Error& e) { + fprintf(stderr, "Error in Redis network listener: %s\n", e.what()); + } + catch (const std::exception& e) { + fprintf(stderr, "Exception in Redis network listener: %s\n", e.what()); + } + } +} + +bool RedisNetworkListener::onNotification(const std::string& payload) +{ + // Handle notifications if needed + return true; +} + +RedisMemberListener::RedisMemberListener(std::string controller_id, std::shared_ptr redis, DB* db) + : RedisListener(controller_id, redis) + , _db(db) +{ + // Additional initialization for member listener if needed +} + +RedisMemberListener::RedisMemberListener( + std::string controller_id, + std::shared_ptr cluster, + DB* db) + : RedisListener(controller_id, cluster) + , _db(db) +{ + // Additional initialization for member listener if needed +} + +RedisMemberListener::~RedisMemberListener() +{ + // Destructor logic if needed +} + +void RedisMemberListener::listen() +{ + std::string key = "member-stream:{" + _controller_id + "}"; + std::string lastID = "0"; + fprintf(stderr, "Listening to Redis member stream: %s\n", key.c_str()); + while (_run) { + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("RedisMemberListener"); + auto span = tracer->StartSpan("RedisMemberListener::listen"); + auto scope = tracer->WithActiveSpan(span); + + try { + nlohmann::json tmp; + std::unordered_map result; + if (_is_cluster) { + _cluster->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end())); + } + else { + _redis->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end())); + } + + if (! result.empty()) { + for (auto element : result) { + for (auto rec : element.second) { + std::string id = rec.first; + auto attrs = rec.second; + + for (auto a : attrs) { + try { + tmp = nlohmann::json::parse(a.second); + nlohmann::json& ov = tmp["old_val"]; + nlohmann::json& nv = tmp["new_val"]; + nlohmann::json oldConfig, newConfig; + if (ov.is_object()) + oldConfig = ov; + if (nv.is_object()) + newConfig = nv; + if (oldConfig.is_object() || newConfig.is_object()) { + _db->_memberChanged(oldConfig, newConfig, true); + } + } + catch (const nlohmann::json::parse_error& e) { + fprintf(stderr, "JSON parse error: %s\n", e.what()); + } + catch (const std::exception& e) { + fprintf(stderr, "Exception in Redis member listener: %s\n", e.what()); + } + } + if (_is_cluster) { + _cluster->xdel(key, id); + } + else { + _redis->xdel(key, id); + } + lastID = id; + } + Metrics::redis_mem_notification++; + } + } + } + catch (sw::redis::Error& e) { + fprintf(stderr, "Error in Redis member listener: %s\n", e.what()); + } + catch (const std::exception& e) { + fprintf(stderr, "Exception in Redis member listener: %s\n", e.what()); + } + } +} + +bool RedisMemberListener::onNotification(const std::string& payload) +{ + return true; +} + +} // namespace ZeroTier + +#endif // ZT_CONTROLLER_USE_LIBPQ \ No newline at end of file diff --git a/nonfree/controller/RedisListener.hpp b/nonfree/controller/RedisListener.hpp new file mode 100644 index 0000000000..e9e4323425 --- /dev/null +++ b/nonfree/controller/RedisListener.hpp @@ -0,0 +1,77 @@ +#ifdef ZT_CONTROLLER_USE_LIBPQ + +#ifndef ZT_CONTROLLER_REDIS_LISTENER_HPP +#define ZT_CONTROLLER_REDIS_LISTENER_HPP + +#include "DB.hpp" +#include "NotificationListener.hpp" +#include "Redis.hpp" + +#include +#include +#include +#include + +namespace ZeroTier { + +class RedisListener : public NotificationListener { + public: + RedisListener(std::string controller_id, std::shared_ptr redis); + RedisListener(std::string controller_id, std::shared_ptr cluster); + + virtual ~RedisListener(); + + virtual void listen() = 0; + virtual bool onNotification(const std::string& payload) override + { + return true; + } + + void start() + { + _run = true; + _listenThread = std::thread(&RedisListener::listen, this); + } + + protected: + std::string _controller_id; + std::shared_ptr _redis; + std::shared_ptr _cluster; + bool _is_cluster = false; + bool _run = false; + + private: + std::thread _listenThread; +}; + +class RedisNetworkListener : public RedisListener { + public: + RedisNetworkListener(std::string controller_id, std::shared_ptr redis, DB* db); + RedisNetworkListener(std::string controller_id, std::shared_ptr cluster, DB* db); + virtual ~RedisNetworkListener(); + + virtual void listen() override; + virtual bool onNotification(const std::string& payload) override; + + private: + DB* _db; +}; + +class RedisMemberListener : public RedisListener { + public: + RedisMemberListener(std::string controller_id, std::shared_ptr redis, DB* db); + RedisMemberListener(std::string controller_id, std::shared_ptr cluster, DB* db); + virtual ~RedisMemberListener(); + + virtual void listen() override; + virtual bool onNotification(const std::string& payload) override; + + private: + DB* _db; +}; + +} // namespace ZeroTier + +#endif // ZT_CONTROLLER_REDIS_LISTENER_HPP + +#endif // ZT_CONTROLLER_USE_LIBPQ \ No newline at end of file diff --git a/nonfree/controller/RedisStatusWriter.cpp b/nonfree/controller/RedisStatusWriter.cpp new file mode 100644 index 0000000000..e74dce4ff5 --- /dev/null +++ b/nonfree/controller/RedisStatusWriter.cpp @@ -0,0 +1,123 @@ +#include "RedisStatusWriter.hpp" + +#include "../../node/Metrics.hpp" +#include "../../osdep/OSUtils.hpp" + +#include +#include + +namespace ZeroTier { + +RedisStatusWriter::RedisStatusWriter(std::shared_ptr redis, std::string controller_id) + : _redis(redis) + , _mode(REDIS_MODE_STANDALONE) +{ +} + +RedisStatusWriter::RedisStatusWriter(std::shared_ptr cluster, std::string controller_id) + : _cluster(cluster) + , _mode(REDIS_MODE_CLUSTER) +{ +} + +RedisStatusWriter::~RedisStatusWriter() +{ + writePending(); +} + +void RedisStatusWriter::updateNodeStatus( + const std::string& network_id, + const std::string& node_id, + const std::string& os, + const std::string& arch, + const std::string& version, + const InetAddress& address, + int64_t last_seen, + const std::string& /* frontend unused */) +{ + std::lock_guard l(_lock); + _pending.push_back({ network_id, node_id, os, arch, version, address, last_seen, "" }); +} + +size_t RedisStatusWriter::queueLength() const +{ + std::lock_guard l(_lock); + return _pending.size(); +} + +void RedisStatusWriter::writePending() +{ + try { + if (_mode == REDIS_MODE_STANDALONE) { + auto tx = _redis->transaction(true, false); + _doWritePending(tx); + } + else if (_mode == REDIS_MODE_CLUSTER) { + auto tx = _cluster->transaction(_controller_id, true, false); + _doWritePending(tx); + } + } + catch (const sw::redis::Error& e) { + // Log the error + fprintf(stderr, "Error writing to Redis: %s\n", e.what()); + } +} + +void RedisStatusWriter::_doWritePending(sw::redis::Transaction& tx) +{ + std::vector toWrite; + { + std::lock_guard l(_lock); + toWrite.swap(_pending); + } + if (toWrite.empty()) { + return; + } + + std::set networksUpdated; + uint64_t updateCount = 0; + for (const auto& entry : _pending) { + char iptmp[64] = { 0 }; + std::string ipAddr = entry.address.toIpString(iptmp); + std::unordered_map record = { + { "id", entry.node_id }, { "address", ipAddr }, { "last_updated", std::to_string(entry.last_seen) }, + { "os", entry.os }, { "arch", entry.arch }, { "version", entry.version } + }; + + tx.zadd("nodes-online:{" + _controller_id + "}", entry.node_id, entry.last_seen) + .zadd("nodes-online2:{" + _controller_id + "}", entry.network_id + "-" + entry.node_id, entry.last_seen) + .zadd("network-nodes-online:{" + _controller_id + "}:" + entry.network_id, entry.node_id, entry.last_seen) + .zadd("active-networks:{" + _controller_id + "}", entry.network_id, entry.last_seen) + .sadd("network-nodes-all:{" + _controller_id + "}:" + entry.network_id, entry.node_id) + .hmset( + "member:{" + _controller_id + "}:" + entry.network_id + ":" + entry.node_id, record.begin(), + record.end()); + networksUpdated.insert(entry.network_id); + ++updateCount; + Metrics::redis_node_checkin++; + } + + // expire records from all-nodes and network-nodes member list + uint64_t expireOld = OSUtils::now() - 300000; + + tx.zremrangebyscore( + "nodes-online:{" + _controller_id + "}", + sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); + tx.zremrangebyscore( + "nodes-online2:{" + _controller_id + "}", + sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); + tx.zremrangebyscore( + "active-networks:{" + _controller_id + "}", + sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); + + for (const auto& nwid : networksUpdated) { + tx.zremrangebyscore( + "network-nodes-online:{" + _controller_id + "}:" + nwid, + sw::redis::RightBoundedInterval(expireOld, sw::redis::BoundType::LEFT_OPEN)); + } + + fprintf(stderr, "%s: Updated online status of %d members\n", _controller_id.c_str(), updateCount); + tx.exec(); +} + +} // namespace ZeroTier \ No newline at end of file diff --git a/nonfree/controller/RedisStatusWriter.hpp b/nonfree/controller/RedisStatusWriter.hpp new file mode 100644 index 0000000000..dc05c77efd --- /dev/null +++ b/nonfree/controller/RedisStatusWriter.hpp @@ -0,0 +1,47 @@ +#ifndef REDIS_STATUS_WRITER_HPP +#define REDIS_STATUS_WRITER_HPP + +#include "Redis.hpp" +#include "StatusWriter.hpp" + +#include +#include +#include + +namespace ZeroTier { + +class RedisStatusWriter : public StatusWriter { + public: + RedisStatusWriter(std::shared_ptr redis, std::string controller_id); + RedisStatusWriter(std::shared_ptr cluster, std::string controller_id); + virtual ~RedisStatusWriter(); + + virtual void updateNodeStatus( + const std::string& network_id, + const std::string& node_id, + const std::string& os, + const std::string& arch, + const std::string& version, + const InetAddress& address, + int64_t last_seen, + const std::string& /* frontend unused */) override; + virtual size_t queueLength() const override; + virtual void writePending() override; + + private: + void _doWritePending(sw::redis::Transaction& tx); + + std::string _controller_id; + + enum RedisMode { REDIS_MODE_STANDALONE, REDIS_MODE_CLUSTER }; + std::shared_ptr _redis; + std::shared_ptr _cluster; + RedisMode _mode = REDIS_MODE_STANDALONE; + + mutable std::mutex _lock; + std::vector _pending; +}; + +} // namespace ZeroTier + +#endif // REDIS_STATUS_WRITER_HPP \ No newline at end of file diff --git a/nonfree/controller/StatusWriter.cpp b/nonfree/controller/StatusWriter.cpp new file mode 100644 index 0000000000..b61ec1251c --- /dev/null +++ b/nonfree/controller/StatusWriter.cpp @@ -0,0 +1 @@ +#include "StatusWriter.hpp" diff --git a/nonfree/controller/StatusWriter.hpp b/nonfree/controller/StatusWriter.hpp new file mode 100644 index 0000000000..500005b96b --- /dev/null +++ b/nonfree/controller/StatusWriter.hpp @@ -0,0 +1,45 @@ +#ifndef STATUS_WRITER_HPP +#define STATUS_WRITER_HPP + +#include "../../node/InetAddress.hpp" + +#include + +namespace ZeroTier { + +/** + * Abstract interface for writing status information somewhere. + * + * Implementations might write to a database, a file, or something else. + */ +class StatusWriter { + public: + virtual ~StatusWriter() = default; + + virtual void updateNodeStatus( + const std::string& network_id, + const std::string& node_id, + const std::string& os, + const std::string& arch, + const std::string& version, + const InetAddress& address, + int64_t last_seen, + const std::string& target) = 0; + virtual size_t queueLength() const = 0; + virtual void writePending() = 0; +}; + +struct PendingStatusEntry { + std::string network_id; + std::string node_id; + std::string os; + std::string arch; + std::string version; + InetAddress address; + int64_t last_seen; + std::string target; +}; + +} // namespace ZeroTier + +#endif // STATUS_WRITER_HPP diff --git a/nonfree/controller/protobuf/member.proto b/nonfree/controller/protobuf/member.proto new file mode 100644 index 0000000000..04a1add556 --- /dev/null +++ b/nonfree/controller/protobuf/member.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package pbmessages; + +message MemberChange { + message Member { + string device_id = 1; + string network_id = 2; + string identity = 3; // Identity of the member + bool authorized = 4; // Whether the member is authorized + repeated string ip_assignments = 5; // List of IP assignments + bool active_bridge = 6; // Whether the member is an active bridge + string tags = 7; // JSON string of tags + string capabilities = 8; // JSON string of capabilities + uint64 creation_time = 9; // Unix timestamp in milliseconds + bool no_auto_assign_ips = 10; // Whether auto IP assignment is disabled + uint64 revision = 11; // Revision number + uint64 last_authorized_time = 12; // Last time the member was authorized + uint64 last_deauthorized_time = 13; // Last time the member was deauthorized + optional string last_authorized_credential_type = 14; // Type of credential used for last authorization + optional string last_authorized_credential = 15; // Credential used for last authorization + int32 version_major = 16; // Major version of the member + int32 version_minor = 17; // Minor version of the member + int32 version_rev = 18; // Patch version of the member + int32 version_protocol = 19; // Protocol version of the member + int32 remote_trace_level = 20; // Remote trace level + optional string remote_trace_target = 21; // Remote trace target + bool sso_exempt = 22; // Whether SSO is exempt + uint64 auth_expiry_time = 23; // Authorization expiry time in milliseconds + } + message MemberChangeMetadata { + string trace_id = 1; + string controller_id = 2; + } + + enum ChangeSource { + UNKNOWN = 0; + CV1 = 1; + CV2 = 2; + CONTROLLER = 3; + } + + optional Member old = 1; + optional Member new = 2; + optional MemberChangeMetadata metadata = 3; + optional ChangeSource change_source = 4; +} diff --git a/nonfree/controller/protobuf/member_status.proto b/nonfree/controller/protobuf/member_status.proto new file mode 100644 index 0000000000..2483905176 --- /dev/null +++ b/nonfree/controller/protobuf/member_status.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package pbmessages; + + + +message MemberStatus { + message MemberStatusMetadata { + string trace_id = 1; + string controller_id = 2; + } + + MemberStatusMetadata metadata = 1; + string network_id = 2; + string member_id = 3; + uint64 timestamp = 4; // Unix timestamp in milliseconds + optional string ip_address = 5; // Optional IP address of the member + optional string os = 6; + optional string arch = 7; + optional string version = 8; +} diff --git a/nonfree/controller/protobuf/network.proto b/nonfree/controller/protobuf/network.proto new file mode 100644 index 0000000000..41bbb73e87 --- /dev/null +++ b/nonfree/controller/protobuf/network.proto @@ -0,0 +1,74 @@ +syntax = "proto3"; + +package pbmessages; + +message NetworkChange { + message NetworkChangeMetadata { + string trace_id = 1; + string controller_id = 2; + } + + message IPRange { + string start_ip = 1; // Start of the IP range + string end_ip = 2; // End of the IP range + } + + message Route { + string target = 1; // Target IP or network + optional string via = 2; // Optional next hop IP + } + + message DNS { + string domain = 1; // Search domain + repeated string nameservers = 2; // List of nameservers + } + + message IPV4AssignMode { + bool zt = 1; // Whether ZeroTier is used for IPv4 assignment + } + + message IPV6AssignMode { + bool six_plane = 1; // Whether 6plane is used for IPv6 assignment + bool rfc4193 = 2; // Whether RFC 4193 is used for IPv6 assignment + bool zt = 3; // Whether ZeroTier is used for IPv6 assignment + } + + message Network { + string network_id = 1; + string capabilities = 2; // JSON string of capabilities + uint64 creation_time = 3; // Unix timestamp in milliseconds + bool enable_broadcast = 4; // Whether broadcast is enabled + repeated IPRange assignment_pools = 5; // List of IP ranges for assignment + uint32 mtu = 6; // Maximum Transmission Unit + uint32 multicast_limit = 7; // Limit for multicast messages + optional string name = 8; // Name of the network + bool is_private = 9; // Whether the network is private + uint32 remote_trace_level = 10; // Remote trace level + optional string remote_trace_target = 11; // Remote trace target + uint64 revision = 12; // Revision number + repeated Route routes = 13; // List of routes + string rules = 14; // JSON string of rules + optional string tags = 15; // JSON string of tags + IPV4AssignMode ipv4_assign_mode = 16; // IPv4 assignment mode + IPV6AssignMode ipv6_assign_mode = 17; // IPv6 assignment mode + optional DNS dns = 18; // DNS configuration + bool sso_enabled = 19; // Whether Single Sign-On is enabled + optional string sso_client_id = 20; // SSO client ID + optional string sso_authorization_endpoint = 21; // SSO authorization endpoint + optional string sso_issuer = 22; // SSO issuer + optional string sso_provider = 23; // SSO provider + string rules_source = 24; // source code for rules + } + + enum ChangeSource { + UNKNOWN = 0; + CV1 = 1; + CV2 = 2; + CONTROLLER = 3; + } + + optional Network old = 1; + optional Network new = 2; + optional NetworkChangeMetadata metadata = 3; + optional ChangeSource change_source = 4; +} diff --git a/osdep/CMakeLists.txt b/osdep/CMakeLists.txt new file mode 100644 index 0000000000..a1f09b3106 --- /dev/null +++ b/osdep/CMakeLists.txt @@ -0,0 +1,65 @@ +cmake_minimum_required(VERSION 3.13) +project(zerotier-osdep LANGUAGES CXX ASM) + +if(APPLE) + enable_language(OBJCXX) +endif() + +set(SRC_FILES + Arp.cpp + Arp.hpp + Binder.hpp + BlockingQueue.hpp + EthernetTap.cpp + EthernetTap.hpp + Http.cpp + Http.hpp + ManagedRoute.cpp + ManagedRoute.hpp + NeighborDiscovery.cpp + NeighborDiscovery.hpp + OSUtils.cpp + OSUtils.hpp + Phy.hpp + PortMapper.cpp + PortMapper.hpp + Thread.hpp +) + +option(EXT_OSDEP "Build with external osdep feature" OFF) + +if(EXT_OSDEP) + add_definitions(-DZT_EXTOSDEP) + list(APPEND SRC_FILES + ExtOsdep.cpp + ExtOsdep.hpp) +endif() + +if(LINUX) + list(APPEND SRC_FILES + LinuxEthernetTap.cpp + LinuxEthernetTap.hpp + LinuxNetLink.cpp + LinuxNetLink.hpp) +endif() + +if(APPLE) + list(APPEND SRC_FILES + MacEthernetTap.cpp + MacEthernetTap.hpp + MacKextEthernetTap.cpp + MacKextEthernetTap.hpp + MacDNSHelper.mm + MacDNSHelper.hpp) +endif() + +add_library(zerotier-osdep STATIC ${SRC_FILES}) +target_link_libraries(zerotier-osdep + PRIVATE + nlohmann_json::nlohmann_json + Threads::Threads + prometheus-cpp-lite + Threads::Threads + miniupnpc::miniupnpc + natpmp +) diff --git a/osdep/ExtOsdep.cpp b/osdep/ExtOsdep.cpp index 4efab5a6c0..7af42e4f59 100644 --- a/osdep/ExtOsdep.cpp +++ b/osdep/ExtOsdep.cpp @@ -18,6 +18,8 @@ #define ZT_TAP_BUF_SIZE 16384 +#ifdef ZT_EXTOSDEP + namespace ZeroTier { static int eodFd = -1; @@ -621,3 +623,5 @@ void ExtOsdepTap::setMtu(unsigned int mtu) } } // namespace ZeroTier + +#endif // ZT_EXTOSDEP \ No newline at end of file diff --git a/osdep/OSUtils.cpp b/osdep/OSUtils.cpp index 2fee8f194d..078adfca94 100644 --- a/osdep/OSUtils.cpp +++ b/osdep/OSUtils.cpp @@ -9,6 +9,7 @@ #include "../node/Constants.hpp" #include "../node/Utils.hpp" +#include #include #include #include @@ -109,7 +110,9 @@ std::vector OSUtils::listDirectory(const char* path, bool includeDi WIN32_FIND_DATAA ffd; if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(), &ffd)) != INVALID_HANDLE_VALUE) { do { - if ((strcmp(ffd.cFileName, ".")) && (strcmp(ffd.cFileName, "..")) && (((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) || (((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) && (includeDirectories)))) + if ((strcmp(ffd.cFileName, ".")) && (strcmp(ffd.cFileName, "..")) + && (((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) + || (((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) && (includeDirectories)))) r.push_back(std::string(ffd.cFileName)); } while (FindNextFileA(hFind, &ffd)); FindClose(hFind); @@ -125,7 +128,8 @@ std::vector OSUtils::listDirectory(const char* path, bool includeDi if (readdir_r(d, &de, &dptr)) break; if (dptr) { - if ((strcmp(dptr->d_name, ".")) && (strcmp(dptr->d_name, "..")) && ((dptr->d_type != DT_DIR) || (includeDirectories))) + if ((strcmp(dptr->d_name, ".")) && (strcmp(dptr->d_name, "..")) + && ((dptr->d_type != DT_DIR) || (includeDirectories))) r.push_back(std::string(dptr->d_name)); } else @@ -149,7 +153,8 @@ long OSUtils::cleanDirectory(const char* path, const int64_t olderThan) char tmp[4096]; if ((hFind = FindFirstFileA((std::string(path) + "\\*").c_str(), &ffd)) != INVALID_HANDLE_VALUE) { do { - if ((strcmp(ffd.cFileName, ".")) && (strcmp(ffd.cFileName, "..")) && ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0)) { + if ((strcmp(ffd.cFileName, ".")) && (strcmp(ffd.cFileName, "..")) + && ((ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0)) { date.HighPart = ffd.ftLastWriteTime.dwHighDateTime; date.LowPart = ffd.ftLastWriteTime.dwLowDateTime; if (date.QuadPart > 0) { @@ -258,7 +263,10 @@ void OSUtils::lockDownFile(const char* path, bool isDir) startupInfo.cb = sizeof(startupInfo); memset(&startupInfo, 0, sizeof(STARTUPINFOA)); memset(&processInfo, 0, sizeof(PROCESS_INFORMATION)); - if (CreateProcessA(NULL, (LPSTR)(std::string("C:\\Windows\\System32\\icacls.exe \"") + path + "\" /inheritance:d /Q").c_str(), NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, NULL, &startupInfo, &processInfo)) { + if (CreateProcessA( + NULL, + (LPSTR)(std::string("C:\\Windows\\System32\\icacls.exe \"") + path + "\" /inheritance:d /Q").c_str(), + NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, NULL, &startupInfo, &processInfo)) { WaitForSingleObject(processInfo.hProcess, INFINITE); CloseHandle(processInfo.hProcess); CloseHandle(processInfo.hThread); @@ -267,7 +275,11 @@ void OSUtils::lockDownFile(const char* path, bool isDir) startupInfo.cb = sizeof(startupInfo); memset(&startupInfo, 0, sizeof(STARTUPINFOA)); memset(&processInfo, 0, sizeof(PROCESS_INFORMATION)); - if (CreateProcessA(NULL, (LPSTR)(std::string("C:\\Windows\\System32\\icacls.exe \"") + path + "\" /remove *S-1-5-32-545 /Q").c_str(), NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, NULL, &startupInfo, &processInfo)) { + if (CreateProcessA( + NULL, + (LPSTR)(std::string("C:\\Windows\\System32\\icacls.exe \"") + path + "\" /remove *S-1-5-32-545 /Q") + .c_str(), + NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, NULL, &startupInfo, &processInfo)) { WaitForSingleObject(processInfo.hProcess, INFINITE); CloseHandle(processInfo.hProcess); CloseHandle(processInfo.hThread); @@ -277,7 +289,11 @@ void OSUtils::lockDownFile(const char* path, bool isDir) startupInfo.cb = sizeof(startupInfo); memset(&startupInfo, 0, sizeof(STARTUPINFOA)); memset(&processInfo, 0, sizeof(PROCESS_INFORMATION)); - if (CreateProcessA(NULL, (LPSTR)(std::string("C:\\Windows\\System32\\icacls.exe \"") + path + "\" /remove:g Everyone /t /c /Q").c_str(), NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, NULL, &startupInfo, &processInfo)) { + if (CreateProcessA( + NULL, + (LPSTR)(std::string("C:\\Windows\\System32\\icacls.exe \"") + path + "\" /remove:g Everyone /t /c /Q") + .c_str(), + NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, NULL, &startupInfo, &processInfo)) { WaitForSingleObject(processInfo.hProcess, INFINITE); CloseHandle(processInfo.hProcess); CloseHandle(processInfo.hThread); @@ -507,6 +523,25 @@ uint64_t OSUtils::jsonInt(const nlohmann::json& jv, const uint64_t dfl) return dfl; } +int64_t OSUtils::jsonUInt(const nlohmann::json& jv, const int64_t dfl) +{ + try { + if (jv.is_number()) { + return (int64_t)jv; + } + else if (jv.is_string()) { + std::string s = jv; + return Utils::strTo64(s.c_str()); + } + else if (jv.is_boolean()) { + return ((bool)jv ? 1LL : 0LL); + } + } + catch (...) { + } + return dfl; +} + double OSUtils::jsonDouble(const nlohmann::json& jv, const double dfl) { try { @@ -614,14 +649,21 @@ std::string OSUtils::jsonBinFromHex(const nlohmann::json& jv) #endif // OMIT_JSON_SUPPORT // Used to convert HTTP header names to ASCII lower case -const unsigned char OSUtils::TOLOWER_TABLE[256] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, - 0x1d, 0x1e, 0x1f, ' ', '!', '"', '#', '$', '%', '&', 0x27, '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', - ':', ';', '<', '=', '>', '?', '@', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '{', '|', '}', '~', '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', - 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, - 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, - 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, - 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff }; +const unsigned char OSUtils::TOLOWER_TABLE[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, + 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, ' ', '!', '"', '#', '$', '%', + '&', 0x27, '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', + '9', ':', ';', '<', '=', '>', '?', '@', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', + '_', '`', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', + 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 0x7f, 0x80, 0x81, 0x82, 0x83, 0x84, + 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, + 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, + 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, + 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, + 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, + 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff +}; } // namespace ZeroTier diff --git a/osdep/OSUtils.hpp b/osdep/OSUtils.hpp index b122a0c946..6a63ff615d 100644 --- a/osdep/OSUtils.hpp +++ b/osdep/OSUtils.hpp @@ -293,6 +293,7 @@ class OSUtils { static nlohmann::json jsonParse(const std::string& buf); static std::string jsonDump(const nlohmann::json& j, int indentation = 1); static uint64_t jsonInt(const nlohmann::json& jv, const uint64_t dfl); + static int64_t jsonUInt(const nlohmann::json& jv, const int64_t dfl); static double jsonDouble(const nlohmann::json& jv, const double dfl); static uint64_t jsonIntHex(const nlohmann::json& jv, const uint64_t dfl); static bool jsonBool(const nlohmann::json& jv, const bool dfl); diff --git a/rustybits/Cargo.lock b/rustybits/Cargo.lock index ba8d1f8367..4ba4788db6 100644 --- a/rustybits/Cargo.lock +++ b/rustybits/Cargo.lock @@ -48,12 +48,18 @@ dependencies = [ ] [[package]] -name = "ansi_term" -version = "0.12.1" +name = "anstream" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" dependencies = [ - "winapi", + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", ] [[package]] @@ -62,6 +68,35 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.60.2", +] + [[package]] name = "anyhow" version = "1.0.99" @@ -76,7 +111,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -85,17 +120,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.5.0" @@ -224,6 +248,56 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http 1.3.1", + "http-body-util", + "hyper 1.7.0", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 2.0.16", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.47.1-rc.27.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + [[package]] name = "bumpalo" version = "3.19.0" @@ -244,19 +318,19 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cbindgen" -version = "0.20.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e3973b165dc0f435831a9e426de67e894de532754ff7a3f307c03ee5dec7dc" +checksum = "975982cdb7ad6a142be15bdf84aea7ec6a9e5d4d797c004d43185b24cfe4e684" dependencies = [ "clap", - "heck 0.3.3", - "indexmap 1.9.3", + "heck", + "indexmap 2.11.0", "log", "proc-macro2", "quote", "serde", "serde_json", - "syn 1.0.109", + "syn", "tempfile", "toml", ] @@ -294,19 +368,37 @@ dependencies = [ [[package]] name = "clap" -version = "2.34.0" +version = "4.5.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882" dependencies = [ - "ansi_term", - "atty", - "bitflags 1.3.2", - "strsim 0.8.0", - "textwrap", - "unicode-width", - "vec_map", + "clap_builder", ] +[[package]] +name = "clap_builder" +version = "4.5.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "const-oid" version = "0.9.6" @@ -418,7 +510,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -441,8 +533,8 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim 0.11.1", - "syn 2.0.106", + "strsim", + "syn", ] [[package]] @@ -453,7 +545,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -509,7 +601,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -519,7 +611,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.106", + "syn", ] [[package]] @@ -539,7 +631,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", "unicode-xid", ] @@ -563,7 +655,18 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", +] + +[[package]] +name = "docker_credential" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d89dfcba45b4afad7450a99b39e751590463e45c04728cf555d36bb66940de8" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", ] [[package]] @@ -669,7 +772,7 @@ checksum = "a1ab991c1362ac86c61ab6f556cff143daa22e5a15e4e189df818b2fd19fe65b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -681,7 +784,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -710,6 +813,17 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "etcetera" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c7b13d0780cb82722fd59f6f57f925e143427e4a75313a6c77243bf5326ae6" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.59.0", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -732,11 +846,23 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "find-msvc-tools" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e178e4fba8a2726903f6ba98a6d221e76f9c12c650d5dc0e6afdc50677b49650" +checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" [[package]] name = "fixedbitset" @@ -842,7 +968,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -1041,30 +1167,12 @@ dependencies = [ "foldhash", ] -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hex" version = "0.4.3" @@ -1089,6 +1197,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "http" version = "0.2.12" @@ -1204,6 +1321,37 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper 1.7.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http 1.3.1", + "hyper 1.7.0", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "hyper-timeout" version = "0.5.2" @@ -1251,6 +1399,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.7.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.63" @@ -1445,6 +1608,12 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -1514,6 +1683,17 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "libredox" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4488594b9328dee448adb906d8b126d9b7deb7cf5c22161ee591610bb1be83c0" +dependencies = [ + "bitflags 2.9.4", + "libc", + "redox_syscall 0.5.17", +] + [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -1621,7 +1801,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -1778,6 +1958,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "openidconnect" version = "3.5.0" @@ -1833,7 +2019,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -1919,11 +2105,36 @@ checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.17", "smallvec", "windows-targets 0.52.6", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -1975,7 +2186,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -2089,7 +2300,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.106", + "syn", ] [[package]] @@ -2117,7 +2328,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.5", +] + +[[package]] +name = "prost" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +dependencies = [ + "bytes", + "prost-derive 0.14.1", ] [[package]] @@ -2126,17 +2347,37 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.5.0", + "heck", "itertools 0.14.0", "log", "multimap", "once_cell", "petgraph", "prettyplease", - "prost", - "prost-types", + "prost 0.13.5", + "prost-types 0.13.5", "regex", - "syn 2.0.106", + "syn", + "tempfile", +] + +[[package]] +name = "prost-build" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost 0.14.1", + "prost-types 0.14.1", + "regex", + "syn", "tempfile", ] @@ -2150,7 +2391,20 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn", +] + +[[package]] +name = "prost-derive" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2159,7 +2413,16 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost", + "prost 0.13.5", +] + +[[package]] +name = "prost-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" +dependencies = [ + "prost 0.14.1", ] [[package]] @@ -2170,7 +2433,7 @@ checksum = "497e1e938f0c09ef9cabe1d49437b4016e03e8f82fbbe5d1c62a9b61b9decae1" dependencies = [ "chrono", "inventory", - "prost", + "prost 0.13.5", "serde", "serde_derive", "serde_json", @@ -2183,10 +2446,10 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07b8bf115b70a7aa5af1fd5d6e9418492e9ccb6e4785e858c938e28d132a884b" dependencies = [ - "heck 0.5.0", - "prost", - "prost-build", - "prost-types", + "heck", + "prost 0.13.5", + "prost-build 0.13.5", + "prost-types 0.13.5", "quote", ] @@ -2197,9 +2460,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8cdde6df0a98311c839392ca2f2f0bcecd545f86a62b4e3c6a49c336e970fe5" dependencies = [ "chrono", - "prost", - "prost-build", - "prost-types", + "prost 0.13.5", + "prost-build 0.13.5", + "prost-types 0.13.5", "prost-wkt", "prost-wkt-build", "regex", @@ -2306,6 +2569,15 @@ dependencies = [ "bitflags 2.9.4", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.17" @@ -2332,7 +2604,7 @@ checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -2388,7 +2660,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -2492,7 +2764,7 @@ dependencies = [ "proc-macro2", "quote", "rustfsm_trait", - "syn 2.0.106", + "syn", ] [[package]] @@ -2549,6 +2821,15 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.12.0" @@ -2575,6 +2856,32 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "rustybits" +version = "0.1.0" +dependencies = [ + "base64 0.21.7", + "bytes", + "cbindgen", + "jwt", + "openidconnect", + "prost-build 0.14.1", + "reqwest", + "serde", + "serde_json", + "temporal-client", + "temporal-sdk", + "temporal-sdk-core-protos", + "testcontainers", + "testcontainers-modules", + "thiserror 1.0.69", + "time", + "tokio", + "tokio-util", + "url", + "uuid", +] + [[package]] name = "ryu" version = "1.0.20" @@ -2703,7 +3010,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -2737,6 +3044,26 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2778,7 +3105,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -2853,20 +3180,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "smeeclient" -version = "0.1.0" -dependencies = [ - "cbindgen", - "serde", - "temporal-client", - "temporal-sdk", - "temporal-sdk-core-protos", - "tokio", - "url", - "uuid", -] - [[package]] name = "socket2" version = "0.5.10" @@ -2918,12 +3231,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - [[package]] name = "strsim" version = "0.11.1" @@ -2931,22 +3238,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] -name = "subtle" -version = "2.6.1" +name = "structmeta" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn", +] [[package]] -name = "syn" -version = "1.0.109" +name = "structmeta-derive" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", - "unicode-ident", + "syn", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.106" @@ -2978,7 +3297,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -3108,7 +3427,7 @@ dependencies = [ "parking_lot", "pid", "pin-project", - "prost", + "prost 0.13.5", "prost-wkt-types", "rand 0.9.2", "ringbuf", @@ -3141,7 +3460,7 @@ dependencies = [ "derive_builder", "derive_more", "opentelemetry", - "prost", + "prost 0.13.5", "serde_json", "temporal-sdk-core-protos", "thiserror 2.0.16", @@ -3159,8 +3478,8 @@ dependencies = [ "anyhow", "base64 0.22.1", "derive_more", - "prost", - "prost-build", + "prost 0.13.5", + "prost-build 0.13.5", "prost-wkt", "prost-wkt-build", "prost-wkt-types", @@ -3180,12 +3499,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] -name = "textwrap" -version = "0.11.0" +name = "testcontainers" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +checksum = "23bb7577dca13ad86a78e8271ef5d322f37229ec83b8d98da6d996c588a1ddb1" dependencies = [ - "unicode-width", + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "docker_credential", + "either", + "etcetera", + "futures", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.16", + "tokio", + "tokio-stream", + "tokio-tar", + "tokio-util", + "url", +] + +[[package]] +name = "testcontainers-modules" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac95cde96549fc19c6bf19ef34cc42bd56e264c1cb97e700e21555be0ecf9e2" +dependencies = [ + "testcontainers", ] [[package]] @@ -3214,7 +3562,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -3225,7 +3573,7 @@ checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -3305,7 +3653,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -3339,6 +3687,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "tokio-util" version = "0.7.16" @@ -3354,13 +3717,45 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.11" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ + "indexmap 2.11.0", "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tonic" version = "0.13.1" @@ -3380,7 +3775,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "rustls-native-certs", "socket2 0.5.10", "tokio", @@ -3400,10 +3795,10 @@ checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" dependencies = [ "prettyplease", "proc-macro2", - "prost-build", - "prost-types", + "prost-build 0.13.5", + "prost-types 0.13.5", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -3456,7 +3851,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -3525,7 +3920,7 @@ checksum = "35f5380909ffc31b4de4f4bdf96b877175a016aa2ca98cee39fcfd8c4d53d952" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -3534,18 +3929,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-xid" version = "0.2.6" @@ -3576,6 +3959,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.18.1" @@ -3599,12 +3988,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - [[package]] name = "version_check" version = "0.9.5" @@ -3658,7 +4041,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.106", + "syn", "wasm-bindgen-shared", ] @@ -3693,7 +4076,7 @@ checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3803,7 +4186,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -3814,7 +4197,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -4082,6 +4465,15 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winnow" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -4104,6 +4496,16 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "xattr" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yoke" version = "0.8.0" @@ -4124,7 +4526,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", "synstructure", ] @@ -4145,7 +4547,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -4165,27 +4567,10 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", "synstructure", ] -[[package]] -name = "zeroidc" -version = "0.1.0" -dependencies = [ - "base64 0.21.7", - "bytes", - "cbindgen", - "jwt", - "openidconnect", - "reqwest", - "serde", - "thiserror 1.0.69", - "time", - "tokio", - "url", -] - [[package]] name = "zeroize" version = "1.8.1" @@ -4222,5 +4607,5 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] diff --git a/rustybits/Cargo.toml b/rustybits/Cargo.toml index 3de3522173..9f718d1911 100644 --- a/rustybits/Cargo.toml +++ b/rustybits/Cargo.toml @@ -1,6 +1,61 @@ -[workspace] -resolver = "2" -members = ["smeeclient", "zeroidc"] +[package] +name = "rustybits" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["staticlib", "rlib"] + +[features] +default = ["zeroidc"] +zeroidc = [] +ztcontroller = [ + "dep:serde", + "dep:serde_json", + "dep:temporal-sdk", + "dep:temporal-client", + "dep:temporal-sdk-core-protos", + "dep:tokio", + "dep:tokio-util", +] + +[dependencies] +serde = { version = "1", features = ["derive"], optional = true } +serde_json = { version = "1", optional = true } +temporal-sdk = { git = "https://github.com/temporalio/sdk-core", branch = "master", optional = true } +temporal-client = { git = "https://github.com/temporalio/sdk-core", branch = "master", optional = true, features = [ + "telemetry", +] } +temporal-sdk-core-protos = { git = "https://github.com/temporalio/sdk-core", branch = "master", optional = true } +tokio = { version = "1.43", features = [ + "full", + "rt", + "macros", +], optional = true } +tokio-util = { version = "0.7", optional = true } +uuid = { version = "1.4", features = ["v4"] } +openidconnect = { version = "3.4", default-features = false, features = [ + "reqwest", + "native-tls", + "accept-rfc3339-timestamps", +] } +base64 = "0.21" +url = "2.3" +reqwest = "0.11" +jwt = { version = "0.16", git = "https://github.com/glimberg/rust-jwt" } +time = { version = "~0.3", features = ["formatting"] } +bytes = "1.3" +thiserror = "1" + +[dev-dependencies] +testcontainers = { version = "0.24", features = ["blocking"] } +testcontainers-modules = { version = "0.12.1", features = [ + "google_cloud_sdk_emulators", +] } + +[build-dependencies] +cbindgen = "0.29" +prost-build = "0.14" [profile.release] strip = "debuginfo" diff --git a/rustybits/smeeclient/build.rs b/rustybits/build.rs similarity index 73% rename from rustybits/smeeclient/build.rs rename to rustybits/build.rs index 73e97bcc6b..d756ef3f3b 100644 --- a/rustybits/smeeclient/build.rs +++ b/rustybits/build.rs @@ -1,19 +1,21 @@ extern crate cbindgen; -use cbindgen::{Config, Language}; +use cbindgen::{Config, Language, MacroExpansionConfig}; use std::env; use std::path::PathBuf; - fn main() { let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); let package_name = env::var("CARGO_PKG_NAME").unwrap(); let output_file = target_dir().join(format!("{package_name}.h")).display().to_string(); + let meconfig = MacroExpansionConfig { bitflags: true, ..Default::default() }; + let config = Config { language: Language::C, cpp_compat: true, - namespace: Some(String::from("smeeclient")), + namespace: Some(String::from("rustybits")), + macro_expansion: meconfig, ..Default::default() }; @@ -29,8 +31,6 @@ fn target_dir() -> PathBuf { if let Ok(target) = env::var("CARGO_TARGET_DIR") { PathBuf::from(target) } else { - PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()) - .join("..") - .join("target") + PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()).join("target") } } diff --git a/rustybits/smeeclient/Cargo.toml b/rustybits/smeeclient/Cargo.toml deleted file mode 100644 index 6f32da35d6..0000000000 --- a/rustybits/smeeclient/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "smeeclient" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["staticlib", "rlib"] - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -serde = { version = "1", features = ["derive"] } -temporal-sdk = { git = "https://github.com/temporalio/sdk-core", branch = "master" } -temporal-client = { git = "https://github.com/temporalio/sdk-core", branch = "master", features = ["telemetry"] } -temporal-sdk-core-protos = { git = "https://github.com/temporalio/sdk-core", branch = "master" } -tokio = { version = "1.43", features = ["full"] } -url = { version = "2" } -uuid = { version = "1.4", features = ["v4"] } - -[build-dependencies] -cbindgen = "0.20" diff --git a/rustybits/smeeclient/rustfmt.toml b/rustybits/smeeclient/rustfmt.toml deleted file mode 120000 index 39f97b043b..0000000000 --- a/rustybits/smeeclient/rustfmt.toml +++ /dev/null @@ -1 +0,0 @@ -../rustfmt.toml \ No newline at end of file diff --git a/rustybits/smeeclient/src/ext.rs b/rustybits/smeeclient/src/ext.rs deleted file mode 100644 index 9db10a375c..0000000000 --- a/rustybits/smeeclient/src/ext.rs +++ /dev/null @@ -1,90 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. - * - * (c) ZeroTier, Inc. - * https://www.zerotier.com/ - */ - -#![allow(clippy::uninlined_format_args, clippy::missing_safety_doc)] - -use std::ffi::CStr; -use std::os::raw::c_char; - -use crate::NetworkJoinedParams; -use crate::SmeeClient; - -#[no_mangle] -pub unsafe extern "C" fn smee_client_new( - temporal_url: *const c_char, - namespace: *const c_char, - task_queue: *const c_char, -) -> *mut SmeeClient { - let url = unsafe { - assert!(!temporal_url.is_null()); - CStr::from_ptr(temporal_url).to_str().unwrap() - }; - - let ns = unsafe { - assert!(!namespace.is_null()); - CStr::from_ptr(namespace).to_str().unwrap() - }; - - let tq = unsafe { - assert!(!task_queue.is_null()); - CStr::from_ptr(task_queue).to_str().unwrap() - }; - - match SmeeClient::new(url, ns, tq) { - Ok(c) => Box::into_raw(Box::new(c)), - Err(e) => { - println!("error creating smee client instance: {}", e); - std::ptr::null_mut() - } - } -} - -#[no_mangle] -pub unsafe extern "C" fn smee_client_delete(ptr: *mut SmeeClient) { - if ptr.is_null() { - return; - } - let smee = unsafe { - assert!(!ptr.is_null()); - Box::from_raw(&mut *ptr) - }; - - smee.shutdown(); -} - -#[no_mangle] -pub unsafe extern "C" fn smee_client_notify_network_joined( - smee_instance: *mut SmeeClient, - network_id: *const c_char, - member_id: *const c_char, -) -> bool { - let nwid = unsafe { - assert!(!network_id.is_null()); - CStr::from_ptr(network_id).to_str().unwrap() - }; - - let mem_id = unsafe { - assert!(!member_id.is_null()); - CStr::from_ptr(member_id).to_str().unwrap() - }; - - let smee = unsafe { - assert!(!smee_instance.is_null()); - &mut *smee_instance - }; - - let params = NetworkJoinedParams::new(nwid, mem_id); - - match smee.notify_network_joined(params) { - Ok(()) => true, - Err(e) => { - println!("error notifying network joined: {0}", e); - false - } - } -} diff --git a/rustybits/zeroidc/src/ext.rs b/rustybits/src/ext.rs similarity index 50% rename from rustybits/zeroidc/src/ext.rs rename to rustybits/src/ext.rs index 8d86cda50c..32687bcaf9 100644 --- a/rustybits/zeroidc/src/ext.rs +++ b/rustybits/src/ext.rs @@ -8,16 +8,59 @@ use std::ffi::{CStr, CString}; use std::os::raw::c_char; +#[cfg(feature = "ztcontroller")] +use tokio::runtime; use url::Url; -use crate::ZeroIDC; +#[cfg(feature = "ztcontroller")] +static mut RT: Option = None; -#[cfg(any( - all(target_os = "linux", target_arch = "x86"), - all(target_os = "linux", target_arch = "x86_64"), - all(target_os = "linux", target_arch = "aarch64"), - target_os = "windows", - target_os = "macos", +#[cfg(feature = "ztcontroller")] +static START: std::sync::Once = std::sync::Once::new(); +#[cfg(feature = "ztcontroller")] +static SHUTDOWN: std::sync::Once = std::sync::Once::new(); + +#[cfg(feature = "ztcontroller")] +#[no_mangle] +pub unsafe extern "C" fn init_async_runtime() { + START.call_once(|| { + let rt = runtime::Builder::new_multi_thread() + .worker_threads(4) + .thread_name("rust-async-worker") + .enable_all() + .build() + .expect("Failed to create tokio runtime"); + + unsafe { RT = Some(rt) }; + }); +} + +#[cfg(feature = "ztcontroller")] +#[no_mangle] +#[allow(static_mut_refs)] +pub unsafe extern "C" fn shutdown_async_runtime() { + SHUTDOWN.call_once(|| { + // Shutdown the tokio runtime + unsafe { + if let Some(rt) = RT.take() { + rt.shutdown_timeout(std::time::Duration::from_secs(5)); + } + } + }); +} + +#[cfg(feature = "zeroidc")] +use crate::zeroidc::ZeroIDC; + +#[cfg(all( + feature = "zeroidc", + any( + all(target_os = "linux", target_arch = "x86"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), + target_os = "windows", + target_os = "macos", + ) ))] #[no_mangle] pub unsafe extern "C" fn zeroidc_new( @@ -66,12 +109,15 @@ pub unsafe extern "C" fn zeroidc_new( } } -#[cfg(any( - all(target_os = "linux", target_arch = "x86"), - all(target_os = "linux", target_arch = "x86_64"), - all(target_os = "linux", target_arch = "aarch64"), - target_os = "windows", - target_os = "macos", +#[cfg(all( + feature = "zeroidc", + any( + all(target_os = "linux", target_arch = "x86"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), + target_os = "windows", + target_os = "macos", + ) ))] #[no_mangle] pub unsafe extern "C" fn zeroidc_delete(ptr: *mut ZeroIDC) { @@ -89,12 +135,15 @@ pub unsafe extern "C" fn zeroidc_delete(ptr: *mut ZeroIDC) { } } -#[cfg(any( - all(target_os = "linux", target_arch = "x86"), - all(target_os = "linux", target_arch = "x86_64"), - all(target_os = "linux", target_arch = "aarch64"), - target_os = "windows", - target_os = "macos", +#[cfg(all( + feature = "zeroidc", + any( + all(target_os = "linux", target_arch = "x86"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), + target_os = "windows", + target_os = "macos", + ) ))] #[no_mangle] pub unsafe extern "C" fn zeroidc_start(ptr: *mut ZeroIDC) { @@ -105,12 +154,15 @@ pub unsafe extern "C" fn zeroidc_start(ptr: *mut ZeroIDC) { idc.start(); } -#[cfg(any( - all(target_os = "linux", target_arch = "x86"), - all(target_os = "linux", target_arch = "x86_64"), - all(target_os = "linux", target_arch = "aarch64"), - target_os = "windows", - target_os = "macos", +#[cfg(all( + feature = "zeroidc", + any( + all(target_os = "linux", target_arch = "x86"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), + target_os = "windows", + target_os = "macos", + ) ))] #[no_mangle] pub unsafe extern "C" fn zeroidc_stop(ptr: *mut ZeroIDC) { @@ -121,12 +173,15 @@ pub unsafe extern "C" fn zeroidc_stop(ptr: *mut ZeroIDC) { idc.stop(); } -#[cfg(any( - all(target_os = "linux", target_arch = "x86"), - all(target_os = "linux", target_arch = "x86_64"), - all(target_os = "linux", target_arch = "aarch64"), - target_os = "windows", - target_os = "macos", +#[cfg(all( + feature = "zeroidc", + any( + all(target_os = "linux", target_arch = "x86"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), + target_os = "windows", + target_os = "macos", + ) ))] #[no_mangle] pub unsafe extern "C" fn zeroidc_is_running(ptr: *mut ZeroIDC) -> bool { @@ -138,6 +193,16 @@ pub unsafe extern "C" fn zeroidc_is_running(ptr: *mut ZeroIDC) -> bool { idc.is_running() } +#[cfg(all( + feature = "zeroidc", + any( + all(target_os = "linux", target_arch = "x86"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), + target_os = "windows", + target_os = "macos", + ) +))] #[no_mangle] pub unsafe extern "C" fn zeroidc_get_exp_time(ptr: *mut ZeroIDC) -> u64 { let id = unsafe { @@ -148,12 +213,15 @@ pub unsafe extern "C" fn zeroidc_get_exp_time(ptr: *mut ZeroIDC) -> u64 { id.get_exp_time() } -#[cfg(any( - all(target_os = "linux", target_arch = "x86"), - all(target_os = "linux", target_arch = "x86_64"), - all(target_os = "linux", target_arch = "aarch64"), - target_os = "windows", - target_os = "macos", +#[cfg(all( + feature = "zeroidc", + any( + all(target_os = "linux", target_arch = "x86"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), + target_os = "windows", + target_os = "macos", + ) ))] #[no_mangle] pub unsafe extern "C" fn zeroidc_set_nonce_and_csrf( @@ -182,12 +250,15 @@ pub unsafe extern "C" fn zeroidc_set_nonce_and_csrf( idc.set_nonce_and_csrf(csrf_token, nonce); } -#[cfg(any( - all(target_os = "linux", target_arch = "x86"), - all(target_os = "linux", target_arch = "x86_64"), - all(target_os = "linux", target_arch = "aarch64"), - target_os = "windows", - target_os = "macos", +#[cfg(all( + feature = "zeroidc", + any( + all(target_os = "linux", target_arch = "x86"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), + target_os = "windows", + target_os = "macos", + ) ))] #[no_mangle] pub unsafe extern "C" fn free_cstr(s: *mut c_char) { @@ -201,12 +272,15 @@ pub unsafe extern "C" fn free_cstr(s: *mut c_char) { } } -#[cfg(any( - all(target_os = "linux", target_arch = "x86"), - all(target_os = "linux", target_arch = "x86_64"), - all(target_os = "linux", target_arch = "aarch64"), - target_os = "windows", - target_os = "macos", +#[cfg(all( + feature = "zeroidc", + any( + all(target_os = "linux", target_arch = "x86"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), + target_os = "windows", + target_os = "macos", + ) ))] #[no_mangle] pub unsafe extern "C" fn zeroidc_get_auth_url(ptr: *mut ZeroIDC) -> *mut c_char { @@ -220,12 +294,15 @@ pub unsafe extern "C" fn zeroidc_get_auth_url(ptr: *mut ZeroIDC) -> *mut c_char s.into_raw() } -#[cfg(any( - all(target_os = "linux", target_arch = "x86"), - all(target_os = "linux", target_arch = "x86_64"), - all(target_os = "linux", target_arch = "aarch64"), - target_os = "windows", - target_os = "macos", +#[cfg(all( + feature = "zeroidc", + any( + all(target_os = "linux", target_arch = "x86"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), + target_os = "windows", + target_os = "macos", + ) ))] #[no_mangle] pub unsafe extern "C" fn zeroidc_token_exchange(idc: *mut ZeroIDC, code: *const c_char) -> *mut c_char { @@ -310,12 +387,15 @@ pub unsafe extern "C" fn zeroidc_network_id_from_state(state: *const c_char) -> s.into_raw() } -#[cfg(any( - all(target_os = "linux", target_arch = "x86"), - all(target_os = "linux", target_arch = "x86_64"), - all(target_os = "linux", target_arch = "aarch64"), - target_os = "windows", - target_os = "macos", +#[cfg(all( + feature = "zeroidc", + any( + all(target_os = "linux", target_arch = "x86"), + all(target_os = "linux", target_arch = "x86_64"), + all(target_os = "linux", target_arch = "aarch64"), + target_os = "windows", + target_os = "macos", + ) ))] #[no_mangle] pub unsafe extern "C" fn zeroidc_kick_refresh_thread(idc: *mut ZeroIDC) { @@ -327,3 +407,85 @@ pub unsafe extern "C" fn zeroidc_kick_refresh_thread(idc: *mut ZeroIDC) { idc.kick_refresh_thread(); } + +#[cfg(feature = "ztcontroller")] +use crate::smeeclient::NetworkJoinedParams; +#[cfg(feature = "ztcontroller")] +use crate::smeeclient::SmeeClient; + +#[cfg(feature = "ztcontroller")] +#[no_mangle] +pub unsafe extern "C" fn smee_client_new( + temporal_url: *const c_char, + namespace: *const c_char, + task_queue: *const c_char, +) -> *mut SmeeClient { + let url = unsafe { + assert!(!temporal_url.is_null()); + CStr::from_ptr(temporal_url).to_str().unwrap() + }; + + let ns = unsafe { + assert!(!namespace.is_null()); + CStr::from_ptr(namespace).to_str().unwrap() + }; + + let tq = unsafe { + assert!(!task_queue.is_null()); + CStr::from_ptr(task_queue).to_str().unwrap() + }; + + match SmeeClient::new(url, ns, tq) { + Ok(c) => Box::into_raw(Box::new(c)), + Err(e) => { + println!("error creating smee client instance: {}", e); + std::ptr::null_mut() + } + } +} + +#[cfg(feature = "ztcontroller")] +#[no_mangle] +pub unsafe extern "C" fn smee_client_delete(ptr: *mut SmeeClient) { + if ptr.is_null() { + return; + } + let smee = unsafe { + assert!(!ptr.is_null()); + Box::from_raw(&mut *ptr) + }; + drop(smee); +} + +#[cfg(feature = "ztcontroller")] +#[no_mangle] +pub unsafe extern "C" fn smee_client_notify_network_joined( + smee_instance: *mut SmeeClient, + network_id: *const c_char, + member_id: *const c_char, +) -> bool { + let nwid = unsafe { + assert!(!network_id.is_null()); + CStr::from_ptr(network_id).to_str().unwrap() + }; + + let mem_id = unsafe { + assert!(!member_id.is_null()); + CStr::from_ptr(member_id).to_str().unwrap() + }; + + let smee = unsafe { + assert!(!smee_instance.is_null()); + &mut *smee_instance + }; + + let params = NetworkJoinedParams::new(nwid, mem_id); + + match smee.notify_network_joined(params) { + Ok(()) => true, + Err(e) => { + println!("error notifying network joined: {0}", e); + false + } + } +} diff --git a/rustybits/src/lib.rs b/rustybits/src/lib.rs new file mode 100644 index 0000000000..a11b77299c --- /dev/null +++ b/rustybits/src/lib.rs @@ -0,0 +1,5 @@ +pub mod ext; +#[cfg(feature = "ztcontroller")] +pub mod smeeclient; +#[cfg(feature = "zeroidc")] +pub mod zeroidc; diff --git a/rustybits/smeeclient/src/lib.rs b/rustybits/src/smeeclient/mod.rs similarity index 85% rename from rustybits/smeeclient/src/lib.rs rename to rustybits/src/smeeclient/mod.rs index 94bf5e8ab4..203444b60d 100644 --- a/rustybits/smeeclient/src/lib.rs +++ b/rustybits/src/smeeclient/mod.rs @@ -5,17 +5,14 @@ * (c) ZeroTier, Inc. * https://www.zerotier.com/ */ - -pub mod ext; - use serde::{Deserialize, Serialize}; use std::str::FromStr; -use std::time::Duration; use temporal_client::{Client, ClientOptionsBuilder, RetryClient, WorkflowClientTrait, WorkflowOptions}; use temporal_sdk_core_protos::{ coresdk::AsJsonPayloadExt, temporal::api::enums::v1::{WorkflowIdConflictPolicy, WorkflowIdReusePolicy}, }; +use tokio::runtime::Handle; use url::Url; use uuid::Uuid; @@ -33,7 +30,7 @@ pub struct NetworkJoinedParams { } impl NetworkJoinedParams { - fn new(network_id: &str, member_id: &str) -> Self { + pub fn new(network_id: &str, member_id: &str) -> Self { Self { network_id: network_id.to_string(), member_id: member_id.to_string(), @@ -42,16 +39,13 @@ impl NetworkJoinedParams { } pub struct SmeeClient { - tokio_rt: tokio::runtime::Runtime, client: RetryClient, task_queue: String, } impl SmeeClient { pub fn new(temporal_url: &str, namespace: &str, task_queue: &str) -> Result> { - // start tokio runtime. Required by temporal - let rt = tokio::runtime::Runtime::new()?; - + let rt = Handle::current(); let c = ClientOptionsBuilder::default() .target_url(Url::from_str(temporal_url).unwrap()) .client_name(CLIENT_NAME) @@ -60,11 +54,7 @@ impl SmeeClient { let con = rt.block_on(async { c.connect(namespace.to_string(), None).await })?; - Ok(Self { - tokio_rt: rt, - client: con, - task_queue: task_queue.to_string(), - }) + Ok(Self { client: con, task_queue: task_queue.to_string() }) } pub fn notify_network_joined(&self, params: NetworkJoinedParams) -> Result<(), Box> { @@ -88,7 +78,8 @@ impl SmeeClient { let workflow_id = Uuid::new_v4(); - self.tokio_rt.block_on(async { + let rt = Handle::current(); + rt.block_on(async { println!("calilng start_workflow"); self.client .start_workflow( @@ -104,8 +95,4 @@ impl SmeeClient { Ok(()) } - - pub fn shutdown(self) { - self.tokio_rt.shutdown_timeout(Duration::from_secs(5)) - } } diff --git a/rustybits/zeroidc/src/error.rs b/rustybits/src/zeroidc/error.rs similarity index 100% rename from rustybits/zeroidc/src/error.rs rename to rustybits/src/zeroidc/error.rs diff --git a/rustybits/zeroidc/src/lib.rs b/rustybits/src/zeroidc/mod.rs similarity index 99% rename from rustybits/zeroidc/src/lib.rs rename to rustybits/src/zeroidc/mod.rs index cb2e4d23f1..973c9babba 100644 --- a/rustybits/zeroidc/src/lib.rs +++ b/rustybits/src/zeroidc/mod.rs @@ -13,7 +13,6 @@ )] pub mod error; -pub mod ext; extern crate base64; extern crate bytes; @@ -21,7 +20,7 @@ extern crate openidconnect; extern crate time; extern crate url; -use crate::error::*; +use crate::zeroidc::error::*; use bytes::Bytes; use jwt::Token; @@ -162,12 +161,12 @@ impl ZeroIDC { Ok(idc) } - fn kick_refresh_thread(&mut self) { + pub fn kick_refresh_thread(&mut self) { let local = Arc::clone(&self.inner); local.lock().unwrap().kick = true; } - fn start(&mut self) { + pub fn start(&mut self) { let local = Arc::clone(&self.inner); if !local.lock().unwrap().running { diff --git a/rustybits/zeroidc.vcxproj b/rustybits/zeroidc.vcxproj index 8334372384..a4318fba52 100644 --- a/rustybits/zeroidc.vcxproj +++ b/rustybits/zeroidc.vcxproj @@ -87,49 +87,49 @@ - cargo build -p zeroidc --release --target=x86_64-pc-windows-msvc + cargo build --release --target=x86_64-pc-windows-msvc cargo clean - cargo clean & cargo build -p zeroidc --release --target=x86_64-pc-windows-msvc + cargo clean & cargo build --release --target=x86_64-pc-windows-msvc NDEBUG;$(NMakePreprocessorDefinitions) - cargo build -p zeroidc --release --target=aarch64-pc-windows-msvc + cargo build --release --target=aarch64-pc-windows-msvc cargo clean - cargo clean & cargo build -p zeroidc --release --target=aarch64-pc-windows-msvc + cargo clean & cargo build --release --target=aarch64-pc-windows-msvc NDEBUG;$(NMakePreprocessorDefinitions) - cargo build -p zeroidc --target=i686-pc-windows-msvc + cargo build --target=i686-pc-windows-msvc cargo clean - cargo clean & cargo build -p zeroidc --target=i686-pc-windows-msvc + cargo clean & cargo build --target=i686-pc-windows-msvc WIN32;_DEBUG;$(NMakePreprocessorDefinitions) - cargo build -p zeroidc --target=x86_64-pc-windows-msvc + cargo build --target=x86_64-pc-windows-msvc cargo clean - cargo clean & cargo build -p zeroidc --target=x86_64-pc-windows-msvc + cargo clean & cargo build --target=x86_64-pc-windows-msvc _DEBUG;$(NMakePreprocessorDefinitions) - cargo build -p zeroidc --target=aarch64-pc-windows-msvc + cargo build --target=aarch64-pc-windows-msvc cargo clean - cargo clean & cargo build -p zeroidc --target=aarch64-pc-windows-msvc + cargo clean & cargo build --target=aarch64-pc-windows-msvc _DEBUG;$(NMakePreprocessorDefinitions) - cargo build --release -p zeroidc --target=i686-pc-windows-msvc + cargo build --release --target=i686-pc-windows-msvc cargo clean - cargo clean & cargo build -p zeroidc --release --target=i686-pc-windows-msvc + cargo clean & cargo build --release --target=i686-pc-windows-msvc WIN32;NDEBUG;$(NMakePreprocessorDefinitions) diff --git a/rustybits/zeroidc/.cargo/config.toml b/rustybits/zeroidc/.cargo/config.toml deleted file mode 100644 index 5326b36476..0000000000 --- a/rustybits/zeroidc/.cargo/config.toml +++ /dev/null @@ -1,11 +0,0 @@ -[target.x86_64-apple-darwin] -rustflags=["-C", "link-arg=-mmacosx-version-min=10.13"] - -[target.aarch64-apple-darwin] -rustflags=["-C", "link-arg=-mmacosx-version-min=10.13"] - -[target.x86_64-pc-windows-msvc] -rustflags = ["-C", "target-feature=+crt-static"] - -[target.i686-pc-windows-msvc] -rustflags = ["-C", "target-feature=+crt-static"] diff --git a/rustybits/zeroidc/.gitattributes b/rustybits/zeroidc/.gitattributes deleted file mode 100644 index ba3a8fca87..0000000000 --- a/rustybits/zeroidc/.gitattributes +++ /dev/null @@ -1,3 +0,0 @@ -# disable autocrlf because it doesn't mix well with -# vendored rust packages -* text=false diff --git a/rustybits/zeroidc/Cargo.toml b/rustybits/zeroidc/Cargo.toml deleted file mode 100644 index e38c80e690..0000000000 --- a/rustybits/zeroidc/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "zeroidc" -version = "0.1.0" -edition = "2021" -build = "build.rs" -publish = false - -[lib] -crate-type = ["staticlib", "rlib"] - -[dependencies] -openidconnect = { version = "3.4", default-features = false, features = [ - "reqwest", - "native-tls", - "accept-rfc3339-timestamps", -] } -base64 = "0.21" -url = "2.3" -reqwest = "0.11" -jwt = { version = "0.16", git = "https://github.com/glimberg/rust-jwt" } -serde = "1.0" -time = { version = "~0.3", features = ["formatting"] } -bytes = "1.3" -thiserror = "1" -tokio = { version = ">=1.24" } - -[build-dependencies] -cbindgen = "0.20" diff --git a/rustybits/zeroidc/build.rs b/rustybits/zeroidc/build.rs deleted file mode 100644 index fbb256fc63..0000000000 --- a/rustybits/zeroidc/build.rs +++ /dev/null @@ -1,36 +0,0 @@ -extern crate cbindgen; - -use cbindgen::{Config, Language}; -use std::env; -use std::path::PathBuf; - -fn main() { - let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); - - let package_name = env::var("CARGO_PKG_NAME").unwrap(); - let output_file = target_dir().join(format!("{package_name}.h")).display().to_string(); - - let config = Config { - language: Language::C, - cpp_compat: true, - namespace: Some(String::from("zeroidc")), - ..Default::default() - }; - - cbindgen::generate_with_config(&crate_dir, config) - .unwrap() - .write_to_file(&output_file); -} - -/// Find the location of the `target/` directory. Note that this may be -/// overridden by `cmake`, so we also need to check the `CARGO_TARGET_DIR` -/// variable. -fn target_dir() -> PathBuf { - if let Ok(target) = env::var("CARGO_TARGET_DIR") { - PathBuf::from(target) - } else { - PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()) - .join("..") - .join("target") - } -} diff --git a/rustybits/zeroidc/rustfmt.toml b/rustybits/zeroidc/rustfmt.toml deleted file mode 120000 index 39f97b043b..0000000000 --- a/rustybits/zeroidc/rustfmt.toml +++ /dev/null @@ -1 +0,0 @@ -../rustfmt.toml \ No newline at end of file diff --git a/service/CMakeLists.txt b/service/CMakeLists.txt new file mode 100644 index 0000000000..359d578b57 --- /dev/null +++ b/service/CMakeLists.txt @@ -0,0 +1,51 @@ +cmake_minimum_required(VERSION 3.13) +project(service) + +file(GLOB service_src_glob ${PROJ_DIR}/service/*.cpp) +file(GLOB service_hdr_glob ${PROJ_DIR}/service/*.hpp) + +add_library(zerotier-service STATIC ${service_src_glob} ${service_hdr_glob}) + +set(INCLUDE_DIRS + "${httplib_SOURCE_DIR}" + ${RUSTYBITS_INCLUDE_DIR} +) + +find_package(OpenSSL REQUIRED) + +set(LINK_LIBS + zerotier-osdep + zerotier-core + prometheus-cpp-lite + nlohmann_json::nlohmann_json + Threads::Threads + opentelemetry-cpp::api + Threads::Threads + OpenSSL::Crypto + OpenSSL::SSL + rustybits +) + +if(ZT1_CENTRAL_CONTROLLER) + add_definitions(-DZT_CONTROLLER_USE_LIBPQ) + list(APPEND INCLUDE_DIRS + ${PostgreSQL_INCLUDE_DIRS} + "${redis++_BUILD_DIR}/src" + ${pqxx_INCLUDE_DIRS} + ) + list(APPEND LINK_LIBS + pqxx + redis++::redis++_static + opentelemetry-cpp::sdk + ) +endif() + +target_include_directories(zerotier-service + PRIVATE + ${INCLUDE_DIRS} +) + + +target_link_libraries(zerotier-service + ${LINK_LIBS} +) diff --git a/service/OneService.cpp b/service/OneService.cpp index 41d7905165..107562cbfd 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -42,9 +42,17 @@ #include "../version.h" #include "OneService.hpp" +#ifdef CMAKE_BUILD +#include +#else #include +#endif #ifdef ZT_OPENTELEMETRY_ENABLED +#include "opentelemetry/baggage/propagation/baggage_propagator.h" +#include "opentelemetry/context/propagation/composite_propagator.h" +#include "opentelemetry/context/propagation/global_propagator.h" +#include "opentelemetry/context/propagation/text_map_propagator.h" #include "opentelemetry/exporters/memory/in_memory_data.h" #include "opentelemetry/exporters/otlp/otlp_grpc_exporter.h" #include "opentelemetry/exporters/otlp/otlp_grpc_log_record_exporter.h" @@ -64,6 +72,7 @@ #include "opentelemetry/sdk/trace/tracer.h" #include "opentelemetry/sdk/trace/tracer_context.h" #include "opentelemetry/sdk/trace/tracer_provider.h" +#include "opentelemetry/trace/propagation/http_trace_context.h" #include "opentelemetry/trace/provider.h" namespace sdktrace = opentelemetry::v1::sdk::trace; @@ -77,7 +86,7 @@ namespace sdkresource = opentelemetry::v1::sdk::resource; #endif #if ZT_SSO_ENABLED -#include +#include #endif #ifdef __WINDOWS__ @@ -126,6 +135,10 @@ using json = nlohmann::json; #include "../nonfree/controller/EmbeddedNetworkController.hpp" #include "../nonfree/controller/PostgreSQL.hpp" #include "../nonfree/controller/Redis.hpp" +#ifdef ZT1_CENTRAL_CONTROLLER +#include "../nonfree/controller/CentralDB.hpp" +#include "../nonfree/controller/ControllerConfig.hpp" +#endif #include "../osdep/EthernetTap.hpp" #ifdef __WINDOWS__ #include "../osdep/WindowsEthernetTap.hpp" @@ -269,7 +282,8 @@ std::string http_log(const httplib::Request& req, const httplib::Response& res) std::string query; for (auto it = req.params.begin(); it != req.params.end(); ++it) { const auto& x = *it; - snprintf(buf, sizeof(buf), "%c%s=%s", (it == req.params.begin()) ? '?' : '&', x.first.c_str(), x.second.c_str()); + snprintf( + buf, sizeof(buf), "%c%s=%s", (it == req.params.begin()) ? '?' : '&', x.first.c_str(), x.second.c_str()); query += buf; } snprintf(buf, sizeof(buf), "%s\n", query.c_str()); @@ -319,8 +333,8 @@ class NetworkState { #if ZT_SSO_ENABLED if (_idc) { - zeroidc::zeroidc_stop(_idc); - zeroidc::zeroidc_delete(_idc); + rustybits::zeroidc_stop(_idc); + rustybits::zeroidc_delete(_idc); _idc = nullptr; } #endif @@ -418,7 +432,8 @@ class NetworkState { assert(_config.centralAuthURL != nullptr); assert(_config.ssoProvider != nullptr); - _idc = zeroidc::zeroidc_new(_config.issuerURL, _config.ssoClientID, _config.centralAuthURL, _config.ssoProvider, _webPort); + _idc = rustybits::zeroidc_new( + _config.issuerURL, _config.ssoClientID, _config.centralAuthURL, _config.ssoProvider, _webPort); if (_idc == nullptr) { fprintf(stderr, "idc is null\n"); @@ -426,15 +441,15 @@ class NetworkState { } } - zeroidc::zeroidc_set_nonce_and_csrf(_idc, _config.ssoState, _config.ssoNonce); + rustybits::zeroidc_set_nonce_and_csrf(_idc, _config.ssoState, _config.ssoNonce); - char* url = zeroidc::zeroidc_get_auth_url(_idc); + char* url = rustybits::zeroidc_get_auth_url(_idc); memcpy(_config.authenticationURL, url, strlen(url)); _config.authenticationURL[strlen(url)] = 0; - zeroidc::free_cstr(url); + rustybits::free_cstr(url); - if (zeroidc::zeroidc_is_running(_idc) && nwc->status == ZT_NETWORK_STATUS_AUTHENTICATION_REQUIRED) { - zeroidc::zeroidc_kick_refresh_thread(_idc); + if (rustybits::zeroidc_is_running(_idc) && nwc->status == ZT_NETWORK_STATUS_AUTHENTICATION_REQUIRED) { + rustybits::zeroidc_kick_refresh_thread(_idc); } #endif } @@ -464,13 +479,13 @@ class NetworkState { return ret; } - ret = zeroidc::zeroidc_token_exchange(_idc, code); - zeroidc::zeroidc_set_nonce_and_csrf(_idc, _config.ssoState, _config.ssoNonce); + ret = rustybits::zeroidc_token_exchange(_idc, code); + rustybits::zeroidc_set_nonce_and_csrf(_idc, _config.ssoState, _config.ssoNonce); - char* url = zeroidc::zeroidc_get_auth_url(_idc); + char* url = rustybits::zeroidc_get_auth_url(_idc); memcpy(_config.authenticationURL, url, strlen(url)); _config.authenticationURL[strlen(url)] = 0; - zeroidc::free_cstr(url); + rustybits::free_cstr(url); #endif return ret; } @@ -482,7 +497,7 @@ class NetworkState { fprintf(stderr, "idc is null\n"); return 0; } - return zeroidc::zeroidc_get_exp_time(_idc); + return rustybits::zeroidc_get_exp_time(_idc); #else return 0; #endif @@ -496,7 +511,7 @@ class NetworkState { std::map > _managedRoutes; OneService::NetworkSettings _settings; #if ZT_SSO_ENABLED - zeroidc::ZeroIDC* _idc; + rustybits::ZeroIDC* _idc; #endif }; @@ -505,9 +520,15 @@ namespace { static const InetAddress NULL_INET_ADDR; // Fake TLS hello for TCP tunnel outgoing connections (TUNNELED mode) -static const char ZT_TCP_TUNNEL_HELLO[9] = { - 0x17, 0x03, 0x03, 0x00, 0x04, (char)ZEROTIER_ONE_VERSION_MAJOR, (char)ZEROTIER_ONE_VERSION_MINOR, (char)((ZEROTIER_ONE_VERSION_REVISION >> 8) & 0xff), (char)(ZEROTIER_ONE_VERSION_REVISION & 0xff) -}; +static const char ZT_TCP_TUNNEL_HELLO[9] = { 0x17, + 0x03, + 0x03, + 0x00, + 0x04, + (char)ZEROTIER_ONE_VERSION_MAJOR, + (char)ZEROTIER_ONE_VERSION_MINOR, + (char)((ZEROTIER_ONE_VERSION_REVISION >> 8) & 0xff), + (char)(ZEROTIER_ONE_VERSION_REVISION & 0xff) }; static std::string _trimString(const std::string& s) { @@ -571,14 +592,9 @@ static void _networkToJson(nlohmann::json& nj, NetworkState& ns) nj["id"] = tmp; nj["nwid"] = tmp; OSUtils::ztsnprintf( - tmp, - sizeof(tmp), - "%.2x:%.2x:%.2x:%.2x:%.2x:%.2x", - (unsigned int)((ns.config().mac >> 40) & 0xff), - (unsigned int)((ns.config().mac >> 32) & 0xff), - (unsigned int)((ns.config().mac >> 24) & 0xff), - (unsigned int)((ns.config().mac >> 16) & 0xff), - (unsigned int)((ns.config().mac >> 8) & 0xff), + tmp, sizeof(tmp), "%.2x:%.2x:%.2x:%.2x:%.2x:%.2x", (unsigned int)((ns.config().mac >> 40) & 0xff), + (unsigned int)((ns.config().mac >> 32) & 0xff), (unsigned int)((ns.config().mac >> 24) & 0xff), + (unsigned int)((ns.config().mac >> 16) & 0xff), (unsigned int)((ns.config().mac >> 8) & 0xff), (unsigned int)(ns.config().mac & 0xff)); nj["mac"] = tmp; nj["name"] = ns.config().name; @@ -746,15 +762,76 @@ static void _moonToJson(nlohmann::json& mj, const World& world) class OneServiceImpl; -static int SnodeVirtualNetworkConfigFunction(ZT_Node* node, void* uptr, void* tptr, uint64_t nwid, void** nuptr, enum ZT_VirtualNetworkConfigOperation op, const ZT_VirtualNetworkConfig* nwconf); +static int SnodeVirtualNetworkConfigFunction( + ZT_Node* node, + void* uptr, + void* tptr, + uint64_t nwid, + void** nuptr, + enum ZT_VirtualNetworkConfigOperation op, + const ZT_VirtualNetworkConfig* nwconf); static void SnodeEventCallback(ZT_Node* node, void* uptr, void* tptr, enum ZT_Event event, const void* metaData); -static void SnodeStatePutFunction(ZT_Node* node, void* uptr, void* tptr, enum ZT_StateObjectType type, const uint64_t id[2], const void* data, int len); -static int SnodeStateGetFunction(ZT_Node* node, void* uptr, void* tptr, enum ZT_StateObjectType type, const uint64_t id[2], void* data, unsigned int maxlen); -static int SnodeWirePacketSendFunction(ZT_Node* node, void* uptr, void* tptr, int64_t localSocket, const struct sockaddr_storage* addr, const void* data, unsigned int len, unsigned int ttl); -static void SnodeVirtualNetworkFrameFunction(ZT_Node* node, void* uptr, void* tptr, uint64_t nwid, void** nuptr, uint64_t sourceMac, uint64_t destMac, unsigned int etherType, unsigned int vlanId, const void* data, unsigned int len); -static int SnodePathCheckFunction(ZT_Node* node, void* uptr, void* tptr, uint64_t ztaddr, int64_t localSocket, const struct sockaddr_storage* remoteAddr); -static int SnodePathLookupFunction(ZT_Node* node, void* uptr, void* tptr, uint64_t ztaddr, int family, struct sockaddr_storage* result); -static void StapFrameHandler(void* uptr, void* tptr, uint64_t nwid, const MAC& from, const MAC& to, unsigned int etherType, unsigned int vlanId, const void* data, unsigned int len); +static void SnodeStatePutFunction( + ZT_Node* node, + void* uptr, + void* tptr, + enum ZT_StateObjectType type, + const uint64_t id[2], + const void* data, + int len); +static int SnodeStateGetFunction( + ZT_Node* node, + void* uptr, + void* tptr, + enum ZT_StateObjectType type, + const uint64_t id[2], + void* data, + unsigned int maxlen); +static int SnodeWirePacketSendFunction( + ZT_Node* node, + void* uptr, + void* tptr, + int64_t localSocket, + const struct sockaddr_storage* addr, + const void* data, + unsigned int len, + unsigned int ttl); +static void SnodeVirtualNetworkFrameFunction( + ZT_Node* node, + void* uptr, + void* tptr, + uint64_t nwid, + void** nuptr, + uint64_t sourceMac, + uint64_t destMac, + unsigned int etherType, + unsigned int vlanId, + const void* data, + unsigned int len); +static int SnodePathCheckFunction( + ZT_Node* node, + void* uptr, + void* tptr, + uint64_t ztaddr, + int64_t localSocket, + const struct sockaddr_storage* remoteAddr); +static int SnodePathLookupFunction( + ZT_Node* node, + void* uptr, + void* tptr, + uint64_t ztaddr, + int family, + struct sockaddr_storage* result); +static void StapFrameHandler( + void* uptr, + void* tptr, + uint64_t nwid, + const MAC& from, + const MAC& to, + unsigned int etherType, + unsigned int vlanId, + const void* data, + unsigned int len); static int ShttpOnMessageBegin(http_parser* parser); static int ShttpOnUrl(http_parser* parser, const char* ptr, size_t length); @@ -770,9 +847,15 @@ static int ShttpOnBody(http_parser* parser, const char* ptr, size_t length); static int ShttpOnMessageComplete(http_parser* parser); #if (HTTP_PARSER_VERSION_MAJOR >= 2) && (HTTP_PARSER_VERSION_MINOR >= 1) -static const struct http_parser_settings HTTP_PARSER_SETTINGS = { ShttpOnMessageBegin, ShttpOnUrl, ShttpOnStatus, ShttpOnHeaderField, ShttpOnValue, ShttpOnHeadersComplete, ShttpOnBody, ShttpOnMessageComplete }; +static const struct http_parser_settings HTTP_PARSER_SETTINGS = { ShttpOnMessageBegin, ShttpOnUrl, + ShttpOnStatus, ShttpOnHeaderField, + ShttpOnValue, ShttpOnHeadersComplete, + ShttpOnBody, ShttpOnMessageComplete }; #else -static const struct http_parser_settings HTTP_PARSER_SETTINGS = { ShttpOnMessageBegin, ShttpOnUrl, ShttpOnHeaderField, ShttpOnValue, ShttpOnHeadersComplete, ShttpOnBody, ShttpOnMessageComplete }; +static const struct http_parser_settings HTTP_PARSER_SETTINGS = { ShttpOnMessageBegin, ShttpOnUrl, + ShttpOnHeaderField, ShttpOnValue, + ShttpOnHeadersComplete, ShttpOnBody, + ShttpOnMessageComplete }; #endif /** @@ -936,6 +1019,10 @@ class OneServiceImpl : public OneService { double _exporterSampleRate; #endif +#ifdef ZT1_CENTRAL_CONTROLLER + ControllerConfig _controllerConfig; +#endif + // end member variables ---------------------------------------------------- OneServiceImpl(const char* hp, unsigned int port) @@ -982,6 +1069,9 @@ class OneServiceImpl : public OneService { , _traceProvider(nullptr) , _exporterEndpoint() , _exporterSampleRate(1.0) +#endif +#ifdef ZT1_CENTRAL_CONTROLLER + , _controllerConfig() #endif { _ports[0] = 0; @@ -1050,29 +1140,53 @@ class OneServiceImpl : public OneService { void initTracing() { if (! _exporterEndpoint.empty() && _exporterSampleRate > 0.0) { - fprintf(stderr, "OpenTelemetry tracing enabled with endpoint %s and sample rate %.2f\n", _exporterEndpoint.c_str(), _exporterSampleRate); + fprintf( + stderr, "OpenTelemetry tracing enabled with endpoint %s and sample rate %.2f\n", + _exporterEndpoint.c_str(), _exporterSampleRate); // Set up OpenTelemetry exporter and tracer provider opentelemetry::v1::exporter::otlp::OtlpGrpcExporterOptions opts; opts.endpoint = _exporterEndpoint + "/v1/traces"; - auto exporter = std::unique_ptr(new opentelemetry::exporter::otlp::OtlpGrpcExporter(opts)); + auto exporter = std::unique_ptr( + new opentelemetry::exporter::otlp::OtlpGrpcExporter(opts)); sdktrace::BatchSpanProcessorOptions batch_options {}; batch_options.schedule_delay_millis = std::chrono::milliseconds(5000); // 5 seconds - auto processor = std::unique_ptr(new sdktrace::BatchSpanProcessor(std::move(exporter), batch_options)); + auto processor = std::unique_ptr( + new sdktrace::BatchSpanProcessor(std::move(exporter), batch_options)); auto processors = std::vector >(); processors.push_back(std::move(processor)); char buf[256]; auto versionString = std::stringstream(); - versionString << ZEROTIER_ONE_VERSION_MAJOR << "." << ZEROTIER_ONE_VERSION_MINOR << "." << ZEROTIER_ONE_VERSION_REVISION; - auto resource_attributes = sdkresource::ResourceAttributes { { "service.version", versionString.str() }, { "service.node_id", _node->identity().address().toString(buf) }, { "service.namespace", "com.zerotier.zerotier-one" } }; + versionString << ZEROTIER_ONE_VERSION_MAJOR << "." << ZEROTIER_ONE_VERSION_MINOR << "." + << ZEROTIER_ONE_VERSION_REVISION; + auto resource_attributes = + sdkresource::ResourceAttributes { { "service.version", versionString.str() }, + { "service.node_id", _node->identity().address().toString(buf) }, + { "service.namespace", "com.zerotier.zerotier-one" } }; auto resource = sdkresource::Resource::Create(resource_attributes); - auto sampler = std::unique_ptr(new sdktrace::TraceIdRatioBasedSampler(_exporterSampleRate)); - auto tracer_context = std::make_unique(std::move(processors), resource, std::move(sampler)); - _traceProvider = opentelemetry::nostd::shared_ptr(new sdktrace::TracerProvider(std::move(tracer_context))); + auto sampler = + std::unique_ptr(new sdktrace::TraceIdRatioBasedSampler(_exporterSampleRate)); + auto tracer_context = + std::make_unique(std::move(processors), resource, std::move(sampler)); + _traceProvider = opentelemetry::nostd::shared_ptr( + new sdktrace::TracerProvider(std::move(tracer_context))); sdktrace::Provider::SetTracerProvider(_traceProvider); + + std::vector > propagators; + propagators.push_back( + std::unique_ptr( + new opentelemetry::trace::propagation::HttpTraceContext())); + propagators.push_back( + std::unique_ptr( + new opentelemetry::baggage::propagation::BaggagePropagator())); + + auto p = opentelemetry::nostd::shared_ptr( + new opentelemetry::context::propagation::CompositePropagator(std::move(propagators))); + + opentelemetry::context::propagation::GlobalTextMapPropagator::SetGlobalPropagator(p); } } @@ -1082,10 +1196,12 @@ class OneServiceImpl : public OneService { // Set up OpenTelemetry metrics exporter // opentelemetry::exporter::otlp::OtlpGrpcExporterOptions opts; // opts.endpoint = _exporterEndpoint + "/v1/metrics"; - // auto exporter = std::unique_ptr(new opentelemetry::exporter::otlp::OtlpGrpcExporter(opts)); - // auto processor = std::unique_ptr(new sdkmetrics::PeriodicExportingMetricReader(std::move(exporter))); - // auto meter_provider = opentelemetry::v1::nostd::shared_ptr(new sdkmetrics::MeterProvider(std::move(processor))); - // sdkmetrics::Provider::SetMeterProvider(meter_provider); + // auto exporter = std::unique_ptr(new + // opentelemetry::exporter::otlp::OtlpGrpcExporter(opts)); auto processor = + // std::unique_ptr(new + // sdkmetrics::PeriodicExportingMetricReader(std::move(exporter))); auto meter_provider = + // opentelemetry::v1::nostd::shared_ptr(new + // sdkmetrics::MeterProvider(std::move(processor))); sdkmetrics::Provider::SetMeterProvider(meter_provider); } } @@ -1095,9 +1211,12 @@ class OneServiceImpl : public OneService { // Set up OpenTelemetry logging exporter // opentelemetry::exporter::otlp::OtlpGrpcExporterOptions opts; // opts.endpoint = _exporterEndpoint + "/v1/logs"; - // auto exporter = std::unique_ptr(new opentelemetry::exporter::otlp::OtlpGrpcExporter(opts)); - // auto processor = std::unique_ptr(new opentelemetry::v1::sdk::logs::SimpleLogRecordProcessor(std::move(exporter))); - // auto logger_provider = opentelemetry::nostd::shared_ptr(new opentelemetry::v1::sdk::logs::LoggerProvider(std::move(processor))); + // auto exporter = std::unique_ptr(new + // opentelemetry::exporter::otlp::OtlpGrpcExporter(opts)); auto processor = + // std::unique_ptr(new + // opentelemetry::v1::sdk::logs::SimpleLogRecordProcessor(std::move(exporter))); auto logger_provider = + // opentelemetry::nostd::shared_ptr(new + // opentelemetry::v1::sdk::logs::LoggerProvider(std::move(processor))); // opentelemetry::logs::Provider::SetLoggerProvider(logger_provider); } } @@ -1117,7 +1236,8 @@ class OneServiceImpl : public OneService { if (! OSUtils::writeFile(authTokenPath.c_str(), _authToken)) { Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; - _fatalErrorMessage = "authtoken.secret could not be written (try running with -U to prevent dropping of privileges)"; + _fatalErrorMessage = "authtoken.secret could not be written (try running with -U to prevent " + "dropping of privileges)"; return _termReason; } else { @@ -1138,7 +1258,8 @@ class OneServiceImpl : public OneService { if (! OSUtils::writeFile(metricsTokenPath.c_str(), _metricsToken)) { Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; - _fatalErrorMessage = "metricstoken.secret could not be written (try running with -U to prevent dropping of privileges)"; + _fatalErrorMessage = "metricstoken.secret could not be written (try running with -U to prevent " + "dropping of privileges)"; return _termReason; } else { @@ -1198,7 +1319,8 @@ class OneServiceImpl : public OneService { if (_ports[0] == 0) { Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; - _fatalErrorMessage = std::string("cannot bind to local control interface port ") + std::to_string(_configuredPort); + _fatalErrorMessage = + std::string("cannot bind to local control interface port ") + std::to_string(_configuredPort); return _termReason; } @@ -1240,7 +1362,8 @@ class OneServiceImpl : public OneService { if (_ports[2]) { char uniqueName[64]; - OSUtils::ztsnprintf(uniqueName, sizeof(uniqueName), "ZeroTier/%.10llx@%u", _node->address(), _ports[2]); + OSUtils::ztsnprintf( + uniqueName, sizeof(uniqueName), "ZeroTier/%.10llx@%u", _node->address(), _ports[2]); _portMapper = new PortMapper(_ports[2], uniqueName); } } @@ -1258,9 +1381,15 @@ class OneServiceImpl : public OneService { // Delete legacy iddb.d if present (cleanup) OSUtils::rmDashRf((_homePath + ZT_PATH_SEPARATOR_S "iddb.d").c_str()); - // Network controller is now enabled by default for desktop and server + // Network controller is now disabled by default for desktop and server #ifdef ZT_NONFREE_CONTROLLER - _controller = new EmbeddedNetworkController(_node, _homePath.c_str(), _controllerDbPath.c_str(), _ports[0], _rc); +#ifdef ZT1_CENTRAL_CONTROLLER + _controller = new EmbeddedNetworkController( + _node, _homePath.c_str(), _controllerDbPath.c_str(), _ports[0], &_controllerConfig); +#else + _controller = + new EmbeddedNetworkController(_node, _homePath.c_str(), _controllerDbPath.c_str(), _ports[0], _rc); +#endif if (! _ssoRedirectURL.empty()) { _controller->setSSORedirectURL(_ssoRedirectURL); } @@ -1271,7 +1400,8 @@ class OneServiceImpl : public OneService { // Join existing networks in networks.d { - std::vector networksDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S "networks.d").c_str())); + std::vector networksDotD( + OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S "networks.d").c_str())); for (std::vector::iterator f(networksDotD.begin()); f != networksDotD.end(); ++f) { std::size_t dot = f->find_last_of('.'); if ((dot == 16) && (f->substr(16) == ".conf")) @@ -1281,7 +1411,8 @@ class OneServiceImpl : public OneService { // Orbit existing moons in moons.d { - std::vector moonsDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S "moons.d").c_str())); + std::vector moonsDotD( + OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S "moons.d").c_str())); for (std::vector::iterator f(moonsDotD.begin()); f != moonsDotD.end(); ++f) { std::size_t dot = f->find_last_of('.'); if ((dot == 16) && (f->substr(16) == ".moon")) @@ -1334,8 +1465,11 @@ class OneServiceImpl : public OneService { } } - // Refresh bindings in case device's interfaces have changed, and also sync routes to update any shadow routes (e.g. shadow default) - if (((now - lastBindRefresh) >= (_node->bondController()->inUse() ? ZT_BINDER_REFRESH_PERIOD / 4 : ZT_BINDER_REFRESH_PERIOD)) || restarted) { + // Refresh bindings in case device's interfaces have changed, and also sync routes to update any shadow + // routes (e.g. shadow default) + if (((now - lastBindRefresh) + >= (_node->bondController()->inUse() ? ZT_BINDER_REFRESH_PERIOD / 4 : ZT_BINDER_REFRESH_PERIOD)) + || restarted) { // If secondary port is not configured to a constant value and we've been offline for a while, // bind a new secondary port. This is a workaround for a "coma" issue caused by buggy NATs that stop // working on one port after a while. @@ -1371,7 +1505,8 @@ class OneServiceImpl : public OneService { #ifdef ZT_USE_MINIUPNPC if (_portMapper) { std::vector mappedAddresses(_portMapper->get()); - for (std::vector::const_iterator ext(mappedAddresses.begin()); ext != mappedAddresses.end(); ++ext) + for (std::vector::const_iterator ext(mappedAddresses.begin()); + ext != mappedAddresses.end(); ++ext) _node->addLocalInterfaceAddress(reinterpret_cast(&(*ext))); } #endif @@ -1397,28 +1532,42 @@ class OneServiceImpl : public OneService { } // Close TCP fallback tunnel if we have direct UDP - if (! _forceTcpRelay && (_tcpFallbackTunnel) && ((now - _lastDirectReceiveFromGlobal) < (ZT_TCP_FALLBACK_AFTER / 2))) { + if (! _forceTcpRelay && (_tcpFallbackTunnel) + && ((now - _lastDirectReceiveFromGlobal) < (ZT_TCP_FALLBACK_AFTER / 2))) { _phy.close(_tcpFallbackTunnel->sock); } // Sync multicast group memberships if ((now - lastTapMulticastGroupCheck) >= ZT_TAP_CHECK_MULTICAST_INTERVAL) { lastTapMulticastGroupCheck = now; - std::vector, std::vector > > > mgChanges; + std::vector< + std::pair, std::vector > > > + mgChanges; { Mutex::Lock _l(_nets_m); mgChanges.reserve(_nets.size() + 1); for (std::map::const_iterator n(_nets.begin()); n != _nets.end(); ++n) { if (n->second.tap()) { - mgChanges.push_back(std::pair, std::vector > >(n->first, std::pair, std::vector >())); - n->second.tap()->scanMulticastGroups(mgChanges.back().second.first, mgChanges.back().second.second); + mgChanges.push_back( + std::pair< + uint64_t, + std::pair, std::vector > >( + n->first, + std::pair, std::vector >())); + n->second.tap()->scanMulticastGroups( + mgChanges.back().second.first, mgChanges.back().second.second); } } } - for (std::vector, std::vector > > >::iterator c(mgChanges.begin()); c != mgChanges.end(); ++c) { - for (std::vector::iterator m(c->second.first.begin()); m != c->second.first.end(); ++m) + for (std::vector, std::vector > > >:: + iterator c(mgChanges.begin()); + c != mgChanges.end(); ++c) { + for (std::vector::iterator m(c->second.first.begin()); + m != c->second.first.end(); ++m) _node->multicastSubscribe((void*)0, c->first, m->mac().toInt(), m->adi()); - for (std::vector::iterator m(c->second.second.begin()); m != c->second.second.end(); ++m) + for (std::vector::iterator m(c->second.second.begin()); + m != c->second.second.end(); ++m) _node->multicastUnsubscribe(c->first, m->mac().toInt(), m->adi()); } } @@ -1426,7 +1575,9 @@ class OneServiceImpl : public OneService { // Clean peers.d periodically if ((now - lastCleanedPeersDb) >= 3600000) { lastCleanedPeersDb = now; - OSUtils::cleanDirectory((_homePath + ZT_PATH_SEPARATOR_S "peers.d").c_str(), now - 2592000000LL); // delete older than 30 days + OSUtils::cleanDirectory( + (_homePath + ZT_PATH_SEPARATOR_S "peers.d").c_str(), + now - 2592000000LL); // delete older than 30 days } const unsigned long delay = (dl > now) ? (unsigned long)(dl - now) : 500; @@ -1460,7 +1611,9 @@ class OneServiceImpl : public OneService { break; } case ZT_EXCEPTION_INVALID_IDENTITY: { - _fatalErrorMessage = "invalid identity loaded from disk. Please remove identity.public and identity.secret from " + _homePath + " and try again"; + _fatalErrorMessage = + "invalid identity loaded from disk. Please remove identity.public and identity.secret from " + + _homePath + " and try again"; break; } case ZT_EXCEPTION_INVALID_SERIALIZED_DATA_INVALID_TYPE: { @@ -1518,14 +1671,17 @@ class OneServiceImpl : public OneService { // LEGACY: support old "trustedpaths" flat file FILE* trustpaths = fopen((_homePath + ZT_PATH_SEPARATOR_S "trustedpaths").c_str(), "r"); if (trustpaths) { - fprintf(stderr, "WARNING: 'trustedpaths' flat file format is deprecated in favor of path definitions in local.conf" ZT_EOL_S); + fprintf( + stderr, "WARNING: 'trustedpaths' flat file format is deprecated in favor of path definitions in " + "local.conf" ZT_EOL_S); char buf[1024]; while (fgets(buf, sizeof(buf), trustpaths)) { int fno = 0; char* saveptr = (char*)0; uint64_t trustedPathId = 0; InetAddress trustedPathNetwork; - for (char* f = Utils::stok(buf, "=\r\n \t", &saveptr); (f); f = Utils::stok((char*)0, "=\r\n \t", &saveptr)) { + for (char* f = Utils::stok(buf, "=\r\n \t", &saveptr); (f); + f = Utils::stok((char*)0, "=\r\n \t", &saveptr)) { if (fno == 0) { trustedPathId = Utils::hexStrToU64(f); } @@ -1536,7 +1692,9 @@ class OneServiceImpl : public OneService { break; ++fno; } - if ((trustedPathId != 0) && ((trustedPathNetwork.ss_family == AF_INET) || (trustedPathNetwork.ss_family == AF_INET6)) && (trustedPathNetwork.netmaskBits() > 0)) { + if ((trustedPathId != 0) + && ((trustedPathNetwork.ss_family == AF_INET) || (trustedPathNetwork.ss_family == AF_INET6)) + && (trustedPathNetwork.netmaskBits() > 0)) { ppc[trustedPathNetwork].trustedPathId = trustedPathId; ppc[trustedPathNetwork].mtu = 0; // use default } @@ -1552,7 +1710,8 @@ class OneServiceImpl : public OneService { try { _localConfig = OSUtils::jsonParse(lcbuf); if (! _localConfig.is_object()) { - fprintf(stderr, "ERROR: unable to parse local.conf (root element is not a JSON object)" ZT_EOL_S); + fprintf( + stderr, "ERROR: unable to parse local.conf (root element is not a JSON object)" ZT_EOL_S); exit(1); } } @@ -1611,17 +1770,21 @@ class OneServiceImpl : public OneService { _exporterEndpoint = OSUtils::jsonString(otel["exporterEndpoint"], ""); _exporterSampleRate = OSUtils::jsonDouble(otel["exporterSampleRate"], 1.0f); if (_exporterEndpoint.empty()) { - fprintf(stderr, "WARNING: OpenTelemetry exporter endpoint is not set. Traces will not be exported." ZT_EOL_S); + fprintf( + stderr, + "WARNING: OpenTelemetry exporter endpoint is not set. Traces will not be exported." ZT_EOL_S); } if (_exporterSampleRate <= 0.0) { - fprintf(stderr, "WARNING: OpenTelemetry exporter sample rate is not set or invalid. Traces will not be exported." ZT_EOL_S); + fprintf( + stderr, "WARNING: OpenTelemetry exporter sample rate is not set or invalid. Traces will not be " + "exported." ZT_EOL_S); } } else { - fprintf(stderr, "WARNING: OpenTelemetry exporter settings are not set. Traces will not be exported." ZT_EOL_S); + fprintf( + stderr, + "WARNING: OpenTelemetry exporter settings are not set. Traces will not be exported." ZT_EOL_S); } -#else - fprintf(stderr, "WARNING: OpenTelemetry support is not enabled. Traces will not be exported." ZT_EOL_S); #endif // Bind to wildcard instead of to specific interfaces (disables full tunnel capability) @@ -1641,8 +1804,71 @@ class OneServiceImpl : public OneService { // Set trusted paths if there are any if (! ppc.empty()) { for (std::map::iterator i(ppc.begin()); i != ppc.end(); ++i) - _node->setPhysicalPathConfiguration(reinterpret_cast(&(i->first)), &(i->second)); + _node->setPhysicalPathConfiguration( + reinterpret_cast(&(i->first)), &(i->second)); + } + +#ifdef ZT1_CENTRAL_CONTROLLER + // Central Controller Mode Settings + json& cc = lc["controller"]; + if (cc.is_object()) { + _controllerConfig.listenMode = OSUtils::jsonString(cc["listenMode"], "pgsql"); + _controllerConfig.statusMode = OSUtils::jsonString(cc["statusMode"], "pgsql"); + + // redis settings + if (cc["redis"].is_object() && _rc == NULL) { + json& redis = cc["redis"]; + _rc = new RedisConfig; + _rc->hostname = OSUtils::jsonString(redis["hostname"], ""); + _rc->port = OSUtils::jsonInt(redis["port"], 6379); + _rc->password = OSUtils::jsonString(redis["password"], ""); + _rc->clusterMode = OSUtils::jsonBool(redis["clusterMode"], false); + _controllerConfig.redisConfig = _rc; + } + else if (cc["redis"].is_object() && _rc != NULL) { + _controllerConfig.redisConfig = _rc; + } + if ((_controllerConfig.listenMode == "redis" || _controllerConfig.statusMode == "redis") + && ! _controllerConfig.redisConfig) { + fprintf( + stderr, + "ERROR: redis listenMode or statusMode requires redis configuration in local.conf" ZT_EOL_S); + exit(1); + } + + // pubsub settings + if (cc["pubsub"].is_object()) { + json& ps = cc["pubsub"]; + _controllerConfig.pubSubConfig = new PubSubConfig(); + _controllerConfig.pubSubConfig->project_id = OSUtils::jsonString(ps["project_id"], ""); + _controllerConfig.pubSubConfig->member_change_recv_topic = + OSUtils::jsonString(ps["member_change_recv_topic"], ""); + _controllerConfig.pubSubConfig->member_change_send_topic = + OSUtils::jsonString(ps["member_change_send_topic"], ""); + _controllerConfig.pubSubConfig->network_change_recv_topic = + OSUtils::jsonString(ps["network_change_recv_topic"], ""); + _controllerConfig.pubSubConfig->network_change_send_topic = + OSUtils::jsonString(ps["network_change_send_topic"], ""); + } + if (_controllerConfig.listenMode == "pubsub" && ! _controllerConfig.pubSubConfig) { + fprintf(stderr, "ERROR: pubsub listenMode requires pubsub configuration in local.conf" ZT_EOL_S); + exit(1); + } + + // bigtable settings + if (cc["bigtable"].is_object()) { + json& bt = cc["bigtable"]; + _controllerConfig.bigTableConfig = new BigTableConfig(); + _controllerConfig.bigTableConfig->project_id = OSUtils::jsonString(bt["project_id"], ""); + _controllerConfig.bigTableConfig->instance_id = OSUtils::jsonString(bt["instance_id"], ""); + _controllerConfig.bigTableConfig->table_id = OSUtils::jsonString(bt["table_id"], ""); + } + if (_controllerConfig.listenMode == "bigtable" && ! _controllerConfig.bigTableConfig) { + fprintf(stderr, "ERROR: bigtable listenMode requires bigtable configuration in local.conf" ZT_EOL_S); + exit(1); + } } +#endif } virtual ReasonForTermination reasonForTermination() const @@ -1712,7 +1938,8 @@ class OneServiceImpl : public OneService { virtual bool setNetworkSettings(const uint64_t nwid, const NetworkSettings& settings) { char nlcpath[4096]; - OSUtils::ztsnprintf(nlcpath, sizeof(nlcpath), "%s" ZT_PATH_SEPARATOR_S "%.16llx.local.conf", _networksPath.c_str(), nwid); + OSUtils::ztsnprintf( + nlcpath, sizeof(nlcpath), "%s" ZT_PATH_SEPARATOR_S "%.16llx.local.conf", _networksPath.c_str(), nwid); FILE* out = fopen(nlcpath, "w"); if (out) { fprintf(out, "allowManaged=%d\n", (int)settings.allowManaged); @@ -1776,15 +2003,22 @@ class OneServiceImpl : public OneService { auto ret = _controlPlane.set_mount_point(appUiPath, appUiDir); _controlPlaneV6.set_mount_point(appUiPath, appUiDir); if (! ret) { - fprintf(stderr, "Mounting app directory failed. Creating it. Path: %s - Dir: %s\n", appUiPath.c_str(), appUiDir); + fprintf( + stderr, "Mounting app directory failed. Creating it. Path: %s - Dir: %s\n", appUiPath.c_str(), + appUiDir); if (! OSUtils::mkdir(appUiDir)) { - fprintf(stderr, "Could not create app directory either. Path: %s - Dir: %s\n", appUiPath.c_str(), appUiDir); + fprintf( + stderr, "Could not create app directory either. Path: %s - Dir: %s\n", appUiPath.c_str(), + appUiDir); } else { ret = _controlPlane.set_mount_point(appUiPath, appUiDir); _controlPlaneV6.set_mount_point(appUiPath, appUiDir); if (! ret) { - fprintf(stderr, "Really could not create and mount directory. Path: %s - Dir: %s\nWeb apps won't work.\n", appUiPath.c_str(), appUiDir); + fprintf( + stderr, + "Really could not create and mount directory. Path: %s - Dir: %s\nWeb apps won't work.\n", + appUiPath.c_str(), appUiDir); } } } @@ -1798,7 +2032,9 @@ class OneServiceImpl : public OneService { if (match.matched) { // fallback char indexHtmlPath[16384]; - snprintf(indexHtmlPath, sizeof(indexHtmlPath), "%s/%s/%s", appUiDir, match.str().c_str(), "index.html"); + snprintf( + indexHtmlPath, sizeof(indexHtmlPath), "%s/%s/%s", appUiDir, match.str().c_str(), + "index.html"); // fprintf(stderr, "fallback path %s\n", indexHtmlPath); std::string indexHtml; @@ -1822,7 +2058,9 @@ class OneServiceImpl : public OneService { // add .html std::string htmlFile; char htmlPath[16384]; - snprintf(htmlPath, sizeof(htmlPath), "%s%s%s", appUiDir, (req.path).substr(appUiPath.length()).c_str(), ".html"); + snprintf( + htmlPath, sizeof(htmlPath), "%s%s%s", appUiDir, (req.path).substr(appUiPath.length()).c_str(), + ".html"); // fprintf(stderr, "path: %s\n", htmlPath); if (OSUtils::readFile(htmlPath, htmlFile)) { res.set_content(htmlFile.c_str(), "text/html"); @@ -2048,7 +2286,10 @@ class OneServiceImpl : public OneService { res.status = 400; return; } - res.status = _node->bondController()->setAllMtuByTuple(mtu, req.matches[2].str().c_str(), req.matches[3].str().c_str()) ? 200 : 400; + res.status = _node->bondController()->setAllMtuByTuple( + mtu, req.matches[2].str().c_str(), req.matches[3].str().c_str()) + ? 200 + : 400; if (res.status == 400) { setContent(req, res, "Unable to find specified link"); return; @@ -2122,7 +2363,9 @@ class OneServiceImpl : public OneService { out["versionMinor"] = ZEROTIER_ONE_VERSION_MINOR; out["versionRev"] = ZEROTIER_ONE_VERSION_REVISION; out["versionBuild"] = ZEROTIER_ONE_VERSION_BUILD; - OSUtils::ztsnprintf(tmp, sizeof(tmp), "%d.%d.%d", ZEROTIER_ONE_VERSION_MAJOR, ZEROTIER_ONE_VERSION_MINOR, ZEROTIER_ONE_VERSION_REVISION); + OSUtils::ztsnprintf( + tmp, sizeof(tmp), "%d.%d.%d", ZEROTIER_ONE_VERSION_MAJOR, ZEROTIER_ONE_VERSION_MINOR, + ZEROTIER_ONE_VERSION_REVISION); out["version"] = tmp; out["clock"] = OSUtils::now(); @@ -2435,7 +2678,9 @@ class OneServiceImpl : public OneService { out["versionMinor"] = ZEROTIER_ONE_VERSION_MINOR; out["versionRev"] = ZEROTIER_ONE_VERSION_REVISION; out["versionBuild"] = ZEROTIER_ONE_VERSION_BUILD; - OSUtils::ztsnprintf(tmp, sizeof(tmp), "%d.%d.%d", ZEROTIER_ONE_VERSION_MAJOR, ZEROTIER_ONE_VERSION_MINOR, ZEROTIER_ONE_VERSION_REVISION); + OSUtils::ztsnprintf( + tmp, sizeof(tmp), "%d.%d.%d", ZEROTIER_ONE_VERSION_MAJOR, ZEROTIER_ONE_VERSION_MINOR, + ZEROTIER_ONE_VERSION_REVISION); out["version"] = tmp; out["clock"] = OSUtils::now(); @@ -2444,7 +2689,8 @@ class OneServiceImpl : public OneService { out["config"] = _localConfig; } json& settings = out["config"]["settings"]; - settings["allowTcpFallbackRelay"] = OSUtils::jsonBool(settings["allowTcpFallbackRelay"], _allowTcpFallbackRelay); + settings["allowTcpFallbackRelay"] = + OSUtils::jsonBool(settings["allowTcpFallbackRelay"], _allowTcpFallbackRelay); settings["forceTcpRelay"] = OSUtils::jsonBool(settings["forceTcpRelay"], _forceTcpRelay); settings["primaryPort"] = OSUtils::jsonInt(settings["primaryPort"], (uint64_t)_primaryPort) & 0xffff; settings["secondaryPort"] = OSUtils::jsonInt(settings["secondaryPort"], (uint64_t)_ports[1]) & 0xffff; @@ -2512,13 +2758,13 @@ class OneServiceImpl : public OneService { // SSO redirect handling std::string state = req.get_param_value("state"); - char* nwid = zeroidc::zeroidc_network_id_from_state(state.c_str()); + char* nwid = rustybits::zeroidc_network_id_from_state(state.c_str()); outData["networkId"] = std::string(nwid); const uint64_t id = Utils::hexStrToU64(nwid); - zeroidc::free_cstr(nwid); + rustybits::free_cstr(nwid); Mutex::Lock l(_nets_m); if (_nets.find(id) != _nets.end()) { @@ -2550,7 +2796,7 @@ class OneServiceImpl : public OneService { res.status = 500; } - zeroidc::free_cstr(ret); + rustybits::free_cstr(ret); } }; _controlPlane.Get(ssoPath, ssoGet); @@ -2575,26 +2821,27 @@ class OneServiceImpl : public OneService { _controlPlane.Get(metricsPath, metricsGet); _controlPlaneV6.Get(metricsPath, metricsGet); - auto exceptionHandler = [&, setContent](const httplib::Request& req, httplib::Response& res, std::exception_ptr ep) { - auto provider = opentelemetry::trace::Provider::GetTracerProvider(); - auto tracer = provider->GetTracer("http_control_plane"); - auto span = tracer->StartSpan("http_control_plane::exceptionHandler"); - auto scope = tracer->WithActiveSpan(span); + auto exceptionHandler = + [&, setContent](const httplib::Request& req, httplib::Response& res, std::exception_ptr ep) { + auto provider = opentelemetry::trace::Provider::GetTracerProvider(); + auto tracer = provider->GetTracer("http_control_plane"); + auto span = tracer->StartSpan("http_control_plane::exceptionHandler"); + auto scope = tracer->WithActiveSpan(span); - char buf[1024]; - auto fmt = "{\"error\": %d, \"description\": \"%s\"}"; - try { - std::rethrow_exception(ep); - } - catch (std::exception& e) { - snprintf(buf, sizeof(buf), fmt, 500, e.what()); - } - catch (...) { - snprintf(buf, sizeof(buf), fmt, 500, "Unknown Exception"); - } - setContent(req, res, buf); - res.status = 500; - }; + char buf[1024]; + auto fmt = "{\"error\": %d, \"description\": \"%s\"}"; + try { + std::rethrow_exception(ep); + } + catch (std::exception& e) { + snprintf(buf, sizeof(buf), fmt, 500, e.what()); + } + catch (...) { + snprintf(buf, sizeof(buf), fmt, 500, "Unknown Exception"); + } + setContent(req, res, buf); + res.status = 500; + }; _controlPlane.set_exception_handler(exceptionHandler); _controlPlaneV6.set_exception_handler(exceptionHandler); @@ -2610,8 +2857,12 @@ class OneServiceImpl : public OneService { _controlPlaneV6.set_pre_routing_handler(authCheck); #if ZT_DEBUG == 1 - _controlPlane.set_logger([](const httplib::Request& req, const httplib::Response& res) { fprintf(stderr, "%s", http_log(req, res).c_str()); }); - _controlPlaneV6.set_logger([](const httplib::Request& req, const httplib::Response& res) { fprintf(stderr, "%s", http_log(req, res).c_str()); }); + _controlPlane.set_logger([](const httplib::Request& req, const httplib::Response& res) { + fprintf(stderr, "%s", http_log(req, res).c_str()); + }); + _controlPlaneV6.set_logger([](const httplib::Request& req, const httplib::Response& res) { + fprintf(stderr, "%s", http_log(req, res).c_str()); + }); #endif if (_primaryPort == 0) { fprintf(stderr, "unable to determine local control port"); @@ -2751,24 +3002,34 @@ class OneServiceImpl : public OneService { std::string defaultBondingPolicyStr(OSUtils::jsonString(settings["defaultBondingPolicy"], "")); int defaultBondingPolicy = _node->bondController()->getPolicyCodeByStr(defaultBondingPolicyStr); _node->bondController()->setBondingLayerDefaultPolicy(defaultBondingPolicy); - _node->bondController()->setBondingLayerDefaultPolicyStr(defaultBondingPolicyStr); // Used if custom policy + _node->bondController()->setBondingLayerDefaultPolicyStr( + defaultBondingPolicyStr); // Used if custom policy // Custom Policies json& customBondingPolicies = settings["policies"]; - for (json::iterator policyItr = customBondingPolicies.begin(); policyItr != customBondingPolicies.end(); ++policyItr) { + for (json::iterator policyItr = customBondingPolicies.begin(); policyItr != customBondingPolicies.end(); + ++policyItr) { // Custom Policy std::string customPolicyStr(policyItr.key()); json& customPolicy = policyItr.value(); std::string basePolicyStr(OSUtils::jsonString(customPolicy["basePolicy"], "")); if (basePolicyStr.empty()) { - fprintf(stderr, "error: no base policy was specified for custom policy (%s)\n", customPolicyStr.c_str()); + fprintf( + stderr, "error: no base policy was specified for custom policy (%s)\n", + customPolicyStr.c_str()); } int basePolicyCode = _node->bondController()->getPolicyCodeByStr(basePolicyStr); if (basePolicyCode == ZT_BOND_POLICY_NONE) { - fprintf(stderr, "error: custom policy (%s) is invalid, unknown base policy (%s).\n", customPolicyStr.c_str(), basePolicyStr.c_str()); + fprintf( + stderr, "error: custom policy (%s) is invalid, unknown base policy (%s).\n", + customPolicyStr.c_str(), basePolicyStr.c_str()); continue; } if (_node->bondController()->getPolicyCodeByStr(customPolicyStr) != ZT_BOND_POLICY_NONE) { - fprintf(stderr, "error: custom policy (%s) will be ignored, cannot use standard policy names for custom policies.\n", customPolicyStr.c_str()); + fprintf( + stderr, + "error: custom policy (%s) will be ignored, cannot use standard policy names for custom " + "policies.\n", + customPolicyStr.c_str()); continue; } // New bond, used as a copy template for new instances @@ -2791,7 +3052,8 @@ class OneServiceImpl : public OneService { // Bond-specific properties newTemplateBond->setUpDelay(OSUtils::jsonInt(customPolicy["upDelay"], -1)); newTemplateBond->setDownDelay(OSUtils::jsonInt(customPolicy["downDelay"], -1)); - newTemplateBond->setFailoverInterval(OSUtils::jsonInt(customPolicy["failoverInterval"], ZT_BOND_FAILOVER_DEFAULT_INTERVAL)); + newTemplateBond->setFailoverInterval( + OSUtils::jsonInt(customPolicy["failoverInterval"], ZT_BOND_FAILOVER_DEFAULT_INTERVAL)); newTemplateBond->setPacketsPerLink(OSUtils::jsonInt(customPolicy["packetsPerLink"], -1)); // Policy-Specific link set @@ -2823,7 +3085,9 @@ class OneServiceImpl : public OneService { failoverToStr = ""; enabled = false; } - _node->bondController()->addCustomLink(customPolicyStr, new Link(linkNameStr, ipvPref, mtu, capacity, enabled, linkMode, failoverToStr)); + _node->bondController()->addCustomLink( + customPolicyStr, + new Link(linkNameStr, ipvPref, mtu, capacity, enabled, linkMode, failoverToStr)); } // Check for new field name first, fall back to legacy field name for backward compatibility std::string linkSelectMethodStr; @@ -2833,7 +3097,11 @@ class OneServiceImpl : public OneService { else { linkSelectMethodStr = OSUtils::jsonString(customPolicy["activeReselect"], "always"); if (customPolicy.contains("activeReselect")) { - fprintf(stderr, "warning: 'activeReselect' is deprecated, please use 'linkSelectMethod' instead in policy '%s'\n", customPolicyStr.c_str()); + fprintf( + stderr, + "warning: 'activeReselect' is deprecated, please use 'linkSelectMethod' instead in policy " + "'%s'\n", + customPolicyStr.c_str()); } } @@ -2850,10 +3118,13 @@ class OneServiceImpl : public OneService { newTemplateBond->setLinkSelectMethod(ZT_BOND_RESELECTION_POLICY_OPTIMIZE); } if (newTemplateBond->getLinkSelectMethod() < 0 || newTemplateBond->getLinkSelectMethod() > 3) { - fprintf(stderr, "warning: invalid value (%s) for linkSelectMethod, assuming mode: always\n", linkSelectMethodStr.c_str()); + fprintf( + stderr, "warning: invalid value (%s) for linkSelectMethod, assuming mode: always\n", + linkSelectMethodStr.c_str()); } if (! _node->bondController()->addCustomPolicy(newTemplateBond)) { - fprintf(stderr, "error: a custom policy of this name (%s) already exists.\n", customPolicyStr.c_str()); + fprintf( + stderr, "error: a custom policy of this name (%s) already exists.\n", customPolicyStr.c_str()); } } // Peer-specific bonding @@ -2863,7 +3134,9 @@ class OneServiceImpl : public OneService { } // Check settings if (defaultBondingPolicyStr.length() && ! defaultBondingPolicy && ! _node->bondController()->inUse()) { - fprintf(stderr, "error: unknown policy (%s) specified by defaultBondingPolicy, bond disabled.\n", defaultBondingPolicyStr.c_str()); + fprintf( + stderr, "error: unknown policy (%s) specified by defaultBondingPolicy, bond disabled.\n", + defaultBondingPolicyStr.c_str()); } } @@ -2873,19 +3146,23 @@ class OneServiceImpl : public OneService { if (_forceTcpRelayTmp && _bondInUse) { fprintf(stderr, "Warning: forceTcpRelay cannot be used with multipath. Disabling forceTcpRelay\n"); } - _allowTcpFallbackRelay = (OSUtils::jsonBool(settings["allowTcpFallbackRelay"], true) && ! _node->bondController()->inUse()); + _allowTcpFallbackRelay = + (OSUtils::jsonBool(settings["allowTcpFallbackRelay"], true) && ! _node->bondController()->inUse()); _forceTcpRelay = (_forceTcpRelayTmp && ! _node->bondController()->inUse()); _enableWebServer = (OSUtils::jsonBool(settings["enableWebServer"], false)); #ifdef ZT_TCP_FALLBACK_RELAY - _fallbackRelayAddress = InetAddress(OSUtils::jsonString(settings["tcpFallbackRelay"], ZT_TCP_FALLBACK_RELAY).c_str()); + _fallbackRelayAddress = + InetAddress(OSUtils::jsonString(settings["tcpFallbackRelay"], ZT_TCP_FALLBACK_RELAY).c_str()); #endif _primaryPort = (unsigned int)OSUtils::jsonInt(settings["primaryPort"], (uint64_t)_primaryPort) & 0xffff; _allowSecondaryPort = OSUtils::jsonBool(settings["allowSecondaryPort"], true); _secondaryPort = (unsigned int)OSUtils::jsonInt(settings["secondaryPort"], 0); _tertiaryPort = (unsigned int)OSUtils::jsonInt(settings["tertiaryPort"], 0); if (_secondaryPort != 0 || _tertiaryPort != 0) { - fprintf(stderr, "WARNING: using manually-specified secondary and/or tertiary ports. This can cause NAT issues." ZT_EOL_S); + fprintf( + stderr, "WARNING: using manually-specified secondary and/or tertiary ports. This can cause NAT " + "issues." ZT_EOL_S); } _portMappingEnabled = OSUtils::jsonBool(settings["portMappingEnabled"], true); _node->setEncryptedHelloEnabled(OSUtils::jsonBool(settings["encryptedHelloEnabled"], false)); @@ -2898,7 +3175,10 @@ class OneServiceImpl : public OneService { unsigned int maxConcurrency = std::thread::hardware_concurrency(); if (_concurrency <= 1 || _concurrency >= maxConcurrency) { unsigned int conservativeDefault = (std::thread::hardware_concurrency() >= 4 ? 2 : 1); - fprintf(stderr, "Concurrency level provided (%d) is invalid, assigning conservative default value of (%d)\n", _concurrency, conservativeDefault); + fprintf( + stderr, + "Concurrency level provided (%d) is invalid, assigning conservative default value of (%d)\n", + _concurrency, conservativeDefault); _concurrency = conservativeDefault; } setUpMultithreading(); @@ -3055,7 +3335,8 @@ class OneServiceImpl : public OneService { #ifdef __APPLE__ // We can't add multiple addresses to an interface on macOs unless we futz with the netmask. // see `man ifconfig`, alias section - // "If the address is on the same subnet as the first network address for this interface, a non-conflicting netmask must be given. Usually 0xffffffff is most appropriate." + // "If the address is on the same subnet as the first network address for this interface, a + // non-conflicting netmask must be given. Usually 0xffffffff is most appropriate." auto same_subnet = [ip](InetAddress i) { return ip->network() == i.network(); }; #endif @@ -3063,7 +3344,9 @@ class OneServiceImpl : public OneService { if (std::find(n.managedIps().begin(), n.managedIps().end(), *ip) == n.managedIps().end()) { #ifdef __APPLE__ // if same subnet as a previously added address - if (std::find_if(n.managedIps().begin(), n.managedIps().end(), same_subnet) != n.managedIps().end() || std::find_if(newManagedIps2.begin(), newManagedIps2.end(), same_subnet) != newManagedIps2.end()) { + if (std::find_if(n.managedIps().begin(), n.managedIps().end(), same_subnet) != n.managedIps().end() + || std::find_if(newManagedIps2.begin(), newManagedIps2.end(), same_subnet) + != newManagedIps2.end()) { if (ip->isV4()) { ip->setPort(32); } @@ -3088,11 +3371,13 @@ class OneServiceImpl : public OneService { } #ifdef __APPLE__ - if (! MacDNSHelper::addIps6(n.config().nwid, n.config().mac, n.tap()->deviceName().c_str(), newManagedIps)) { + if (! MacDNSHelper::addIps6( + n.config().nwid, n.config().mac, n.tap()->deviceName().c_str(), newManagedIps)) { fprintf(stderr, "ERROR: unable to add v6 addresses to system configuration" ZT_EOL_S); } - if (! MacDNSHelper::addIps4(n.config().nwid, n.config().mac, n.tap()->deviceName().c_str(), newManagedIps)) { + if (! MacDNSHelper::addIps4( + n.config().nwid, n.config().mac, n.tap()->deviceName().c_str(), newManagedIps)) { fprintf(stderr, "ERROR: unable to add v4 addresses to system configuration" ZT_EOL_S); } #endif @@ -3103,7 +3388,9 @@ class OneServiceImpl : public OneService { // Get tap device name (use LUID in hex on Windows) and IP addresses. #if defined(__WINDOWS__) && ! defined(ZT_SDK) char tapdevbuf[64]; - OSUtils::ztsnprintf(tapdevbuf, sizeof(tapdevbuf), "%.16llx", (unsigned long long)((WindowsEthernetTap*)(n.tap().get()))->luid().Value); + OSUtils::ztsnprintf( + tapdevbuf, sizeof(tapdevbuf), "%.16llx", + (unsigned long long)((WindowsEthernetTap*)(n.tap().get()))->luid().Value); std::string tapdev(tapdevbuf); #else std::string tapdev(n.tap()->deviceName()); @@ -3121,7 +3408,8 @@ class OneServiceImpl : public OneService { // Make sure we are allowed to set this managed route, and that 'via' is not our IP. The latter // avoids setting routes via the router on the router. - if ((! checkIfManagedIsAllowed(n, *target)) || ((via->ss_family == target->ss_family) && (matchIpOnly(myIps, *via)))) + if ((! checkIfManagedIsAllowed(n, *target)) + || ((via->ss_family == target->ss_family) && (matchIpOnly(myIps, *via)))) continue; // Find an IP on the interface that can be a source IP, abort if no IPs assigned. @@ -3129,7 +3417,8 @@ class OneServiceImpl : public OneService { unsigned int mostMatchingPrefixBits = 0; for (std::set::const_iterator i(myIps.begin()); i != myIps.end(); ++i) { const unsigned int matchingPrefixBits = i->matchingPrefixBits(*target); - if (matchingPrefixBits >= mostMatchingPrefixBits && ((target->isV4() && i->isV4()) || (target->isV6() && i->isV6()))) { + if (matchingPrefixBits >= mostMatchingPrefixBits + && ((target->isV4() && i->isV4()) || (target->isV6() && i->isV6()))) { mostMatchingPrefixBits = matchingPrefixBits; src = &(*i); } @@ -3160,7 +3449,8 @@ class OneServiceImpl : public OneService { #endif } - for (std::map >::iterator r(n.managedRoutes().begin()); r != n.managedRoutes().end();) { + for (std::map >::iterator r(n.managedRoutes().begin()); + r != n.managedRoutes().end();) { if (haveRouteTargets.find(r->first) == haveRouteTargets.end()) n.managedRoutes().erase(r++); else @@ -3170,11 +3460,13 @@ class OneServiceImpl : public OneService { // Sync device-local managed routes first, then indirect results. That way // we don't get destination unreachable for routes that are via things // that do not yet have routes in the system. - for (std::map >::iterator r(n.managedRoutes().begin()); r != n.managedRoutes().end(); ++r) { + for (std::map >::iterator r(n.managedRoutes().begin()); + r != n.managedRoutes().end(); ++r) { if (! r->second->via()) r->second->sync(); } - for (std::map >::iterator r(n.managedRoutes().begin()); r != n.managedRoutes().end(); ++r) { + for (std::map >::iterator r(n.managedRoutes().begin()); + r != n.managedRoutes().end(); ++r) { if (r->second->via() && (! r->second->target().isDefaultRoute() || _node->online())) { r->second->sync(); } @@ -3208,7 +3500,13 @@ class OneServiceImpl : public OneService { // Handlers for Node and Phy<> callbacks // ========================================================================= - inline void phyOnDatagram(PhySocket* sock, void** uptr, const struct sockaddr* localAddr, const struct sockaddr* from, void* data, unsigned long len) + inline void phyOnDatagram( + PhySocket* sock, + void** uptr, + const struct sockaddr* localAddr, + const struct sockaddr* from, + void* data, + unsigned long len) { if (_forceTcpRelay) { return; @@ -3218,7 +3516,9 @@ class OneServiceImpl : public OneService { if ((len >= 16) && (reinterpret_cast(from)->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) { _lastDirectReceiveFromGlobal = now; } - const ZT_ResultCode rc = _node->processWirePacket(nullptr, now, reinterpret_cast(sock), reinterpret_cast(from), data, len, &_nextBackgroundTaskDeadline); + const ZT_ResultCode rc = _node->processWirePacket( + nullptr, now, reinterpret_cast(sock), reinterpret_cast(from), data, + len, &_nextBackgroundTaskDeadline); if (ZT_ResultCode_isFatal(rc)) { char tmp[256]; OSUtils::ztsnprintf(tmp, sizeof(tmp), "fatal error code from processWirePacket: %d", (int)rc); @@ -3254,7 +3554,8 @@ class OneServiceImpl : public OneService { } } - inline void phyOnTcpAccept(PhySocket* sockL, PhySocket* sockN, void** uptrL, void** uptrN, const struct sockaddr* from) + inline void + phyOnTcpAccept(PhySocket* sockL, PhySocket* sockN, void** uptrL, void** uptrN, const struct sockaddr* from) { if (! from) { _phy.close(sockN, false); @@ -3262,7 +3563,8 @@ class OneServiceImpl : public OneService { } else { #ifdef ZT_SDK - // Immediately close new local connections. The intention is to prevent the backplane from being accessed when operating as libzt + // Immediately close new local connections. The intention is to prevent the backplane from being accessed + // when operating as libzt if (! allowHttpBackplaneManagement && ((InetAddress*)from)->ipScope() == InetAddress::IP_SCOPE_LOOPBACK) { _phy.close(sockN, false); return; @@ -3296,7 +3598,8 @@ class OneServiceImpl : public OneService { } { Mutex::Lock _l(_tcpConnections_m); - _tcpConnections.erase(std::remove(_tcpConnections.begin(), _tcpConnections.end(), tc), _tcpConnections.end()); + _tcpConnections.erase( + std::remove(_tcpConnections.begin(), _tcpConnections.end(), tc), _tcpConnections.end()); } delete tc; } @@ -3325,7 +3628,8 @@ class OneServiceImpl : public OneService { tc->readq.append((const char*)data, len); while (tc->readq.length() >= 5) { const char* data = tc->readq.data(); - const unsigned long mlen = (((((unsigned long)data[3]) & 0xff) << 8) | (((unsigned long)data[4]) & 0xff)); + const unsigned long mlen = + (((((unsigned long)data[3]) & 0xff) << 8) | (((unsigned long)data[4]) & 0xff)); if (tc->readq.length() >= (mlen + 5)) { InetAddress from; @@ -3339,7 +3643,10 @@ class OneServiceImpl : public OneService { switch (data[0]) { case 4: // IPv4 if (plen >= 7) { - from.set((const void*)(data + 1), 4, ((((unsigned int)data[5]) & 0xff) << 8) | (((unsigned int)data[6]) & 0xff)); + from.set( + (const void*)(data + 1), 4, + ((((unsigned int)data[5]) & 0xff) << 8) + | (((unsigned int)data[6]) & 0xff)); data += 7; // type + 4 byte IP + 2 byte port plen -= 7; } @@ -3350,7 +3657,10 @@ class OneServiceImpl : public OneService { break; case 6: // IPv6 if (plen >= 19) { - from.set((const void*)(data + 1), 16, ((((unsigned int)data[17]) & 0xff) << 8) | (((unsigned int)data[18]) & 0xff)); + from.set( + (const void*)(data + 1), 16, + ((((unsigned int)data[17]) & 0xff) << 8) + | (((unsigned int)data[18]) & 0xff)); data += 19; // type + 16 byte IP + 2 byte port plen -= 19; } @@ -3370,10 +3680,13 @@ class OneServiceImpl : public OneService { if (from) { InetAddress fakeTcpLocalInterfaceAddress((uint32_t)0xffffffff, 0xffff); - const ZT_ResultCode rc = _node->processWirePacket((void*)0, OSUtils::now(), -1, reinterpret_cast(&from), data, plen, &_nextBackgroundTaskDeadline); + const ZT_ResultCode rc = _node->processWirePacket( + (void*)0, OSUtils::now(), -1, reinterpret_cast(&from), + data, plen, &_nextBackgroundTaskDeadline); if (ZT_ResultCode_isFatal(rc)) { char tmp[256]; - OSUtils::ztsnprintf(tmp, sizeof(tmp), "fatal error code from processWirePacket: %d", (int)rc); + OSUtils::ztsnprintf( + tmp, sizeof(tmp), "fatal error code from processWirePacket: %d", (int)rc); Mutex::Lock _l(_termReason_m); _termReason = ONE_UNRECOVERABLE_ERROR; _fatalErrorMessage = tmp; @@ -3442,62 +3755,64 @@ class OneServiceImpl : public OneService { inline void phyOnUnixData(PhySocket* sock, void** uptr, void* data, unsigned long len) { #ifdef ZT_EXTOSDEP - if (ExtOsdep::mgmtRecv(*uptr, data, len, [&](unsigned method, const std::string& path, const std::string& data, std::string& resp) { - // fprintf(stderr, "mgmtRecv: %u %s %s\n", method, path.c_str(), data.c_str()); - httplib::Request req; - httplib::Response res; - req.path = "/" + path; - if (method == 1) - req.method = "GET"; - else if (method == 3) - req.method = "POST"; - else if (method == 0) - req.method = "DELETE"; - struct S : public httplib::Stream { - const char* ptr; - unsigned size; - S(const std::string& s) : ptr(s.c_str()), size(s.size()) - { - } - virtual bool is_readable() const - { - return true; - } - virtual bool is_writable() const - { - return true; - } - virtual ssize_t read(char* p, size_t sz) - { - // fprintf(stderr, "S::read %d\n", (int)size); - if (sz > (size_t)size) - sz = size; - memcpy(p, ptr, sz); - size -= (unsigned)sz; - ptr += sz; - return (ssize_t)sz; - } - virtual ssize_t write(const char* ptr, size_t size) - { - // fprintf(stderr, "S::write %d\n", (int)size); - return size; - } - virtual void get_remote_ip_and_port(std::string& ip, int& port) const - { - } - virtual void get_local_ip_and_port(std::string& ip, int& port) const {}; - virtual socket_t socket() const - { - return 0; - } - }; - S s(data); - - bool x = _controlPlane.routing(req, res, s); - // fprintf(stderr, "mgmtRecv: done, x %d status %u body %s\n", x, res.status, res.body.c_str()); - resp = res.body; - return res.status; - })) + if (ExtOsdep::mgmtRecv( + *uptr, data, len, + [&](unsigned method, const std::string& path, const std::string& data, std::string& resp) { + // fprintf(stderr, "mgmtRecv: %u %s %s\n", method, path.c_str(), data.c_str()); + httplib::Request req; + httplib::Response res; + req.path = "/" + path; + if (method == 1) + req.method = "GET"; + else if (method == 3) + req.method = "POST"; + else if (method == 0) + req.method = "DELETE"; + struct S : public httplib::Stream { + const char* ptr; + unsigned size; + S(const std::string& s) : ptr(s.c_str()), size(s.size()) + { + } + virtual bool is_readable() const + { + return true; + } + virtual bool is_writable() const + { + return true; + } + virtual ssize_t read(char* p, size_t sz) + { + // fprintf(stderr, "S::read %d\n", (int)size); + if (sz > (size_t)size) + sz = size; + memcpy(p, ptr, sz); + size -= (unsigned)sz; + ptr += sz; + return (ssize_t)sz; + } + virtual ssize_t write(const char* ptr, size_t size) + { + // fprintf(stderr, "S::write %d\n", (int)size); + return size; + } + virtual void get_remote_ip_and_port(std::string& ip, int& port) const + { + } + virtual void get_local_ip_and_port(std::string& ip, int& port) const {}; + virtual socket_t socket() const + { + return 0; + } + }; + S s(data); + + bool x = _controlPlane.routing(req, res, s); + // fprintf(stderr, "mgmtRecv: done, x %d status %u body %s\n", x, res.status, res.body.c_str()); + resp = res.body; + return res.status; + })) _phy.setNotifyWritable(sock, true); #endif } @@ -3509,7 +3824,11 @@ class OneServiceImpl : public OneService { #endif } - inline int nodeVirtualNetworkConfigFunction(uint64_t nwid, void** nuptr, enum ZT_VirtualNetworkConfigOperation op, const ZT_VirtualNetworkConfig* nwc) + inline int nodeVirtualNetworkConfigFunction( + uint64_t nwid, + void** nuptr, + enum ZT_VirtualNetworkConfigOperation op, + const ZT_VirtualNetworkConfig* nwc) { Mutex::Lock _l(_nets_m); NetworkState& n = _nets[nwid]; @@ -3522,11 +3841,17 @@ class OneServiceImpl : public OneService { char friendlyName[128]; OSUtils::ztsnprintf(friendlyName, sizeof(friendlyName), "ZeroTier One [%.16llx]", nwid); - n.setTap(EthernetTap::newInstance(nullptr, _concurrency, _cpuPinningEnabled, _homePath.c_str(), MAC(nwc->mac), nwc->mtu, (unsigned int)ZT_IF_METRIC, nwid, friendlyName, StapFrameHandler, (void*)this)); + n.setTap( + EthernetTap::newInstance( + nullptr, _concurrency, _cpuPinningEnabled, _homePath.c_str(), MAC(nwc->mac), nwc->mtu, + (unsigned int)ZT_IF_METRIC, nwid, friendlyName, StapFrameHandler, (void*)this)); *nuptr = (void*)&n; char nlcpath[256]; - OSUtils::ztsnprintf(nlcpath, sizeof(nlcpath), "%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf", _homePath.c_str(), nwid); + OSUtils::ztsnprintf( + nlcpath, sizeof(nlcpath), + "%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf", + _homePath.c_str(), nwid); std::string nlcbuf; if (OSUtils::readFile(nlcpath, nlcbuf)) { Dictionary<4096> nc; @@ -3548,7 +3873,8 @@ class OneServiceImpl : public OneService { size_t pos = 0; while (true) { size_t nextPos = addresses.find(',', pos); - std::string address = addresses.substr(pos, (nextPos == std::string::npos ? addresses.size() : nextPos) - pos); + std::string address = addresses.substr( + pos, (nextPos == std::string::npos ? addresses.size() : nextPos) - pos); n.addToAllowManagedWhiteList(InetAddress(address.c_str())); if (nextPos == std::string::npos) break; @@ -3593,7 +3919,8 @@ class OneServiceImpl : public OneService { // without WindowsEthernetTap::isInitialized() returning true, the won't actually // be online yet and setting managed routes on it will fail. const int MAX_SLEEP_COUNT = 500; - for (int i = 0; ! ((WindowsEthernetTap*)(n.tap().get()))->isInitialized() && i < MAX_SLEEP_COUNT; i++) { + for (int i = 0; ! ((WindowsEthernetTap*)(n.tap().get()))->isInitialized() && i < MAX_SLEEP_COUNT; + i++) { Sleep(10); } #endif @@ -3623,7 +3950,10 @@ class OneServiceImpl : public OneService { #endif if (op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY) { char nlcpath[256]; - OSUtils::ztsnprintf(nlcpath, sizeof(nlcpath), "%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf", _homePath.c_str(), nwid); + OSUtils::ztsnprintf( + nlcpath, sizeof(nlcpath), + "%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.local.conf", + _homePath.c_str(), nwid); OSUtils::rm(nlcpath); } } @@ -3765,15 +4095,18 @@ class OneServiceImpl : public OneService { break; case ZT_STATE_OBJECT_MOON: OSUtils::ztsnprintf(dirname, sizeof(dirname), "%s" ZT_PATH_SEPARATOR_S "moons.d", _homePath.c_str()); - OSUtils::ztsnprintf(p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "%.16llx.moon", dirname, (unsigned long long)id[0]); + OSUtils::ztsnprintf( + p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "%.16llx.moon", dirname, (unsigned long long)id[0]); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: OSUtils::ztsnprintf(dirname, sizeof(dirname), "%s" ZT_PATH_SEPARATOR_S "networks.d", _homePath.c_str()); - OSUtils::ztsnprintf(p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "%.16llx.conf", dirname, (unsigned long long)id[0]); + OSUtils::ztsnprintf( + p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "%.16llx.conf", dirname, (unsigned long long)id[0]); break; case ZT_STATE_OBJECT_PEER: OSUtils::ztsnprintf(dirname, sizeof(dirname), "%s" ZT_PATH_SEPARATOR_S "peers.d", _homePath.c_str()); - OSUtils::ztsnprintf(p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "%.10llx.peer", dirname, (unsigned long long)id[0]); + OSUtils::ztsnprintf( + p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "%.10llx.peer", dirname, (unsigned long long)id[0]); break; default: return; @@ -3917,13 +4250,19 @@ class OneServiceImpl : public OneService { OSUtils::ztsnprintf(p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "planet", _homePath.c_str()); break; case ZT_STATE_OBJECT_MOON: - OSUtils::ztsnprintf(p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "moons.d" ZT_PATH_SEPARATOR_S "%.16llx.moon", _homePath.c_str(), (unsigned long long)id[0]); + OSUtils::ztsnprintf( + p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "moons.d" ZT_PATH_SEPARATOR_S "%.16llx.moon", + _homePath.c_str(), (unsigned long long)id[0]); break; case ZT_STATE_OBJECT_NETWORK_CONFIG: - OSUtils::ztsnprintf(p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.conf", _homePath.c_str(), (unsigned long long)id[0]); + OSUtils::ztsnprintf( + p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.conf", + _homePath.c_str(), (unsigned long long)id[0]); break; case ZT_STATE_OBJECT_PEER: - OSUtils::ztsnprintf(p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "peers.d" ZT_PATH_SEPARATOR_S "%.10llx.peer", _homePath.c_str(), (unsigned long long)id[0]); + OSUtils::ztsnprintf( + p, sizeof(p), "%s" ZT_PATH_SEPARATOR_S "peers.d" ZT_PATH_SEPARATOR_S "%.10llx.peer", + _homePath.c_str(), (unsigned long long)id[0]); break; default: return -1; @@ -3949,18 +4288,26 @@ class OneServiceImpl : public OneService { return -1; } - inline int nodeWirePacketSendFunction(const int64_t localSocket, const struct sockaddr_storage* addr, const void* data, unsigned int len, unsigned int ttl) + inline int nodeWirePacketSendFunction( + const int64_t localSocket, + const struct sockaddr_storage* addr, + const void* data, + unsigned int len, + unsigned int ttl) { #ifdef ZT_TCP_FALLBACK_RELAY if (_allowTcpFallbackRelay) { if (addr->ss_family == AF_INET) { // TCP fallback tunnel support, currently IPv4 only - if ((len >= 16) && (reinterpret_cast(addr)->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) { + if ((len >= 16) + && (reinterpret_cast(addr)->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) { // Engage TCP tunnel fallback if we haven't received anything valid from a global // IP address in ZT_TCP_FALLBACK_AFTER milliseconds. If we do start getting // valid direct traffic we'll stop using it and close the socket after a while. const int64_t now = OSUtils::now(); - if (_forceTcpRelay || (((now - _lastDirectReceiveFromGlobal) > ZT_TCP_FALLBACK_AFTER) && ((now - _lastRestart) > ZT_TCP_FALLBACK_AFTER))) { + if (_forceTcpRelay + || (((now - _lastDirectReceiveFromGlobal) > ZT_TCP_FALLBACK_AFTER) + && ((now - _lastRestart) > ZT_TCP_FALLBACK_AFTER))) { if (_tcpFallbackTunnel) { bool flushNow = false; { @@ -3977,8 +4324,14 @@ class OneServiceImpl : public OneService { _tcpFallbackTunnel->writeq.push_back((char)((mlen >> 8) & 0xff)); _tcpFallbackTunnel->writeq.push_back((char)(mlen & 0xff)); _tcpFallbackTunnel->writeq.push_back((char)4); // IPv4 - _tcpFallbackTunnel->writeq.append(reinterpret_cast(reinterpret_cast(&(reinterpret_cast(addr)->sin_addr.s_addr))), 4); - _tcpFallbackTunnel->writeq.append(reinterpret_cast(reinterpret_cast(&(reinterpret_cast(addr)->sin_port))), 2); + _tcpFallbackTunnel->writeq.append( + reinterpret_cast(reinterpret_cast( + &(reinterpret_cast(addr)->sin_addr.s_addr))), + 4); + _tcpFallbackTunnel->writeq.append( + reinterpret_cast(reinterpret_cast( + &(reinterpret_cast(addr)->sin_port))), + 2); _tcpFallbackTunnel->writeq.append((const char*)data, len); } } @@ -3987,7 +4340,10 @@ class OneServiceImpl : public OneService { phyOnTcpWritable(_tcpFallbackTunnel->sock, &tmpptr); } } - else if (_forceTcpRelay || (((now - _lastSendToGlobalV4) < ZT_TCP_FALLBACK_AFTER) && ((now - _lastSendToGlobalV4) > (ZT_PING_CHECK_INTERVAL / 2)))) { + else if ( + _forceTcpRelay + || (((now - _lastSendToGlobalV4) < ZT_TCP_FALLBACK_AFTER) + && ((now - _lastSendToGlobalV4) > (ZT_PING_CHECK_INTERVAL / 2)))) { const InetAddress addr(_fallbackRelayAddress); TcpConnection* tc = new TcpConnection(); { @@ -4001,7 +4357,8 @@ class OneServiceImpl : public OneService { tc->sock = (PhySocket*)0; // set in connect handler tc->messageSize = 0; bool connected = false; - _phy.tcpConnect(reinterpret_cast(&addr), connected, (void*)tc, true); + _phy.tcpConnect( + reinterpret_cast(&addr), connected, (void*)tc, true); } } _lastSendToGlobalV4 = now; @@ -4017,7 +4374,8 @@ class OneServiceImpl : public OneService { // Even when relaying we still send via UDP. This way if UDP starts // working we can instantly "fail forward" to it and stop using TCP // proxy fallback, which is slow. - if ((localSocket != -1) && (localSocket != 0) && (_binder.isUdpSocketValid((PhySocket*)((uintptr_t)localSocket)))) { + if ((localSocket != -1) && (localSocket != 0) + && (_binder.isUdpSocketValid((PhySocket*)((uintptr_t)localSocket)))) { if ((ttl) && (addr->ss_family == AF_INET)) { _phy.setIp4UdpTtl((PhySocket*)((uintptr_t)localSocket), ttl); } @@ -4032,7 +4390,15 @@ class OneServiceImpl : public OneService { } } - inline void nodeVirtualNetworkFrameFunction(uint64_t nwid, void** nuptr, uint64_t sourceMac, uint64_t destMac, unsigned int etherType, unsigned int vlanId, const void* data, unsigned int len) + inline void nodeVirtualNetworkFrameFunction( + uint64_t nwid, + void** nuptr, + uint64_t sourceMac, + uint64_t destMac, + unsigned int etherType, + unsigned int vlanId, + const void* data, + unsigned int len) { NetworkState* n = reinterpret_cast(*nuptr); if ((! n) || (! n->tap())) { @@ -4041,7 +4407,8 @@ class OneServiceImpl : public OneService { n->tap()->put(MAC(sourceMac), MAC(destMac), etherType, data, len); } - inline int nodePathCheckFunction(uint64_t ztaddr, const int64_t localSocket, const struct sockaddr_storage* remoteAddr) + inline int + nodePathCheckFunction(uint64_t ztaddr, const int64_t localSocket, const struct sockaddr_storage* remoteAddr) { // Make sure we're not trying to do ZeroTier-over-ZeroTier { @@ -4065,7 +4432,8 @@ class OneServiceImpl : public OneService { * revisit if we see recursion problems. */ // Check blacklists - const Hashtable >* blh = (const Hashtable >*)0; + const Hashtable >* blh = + (const Hashtable >*)0; const std::vector* gbl = (const std::vector*)0; if (remoteAddr->ss_family == AF_INET) { blh = &_v4Blacklists; @@ -4096,7 +4464,8 @@ class OneServiceImpl : public OneService { inline int nodePathLookupFunction(uint64_t ztaddr, int family, struct sockaddr_storage* result) { - const Hashtable >* lh = (const Hashtable >*)0; + const Hashtable >* lh = + (const Hashtable >*)0; if (family < 0) lh = (_node->prng() & 1) ? &_v4Hints : &_v6Hints; else if (family == AF_INET) @@ -4114,9 +4483,18 @@ class OneServiceImpl : public OneService { return 0; } - inline void tapFrameHandler(uint64_t nwid, const MAC& from, const MAC& to, unsigned int etherType, unsigned int vlanId, const void* data, unsigned int len) + inline void tapFrameHandler( + uint64_t nwid, + const MAC& from, + const MAC& to, + unsigned int etherType, + unsigned int vlanId, + const void* data, + unsigned int len) { - _node->processVirtualNetworkFrame((void*)0, OSUtils::now(), nwid, from.toInt(), to.toInt(), etherType, vlanId, data, len, &_nextBackgroundTaskDeadline); + _node->processVirtualNetworkFrame( + (void*)0, OSUtils::now(), nwid, from.toInt(), to.toInt(), etherType, vlanId, data, len, + &_nextBackgroundTaskDeadline); } inline void onHttpResponseFromClient(TcpConnection* tc) @@ -4161,7 +4539,8 @@ class OneServiceImpl : public OneService { { Mutex::Lock _l(_localConfig_m); - for (std::vector::const_iterator p(_interfacePrefixBlacklist.begin()); p != _interfacePrefixBlacklist.end(); ++p) { + for (std::vector::const_iterator p(_interfacePrefixBlacklist.begin()); + p != _interfacePrefixBlacklist.end(); ++p) { if (! strncmp(p->c_str(), ifname, p->length())) return false; } @@ -4257,7 +4636,14 @@ class OneServiceImpl : public OneService { } }; -static int SnodeVirtualNetworkConfigFunction(ZT_Node* node, void* uptr, void* tptr, uint64_t nwid, void** nuptr, enum ZT_VirtualNetworkConfigOperation op, const ZT_VirtualNetworkConfig* nwconf) +static int SnodeVirtualNetworkConfigFunction( + ZT_Node* node, + void* uptr, + void* tptr, + uint64_t nwid, + void** nuptr, + enum ZT_VirtualNetworkConfigOperation op, + const ZT_VirtualNetworkConfig* nwconf) { return reinterpret_cast(uptr)->nodeVirtualNetworkConfigFunction(nwid, nuptr, op, nwconf); } @@ -4265,31 +4651,86 @@ static void SnodeEventCallback(ZT_Node* node, void* uptr, void* tptr, enum ZT_Ev { reinterpret_cast(uptr)->nodeEventCallback(event, metaData); } -static void SnodeStatePutFunction(ZT_Node* node, void* uptr, void* tptr, enum ZT_StateObjectType type, const uint64_t id[2], const void* data, int len) +static void SnodeStatePutFunction( + ZT_Node* node, + void* uptr, + void* tptr, + enum ZT_StateObjectType type, + const uint64_t id[2], + const void* data, + int len) { reinterpret_cast(uptr)->nodeStatePutFunction(type, id, data, len); } -static int SnodeStateGetFunction(ZT_Node* node, void* uptr, void* tptr, enum ZT_StateObjectType type, const uint64_t id[2], void* data, unsigned int maxlen) +static int SnodeStateGetFunction( + ZT_Node* node, + void* uptr, + void* tptr, + enum ZT_StateObjectType type, + const uint64_t id[2], + void* data, + unsigned int maxlen) { return reinterpret_cast(uptr)->nodeStateGetFunction(type, id, data, maxlen); } -static int SnodeWirePacketSendFunction(ZT_Node* node, void* uptr, void* tptr, int64_t localSocket, const struct sockaddr_storage* addr, const void* data, unsigned int len, unsigned int ttl) +static int SnodeWirePacketSendFunction( + ZT_Node* node, + void* uptr, + void* tptr, + int64_t localSocket, + const struct sockaddr_storage* addr, + const void* data, + unsigned int len, + unsigned int ttl) { return reinterpret_cast(uptr)->nodeWirePacketSendFunction(localSocket, addr, data, len, ttl); } -static void SnodeVirtualNetworkFrameFunction(ZT_Node* node, void* uptr, void* tptr, uint64_t nwid, void** nuptr, uint64_t sourceMac, uint64_t destMac, unsigned int etherType, unsigned int vlanId, const void* data, unsigned int len) +static void SnodeVirtualNetworkFrameFunction( + ZT_Node* node, + void* uptr, + void* tptr, + uint64_t nwid, + void** nuptr, + uint64_t sourceMac, + uint64_t destMac, + unsigned int etherType, + unsigned int vlanId, + const void* data, + unsigned int len) { - reinterpret_cast(uptr)->nodeVirtualNetworkFrameFunction(nwid, nuptr, sourceMac, destMac, etherType, vlanId, data, len); + reinterpret_cast(uptr)->nodeVirtualNetworkFrameFunction( + nwid, nuptr, sourceMac, destMac, etherType, vlanId, data, len); } -static int SnodePathCheckFunction(ZT_Node* node, void* uptr, void* tptr, uint64_t ztaddr, int64_t localSocket, const struct sockaddr_storage* remoteAddr) +static int SnodePathCheckFunction( + ZT_Node* node, + void* uptr, + void* tptr, + uint64_t ztaddr, + int64_t localSocket, + const struct sockaddr_storage* remoteAddr) { return reinterpret_cast(uptr)->nodePathCheckFunction(ztaddr, localSocket, remoteAddr); } -static int SnodePathLookupFunction(ZT_Node* node, void* uptr, void* tptr, uint64_t ztaddr, int family, struct sockaddr_storage* result) +static int SnodePathLookupFunction( + ZT_Node* node, + void* uptr, + void* tptr, + uint64_t ztaddr, + int family, + struct sockaddr_storage* result) { return reinterpret_cast(uptr)->nodePathLookupFunction(ztaddr, family, result); } -static void StapFrameHandler(void* uptr, void* tptr, uint64_t nwid, const MAC& from, const MAC& to, unsigned int etherType, unsigned int vlanId, const void* data, unsigned int len) +static void StapFrameHandler( + void* uptr, + void* tptr, + uint64_t nwid, + const MAC& from, + const MAC& to, + unsigned int etherType, + unsigned int vlanId, + const void* data, + unsigned int len) { reinterpret_cast(uptr)->tapFrameHandler(nwid, from, to, etherType, vlanId, data, len); } diff --git a/tmp/local.conf b/tmp/local.conf new file mode 100644 index 0000000000..9555b24d37 --- /dev/null +++ b/tmp/local.conf @@ -0,0 +1,33 @@ +{ + "settings": { + "controllerDbPath": "postgres:host=postgres port=5432 dbname=central user=testuser password=test_password application_name=controller-203e4472cf sslmode=prefer sslcert= sslkey= sslrootcert=", + "portMappingEnabled": true, + "softwareUpdate": "disable", + "interfacePrefixBlacklist": [ + "inot", + "nat64" + ], + "lowBandwidthMode": false, + "ssoRedirectURL": "", + "allowManagementFrom": ["127.0.0.1", "::1", "10.0.0.0/8"], + "otel": { + "exporterEndpoint": "jaeger:4317", + "exporterSampleRate": 1 + } + , "redis": null + }, + "controller": { + "listenMode": "pubsub", + "statusMode": "bigtable" + , "redis": null + , "bigtable": { + "project_id": "arbitrary-project", + "instance_id": "arbitrary-instance", + "table_id": "member_status" + } + + , "pubsub": { + "project_id": "arbitrary-project" + } + } +} diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index e072660f42..2046974cb3 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -423,7 +423,7 @@ true - wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;zeroidc.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) + wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;rustybits.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) false $(SolutionDir)\..\rustybits\target\i686-pc-windows-msvc\debug\;%(AdditionalLibraryDirectories) @@ -466,7 +466,7 @@ true - wbemuuid.lib;wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;zeroidc.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) + wbemuuid.lib;wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;rustybits.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) false "notelemetry.obj" %(AdditionalOptions) $(SolutionDir)..\rustybits\target\x86_64-pc-windows-msvc\debug\;%(AdditionalLibraryDirectories) @@ -489,7 +489,7 @@ true - wbemuuid.lib;wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;zeroidc.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) + wbemuuid.lib;wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;rustybits.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) false $(SolutionDir)..\rustybits\target\aarch64-pc-windows-msvc\debug\;%(AdditionalLibraryDirectories) @@ -569,7 +569,7 @@ false true true - wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;zeroidc.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) + wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;rustybits.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) false $(SolutionDir)..\rustybits\target\i686-pc-windows-msvc\release\;%(AdditionalLibraryDirectories) @@ -607,7 +607,7 @@ false true true - wbemuuid.lib;wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;zeroidc.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) + wbemuuid.lib;wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;rustybits.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) false $(SolutionDir)..\rustybits\target\x86_64-pc-windows-msvc\release\;%(AdditionalLibraryDirectories) @@ -645,7 +645,7 @@ false true true - wbemuuid.lib;wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;zeroidc.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) + wbemuuid.lib;wsock32.lib;ws2_32.lib;Iphlpapi.lib;Rpcrt4.lib;rustybits.lib;bcrypt.lib;userenv.lib;crypt32.lib;secur32.lib;ncrypt.lib;ntdll.lib;%(AdditionalDependencies) false $(SolutionDir)..\rustybits\target\aarch64-pc-windows-msvc\release\;%(AdditionalLibraryDirectories)