From 9527e8e5da429fe8d985b7215ef3c0c688c1ccdb Mon Sep 17 00:00:00 2001 From: Michael Lefebvre Date: Wed, 7 Jan 2026 09:25:20 -0500 Subject: [PATCH] Remove fabrics_ext, because this module would break the promise of seemless interopability. Co-authored-by: Michael Lefebvre Signed-off-by: Michael Lefebvre --- lib/fabrics/include/mxl/fabrics.h | 3 - lib/fabrics/include/mxl/fabrics_ext.h | 85 ------ lib/fabrics/ofi/src/fabrics.cpp | 44 --- lib/fabrics/ofi/src/internal/Region.cpp | 27 -- lib/fabrics/ofi/src/internal/Region.hpp | 25 +- lib/tests/fabrics/ofi/Util.hpp | 70 ++--- lib/tests/fabrics/ofi/test_Region.cpp | 35 --- lib/tests/fabrics/test_basics.cpp | 357 +----------------------- tools/mxl-fabrics-demo/demo.cpp | 2 +- 9 files changed, 41 insertions(+), 607 deletions(-) delete mode 100644 lib/fabrics/include/mxl/fabrics_ext.h diff --git a/lib/fabrics/include/mxl/fabrics.h b/lib/fabrics/include/mxl/fabrics.h index 9a4fa7f92..dd00dcbdf 100644 --- a/lib/fabrics/include/mxl/fabrics.h +++ b/lib/fabrics/include/mxl/fabrics.h @@ -233,9 +233,6 @@ extern "C" * \param in_startSlice The start slice in the slice range to transfer. This is inclusive. * \param in_endSlice The end slice in the slice range to transfer. This is exclusive. * \return The result code. \see mxlStatus - * \note This function assumes: (1) the underlying buffer layout matches the MXL grain data layout, and (2) ring buffer entries for local and - * remote regions can be calculated via modulo operation: `grainIndex % regions.size()`.If these assumptions do not hold, use - * mxlFabricsExtInitiatorTransferGrain() instead, see fabrics_ext.h. */ MXL_EXPORT mxlStatus mxlFabricsInitiatorTransferGrain(mxlFabricsInitiator in_initiator, uint64_t in_grainIndex, uint16_t in_startSlice, diff --git a/lib/fabrics/include/mxl/fabrics_ext.h b/lib/fabrics/include/mxl/fabrics_ext.h deleted file mode 100644 index 5d4089844..000000000 --- a/lib/fabrics/include/mxl/fabrics_ext.h +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-FileCopyrightText: 2025 Contributors to the Media eXchange Layer project. -// -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -/** \brief This module enables users to integrate custom buffers with the MXL Fabrics API. - *Custom buffers can be registered with MXL Fabrics by defining their memory regions and describing the flow using mxlFabricsExtRegionsConfig. - * Each memory region maps to a single grain. Users can then instantiate an mxlFabricsRegions object by invoking mxlFabricsExtGetRegions. When custom - * regions use the same data layout as MXL, users can directly leverage the standard fabrics.h API for data transfers and reads. However, if the - * layout differs, users must use `mxlFabricsExtInitiatorTransferGrain` instead of `mxlFabricsInitiatorTransferGrain` for data transfer operations. - * The rest of the API remains valid. - */ - -#include -#include -#include - -#ifdef __cplusplus -extern "C" -{ -#endif - - /** Configuration for a user defined memory region location. - */ - typedef struct mxlFabricsExtMemoryRegionLocation_t - { - mxlPayloadLocation type; /**< Memory type of the payload. */ - uint64_t deviceId; /**< Device Index when device memory is used, otherwise it is ignored. */ - } mxlFabricsExtMemoryRegionLocation; - - /** Configuration for a user defined memory region. - */ - typedef struct mxlFabricsExtMemoryRegion_t - { - uintptr_t addr; /**< Start address of the contiguous memory region. */ - size_t size; /**< Size of that memory region */ - mxlFabricsExtMemoryRegionLocation loc; /**< Location information for that memory region. */ - } mxlFabricsExtMemoryRegion; - - /** User configuration for a collection of user defined memory regions. - */ - typedef struct mxlFabricsExtRegionsConfig_t - { - mxlFabricsExtMemoryRegion const* regions; /**< Pointer to an array of memory regions. */ - size_t regionsCount; /**< The number of memory regions in the array. */ - - uint32_t sliceSize[MXL_MAX_PLANES_PER_GRAIN]; /**< The size of a single slice in bytes. */ - - mxlDataFormat format; /**< The data format representing these regions. */ - } mxlFabricsExtRegionsConfig; - - /** - * Create a regions object from a list of memory region groups. - * \param in_config User configuiration for the memory regions. - * \param out_regions Returns a pointer to the created regions object. The user is responsible for freeing this object by calling - * `mxlFabricsRegionsFree()`. - * \return MXL_STATUS_OK if the regions object was successfully created. - */ - MXL_EXPORT - mxlStatus mxlFabricsExtGetRegions(mxlFabricsExtRegionsConfig const* in_config, mxlFabricsRegions* out_regions); - - /** - * Enqueue a transfer operation to a specific target. This function is always non-blocking. The transfer operation might be started right - * away, but is only guaranteed to have completed after mxlFabricsInitiatorMakeProgress*() no longer returns MXL_ERR_NOT_READY. - * \param in_initiator A valid fabrics initiator - * \param in_targetInfo The target information of the specific target. This should be the same as the one returned from "mxlFabricsTargetSetup". - * \param in_localIndex The index of the memory region (local) to transfer. The ordering was given when mxlFabricsRegions object was created. - * \param in_remoteIndex The index of the memory region (remote) to receive the transfer at the target side. The ordering was given when - * mxlFabricsRegions object was created. - * \param in_payloadOffset Offset in bytes inside the remote memory region before the payload starts. This is typically the header if any. In MXL, - * this corresponds to MXL_GRAIN_PAYLOAD_OFFSET. - * \param in_startSlice The start slice in the slice range to transfer. This is inclusive. - * \param in_endSlice The end slice in the slice range to transfer. This is exclusive. - * \return The result code. \see mxlStatus - * \note This function is useful when the underlying buffer layout does not match the MXL grain data layout. Otherwise \see - * mxlFabricsInitiatorTransferGrain() - */ - MXL_EXPORT - mxlStatus mxlFabricsExtInitiatorTransferGrain(mxlFabricsInitiator in_initiator, mxlFabricsTargetInfo const in_targetInfo, uint64_t in_localIndex, - uint64_t in_remoteIndex, uint64_t in_payloadOffset, uint16_t in_startSlice, uint16_t in_endSlice); - -#ifdef __cplusplus -} -#endif diff --git a/lib/fabrics/ofi/src/fabrics.cpp b/lib/fabrics/ofi/src/fabrics.cpp index 169c28bb1..f0285ce3e 100644 --- a/lib/fabrics/ofi/src/fabrics.cpp +++ b/lib/fabrics/ofi/src/fabrics.cpp @@ -561,47 +561,3 @@ mxlStatus mxlFabricsFreeTargetInfo(mxlFabricsTargetInfo in_info) }, "Failed to free target info object"); } - -extern "C" MXL_EXPORT -mxlStatus mxlFabricsExtGetRegions(mxlFabricsExtRegionsConfig const* in_config, mxlFabricsRegions* out_regions) -{ - if ((in_config == nullptr) || (out_regions == nullptr)) - { - return MXL_ERR_INVALID_ARG; - } - - return ofi::try_run( - [&]() - { - auto regions = ofi::mxlFabricsRegionsFromUser(*in_config); - - // We are leaking the ownership, the user is responsible for calling mxlFabricsRegionsFree to free the memory. - auto regionPtr = std::make_unique(regions).release(); - - *out_regions = regionPtr->toAPI(); - - return MXL_STATUS_OK; - }, - "Failed to create regions object"); -} - -extern "C" MXL_EXPORT -mxlStatus mxlFabricsExtInitiatorTransferGrain(mxlFabricsInitiator in_initiator, mxlFabricsTargetInfo const in_targetInfo, uint64_t in_localIndex, - uint64_t in_remoteIndex, uint64_t in_payloadOffset, uint16_t in_startSlice, uint16_t in_endSlice) -{ - if ((in_initiator == nullptr) || (in_targetInfo == nullptr)) - { - return MXL_ERR_INVALID_ARG; - } - - return ofi::try_run( - [&]() - { - auto targetInfo = ofi::TargetInfo::fromAPI(in_targetInfo); - ofi::InitiatorWrapper::fromAPI(in_initiator) - ->transferGrainToTarget(targetInfo->id, in_localIndex, in_remoteIndex, in_payloadOffset, in_startSlice, in_endSlice); - - return MXL_STATUS_OK; - }, - "Failed to transfer grain to target"); -} diff --git a/lib/fabrics/ofi/src/internal/Region.cpp b/lib/fabrics/ofi/src/internal/Region.cpp index 2e374faef..5b9f451a5 100644 --- a/lib/fabrics/ofi/src/internal/Region.cpp +++ b/lib/fabrics/ofi/src/internal/Region.cpp @@ -60,16 +60,6 @@ namespace mxl::lib::fabrics::ofi return std::holds_alternative(_inner); } - Region::Location Region::Location::fromAPI(mxlFabricsExtMemoryRegionLocation loc) - { - switch (loc.type) - { - case MXL_PAYLOAD_LOCATION_HOST_MEMORY: return Location::host(); - case MXL_PAYLOAD_LOCATION_DEVICE_MEMORY: return Location::cuda(static_cast(loc.deviceId)); - default: throw Exception::invalidArgument("Invalid memory region type"); - } - } - std::string Region::Location::toString() const noexcept { return std::visit( @@ -183,21 +173,4 @@ namespace mxl::lib::fabrics::ofi throw Exception::make(MXL_ERR_UNKNOWN, "Unsupported flow fromat {}", flow.flowInfo()->config.common.format); } } - - MxlRegions mxlFabricsRegionsFromUser(mxlFabricsExtRegionsConfig const& config) - { - auto outRegions = std::vector{}; - for (std::size_t i = 0; i < config.regionsCount; i++) - { - outRegions.emplace_back(config.regions[i].addr, config.regions[i].size, Region::Location::fromAPI(config.regions[i].loc)); - } - if (config.format == MXL_DATA_FORMAT_VIDEO) - { - return {std::move(outRegions), DataLayout::fromVideo(std::to_array(config.sliceSize))}; - } - else - { - throw Exception::invalidArgument("Unsupported data format {}", static_cast(config.format)); - } - } } diff --git a/lib/fabrics/ofi/src/internal/Region.hpp b/lib/fabrics/ofi/src/internal/Region.hpp index 154e8898f..124c2c4e3 100644 --- a/lib/fabrics/ofi/src/internal/Region.hpp +++ b/lib/fabrics/ofi/src/internal/Region.hpp @@ -13,7 +13,6 @@ #include #include #include "mxl/fabrics.h" -#include "mxl/fabrics_ext.h" #include "DataLayout.hpp" namespace mxl::lib::fabrics::ofi @@ -41,11 +40,6 @@ namespace mxl::lib::fabrics::ofi [[nodiscard]] static Location cuda(int deviceId) noexcept; - /** \brief Convert between external and internal versions of this type - */ - [[nodiscard]] - static Location fromAPI(mxlFabricsExtMemoryRegionLocation loc); - /** \brief Return the device id. For host location 0 is returned. */ [[nodiscard]] @@ -203,11 +197,15 @@ namespace mxl::lib::fabrics::ofi class MxlRegions { public: + MxlRegions(std::vector regions, DataLayout dataLayout) + : _regions(std::move(regions)) + , _layout(std::move(dataLayout)) + {} + /** \brief Convert between external and internal versions of this type */ [[nodiscard]] static MxlRegions* fromAPI(mxlFabricsRegions regions) noexcept; - /** \copydoc fromAPI() */ [[nodiscard]] mxlFabricsRegions toAPI() noexcept; @@ -222,13 +220,6 @@ namespace mxl::lib::fabrics::ofi private: friend MxlRegions mxlFabricsRegionsFromFlow(FlowData const& flow); - friend MxlRegions mxlFabricsRegionsFromUser(mxlFabricsExtRegionsConfig const& config); - - private: - MxlRegions(std::vector regions, DataLayout dataLayout) - : _regions(std::move(regions)) - , _layout(std::move(dataLayout)) - {} private: std::vector _regions; @@ -240,10 +231,4 @@ namespace mxl::lib::fabrics::ofi */ [[nodiscard]] MxlRegions mxlFabricsRegionsFromFlow(FlowData const& flow); - - /** \brief Convert user-provided memory regions to MxlRegions. - * Used to convert mxlFabricsMemoryRegion arrays provided by the user. - */ - [[nodiscard]] - MxlRegions mxlFabricsRegionsFromUser(mxlFabricsExtRegionsConfig const& config); } diff --git a/lib/tests/fabrics/ofi/Util.hpp b/lib/tests/fabrics/ofi/Util.hpp index f9bb07f86..8684e3f2a 100644 --- a/lib/tests/fabrics/ofi/Util.hpp +++ b/lib/tests/fabrics/ofi/Util.hpp @@ -11,7 +11,7 @@ #include #include #include "mxl/fabrics.h" -#include "mxl/fabrics_ext.h" +#include "DataLayout.hpp" #include "Domain.hpp" #include "Region.hpp" @@ -72,8 +72,14 @@ namespace mxl::lib::fabrics::ofi return domain; } + inline MxlRegions getEmptyVideoMxlRegions() + { + return MxlRegions({}, DataLayout::fromVideo({8, 0, 0, 0})); + } + inline std::pair getHostRegionGroups() { + /// Warning: Do not modify the values below, you will break many tests auto innerRegions = std::vector>{ std::vector(256), std::vector(512), @@ -81,55 +87,25 @@ namespace mxl::lib::fabrics::ofi std::vector(2048), }; - /// Warning: Do not modify the values below, you will break many tests - // clang-format off - auto mxlFabricsRegions = std::vector{ - mxlFabricsExtMemoryRegion{ - .addr = reinterpret_cast(innerRegions[0].data()), - .size = innerRegions[0].size(), - .loc = {.type = MXL_PAYLOAD_LOCATION_HOST_MEMORY, .deviceId = 0}, - }, - mxlFabricsExtMemoryRegion{ - .addr = reinterpret_cast(innerRegions[1].data()), - .size = innerRegions[1].size(), - .loc = {.type = MXL_PAYLOAD_LOCATION_HOST_MEMORY, .deviceId = 0}, - }, - mxlFabricsExtMemoryRegion{ - .addr = reinterpret_cast(innerRegions[2].data()), - .size = innerRegions[2].size(), - .loc = {.type = MXL_PAYLOAD_LOCATION_HOST_MEMORY, .deviceId = 0}, - }, - mxlFabricsExtMemoryRegion{ - .addr = reinterpret_cast(innerRegions[3].data()), - .size = innerRegions[3].size(), - .loc = {.type = MXL_PAYLOAD_LOCATION_HOST_MEMORY, .deviceId = 0}, - } - }; - - - auto config = mxlFabricsExtRegionsConfig{.regions=mxlFabricsRegions.data(), .regionsCount = mxlFabricsRegions.size(), .sliceSize={8,0,0,0},.format=MXL_DATA_FORMAT_VIDEO}; - - // clang-format on + std::vector regions; + for (auto const& innerRegion : innerRegions) + { + regions.emplace_back(*innerRegion.data(), innerRegion.size()); + } - return {mxlFabricsRegionsFromUser(config), innerRegions}; + auto mxlRegions = MxlRegions(regions, DataLayout::fromVideo({8, 0, 0, 0})); + return {mxlRegions, innerRegions}; } - inline std::pair getUserMxlRegions() + inline MxlRegions getMxlRegions(std::vector> const& innerRegions, + DataLayout dataLayout = DataLayout::fromVideo({8, 0, 0, 0})) { - auto regions = InnerRegions{InnerRegion(256)}; - auto memoryRegions = std::vector{ - mxlFabricsExtMemoryRegion{.addr = reinterpret_cast(regions[0].data()), - .size = regions[0].size(), - .loc = {.type = MXL_PAYLOAD_LOCATION_HOST_MEMORY, .deviceId = 0}}, - }; - auto config = mxlFabricsExtRegionsConfig{ - .regions = memoryRegions.data(), .regionsCount = memoryRegions.size(), .sliceSize = {8, 0, 0, 0}, - .format = MXL_DATA_FORMAT_VIDEO - }; - - mxlFabricsRegions outRegions; - mxlFabricsExtGetRegions(&config, &outRegions); - - return {outRegions, regions}; + std::vector regions; + for (auto const& innerRegion : innerRegions) + { + regions.emplace_back(*innerRegion.data(), innerRegion.size()); + } + return {regions, dataLayout}; } + } diff --git a/lib/tests/fabrics/ofi/test_Region.cpp b/lib/tests/fabrics/ofi/test_Region.cpp index 00f9aeaf3..ba57a4ab6 100644 --- a/lib/tests/fabrics/ofi/test_Region.cpp +++ b/lib/tests/fabrics/ofi/test_Region.cpp @@ -36,38 +36,3 @@ TEST_CASE("ofi: RegionGroup view and iovec conversion", "[ofi][RegionGroup]") REQUIRE(iovecs[1].iov_base == reinterpret_cast(0x2000)); REQUIRE(iovecs[1].iov_len == 128); } - -TEST_CASE("ofi: RegionGroups fromGroups and view", "[ofi][RegionGroups]") -{ - // clang-format off - auto inputRegions = std::array{ - mxlFabricsExtMemoryRegion{ - .addr = 0x3000, - .size = 256, - .loc = {.type = MXL_PAYLOAD_LOCATION_HOST_MEMORY, .deviceId = 0}, - }}; - // clang-format on - - auto config = mxlFabricsExtRegionsConfig{ - .regions = inputRegions.data(), - .regionsCount = 1, - .sliceSize = {8, 0, 0, 0}, - .format = MXL_DATA_FORMAT_VIDEO, - }; - - auto mxlFabricsRegions = mxlFabricsRegionsFromUser(config); - - REQUIRE(mxlFabricsRegions.regions().size() == 1); - - auto const& region = mxlFabricsRegions.regions()[0]; - REQUIRE(region.base == 0x3000); - REQUIRE(region.size == 256); - REQUIRE(region.loc.isHost()); - - auto dataLayout = mxlFabricsRegions.dataLayout(); - - REQUIRE(dataLayout.isVideo()); - - auto videoLayout = dataLayout.asVideo(); - REQUIRE(videoLayout.sliceSizes[0] == 8); -} diff --git a/lib/tests/fabrics/test_basics.cpp b/lib/tests/fabrics/test_basics.cpp index b72fffa44..a2daebe65 100644 --- a/lib/tests/fabrics/test_basics.cpp +++ b/lib/tests/fabrics/test_basics.cpp @@ -12,11 +12,11 @@ #include #include #include -#include "mxl-internal/Logging.hpp" -#include "mxl/dataformat.h" #include "mxl/flow.h" #include "mxl/mxl.h" +#include "ofi/Util.hpp" #include "../Utils.hpp" +#include "Region.hpp" #ifdef MXL_FABRICS_OFI // clang-format off @@ -27,11 +27,6 @@ TEST_CASE("Fabrics basic creation/destroy", "[fabrics][basics]") { - // Create an empty region, because management tests does not need to do transfers - auto regionsConfig = mxlFabricsExtRegionsConfig{.regions = nullptr, .regionsCount = 0, .sliceSize = {}, .format = MXL_DATA_FORMAT_UNSPECIFIED}; - mxlFabricsRegions regions; - mxlFabricsExtGetRegions(®ionsConfig, ®ions); - auto instance = mxlCreateInstance("/dev/shm/", ""); mxlFabricsInstance fabrics; @@ -59,10 +54,11 @@ TEST_CASE("Fabrics basic creation/destroy", "[fabrics][basics]") TEST_CASE("Fabrics connection oriented activation tests", "[fabrics][connected][activation]") { + namespace ofi = mxl::lib::fabrics::ofi; + // Create an empty region, because management tests does not need to do transfers - auto regionsConfig = mxlFabricsExtRegionsConfig{.regions = nullptr, .regionsCount = 0, .sliceSize = {}, .format = MXL_DATA_FORMAT_VIDEO}; - mxlFabricsRegions regions; - mxlFabricsExtGetRegions(®ionsConfig, ®ions); + auto mxlRegions = ofi::getEmptyVideoMxlRegions(); + auto regions = mxlRegions.toAPI(); auto instance = mxlCreateInstance("/dev/shm/", ""); @@ -167,10 +163,11 @@ TEST_CASE("Fabrics connection oriented activation tests", "[fabrics][connected][ TEST_CASE("Fabrics connectionless activation tests", "[fabrics][connectionless][activation]") { + namespace ofi = mxl::lib::fabrics::ofi; + // Create an empty region, because management tests does not need to do transfers - auto regionsConfig = mxlFabricsExtRegionsConfig{.regions = nullptr, .regionsCount = 0, .sliceSize = {}, .format = MXL_DATA_FORMAT_VIDEO}; - mxlFabricsRegions regions; - mxlFabricsExtGetRegions(®ionsConfig, ®ions); + auto mxlRegions = ofi::getEmptyVideoMxlRegions(); + auto regions = mxlRegions.toAPI(); auto instance = mxlCreateInstance("/dev/shm/", ""); @@ -190,11 +187,6 @@ TEST_CASE("Fabrics connectionless activation tests", "[fabrics][connectionless][ .deviceSupport = false }; mxlFabricsTargetInfo targetInfo; - SECTION("target setup") - { - REQUIRE(mxlFabricsTargetSetup(target, &targetConfig, &targetInfo) == MXL_STATUS_OK); - REQUIRE(mxlFabricsFreeTargetInfo(targetInfo) == MXL_STATUS_OK); - } REQUIRE(mxlFabricsTargetSetup(target, &targetConfig, &targetInfo) == MXL_STATUS_OK); auto initiatorConfig = mxlFabricsInitiatorConfig{ @@ -203,10 +195,6 @@ TEST_CASE("Fabrics connectionless activation tests", "[fabrics][connectionless][ .regions = regions, .deviceSupport = false }; - SECTION("initiator setup") - { - REQUIRE(mxlFabricsInitiatorSetup(initiator, &initiatorConfig) == MXL_STATUS_OK); - } REQUIRE(mxlFabricsInitiatorSetup(initiator, &initiatorConfig) == MXL_STATUS_OK); SECTION("initiator add/remove target") @@ -277,326 +265,6 @@ TEST_CASE("Fabrics connectionless activation tests", "[fabrics][connectionless][ REQUIRE(mxlDestroyInstance(instance) == MXL_STATUS_OK); } -TEST_CASE("Fabrics: transfer grain with user buffers", "[fabrics][transfer][user]") -{ - namespace ofi = mxl::lib::fabrics::ofi; - - auto instance = mxlCreateInstance("/dev/shm/", ""); - - mxlFabricsInstance fabrics; - REQUIRE(mxlFabricsCreateInstance(instance, &fabrics) == MXL_STATUS_OK); - - mxlFabricsTarget target; - REQUIRE(mxlFabricsCreateTarget(fabrics, &target) == MXL_STATUS_OK); - - mxlFabricsInitiator initiator; - REQUIRE(mxlFabricsCreateInitiator(fabrics, &initiator) == MXL_STATUS_OK); - - // Setup target regions - mxlFabricsRegions mxlTargetRegions; - auto targetRegion = std::vector(1e6); - auto targetRegions = std::vector{ - { - .addr = reinterpret_cast(targetRegion.data()), - .size = targetRegion.size(), - .loc = {.type = MXL_PAYLOAD_LOCATION_HOST_MEMORY, .deviceId = 0}, - } - }; - auto config = mxlFabricsExtRegionsConfig{ - .regions = targetRegions.data(), .regionsCount = targetRegions.size(), .sliceSize = {720}, .format = MXL_DATA_FORMAT_VIDEO}; - mxlFabricsExtGetRegions(&config, &mxlTargetRegions); - - mxlFabricsRegions mxlInitiatorRegions; - auto initiatorRegion = std::vector(1e6); - auto initiatorRegions = std::vector{ - { - .addr = reinterpret_cast(initiatorRegion.data()), - .size = initiatorRegion.size(), - .loc = {.type = MXL_PAYLOAD_LOCATION_HOST_MEMORY, .deviceId = 0}, - } - }; - mxlFabricsExtGetRegions(&config, &mxlInitiatorRegions); - - SECTION("RC") - { - auto targetConfig = mxlFabricsTargetConfig{ - .endpointAddress = mxlFabricsEndpointAddress{.node = "127.0.0.1", .service = "0"}, - .provider = MXL_FABRICS_PROVIDER_TCP, - .regions = mxlTargetRegions, - .deviceSupport = false - }; - - mxlFabricsTargetInfo targetInfo; - REQUIRE(mxlFabricsTargetSetup(target, &targetConfig, &targetInfo) == MXL_STATUS_OK); - - // Setup initiator regions - - auto initiatorConfig = mxlFabricsInitiatorConfig{ - .endpointAddress = mxlFabricsEndpointAddress{.node = "127.0.0.1", .service = "0"}, - .provider = MXL_FABRICS_PROVIDER_TCP, - .regions = mxlInitiatorRegions, - .deviceSupport = false - }; - REQUIRE(mxlFabricsInitiatorSetup(initiator, &initiatorConfig) == MXL_STATUS_OK); - REQUIRE(mxlFabricsInitiatorAddTarget(initiator, targetInfo) == MXL_STATUS_OK); - - // try to connect them for 5 seconds - auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - std::uint16_t dummyIndex; - std::uint16_t dummyValidSlices; - do - { - mxlFabricsTargetReadGrainNonBlocking(target, &dummyIndex, &dummyValidSlices); // make progress on target - - auto status = mxlFabricsInitiatorMakeProgressNonBlocking(initiator); - if (status != MXL_STATUS_OK && status != MXL_ERR_NOT_READY) - { - FAIL("Something went wrong in the initiator: " + std::to_string(status)); - } - - if (status == MXL_STATUS_OK) - { - // connected - break; - } - if (std::chrono::steady_clock::now() > deadline) - { - FAIL("Failed to connect in 5 seconds"); - } - } - while (true); - - SECTION("non-blocking") - { - // try to post a transfer within 5 seconds - deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - do - { - mxlFabricsTargetReadGrainNonBlocking(target, &dummyIndex, &dummyValidSlices); // target make progress - mxlFabricsInitiatorMakeProgressNonBlocking(initiator); - if (mxlFabricsInitiatorTransferGrain(initiator, 0, 0, 1) == MXL_STATUS_OK) - { - // transfer started - break; - } - - if (std::chrono::steady_clock::now() > deadline) - { - FAIL("Failed to start transfer in 5 seconds"); - } - } - while (std::chrono::steady_clock::now() < deadline); - - // Wait up to 5 seconds for the transfer to complete - deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - do - { - mxlFabricsInitiatorMakeProgressNonBlocking(initiator); - auto status = mxlFabricsTargetReadGrainNonBlocking(target, &dummyIndex, &dummyValidSlices); - if (status == MXL_ERR_INTERRUPTED) - { - FAIL("Peer disconnected before the transfer completed"); - } - if (status == MXL_STATUS_OK) - { - // transfer complete - return; - } - } - while (std::chrono::steady_clock::now() < deadline); - - FAIL("Failed to complete transfer in 5 seconds"); - } - - SECTION("blocking") - { - // try to post a transfer within 5 seconds - deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - do - { - mxlFabricsTargetReadGrain(target, &dummyIndex, &dummyValidSlices, std::chrono::milliseconds(20).count()); // target make progress - mxlFabricsInitiatorMakeProgressBlocking(initiator, std::chrono::milliseconds(20).count()); - if (mxlFabricsInitiatorTransferGrain(initiator, 0, 0, 1) == MXL_STATUS_OK) - { - // transfer started - break; - } - - if (std::chrono::steady_clock::now() > deadline) - { - FAIL("Failed to start transfer in 5 seconds"); - } - } - while (std::chrono::steady_clock::now() < deadline); - - // Wait up to 5 seconds for the transfer to complete - deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - do - { - mxlFabricsInitiatorMakeProgressBlocking(initiator, std::chrono::milliseconds(20).count()); - auto status = mxlFabricsTargetReadGrain(target, &dummyIndex, &dummyValidSlices, std::chrono::milliseconds(20).count()); - if (status == MXL_ERR_INTERRUPTED) - { - FAIL("Peer disconnected before the transfer completed"); - } - if (status == MXL_STATUS_OK) - { - // transfer complete - return; - } - } - while (std::chrono::steady_clock::now() < deadline); - - FAIL("Failed to complete transfer in 5 seconds"); - } - - mxlFabricsFreeTargetInfo(targetInfo); - } - - SECTION("RDM") - { - auto targetConfig = mxlFabricsTargetConfig{ - .endpointAddress = mxlFabricsEndpointAddress{.node = "target", .service = "0"}, - .provider = MXL_FABRICS_PROVIDER_SHM, - .regions = mxlTargetRegions, - .deviceSupport = false - }; - - mxlFabricsTargetInfo targetInfo; - REQUIRE(mxlFabricsTargetSetup(target, &targetConfig, &targetInfo) == MXL_STATUS_OK); - - // Setup initiator regions - - auto initiatorConfig = mxlFabricsInitiatorConfig{ - .endpointAddress = mxlFabricsEndpointAddress{.node = "initiator", .service = "0"}, - .provider = MXL_FABRICS_PROVIDER_SHM, - .regions = mxlInitiatorRegions, - .deviceSupport = false - }; - REQUIRE(mxlFabricsInitiatorSetup(initiator, &initiatorConfig) == MXL_STATUS_OK); - REQUIRE(mxlFabricsInitiatorAddTarget(initiator, targetInfo) == MXL_STATUS_OK); - - // try to connect them for 5 seconds - auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - std::uint16_t dummyIndex; - std::uint16_t dummyValidSlices; - do - { - mxlFabricsTargetReadGrainNonBlocking(target, &dummyIndex, &dummyValidSlices); // make progress on target - - auto status = mxlFabricsInitiatorMakeProgressNonBlocking(initiator); - if (status != MXL_STATUS_OK && status != MXL_ERR_NOT_READY) - { - FAIL("Something went wrong in the initiator: " + std::to_string(status)); - } - - if (status == MXL_STATUS_OK) - { - // connected - break; - } - if (std::chrono::steady_clock::now() > deadline) - { - FAIL("Failed to connect in 5 seconds"); - } - } - while (true); - - SECTION("non-blocking") - { - // try to post a transfer within 5 seconds - deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - do - { - mxlFabricsTargetReadGrainNonBlocking(target, &dummyIndex, &dummyValidSlices); // target make progress - mxlFabricsInitiatorMakeProgressNonBlocking(initiator); - if (mxlFabricsInitiatorTransferGrain(initiator, 0, 0, 1) == MXL_STATUS_OK) - { - // transfer started - break; - } - - if (std::chrono::steady_clock::now() > deadline) - { - FAIL("Failed to start transfer in 5 seconds"); - } - } - while (std::chrono::steady_clock::now() < deadline); - - // Wait up to 5 seconds for the transfer to complete - deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - do - { - mxlFabricsInitiatorMakeProgressNonBlocking(initiator); - auto status = mxlFabricsTargetReadGrainNonBlocking(target, &dummyIndex, &dummyValidSlices); - if (status == MXL_ERR_INTERRUPTED) - { - FAIL("Peer disconnected before the transfer completed"); - } - if (status == MXL_STATUS_OK) - { - // transfer complete - return; - } - } - while (std::chrono::steady_clock::now() < deadline); - - FAIL("Failed to complete transfer in 5 seconds"); - } - - SECTION("blocking") - { - // try to post a transfer within 5 seconds - deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - do - { - mxlFabricsTargetReadGrain(target, &dummyIndex, &dummyValidSlices, std::chrono::milliseconds(20).count()); // target make progress - mxlFabricsInitiatorMakeProgressBlocking(initiator, std::chrono::milliseconds(20).count()); - if (mxlFabricsInitiatorTransferGrain(initiator, 0, 0, 1) == MXL_STATUS_OK) - { - // transfer started - break; - } - - if (std::chrono::steady_clock::now() > deadline) - { - FAIL("Failed to start transfer in 5 seconds"); - } - } - while (std::chrono::steady_clock::now() < deadline); - - // Wait up to 5 seconds for the transfer to complete - deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - do - { - mxlFabricsInitiatorMakeProgressBlocking(initiator, std::chrono::milliseconds(20).count()); - auto status = mxlFabricsTargetReadGrain(target, &dummyIndex, &dummyValidSlices, std::chrono::milliseconds(20).count()); - if (status == MXL_ERR_INTERRUPTED) - { - FAIL("Peer disconnected before the transfer completed"); - } - if (status == MXL_STATUS_OK) - { - // transfer complete - return; - } - } - while (std::chrono::steady_clock::now() < deadline); - - FAIL("Failed to complete transfer in 5 seconds"); - } - - mxlFabricsFreeTargetInfo(targetInfo); - } - - REQUIRE(mxlFabricsRegionsFree(mxlInitiatorRegions) == MXL_STATUS_OK); - REQUIRE(mxlFabricsRegionsFree(mxlTargetRegions) == MXL_STATUS_OK); - REQUIRE(mxlFabricsDestroyInitiator(fabrics, initiator) == MXL_STATUS_OK); - REQUIRE(mxlFabricsDestroyTarget(fabrics, target) == MXL_STATUS_OK); - REQUIRE(mxlFabricsDestroyInstance(fabrics) == MXL_STATUS_OK); - REQUIRE(mxlDestroyInstance(instance) == MXL_STATUS_OK); -} - TEST_CASE_PERSISTENT_FIXTURE(mxl::tests::mxlDomainFixture, "Fabrics: Transfer Grain with flows", "[Fabrics][Transfer][Flows]") { namespace ofi = mxl::lib::fabrics::ofi; @@ -1322,9 +990,8 @@ TEST_CASE("Fabrics: TargetInfo serialize/deserialize", "[fabrics][ofi][target-in REQUIRE(mxlFabricsCreateInstance(instance, &fabrics) == MXL_STATUS_OK); REQUIRE(mxlFabricsCreateTarget(fabrics, &target) == MXL_STATUS_OK); - auto regionsConfig = mxlFabricsExtRegionsConfig{.regions = nullptr, .regionsCount = 0, .sliceSize = {}, .format = MXL_DATA_FORMAT_VIDEO}; - mxlFabricsRegions regions; - mxlFabricsExtGetRegions(®ionsConfig, ®ions); + auto mxlRegions = ofi::getEmptyVideoMxlRegions(); + auto regions = mxlRegions.toAPI(); auto config = mxlFabricsTargetConfig{ .endpointAddress = mxlFabricsEndpointAddress{.node = "127.0.0.1", .service = "0"}, diff --git a/tools/mxl-fabrics-demo/demo.cpp b/tools/mxl-fabrics-demo/demo.cpp index 522b31384..7622dae70 100644 --- a/tools/mxl-fabrics-demo/demo.cpp +++ b/tools/mxl-fabrics-demo/demo.cpp @@ -401,7 +401,7 @@ class AppTarget mxlFlowConfigInfo configInfo; bool flowCreated = false; - status = mxlCreateFlowWriter(_instance, flowDescriptor.c_str(), "", &_writer, &configInfo, &flowCreated); + status = mxlCreateFlowWriter(_instance, flowDescriptor.c_str(), flowOptions.c_str(), &_writer, &configInfo, &flowCreated); if (status != MXL_STATUS_OK) { MXL_ERROR("Failed to create flow writer with status '{}'", static_cast(status));