diff --git a/common/include/villas/kernel/devices/device.hpp b/common/include/villas/kernel/devices/device.hpp new file mode 100644 index 000000000..896cab042 --- /dev/null +++ b/common/include/villas/kernel/devices/device.hpp @@ -0,0 +1,33 @@ +/* Interface for Linux/Unix devices. + * + * Author: Pascal Bauer + * + * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +namespace villas { +namespace kernel { +namespace devices { + +class Device { +public: + virtual ~Device(){}; + + virtual std::optional> driver() const = 0; + virtual std::optional iommu_group() const = 0; + virtual std::string name() const = 0; + virtual std::filesystem::path override_path() const = 0; + virtual std::filesystem::path path() const = 0; + virtual void probe() const = 0; +}; + +} // namespace devices +} // namespace kernel +} // namespace villas diff --git a/common/include/villas/kernel/devices/driver.hpp b/common/include/villas/kernel/devices/driver.hpp new file mode 100644 index 000000000..7578f1e66 --- /dev/null +++ b/common/include/villas/kernel/devices/driver.hpp @@ -0,0 +1,29 @@ +/* Interface for device drivers. OS/platform independend. + * Implemented for Linux/Unix drivers in linux_driver.hpp + * + * Author: Pascal Bauer + * + * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +namespace villas { +namespace kernel { +namespace devices { + +class Device; + +class Driver { +public: + virtual void attach(const Device &device) const = 0; + virtual void bind(const Device &device) const = 0; + virtual std::string name() const = 0; + virtual void override(const Device &device) const = 0; + virtual void unbind(const Device &device) const = 0; +}; + +} // namespace devices +} // namespace kernel +} // namespace villas \ No newline at end of file diff --git a/common/include/villas/kernel/devices/ip_device.hpp b/common/include/villas/kernel/devices/ip_device.hpp new file mode 100644 index 000000000..351f6cae4 --- /dev/null +++ b/common/include/villas/kernel/devices/ip_device.hpp @@ -0,0 +1,35 @@ +/* IpDevice: Linux/Unix device which represents an IP component of a FPGA. + * + * Author: Pascal Bauer + * + * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +namespace villas { +namespace kernel { +namespace devices { + +class IpDevice : public PlatformDevice { +public: + static IpDevice from(const std::filesystem::path unsafe_path); + static bool is_path_valid(const std::filesystem::path unsafe_path); + +private: + IpDevice() = delete; + IpDevice(const std::filesystem::path valid_path) //! Dont allow unvalidated paths + : PlatformDevice(valid_path){}; + +public: + size_t addr() const; + std::string ip_name() const; +}; + +} // namespace devices +} // namespace kernel +} // namespace villas diff --git a/common/include/villas/kernel/devices/linux_driver.hpp b/common/include/villas/kernel/devices/linux_driver.hpp new file mode 100644 index 000000000..71dbce8d7 --- /dev/null +++ b/common/include/villas/kernel/devices/linux_driver.hpp @@ -0,0 +1,52 @@ +/* Implementation of driver interface for Linux/Unix based operation system drivers. + * + * Author: Pascal Bauer + * + * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include + +namespace villas { +namespace kernel { +namespace devices { + +class LinuxDriver : public Driver { +private: + static constexpr char BIND_DEFAULT[] = "bind"; + static constexpr char UNBIND_DEFAULT[] = "unbind"; + +public: + const std::filesystem::path path; + +private: + const std::filesystem::path bind_path; + const std::filesystem::path unbind_path; + +public: + LinuxDriver(const std::filesystem::path path) + : LinuxDriver(path, path / std::filesystem::path(BIND_DEFAULT), + path / std::filesystem::path(UNBIND_DEFAULT)){}; + + LinuxDriver(const std::filesystem::path path, + const std::filesystem::path bind_path, + const std::filesystem::path unbind_path) + : path(path), bind_path(bind_path), unbind_path(unbind_path){}; + +public: + void attach(const Device &device) const override; + void bind(const Device &device) const override; + std::string name() const override; + void override(const Device &device) const override; + void unbind(const Device &device) const override; +}; + +} // namespace devices +} // namespace kernel +} // namespace villas \ No newline at end of file diff --git a/common/include/villas/kernel/devices/pci_device.hpp b/common/include/villas/kernel/devices/pci_device.hpp index 65794644b..8f6a10cf5 100644 --- a/common/include/villas/kernel/devices/pci_device.hpp +++ b/common/include/villas/kernel/devices/pci_device.hpp @@ -15,6 +15,8 @@ #include +#include + namespace villas { namespace kernel { namespace devices { @@ -58,7 +60,11 @@ struct Region { unsigned long long flags; }; -class PciDevice { +class PciDevice : public Device { +private: + static constexpr char PROBE_DEFAULT[] = "/sys/bus/pci/drivers_probe"; + static constexpr char OVERRIDE_DEFAULT[] = "driver_override"; + public: PciDevice(Id i, Slot s) : id(i), slot(s), log(Log::get("kernel:pci")) {} @@ -68,15 +74,17 @@ class PciDevice { bool operator==(const PciDevice &other); - // Get currently loaded driver for device - std::string getDriver() const; + // Implement device interface + std::optional> driver() const override; + std::optional iommu_group() const override; + std::string name() const override; + std::filesystem::path override_path() const override; + std::filesystem::path path() const override; + void probe() const override; // Bind a new LKM to the PCI device bool attachDriver(const std::string &driver) const; - // Return the IOMMU group of this PCI device or -1 if the device is not in a group - int getIommuGroup() const; - std::list getRegions() const; // Write 32-bit BAR value from to the PCI configuration space diff --git a/common/include/villas/kernel/devices/platform_device.hpp b/common/include/villas/kernel/devices/platform_device.hpp new file mode 100644 index 000000000..bbe182548 --- /dev/null +++ b/common/include/villas/kernel/devices/platform_device.hpp @@ -0,0 +1,51 @@ +/* Platform Device: Platform bus based Linux/Unix device. + * + * Author: Pascal Bauer + * + * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +namespace villas { +namespace kernel { +namespace devices { + +class PlatformDevice : public Device { +private: + static constexpr char PROBE_DEFAULT[] = "/sys/bus/platform/drivers_probe"; + static constexpr char OVERRIDE_DEFAULT[] = "driver_override"; + +private: + const std::filesystem::path m_path; + const std::filesystem::path m_probe_path; + const std::filesystem::path m_override_path; + +public: + PlatformDevice(const std::filesystem::path path) + : PlatformDevice(path, std::filesystem::path(PROBE_DEFAULT), + path / std::filesystem::path(OVERRIDE_DEFAULT)){}; + + PlatformDevice(const std::filesystem::path path, + const std::filesystem::path probe_path, + const std::filesystem::path override_path) + : m_path(path), m_probe_path(probe_path), + m_override_path(override_path){}; + + // Implement device interface + std::optional> driver() const override; + std::optional iommu_group() const override; + std::string name() const override; + std::filesystem::path override_path() const override; + std::filesystem::path path() const override; + void probe() const override; +}; + +} // namespace devices +} // namespace kernel +} // namespace villas \ No newline at end of file diff --git a/common/include/villas/utils.hpp b/common/include/villas/utils.hpp index a65fb6cba..18445d891 100644 --- a/common/include/villas/utils.hpp +++ b/common/include/villas/utils.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -211,6 +212,9 @@ template struct overloaded : Ts... { // explicit deduction guide (not needed as of C++20) template overloaded(Ts...) -> overloaded; +void write_to_file(std::string data, const std::filesystem::path file); +std::vector read_names_in_directory(const std::filesystem::path &directory); + namespace base64 { using byte = std::uint8_t; diff --git a/common/lib/CMakeLists.txt b/common/lib/CMakeLists.txt index 116d786e6..74581897c 100644 --- a/common/lib/CMakeLists.txt +++ b/common/lib/CMakeLists.txt @@ -39,7 +39,10 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL Linux) target_sources(villas-common PRIVATE + kernel/devices/ip_device.cpp + kernel/devices/linux_driver.cpp kernel/devices/pci_device.cpp + kernel/devices/platform_device.cpp kernel/vfio_device.cpp kernel/vfio_group.cpp kernel/vfio_container.cpp diff --git a/common/lib/kernel/devices/ip_device.cpp b/common/lib/kernel/devices/ip_device.cpp new file mode 100644 index 000000000..0b850e654 --- /dev/null +++ b/common/lib/kernel/devices/ip_device.cpp @@ -0,0 +1,54 @@ +/* IpDevice + * + * Author: Pascal Bauer + * + * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include +#include + +using villas::kernel::devices::IpDevice; + +IpDevice IpDevice::from(const std::filesystem::path unsafe_path) { + if (!is_path_valid(unsafe_path)) + throw RuntimeError( + "Path {} failed validation as IpDevicePath [adress in hex].[name] ", + unsafe_path.u8string()); + return IpDevice(unsafe_path); +} + +std::string IpDevice::ip_name() const { + int pos = name().find('.'); + return name().substr(pos + 1); +} + +size_t IpDevice::addr() const { + size_t pos = name().find('.'); + std::string addr_hex = name().substr(0, pos); + + // Convert from hex-string to number + std::stringstream ss; + ss << std::hex << addr_hex; + size_t addr = 0; + ss >> addr; + + return addr; +} + +bool IpDevice::is_path_valid(const std::filesystem::path unsafe_path) { + std::string assumed_device_name = unsafe_path.filename(); + + // Match format of hexaddr.devicename + if (!std::regex_match(assumed_device_name, + std::regex(R"([0-9A-Fa-f]+\..*)"))) { + return false; + } + + return true; +} diff --git a/common/lib/kernel/devices/linux_driver.cpp b/common/lib/kernel/devices/linux_driver.cpp new file mode 100644 index 000000000..0cbc39b18 --- /dev/null +++ b/common/lib/kernel/devices/linux_driver.cpp @@ -0,0 +1,40 @@ +/* LinuxDriver + * + * Author: Pascal Bauer + * + * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include +#include + +using villas::kernel::devices::Device, villas::kernel::devices::LinuxDriver; +using villas::utils::write_to_file; + +void LinuxDriver::attach(const Device &device) const { + if (device.driver().has_value()) { + device.driver().value()->unbind(device); + } + this->override(device); + device.probe(); +} + +void LinuxDriver::bind(const Device &device) const { + write_to_file(device.name(), this->bind_path); +} + +std::string LinuxDriver::name() const { + size_t pos = path.u8string().rfind('/'); + return path.u8string().substr(pos + 1); +} + +void LinuxDriver::override(const Device &device) const { + write_to_file(this->name(), device.override_path()); +} + +void LinuxDriver::unbind(const Device &device) const { + write_to_file(device.name(), this->unbind_path); +} diff --git a/common/lib/kernel/devices/pci_device.cpp b/common/lib/kernel/devices/pci_device.cpp index ef8a8b5d0..2324c6013 100644 --- a/common/lib/kernel/devices/pci_device.cpp +++ b/common/lib/kernel/devices/pci_device.cpp @@ -18,10 +18,12 @@ #include #include +#include #include #include using namespace villas::kernel::devices; +using villas::utils::write_to_file; #define PCI_BASE_ADDRESS_N(n) (PCI_BASE_ADDRESS_0 + sizeof(uint32_t) * (n)) @@ -311,8 +313,7 @@ std::list PciDevice::getRegions() const { return regions; } - -std::string PciDevice::getDriver() const { +std::optional> PciDevice::driver() const { int ret; char sysfs[1024], syml[1024]; memset(syml, 0, sizeof(syml)); @@ -323,13 +324,15 @@ std::string PciDevice::getDriver() const { struct stat st; ret = stat(sysfs, &st); if (ret) - return ""; + return std::nullopt; ret = readlink(sysfs, syml, sizeof(syml)); if (ret < 0) throw SystemError("Failed to follow link: {}", sysfs); - return basename(syml); + auto driver = std::make_optional(std::make_unique( + "/sys/bus/pci/drivers/" + std::string(basename(syml)))); + return driver; } bool PciDevice::attachDriver(const std::string &driver) const { @@ -423,7 +426,7 @@ void PciDevice::writeBar(uint32_t addr, unsigned barNum) { file.write(reinterpret_cast(&addr), sizeof(addr)); } -int PciDevice::getIommuGroup() const { +std::optional PciDevice::iommu_group() const { int ret; char *group; @@ -437,11 +440,11 @@ int PciDevice::getIommuGroup() const { ret = readlink(sysfs, link, sizeof(link)); if (ret < 0) - return -1; + return std::nullopt; group = basename(link); - return atoi(group); + return std::make_optional(atoi(group)); } std::fstream PciDevice::openSysFs(const std::string &subPath, @@ -457,3 +460,33 @@ std::fstream PciDevice::openSysFs(const std::string &subPath, return file; } + +// TODO: test +std::string PciDevice::name() const { + char sysfs[1024]; + + snprintf(sysfs, sizeof(sysfs), "%04x:%02x:%02x.%x", slot.domain, slot.bus, + slot.device, slot.function); + + return std::string(sysfs); +} + +// TODO: test +std::filesystem::path PciDevice::path() const { + char sysfs[1024]; + + snprintf(sysfs, sizeof(sysfs), "%04x:%02x:%02x.%x", slot.domain, slot.bus, + slot.device, slot.function); + + return sysfs; +} + +// TODO: test +std::filesystem::path PciDevice::override_path() const { + + return this->path() / OVERRIDE_DEFAULT; +} + +void PciDevice::probe() const { + write_to_file(this->name(), this->PROBE_DEFAULT); +} diff --git a/common/lib/kernel/devices/platform_device.cpp b/common/lib/kernel/devices/platform_device.cpp new file mode 100644 index 000000000..e899a2d75 --- /dev/null +++ b/common/lib/kernel/devices/platform_device.cpp @@ -0,0 +1,50 @@ +/* Platform Device + * + * Author: Pascal Bauer + * + * SPDX-FileCopyrightText: 2023-2024 Pascal Bauer + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +using villas::kernel::devices::Driver, villas::kernel::devices::LinuxDriver; +using villas::kernel::devices::PlatformDevice; +using villas::utils::write_to_file; + +std::optional> PlatformDevice::driver() const { + std::filesystem::path driver_symlink = + this->m_path / std::filesystem::path("driver"); + + if (!std::filesystem::is_symlink(driver_symlink)) + return std::nullopt; + + std::filesystem::path driver_path = + std::filesystem::canonical(driver_symlink); + return std::make_optional(std::make_unique(driver_path)); +} + +std::optional PlatformDevice::iommu_group() const { + std::filesystem::path symlink = + std::filesystem::path(this->m_path.u8string() + "/iommu_group"); + + std::filesystem::path link = std::filesystem::read_symlink(symlink); + std::string delimiter = "iommu_groups/"; + int pos = link.u8string().find(delimiter); + int iommu_group = std::stoi(link.u8string().substr(pos + delimiter.length())); + return std::make_optional(iommu_group); +} + +std::filesystem::path PlatformDevice::path() const { return this->m_path; }; + +void PlatformDevice::probe() const { + write_to_file(this->name(), this->m_probe_path); +} + +std::filesystem::path PlatformDevice::override_path() const { + return this->m_override_path; +} + +std::string PlatformDevice::name() const { return this->m_path.filename(); } \ No newline at end of file diff --git a/common/lib/kernel/vfio_container.cpp b/common/lib/kernel/vfio_container.cpp index 76923aa08..595edf9fd 100644 --- a/common/lib/kernel/vfio_container.cpp +++ b/common/lib/kernel/vfio_container.cpp @@ -197,7 +197,7 @@ std::shared_ptr Container::attachDevice(devices::PciDevice &pdev) { throw RuntimeError("Failed to load kernel driver: vfio_pci"); // Bind PCI card to vfio-pci driver if not already bound - if (pdev.getDriver() != kernelDriver) { + if ( !(pdev.driver().has_value()) || pdev.driver().value()->name() != kernelDriver) { log->debug("Bind PCI card to kernel driver '{}'", kernelDriver); pdev.attachDriver(kernelDriver); } @@ -215,7 +215,7 @@ std::shared_ptr Container::attachDevice(devices::PciDevice &pdev) { } // Get IOMMU group of device - int index = isIommuEnabled() ? pdev.getIommuGroup() : 0; + int index = isIommuEnabled() ? pdev.iommu_group().value() : 0; if (index < 0) { ret = kernel::getCmdlineParam("intel_iommu", iommu_state, sizeof(iommu_state)); diff --git a/common/lib/utils.cpp b/common/lib/utils.cpp index f495d8121..b137bf25e 100644 --- a/common/lib/utils.cpp +++ b/common/lib/utils.cpp @@ -17,7 +17,11 @@ #include #include #include +#include #include +#include +#include +#include #include #include @@ -351,5 +355,27 @@ bool isPrivileged() { return true; } +void write_to_file(std::string data, const std::filesystem::path file) { + villas::Log::get("Filewriter")->debug("{} > {}", data, file.u8string()); + std::ofstream outputFile(file.u8string()); + + if (outputFile.is_open()) { + outputFile << data; + outputFile.close(); + } else { + throw std::filesystem::filesystem_error("Cannot open outputfile", + std::error_code()); + } +} + +std::vector +read_names_in_directory(const std::filesystem::path &directory) { + std::vector names; + for (auto const &dir_entry : std::filesystem::directory_iterator{directory}) { + names.push_back(dir_entry.path().filename()); + } + return names; +} + } // namespace utils } // namespace villas