From 48a0989d6aac021ef06955e5d3c812fdb298534c Mon Sep 17 00:00:00 2001 From: noabauma Date: Wed, 2 Jul 2025 16:09:39 +0200 Subject: [PATCH 1/7] update ci yml files --- .github/workflows/master-test-workflow.yml | 2 +- .github/workflows/pr-development-workflow.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master-test-workflow.yml b/.github/workflows/master-test-workflow.yml index ee8e10f..d7caad9 100644 --- a/.github/workflows/master-test-workflow.yml +++ b/.github/workflows/master-test-workflow.yml @@ -41,7 +41,7 @@ jobs: submodules: 'true' - name: Setup - run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true + run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true - name: Compile run: source /home/hicr/.bashrc && meson compile -C build diff --git a/.github/workflows/pr-development-workflow.yml b/.github/workflows/pr-development-workflow.yml index 0cea673..d701f92 100644 --- a/.github/workflows/pr-development-workflow.yml +++ b/.github/workflows/pr-development-workflow.yml @@ -231,7 +231,7 @@ jobs: docker run --name taskr --shm-size=1024M --privileged -v $PWD:/home/hicr/taskr -w /home/hicr/taskr -td ${{ env.DOCKERIMAGE }}:${{ inputs.arch }}-latest bash - name: Setup - run: docker exec -u hicr taskr bash -c "meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true" + run: docker exec -u hicr taskr bash -c "meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true" - name: Compile run: docker exec -u hicr taskr bash -c "meson compile -C build" @@ -271,7 +271,7 @@ jobs: submodules: 'true' - name: Setup - run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true + run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true - name: Compile run: source /home/hicr/.bashrc && meson compile -C build From fc77df1cb2419d1b08e63c81105b874f1dfe9637 Mon Sep 17 00:00:00 2001 From: noabauma Date: Wed, 2 Jul 2025 17:05:45 +0200 Subject: [PATCH 2/7] adding the command for building pybind11 in the CI --- .github/workflows/master-test-workflow.yml | 4 ++++ .github/workflows/pr-development-workflow.yml | 3 +++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/master-test-workflow.yml b/.github/workflows/master-test-workflow.yml index d7caad9..1cc45e8 100644 --- a/.github/workflows/master-test-workflow.yml +++ b/.github/workflows/master-test-workflow.yml @@ -40,6 +40,10 @@ jobs: with: submodules: 'true' + - name: Build Pybind11 + run: cd extern/pybind11 && mkdir -p build && cd build && cmake .. && make check -j"$(nproc)" && mkdir -p mock_install/share/pkgconfig && echo -e "prefix=$(pwd)/..\nincludedir=\${prefix}/include\n\nName: pybind11\nDescription: Seamless operability between C++11 and Python\nVersion: 2.13.6\nCflags: -I\${includedir}" > mock_install/share/pkgconfig/pybind11.pc && echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/mock_install/share/pkgconfig" >> $GITHUB_ENV + + - name: Setup run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true diff --git a/.github/workflows/pr-development-workflow.yml b/.github/workflows/pr-development-workflow.yml index d701f92..674a97e 100644 --- a/.github/workflows/pr-development-workflow.yml +++ b/.github/workflows/pr-development-workflow.yml @@ -270,6 +270,9 @@ jobs: with: submodules: 'true' + - name: Build Pybind11 + run: cd extern/pybind11 && mkdir -p build && cd build && cmake .. && make check -j"$(nproc)" && mkdir -p mock_install/share/pkgconfig && echo -e "prefix=$(pwd)/..\nincludedir=\${prefix}/include\n\nName: pybind11\nDescription: Seamless operability between C++11 and Python\nVersion: 2.13.6\nCflags: -I\${includedir}" > mock_install/share/pkgconfig/pybind11.pc && echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/mock_install/share/pkgconfig" >> $GITHUB_ENV + - name: Setup run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true From 1b86113cc09cb2ad2b4c3673787ada6346d5fea3 Mon Sep 17 00:00:00 2001 From: noabauma Date: Wed, 2 Jul 2025 17:07:30 +0200 Subject: [PATCH 3/7] adding the command for building pybind11 in the CI --- .github/workflows/master-test-workflow.yml | 19 +++++++++++++++- .github/workflows/pr-development-workflow.yml | 22 +++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master-test-workflow.yml b/.github/workflows/master-test-workflow.yml index 1cc45e8..93a6f12 100644 --- a/.github/workflows/master-test-workflow.yml +++ b/.github/workflows/master-test-workflow.yml @@ -41,9 +41,26 @@ jobs: submodules: 'true' - name: Build Pybind11 - run: cd extern/pybind11 && mkdir -p build && cd build && cmake .. && make check -j"$(nproc)" && mkdir -p mock_install/share/pkgconfig && echo -e "prefix=$(pwd)/..\nincludedir=\${prefix}/include\n\nName: pybind11\nDescription: Seamless operability between C++11 and Python\nVersion: 2.13.6\nCflags: -I\${includedir}" > mock_install/share/pkgconfig/pybind11.pc && echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/mock_install/share/pkgconfig" >> $GITHUB_ENV + run: | + cd extern/pybind11 + mkdir -p build + cd build + cmake .. + make check -j"$(nproc)" + + mkdir -p mock_install/share/pkgconfig + cat < mock_install/share/pkgconfig/pybind11.pc + prefix=$(pwd)/.. + includedir=\${prefix}/include + Name: pybind11 + Description: Seamless operability between C++11 and Python + Version: 2.13.6 + Cflags: -I\${includedir} + EOF + echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/mock_install/share/pkgconfig" >> $GITHUB_ENV + - name: Setup run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true diff --git a/.github/workflows/pr-development-workflow.yml b/.github/workflows/pr-development-workflow.yml index 674a97e..426c0ff 100644 --- a/.github/workflows/pr-development-workflow.yml +++ b/.github/workflows/pr-development-workflow.yml @@ -271,8 +271,26 @@ jobs: submodules: 'true' - name: Build Pybind11 - run: cd extern/pybind11 && mkdir -p build && cd build && cmake .. && make check -j"$(nproc)" && mkdir -p mock_install/share/pkgconfig && echo -e "prefix=$(pwd)/..\nincludedir=\${prefix}/include\n\nName: pybind11\nDescription: Seamless operability between C++11 and Python\nVersion: 2.13.6\nCflags: -I\${includedir}" > mock_install/share/pkgconfig/pybind11.pc && echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/mock_install/share/pkgconfig" >> $GITHUB_ENV - + run: | + cd extern/pybind11 + mkdir -p build + cd build + cmake .. + make check -j"$(nproc)" + + mkdir -p mock_install/share/pkgconfig + cat < mock_install/share/pkgconfig/pybind11.pc + prefix=$(pwd)/.. + includedir=\${prefix}/include + + Name: pybind11 + Description: Seamless operability between C++11 and Python + Version: 2.13.6 + Cflags: -I\${includedir} + EOF + + echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/mock_install/share/pkgconfig" >> $GITHUB_ENV + - name: Setup run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true From d37d75c19b709c6c41e2bfafe6730a2e31262e16 Mon Sep 17 00:00:00 2001 From: noabauma Date: Thu, 3 Jul 2025 13:41:17 +0200 Subject: [PATCH 4/7] minor corrections --- .github/workflows/master-test-workflow.yml | 2 +- .github/workflows/pr-development-workflow.yml | 2 +- include/pytaskr/meson.build | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master-test-workflow.yml b/.github/workflows/master-test-workflow.yml index 93a6f12..8f63c2e 100644 --- a/.github/workflows/master-test-workflow.yml +++ b/.github/workflows/master-test-workflow.yml @@ -45,7 +45,7 @@ jobs: cd extern/pybind11 mkdir -p build cd build - cmake .. + cmake .. -DPython_EXECUTABLE=/usr/bin/python3 make check -j"$(nproc)" mkdir -p mock_install/share/pkgconfig diff --git a/.github/workflows/pr-development-workflow.yml b/.github/workflows/pr-development-workflow.yml index 426c0ff..cb6d960 100644 --- a/.github/workflows/pr-development-workflow.yml +++ b/.github/workflows/pr-development-workflow.yml @@ -275,7 +275,7 @@ jobs: cd extern/pybind11 mkdir -p build cd build - cmake .. + cmake .. -DPython_EXECUTABLE=/usr/bin/python3 make check -j"$(nproc)" mkdir -p mock_install/share/pkgconfig diff --git a/include/pytaskr/meson.build b/include/pytaskr/meson.build index 905bc9e..857bda5 100644 --- a/include/pytaskr/meson.build +++ b/include/pytaskr/meson.build @@ -2,7 +2,10 @@ # Manually compile like this: # c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3-config --includes) -Iextern/pybind11/include example.cpp -o example$(python3-config --extension-suffix) -py = import('python').find_installation(pure: false) +py = import('python').find_installation( + python_executable: '/usr/bin/python3', + pure: false +) pybind11_dep = dependency('pybind11', required: true) py.extension_module('taskr', From ae5a35e65a8d4852d0c2c88df76f79073aa112c4 Mon Sep 17 00:00:00 2001 From: noabauma Date: Mon, 7 Jul 2025 14:48:06 +0200 Subject: [PATCH 5/7] meson updated the yml codes --- .github/workflows/master-test-workflow.yml | 2 ++ .github/workflows/pr-development-workflow.yml | 2 ++ examples/matmul/python/matmul.cpp | 14 +++++++++++++- examples/matmul/python/matmul.py | 4 +--- examples/simple/python/main.py | 8 +++++--- examples/simple/python/simple.py | 3 +-- include/pytaskr/meson.build | 12 ++++++++---- include/pytaskr/pytaskr.cpp | 6 +++--- 8 files changed, 35 insertions(+), 16 deletions(-) diff --git a/.github/workflows/master-test-workflow.yml b/.github/workflows/master-test-workflow.yml index 8f63c2e..17e8b86 100644 --- a/.github/workflows/master-test-workflow.yml +++ b/.github/workflows/master-test-workflow.yml @@ -60,6 +60,8 @@ jobs: EOF echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/mock_install/share/pkgconfig" >> $GITHUB_ENV + + export LD_PRELOAD=/usr/local/lib/libnosv.so - name: Setup run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true diff --git a/.github/workflows/pr-development-workflow.yml b/.github/workflows/pr-development-workflow.yml index cb6d960..b85252f 100644 --- a/.github/workflows/pr-development-workflow.yml +++ b/.github/workflows/pr-development-workflow.yml @@ -290,6 +290,8 @@ jobs: EOF echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$(pwd)/mock_install/share/pkgconfig" >> $GITHUB_ENV + + export LD_PRELOAD=/usr/local/lib/libnosv.so - name: Setup run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngine=mpi -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true diff --git a/examples/matmul/python/matmul.cpp b/examples/matmul/python/matmul.cpp index ccc17c6..2ef0c04 100644 --- a/examples/matmul/python/matmul.cpp +++ b/examples/matmul/python/matmul.cpp @@ -19,6 +19,10 @@ #include +#ifdef ENABLE_INSTRUMENTATION + #include +#endif + #define mytype float /** @@ -26,7 +30,11 @@ */ void matmul(taskr::Task *) { - const size_t N = 1000; +#ifdef ENABLE_INSTRUMENTATION + INSTRUMENTATION_VMARKER_SET(MARK_COLOR_RED); +#endif + + const size_t N = 200; // Allocate memory volatile mytype *A = (mytype *)calloc(1, N * N * sizeof(mytype)); @@ -56,6 +64,10 @@ void matmul(taskr::Task *) free((mytype *)A); free((mytype *)B); free((mytype *)C); + +#ifdef ENABLE_INSTRUMENTATION + INSTRUMENTATION_VMARKER_RESET(); +#endif } PYBIND11_MODULE(cpp_matmul, m) diff --git a/examples/matmul/python/matmul.py b/examples/matmul/python/matmul.py index daf8ebb..3af8722 100644 --- a/examples/matmul/python/matmul.py +++ b/examples/matmul/python/matmul.py @@ -45,15 +45,13 @@ def matmul_cpp_Driver(runtime): - - def matmul_numpy_Driver(runtime): # Initializing taskr runtime.initialize() def matmul_numpy(task): N = 1000 - A = np.zeros((N,N)) + A = np.empty((N,N)) B = np.empty((N,N)) C = np.empty((N,N)) diff --git a/examples/simple/python/main.py b/examples/simple/python/main.py index 63efd91..696f2ec 100644 --- a/examples/simple/python/main.py +++ b/examples/simple/python/main.py @@ -13,13 +13,15 @@ See the License for the specific language governing permissions and limitations under the License. """ - import taskr + import simple +NWORKERS = 4 + def main(): - # Initialize taskr with the wanted compute manager backend and number of PUs - t = taskr.taskr(taskr.HiCRBackend.nosv, 2) + # Initialize taskr with the wanted HiCR backend and number of Workers + t = taskr.taskr(backend=taskr.HiCRBackend.nosv, num_workers=NWORKERS) # Get the runtime runtime = t.get_runtime() diff --git a/examples/simple/python/simple.py b/examples/simple/python/simple.py index bcf9ef9..84f283a 100644 --- a/examples/simple/python/simple.py +++ b/examples/simple/python/simple.py @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. """ - import taskr NTASKS = 2 @@ -22,8 +21,8 @@ def simple(runtime): # Initializing taskr runtime.initialize() + # Create tasks fc = lambda task : print(f"Hello, I am task {task.getLabel()}") - taskfc = taskr.Function(fc) # Adding to tasks to taskr diff --git a/include/pytaskr/meson.build b/include/pytaskr/meson.build index 857bda5..eefc64c 100644 --- a/include/pytaskr/meson.build +++ b/include/pytaskr/meson.build @@ -2,10 +2,14 @@ # Manually compile like this: # c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3-config --includes) -Iextern/pybind11/include example.cpp -o example$(python3-config --extension-suffix) -py = import('python').find_installation( - python_executable: '/usr/bin/python3', - pure: false -) +python_mod = import('python') + +py = python_mod.find_installation(pure: false) + +if not py.found() + py = python_mod.find_installation('/usr/bin/python3', pure: false) +endif + pybind11_dep = dependency('pybind11', required: true) py.extension_module('taskr', diff --git a/include/pytaskr/pytaskr.cpp b/include/pytaskr/pytaskr.cpp index 122e72a..cd229c5 100644 --- a/include/pytaskr/pytaskr.cpp +++ b/include/pytaskr/pytaskr.cpp @@ -48,7 +48,7 @@ PYBIND11_MODULE(taskr, m) .def("setServiceWorkerCallbackHandler", &Runtime::setServiceWorkerCallbackHandler) .def("setTaskWorkerCallbackHandler", &Runtime::setTaskWorkerCallbackHandler) .def("initialize", &Runtime::initialize) - .def("addTask", &Runtime::addTask, py::keep_alive<1, 2>()) // keep_alive as the task should be alive until runtime's destructor + .def("addTask", &Runtime::addTask, py::keep_alive<1, 2>(), py::arg("task")) // keep_alive as the task should be alive until runtime's destructor .def("resumeTask", &Runtime::resumeTask) .def("run", &Runtime::run, py::call_guard()) .def("await_", &Runtime::await, py::call_guard()) // Release GIL is important otherwise non-finished tasks are getting blocked @@ -57,12 +57,12 @@ PYBIND11_MODULE(taskr, m) .def("addService", &Runtime::addService); // TaskR's Function class - py::class_(m, "Function").def(py::init()); + py::class_(m, "Function").def(py::init(), py::arg("fc")); // TaskR's Task class py::class_(m, "Task") .def(py::init(), py::arg("fc"), py::arg("workerAffinity") = -1) - .def(py::init(), py::arg("label"), py::arg("fc"), py::arg("workerAffinity") = -1) + .def(py::init(), py::arg("label"), py::arg("taskfc"), py::arg("workerAffinity") = -1) .def("getLabel", &Task::getLabel) .def("setLabel", &Task::setLabel) .def("getWorkerAffinity", &Task::getWorkerAffinity) From b9d45a4d7e348fda6e8fe0ef30e8ddf0ac0977a2 Mon Sep 17 00:00:00 2001 From: noabauma Date: Mon, 7 Jul 2025 14:54:08 +0200 Subject: [PATCH 6/7] making the numpy example optional (easier for the CI's) --- examples/matmul/python/main.py | 2 +- examples/matmul/python/matmul.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/matmul/python/main.py b/examples/matmul/python/main.py index 1906da7..88d987d 100644 --- a/examples/matmul/python/main.py +++ b/examples/matmul/python/main.py @@ -27,7 +27,7 @@ def main(): # Running matmul example matmul_cpp_Driver(runtime) - matmul_numpy_Driver(runtime) + # matmul_numpy_Driver(runtime) if __name__ == "__main__": diff --git a/examples/matmul/python/matmul.py b/examples/matmul/python/matmul.py index 3af8722..cfdf2c3 100644 --- a/examples/matmul/python/matmul.py +++ b/examples/matmul/python/matmul.py @@ -15,7 +15,6 @@ """ import time -import numpy as np import taskr import cpp_matmul @@ -46,6 +45,7 @@ def matmul_cpp_Driver(runtime): def matmul_numpy_Driver(runtime): + import numpy as np # Initializing taskr runtime.initialize() From 5f67b87e38bd1810674eb6a812de6de7a204dec8 Mon Sep 17 00:00:00 2001 From: noabauma Date: Mon, 7 Jul 2025 15:12:12 +0200 Subject: [PATCH 7/7] adding pytest installation to the CI command --- .github/workflows/master-test-workflow.yml | 2 ++ .github/workflows/pr-development-workflow.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/master-test-workflow.yml b/.github/workflows/master-test-workflow.yml index 17e8b86..30aa7fc 100644 --- a/.github/workflows/master-test-workflow.yml +++ b/.github/workflows/master-test-workflow.yml @@ -42,6 +42,8 @@ jobs: - name: Build Pybind11 run: | + /usr/bin/python3 -m pip install --upgrade pip pytest + cd extern/pybind11 mkdir -p build cd build diff --git a/.github/workflows/pr-development-workflow.yml b/.github/workflows/pr-development-workflow.yml index b85252f..572f663 100644 --- a/.github/workflows/pr-development-workflow.yml +++ b/.github/workflows/pr-development-workflow.yml @@ -272,6 +272,8 @@ jobs: - name: Build Pybind11 run: | + /usr/bin/python3 -m pip install --upgrade pip pytest + cd extern/pybind11 mkdir -p build cd build