From 3dc85ae009fe0a67722b92fe1dabd6e251039531 Mon Sep 17 00:00:00 2001 From: bennahugo Date: Tue, 23 Nov 2021 08:00:06 +0200 Subject: [PATCH 01/12] Fixes #801 casarest compilation failures --- Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 612d03c1..243841f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -124,10 +124,11 @@ RUN makems WSRT_makems.cfg ## BUILD CASArest from source ##################################################################### WORKDIR /src -RUN git clone https://github.com/casacore/casarest -WORKDIR /src/casarest +RUN wget https://github.com/casacore/casarest/archive/v1.7.0.tar.gz +RUN tar xvf v1.7.0.tar.gz +WORKDIR /src/casarest-1.7.0 RUN mkdir -p build -WORKDIR /src/casarest/build +WORKDIR /src/casarest-1.7.0/build RUN cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release ../ RUN make -j 4 RUN make install @@ -168,7 +169,7 @@ RUN python3 -m pip install . # Cattery WORKDIR /src RUN wget https://github.com/ska-sa/meqtrees-cattery/archive/v1.7.0.tar.gz -RUN tar -xvf v1.7.0.tar.gz +RUN tar -xvf v1.7.0.tar.gz.1 WORKDIR /src/meqtrees-cattery-1.7.0 RUN python3 -m pip install . From f80f5ed176634fa40df5336ef7027123cba8995c Mon Sep 17 00:00:00 2001 From: Benjamin Hugo Date: Wed, 20 Mar 2024 16:36:27 +0200 Subject: [PATCH 02/12] Fix readme parsing errors and toml specifier --- README.rst | 30 ++++++++++++++---------------- pyproject.toml | 2 +- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/README.rst b/README.rst index e4e2a217..ecf2e31e 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. (Users / Recommended - Docker based) Run via. Stimela >= 0.2.9 -========================================================== +=============================================================== We recommend running the imaging package through the Stimela framework , built on a widely supported containerization framework, called Docker. This package is on PiPY and and is purely python-based, requiring no dependencies other than Docker. It gives the user instantanious access to other commonly used packages such as Meqtrees, CASA, etc. @@ -107,12 +107,12 @@ in an empty new virtual environment.** deactivate (Users/Optional) FitsBeam, Montblanc, Moresane, Killms support -========================================================== +================================================================ Optional requirements like the FITS beam can be installed by specifying them in brackets. E.g. pip install "/src/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support,kms-support]" (Users/Troubleshooting) Configure max shared memory -========================================================== +================================================================ Running DDFacet on large images requires a lot of shared memory. Most systems limit the amount of shared memory to about 10%. To increase this limit add the following line to your ``/etc/default/tmpfs`` file:: SHM_SIZE=100% @@ -137,21 +137,20 @@ cmake.define = {ENABLE_NATIVE_TUNING = "ON", ENABLE_FAST_MATH = "ON", ENABLE_PYT (Developers/Recommended): setting up your dev environment ========================================================== -**NOTE:Setup your virtual environment just as specified in the user section above. Ensure you activate! - WARNING: you may encounter issues if you have previously installed numpy in the environment - build isolation - will fail if the numpy you have installed is older than the build system pulls during isolation. - You may need to use --no-build-isolation when installing or, even better, ensure that you install DDF and KillMS - in an empty new virtual environment. -** +**NOTE:Setup your virtual environment just as specified in the user section above. Ensure you activate!** +**WARNING: you may encounter issues if you have previously installed numpy in the environment - build isolation** +**will fail if the numpy you have installed is older than the build system pulls during isolation.** +**You may need to use --no-build-isolation when installing or, even better, ensure that you install DDF and KillMS** +**in an empty new virtual environment.** To setup your local development environment navigate clone DDFacet and run:: (ddfvenv) $ git clone https://github.com/cyriltasse/DDFacet (ddfvenv) $ pip install -e DDFacet/ -**IMPORTANT NOTE: You may need to remove the development version before running PIP when installing. If you -are switching between release and debug versions of the backend -- or recompiling in a different configuration -- -you should remove the DDFacet/DDFacet/cbuild directory and everything in it** +**IMPORTANT NOTE: You may need to remove the development version before running PIP when installing. If you** +**are switching between release and debug versions of the backend -- or recompiling in a different configuration -- ** +**you should remove the DDFacet/DDFacet/cbuild directory and everything in it** Note that Python3.8 support is deprecated and editable installation is only tested to work on Python 3.10. @@ -170,13 +169,12 @@ Note that Python3.8 support is deprecated and editable installation is only test **Important: if you ran ``git submodule update --init --recursive`` before you may need to remove the cached SkyModel before building the docker image with ``git rm --cached SkyModel``** (Developers/Debugging) Build a few libraries (by hand with custom flags) -========================================================== +========================================================================= You can build against custom versions of libraries such is libPython and custom numpy versions. To do this modify pyproject.toml. Find and modify the following lines:: ``` cmake.build-type = "ReleaseWithDebugSymbols" # can be set to Debug e.g. -cmake.define = {ENABLE_NATIVE_TUNING = "OFF", ENABLE_FAST_MATH = "ON", ENABLE_PYTHON_2 = "OFF", ENABLE_PYTHON_3 = "ON"} # can be tuned to enable processor - # specific marching +cmake.define = {ENABLE_NATIVE_TUNING = "OFF", ENABLE_FAST_MATH = "ON", ENABLE_PYTHON_2 = "OFF", ENABLE_PYTHON_3 = "ON"} # can be tuned to enable processor specific marching ``` You can also specify path settings for other libraries if you have custom built, e.g. numpy through these ```cmake.define``` @@ -190,7 +188,7 @@ Add this to your ``.bashrc``:: export DDFACET_TEST_OUTPUT_DIR=[folder where you want the acceptance test output to be dumped] To test your branch against the master branch using Jenkins ---------------------------------------------------------- +------------------------------------------------------------ Most of the core use cases will in the nearby future have reference images and an automated acceptance test. Please **do not** commit against cyriltasse/master. The correct strategy is to branch/fork and do a pull request on Github diff --git a/pyproject.toml b/pyproject.toml index c363df2b..b5dcc4c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ authors = [{ name = "Cyril Tasse", email = "cyril.tasse@obspm.fr" }] maintainers = [{ name = "Benjamin Hugo", email = "bhugo@sarao.ac.za" }] urls = { Homepage = "http://github.com/saopicc/DDFacet"} classifiers = [ - "Development Status :: 5 - Production / Stable", + "Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: Science/Research", "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", From 0fff879d8b135b301c619a5ffa56b0777fb446a1 Mon Sep 17 00:00:00 2001 From: Martin Hardcastle Date: Wed, 26 Jun 2024 15:31:59 +0100 Subject: [PATCH 03/12] Skip weight calculation if flagged to do so --- DDFacet/Data/ClassVisServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DDFacet/Data/ClassVisServer.py b/DDFacet/Data/ClassVisServer.py index d7035403..e6d9ad74 100644 --- a/DDFacet/Data/ClassVisServer.py +++ b/DDFacet/Data/ClassVisServer.py @@ -593,7 +593,7 @@ def _handler_LoadVisChunk(self, dictname, iMS, iChunk): print(ModColor.Str("This chunk is all flagged or has zero weight."), file=log) return - if DATA["sort_index"] is not None: # and DATA["Weights"] is not 1: # OMS 2023/12 they're not "1" ever and this seems a bug + if DATA["sort_index"] is not None and DATA["Weights"] is not 1: # '1' is used as a flag by GetVisWeights DATA["Weights"] = DATA["Weights"][DATA["sort_index"]] self.computeBDAInBackground(dictname, ms, DATA, From 9fc1707a6c123a7b4a208763c8a7ce850c7900e2 Mon Sep 17 00:00:00 2001 From: Martin Hardcastle Date: Wed, 26 Jun 2024 15:33:27 +0100 Subject: [PATCH 04/12] Skip weight calculation if flagged to do so --- DDFacet/Data/ClassVisServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DDFacet/Data/ClassVisServer.py b/DDFacet/Data/ClassVisServer.py index e6d9ad74..29b922ee 100644 --- a/DDFacet/Data/ClassVisServer.py +++ b/DDFacet/Data/ClassVisServer.py @@ -593,7 +593,7 @@ def _handler_LoadVisChunk(self, dictname, iMS, iChunk): print(ModColor.Str("This chunk is all flagged or has zero weight."), file=log) return - if DATA["sort_index"] is not None and DATA["Weights"] is not 1: # '1' is used as a flag by GetVisWeights + if DATA["sort_index"] is not None and DATA["Weights"]!=1: # '1' is used as a flag by GetVisWeights DATA["Weights"] = DATA["Weights"][DATA["sort_index"]] self.computeBDAInBackground(dictname, ms, DATA, From d4f6eca6a7cd72a21fdd95fdbed4141b76c97671 Mon Sep 17 00:00:00 2001 From: Martin Hardcastle Date: Thu, 10 Oct 2024 15:40:13 +0100 Subject: [PATCH 05/12] Second attempt at fixing weight logic --- DDFacet/Data/ClassVisServer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DDFacet/Data/ClassVisServer.py b/DDFacet/Data/ClassVisServer.py index 29b922ee..e239f2d6 100644 --- a/DDFacet/Data/ClassVisServer.py +++ b/DDFacet/Data/ClassVisServer.py @@ -593,7 +593,8 @@ def _handler_LoadVisChunk(self, dictname, iMS, iChunk): print(ModColor.Str("This chunk is all flagged or has zero weight."), file=log) return - if DATA["sort_index"] is not None and DATA["Weights"]!=1: # '1' is used as a flag by GetVisWeights + if DATA["sort_index"] is not None and isinstance(DATA["Weights"],np.array): + # a value of '1' is used as a flag by GetVisWeights, but we can't compare with 1 because the weights could be and generally will be an array... DATA["Weights"] = DATA["Weights"][DATA["sort_index"]] self.computeBDAInBackground(dictname, ms, DATA, From 7dbe59183edc6bf5a674a070ca96e07b15c46349 Mon Sep 17 00:00:00 2001 From: Martin Hardcastle Date: Thu, 10 Oct 2024 16:20:31 +0100 Subject: [PATCH 06/12] a vital two letters --- DDFacet/Data/ClassVisServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DDFacet/Data/ClassVisServer.py b/DDFacet/Data/ClassVisServer.py index e239f2d6..ceb71fb5 100644 --- a/DDFacet/Data/ClassVisServer.py +++ b/DDFacet/Data/ClassVisServer.py @@ -593,7 +593,7 @@ def _handler_LoadVisChunk(self, dictname, iMS, iChunk): print(ModColor.Str("This chunk is all flagged or has zero weight."), file=log) return - if DATA["sort_index"] is not None and isinstance(DATA["Weights"],np.array): + if DATA["sort_index"] is not None and isinstance(DATA["Weights"],np.ndarray): # a value of '1' is used as a flag by GetVisWeights, but we can't compare with 1 because the weights could be and generally will be an array... DATA["Weights"] = DATA["Weights"][DATA["sort_index"]] From 6da6600c6d81f998caf7ac50d4994a2885e0e7e6 Mon Sep 17 00:00:00 2001 From: Martin Hardcastle Date: Sun, 20 Oct 2024 21:08:54 +0100 Subject: [PATCH 07/12] Remove pybdsf version fix --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b5dcc4c4..2fbb9d41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ classifiers = [ dft-support = ["montblanc >= 0.6.1, <= 0.7.3.1; python_version >= '3.8' and python_version < '3.11'"] moresane-support = ["pymoresane >= 0.3.0; python_version >= '3.8' and python_version < '3.11'"] fits-beam-support = ["meqtrees-cattery <= 1.7.9; python_version >= '3.8' and python_version < '3.11'"] -kms-support = ["bdsf > 1.8.15,<=1.10.1; python_version >= '3.8' and python_version < '3.11'"] +kms-support = ["bdsf > 1.8.15; python_version >= '3.8' and python_version < '3.11'"] alternate-data-backends = ["dask-ms[xarray]<=0.2.20; python_version >= '3.8' and python_version < '3.11'", "xarray<=2023.12.0; python_version >= '3.8' and python_version < '3.11'"] testing-requirements = ["nose >= 1.3.7; python_version >= '3' and python_version < '3.9'", From 1ec4cd97eadf4febee332bc7faafec958bc4a781 Mon Sep 17 00:00:00 2001 From: Martin Hardcastle Date: Sun, 24 Nov 2024 10:58:39 +0000 Subject: [PATCH 08/12] Attempt at affinity-aware APP --- DDFacet/Other/AsyncProcessPool.py | 75 ++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/DDFacet/Other/AsyncProcessPool.py b/DDFacet/Other/AsyncProcessPool.py index 3876a39f..3862aa63 100644 --- a/DDFacet/Other/AsyncProcessPool.py +++ b/DDFacet/Other/AsyncProcessPool.py @@ -166,6 +166,11 @@ def __init__ (self): self._events = {} self._results_map = {} self._job_counters = JobCounterPool() + # record these on first call + self._cpucount = psutil.cpu_count() + self._process = psutil.Process() + self.inherited_affinity = self._process.cpu_affinity() + self.available_cores = len(self.inherited_affinity) def __del__(self): self.shutdown() @@ -185,27 +190,43 @@ def init(self, ncpu=None, affinity=None, parent_affinity=0, num_io_processes=1, Returns: """ + # Take 0 to disable affinity + if affinity == 0: + print(ModColor.Str("Affinity 0 requested, interpreting as disable"),file=log) + affinity = None + self.affinity = affinity self.verbose = verbose self.pause_on_start = pause_on_start if isinstance(self.affinity, int): - self.cpustep = abs(self.affinity) or 1 - maxcpu = psutil.cpu_count() // self.cpustep - self.ncpu = ncpu or maxcpu - self.parent_affinity = parent_affinity + # check whether we are in an environment (e.g. Slurm) where affinity is + # already set. If we are, ignore arguments and DTRT + if self.available_cores < self._cpucount: + print(ModColor.Str("Warning: inherited affinity is for %d CPUs out of %d only" % (self.available_cores,self._cpucount)), file=log) + if ncpu and ncpu>self.available_cores: + raise RuntimeError("NCPU requested is %d but only %d threads are available" % (ncpu,self.available_cores)) + self.ncpu = ncpu or self.available_cores + maxcpu = self.ncpu + self.affinity = self.inherited_affinity[:self.ncpu] + self.parent_affinity = self.inherited_affinity[0] + else: + # Assume we have all of the machine available + self.cpustep = abs(self.affinity) or 1 + maxcpu = self._cpucount // self.cpustep + self.ncpu = ncpu or maxcpu + self.parent_affinity = parent_affinity elif isinstance(self.affinity, list): if any(map(lambda x: x < 0, self.affinity)): raise RuntimeError("Affinities must be list of positive numbers") - if psutil.cpu_count() < max(self.affinity): - raise RuntimeError("There are %d virtual threads on this system. Some elements of the affinity map are " - "higher than this. Check parset." % psutil.cpu_count()) - self.ncpu = ncpu or len(self.affinity) - if self.ncpu != len(self.affinity): - print(ModColor.Str("Warning: NCPU does not match affinity list length. Falling back to " - "NCPU=%d" % len(self.affinity)), file=log) - self.ncpu = self.ncpu if self.ncpu == len(self.affinity) else len(self.affinity) + if set(self.affinity)<=set(self.inherited_affinity): + if self.ncpu != len(self.affinity): + print(ModColor.Str("Warning: NCPU does not match affinity list length. Falling back to " + "NCPU=%d" % len(self.affinity)), file=log) + self.ncpu=len(self.affinity) + else: + raise RuntimeError("Requested affinity %s is not a subset of available affinity %s" % (str(self.affinity),str(self.inherited_affinity))) maxcpu = max(self.affinity) + 1 # zero indexed list self.parent_affinity = parent_affinity elif isinstance(self.affinity, str) and str(self.affinity) == "enable_ht": @@ -260,11 +281,11 @@ def init(self, ncpu=None, affinity=None, parent_affinity=0, num_io_processes=1, self.parent_affinity = 0 # none unused (HT is probably disabled BIOS level) else: self.parent_affinity = unused[0] # grab the first unused vthread - elif isinstance(self.affinity, str) and str(self.affinity) == "disable": + elif self.affinity is None or isinstance(self.affinity, str) and str(self.affinity) == "disable": self.affinity = None self.parent_affinity = None self.cpustep = 1 - maxcpu = psutil.cpu_count() + maxcpu = len(self.inherited_affinity) self.ncpu = ncpu or maxcpu else: raise RuntimeError("Invalid option for Parallel.Affinity. Expected cpu step (int), list, " @@ -272,8 +293,10 @@ def init(self, ncpu=None, affinity=None, parent_affinity=0, num_io_processes=1, if self.parent_affinity is None: print("Parent and I/O affinities not specified, leaving unset", file=log) else: + if parent_affinity not in self.inherited_affinity: + raise RuntimeError("Parent affinity requested (%d) is not in available list of cores" % parent_affinity) print(ModColor.Str("Fixing parent process to vthread %d" % self.parent_affinity, col="green"), file=log) - psutil.Process().cpu_affinity(range(self.ncpu) if not self.parent_affinity else [self.parent_affinity]) + self._process.cpu_affinity([self.parent_affinity]) # if NCPU is 0, set to number of CPUs on system if not self.ncpu: @@ -440,7 +463,7 @@ def _startBulba (self): proc_id = "comp%02d" % i self._compute_workers.append( multiprocessing.Process(name=proc_id, target=self._start_worker, - args=(self, proc_id, [core], self._compute_queue, + args=(self, proc_id, [core] if self.affinity else None, self._compute_queue, self.pause_on_start))) for i, queue in enumerate(self._io_queues): proc_id = "io%02d" % i @@ -803,13 +826,13 @@ def shutdown(self): print("shutdown complete", file=log) @staticmethod - def _start_worker (object, proc_id, affinity, worker_queue, pause_on_start=False): + def _start_worker (self, proc_id, affinity, worker_queue, pause_on_start=False): """ - Helper method for worker process startup. ets up affinity, and calls _run_worker method on - object with the specified work queue. + Helper method for worker process startup. sets up affinity, and calls _run_worker method on + self with the specified work queue. Args: - object: + self: proc_id: affinity: work_queue: @@ -824,10 +847,12 @@ def _start_worker (object, proc_id, affinity, worker_queue, pause_on_start=False _pyArrays.pySetOMPDynamicNumThreads(1) AsyncProcessPool.proc_id = proc_id logger.subprocess_id = proc_id - if affinity: - psutil.Process().cpu_affinity(affinity) - object._run_worker(worker_queue) - if object.verbose: + if self.affinity: # shouldn't mess with affinity if it was disabled + if self.verbose: + print(ModColor.Str("exiting worker pid %d"%os.getpid()), file=log) + psutils.Process().cpu_affinity(affinity) + self._run_worker(worker_queue) + if self.verbose: print(ModColor.Str("exiting worker pid %d"%os.getpid()), file=log) @@ -931,7 +956,7 @@ def _init_default(): global APP if APP is None: APP = AsyncProcessPool() - APP.init(psutil.cpu_count(), affinity=0, num_io_processes=1, verbose=0) + APP.init(len(APP.inherited_affinity), affinity=0, num_io_processes=1, verbose=0) _init_default() From 8d11bb3f059412f58c577512182b8256b8f4208d Mon Sep 17 00:00:00 2001 From: Martin Hardcastle Date: Sun, 24 Nov 2024 11:48:42 +0000 Subject: [PATCH 09/12] fix process start --- DDFacet/Other/AsyncProcessPool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DDFacet/Other/AsyncProcessPool.py b/DDFacet/Other/AsyncProcessPool.py index 3862aa63..e5359ac9 100644 --- a/DDFacet/Other/AsyncProcessPool.py +++ b/DDFacet/Other/AsyncProcessPool.py @@ -849,8 +849,8 @@ def _start_worker (self, proc_id, affinity, worker_queue, pause_on_start=False): logger.subprocess_id = proc_id if self.affinity: # shouldn't mess with affinity if it was disabled if self.verbose: - print(ModColor.Str("exiting worker pid %d"%os.getpid()), file=log) - psutils.Process().cpu_affinity(affinity) + print(ModColor.Str("setting worker pid %d affinity to %d"% (os.getpid(),affinity), file=log) + psutil.Process().cpu_affinity(affinity) self._run_worker(worker_queue) if self.verbose: print(ModColor.Str("exiting worker pid %d"%os.getpid()), file=log) From 395f9219e852a26b851b5b5caaa85b79277c7ab7 Mon Sep 17 00:00:00 2001 From: Martin Hardcastle Date: Sun, 24 Nov 2024 12:05:34 +0000 Subject: [PATCH 10/12] fix process start --- DDFacet/Other/AsyncProcessPool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DDFacet/Other/AsyncProcessPool.py b/DDFacet/Other/AsyncProcessPool.py index e5359ac9..e2c3ed20 100644 --- a/DDFacet/Other/AsyncProcessPool.py +++ b/DDFacet/Other/AsyncProcessPool.py @@ -849,7 +849,7 @@ def _start_worker (self, proc_id, affinity, worker_queue, pause_on_start=False): logger.subprocess_id = proc_id if self.affinity: # shouldn't mess with affinity if it was disabled if self.verbose: - print(ModColor.Str("setting worker pid %d affinity to %d"% (os.getpid(),affinity), file=log) + print(ModColor.Str("setting worker pid %d affinity to %d"% (os.getpid(),affinity)), file=log) psutil.Process().cpu_affinity(affinity) self._run_worker(worker_queue) if self.verbose: From 4b54f0442c3e80b35b115e6160e56809b3c0a92a Mon Sep 17 00:00:00 2001 From: Martin Hardcastle Date: Sun, 24 Nov 2024 12:40:18 +0000 Subject: [PATCH 11/12] Fix parent and debug --- DDFacet/Other/AsyncProcessPool.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/DDFacet/Other/AsyncProcessPool.py b/DDFacet/Other/AsyncProcessPool.py index e2c3ed20..57648f14 100644 --- a/DDFacet/Other/AsyncProcessPool.py +++ b/DDFacet/Other/AsyncProcessPool.py @@ -210,13 +210,13 @@ def init(self, ncpu=None, affinity=None, parent_affinity=0, num_io_processes=1, self.ncpu = ncpu or self.available_cores maxcpu = self.ncpu self.affinity = self.inherited_affinity[:self.ncpu] - self.parent_affinity = self.inherited_affinity[0] + parent_affinity = self.inherited_affinity[0] else: # Assume we have all of the machine available self.cpustep = abs(self.affinity) or 1 maxcpu = self._cpucount // self.cpustep self.ncpu = ncpu or maxcpu - self.parent_affinity = parent_affinity + self.parent_affinity = parent_affinity elif isinstance(self.affinity, list): if any(map(lambda x: x < 0, self.affinity)): raise RuntimeError("Affinities must be list of positive numbers") @@ -463,7 +463,7 @@ def _startBulba (self): proc_id = "comp%02d" % i self._compute_workers.append( multiprocessing.Process(name=proc_id, target=self._start_worker, - args=(self, proc_id, [core] if self.affinity else None, self._compute_queue, + args=(self, proc_id, [core], self._compute_queue, self.pause_on_start))) for i, queue in enumerate(self._io_queues): proc_id = "io%02d" % i @@ -849,7 +849,7 @@ def _start_worker (self, proc_id, affinity, worker_queue, pause_on_start=False): logger.subprocess_id = proc_id if self.affinity: # shouldn't mess with affinity if it was disabled if self.verbose: - print(ModColor.Str("setting worker pid %d affinity to %d"% (os.getpid(),affinity)), file=log) + print(ModColor.Str("setting worker pid %d affinity to %s"% (os.getpid(),str(affinity))), file=log) psutil.Process().cpu_affinity(affinity) self._run_worker(worker_queue) if self.verbose: From 905e11dbb6689ea4e08cee1de633b7bccb57f9d5 Mon Sep 17 00:00:00 2001 From: Martin Hardcastle Date: Sun, 24 Nov 2024 13:12:00 +0000 Subject: [PATCH 12/12] More debug --- DDFacet/Other/AsyncProcessPool.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DDFacet/Other/AsyncProcessPool.py b/DDFacet/Other/AsyncProcessPool.py index 57648f14..5bde4c41 100644 --- a/DDFacet/Other/AsyncProcessPool.py +++ b/DDFacet/Other/AsyncProcessPool.py @@ -296,7 +296,7 @@ def init(self, ncpu=None, affinity=None, parent_affinity=0, num_io_processes=1, if parent_affinity not in self.inherited_affinity: raise RuntimeError("Parent affinity requested (%d) is not in available list of cores" % parent_affinity) print(ModColor.Str("Fixing parent process to vthread %d" % self.parent_affinity, col="green"), file=log) - self._process.cpu_affinity([self.parent_affinity]) + #self._process.cpu_affinity([self.parent_affinity]) # if NCPU is 0, set to number of CPUs on system if not self.ncpu: @@ -847,10 +847,12 @@ def _start_worker (self, proc_id, affinity, worker_queue, pause_on_start=False): _pyArrays.pySetOMPDynamicNumThreads(1) AsyncProcessPool.proc_id = proc_id logger.subprocess_id = proc_id - if self.affinity: # shouldn't mess with affinity if it was disabled + if affinity and self.affinity: # shouldn't mess with affinity if it was disabled if self.verbose: print(ModColor.Str("setting worker pid %d affinity to %s"% (os.getpid(),str(affinity))), file=log) psutil.Process().cpu_affinity(affinity) + if self.verbose: + print(ModColor.Str("worker pid %d affinity is now %s"% (os.getpid(),str(psutil.Process().cpu_affinity()))), file=log) self._run_worker(worker_queue) if self.verbose: print(ModColor.Str("exiting worker pid %d"%os.getpid()), file=log)