Merge pull request #14 from gjbex/development

gjbex · web-flow · commit c965d58cb655 · 2025-04-23T16:57:33.000+02:00
Development
diff --git a/environment.yml b/environment.yml
@@ -17,4 +17,5 @@ dependencies:
   - openssl
   - fastapi
   - fire
+  - hydra-core
 prefix: /home/gjb/mambaforge/envs/python_for_systems_programming
diff --git a/hands-on/shell_interaction.ipynb b/hands-on/shell_interaction.ipynb
@@ -51,20 +51,13 @@
     "sh.ls('-l')"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The output can be used by assigning the command to a variable, and using the result's `stdout` attribute.  Note that the latter is a sequence of bytes, so it has to be decoded into a UTF-8 string for further processing."
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "cmd = sh.ls('-l', '-a', _encoding='UTF-8')"
+    "result = sh.ls('-l', '-a')"
    ]
   },
   {
@@ -73,7 +66,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "lines = cmd.stdout.decode(encoding='utf8').split('\\n')"
+    "lines = result.split('\\n')"
    ]
   },
   {
@@ -109,7 +102,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sh.ls()"
+    "for file in (line.strip() for line in sh.ls().split()):\n",
+    "    print(file)"
    ]
   },
   {
@@ -179,7 +173,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sh.cat('tmp/date_file.txt')"
+    "print(sh.cat('tmp/date_file.txt'))"
    ]
   },
   {
@@ -228,14 +222,14 @@
    },
    "outputs": [],
    "source": [
-    "sh.grep(sh.ls('-l'), r'\\.ipynb$')"
+    "print(sh.grep('-e', r'\\.ipynb$', _in=sh.ls('-l')))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Pipe the output of `cut` into `sort`."
+    "Pipe the output of `cut` into `sort`.  Also use the `_iter` argument to create a generator over standard output."
    ]
   },
   {
@@ -246,7 +240,8 @@
    },
    "outputs": [],
    "source": [
-    "sh.sort(sh.cut('-d', ' ', '-f', '5', 'tmp/date_file.txt'), '-r')"
+    "for line in sh.sort('-r', _in=sh.cut('-d', ' ', '-f', '5', 'tmp/date_file.txt'), _iter=True):\n",
+    "    print(line.strip())"
    ]
   },
   {
@@ -315,10 +310,12 @@
    },
    "outputs": [],
    "source": [
+    "process = sh.sleep(10, _bg=True, _bg_exc=False, _timeout=3)\n",
     "try:\n",
-    "    process = sh.sleep(10, _bg=True, _timeout=3)\n",
-    "except TimeoutError as error:\n",
-    "    print(error)"
+    "    process.wait()\n",
+    "except sh.TimeoutException as error:\n",
+    "    print('process timed out')\n",
+    "    print(error.exit_code)"
    ]
   },
   {
@@ -578,7 +575,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "process = subprocess.run('ls *.py', stdout=subprocess.PIPE, encoding='utf8', shell=True)"
+    "process = subprocess.run('ls *.ipynb', stdout=subprocess.PIPE, encoding='utf8', shell=True)"
    ]
   },
   {
@@ -653,7 +650,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "scrolled": true
+   },
    "outputs": [],
    "source": [
     "process.returncode"
@@ -676,7 +675,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,
diff --git a/python_for_systems_programming.pptx b/python_for_systems_programming.pptx
diff --git a/python_for_systems_programming_linux64_conda_specs.txt b/python_for_systems_programming_linux64_conda_specs.txt
@@ -4,7 +4,7 @@
 @EXPLICIT
 https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2
 https://repo.anaconda.com/pkgs/main/linux-64/blas-1.0-openblas.conda
-https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.6.2-hbcca054_0.conda
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2
@@ -53,7 +53,7 @@ https://repo.anaconda.com/pkgs/main/linux-64/lz4-c-1.9.4-h6a678d5_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/mpg123-1.32.6-h59595ed_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/nspr-4.35-h27087fc_0.conda
-https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.5-hd590300_0.conda
+https://conda.anaconda.org/conda-forge/linux-64/openssl-3.1.6-h4ab18f5_0.conda
 https://repo.anaconda.com/pkgs/main/linux-64/pcre-8.45-h295c915_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/pixman-0.43.2-h59595ed_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/pthread-stubs-0.4-h36c2ea0_1001.tar.bz2
@@ -109,13 +109,14 @@ https://conda.anaconda.org/conda-forge/linux-64/xcb-util-renderutil-0.3.9-h166bd
 https://conda.anaconda.org/conda-forge/linux-64/xcb-util-wm-0.4.1-h516909a_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/xorg-libx11-1.8.4-h0b41bf4_0.conda
 https://repo.anaconda.com/pkgs/main/linux-64/zstd-1.5.5-hc292b87_0.conda
+https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2
 https://conda.anaconda.org/conda-forge/noarch/attrs-23.2.0-pyh71513ae_0.conda
 https://repo.anaconda.com/pkgs/main/noarch/backcall-0.2.0-pyhd3eb1b0_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/bcrypt-4.1.2-py311h46250e7_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/brotli-1.1.0-hd590300_1.conda
 https://conda.anaconda.org/conda-forge/linux-64/brotli-python-1.1.0-py311hb755f60_1.conda
 https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2
-https://conda.anaconda.org/conda-forge/noarch/certifi-2024.2.2-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/certifi-2024.6.2-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.3.2-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/click-8.1.7-unix_pyh707e725_0.conda
 https://conda.anaconda.org/conda-forge/noarch/cycler-0.12.1-pyhd8ed1ab_0.conda
@@ -225,6 +226,7 @@ https://conda.anaconda.org/conda-forge/linux-64/libsndfile-1.2.2-hc60ed4a_1.cond
 https://repo.anaconda.com/pkgs/main/linux-64/libwebp-1.2.4-h11a3e52_1.conda
 https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.1.7-pyhd8ed1ab_0.conda
 https://repo.anaconda.com/pkgs/main/linux-64/numpy-1.23.5-py311h5585df3_0.conda
+https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.0-hfec8fc6_2.conda
 https://conda.anaconda.org/conda-forge/noarch/overrides-7.7.0-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/pexpect-4.9.0-pyhd8ed1ab_0.conda
@@ -249,6 +251,7 @@ https://conda.anaconda.org/conda-forge/noarch/fqdn-1.5.1-pyhd8ed1ab_0.tar.bz2
 https://conda.anaconda.org/conda-forge/linux-64/gstreamer-1.22.0-h25f0c4b_2.conda
 https://conda.anaconda.org/conda-forge/linux-64/harfbuzz-6.0.0-h8e241bc_0.conda
 https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.5-pyhd8ed1ab_0.conda
+https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/importlib_metadata-7.1.0-hd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/jsonschema-specifications-2023.12.1-pyhd8ed1ab_0.conda
 https://conda.anaconda.org/conda-forge/noarch/jupyter_server_terminals-0.5.3-pyhd8ed1ab_0.conda
diff --git a/source-code/hydra/README.md b/source-code/hydra/README.md
@@ -6,6 +6,7 @@ in the context of scientific software.
 It support configuration file handling, command line arguments, logging,
 multiruns and so on.
 
+
 ## What is it?
 
 1. `gen_rand.py`: Python script to write random numbers to standard
@@ -14,23 +15,50 @@ multiruns and so on.
    1. `config.yaml`: configuration file with the defaults.
    1. `distr/gauss.yaml`: configuration file for the Gaussian distirubtion.
    1. `distr/uniform.yaml`: configuration file for the uniform distirubtion.
+   1. `file_config.yaml`: configuration file with the output file name.
+1. `debug.py`: Python script that simply prints the configuration settings for
+   debugging purposes.
+
 
 ## How to use it?
+
 Run with configuratino file settings:
 ```bash
 $ ./gen_rand.py
 ```
+
 To increase the number of random values:
 ```bash
-$ ./gen_rand.py 10
+$ ./gen_rand.py  n=10
 ```
 
 To use a uniform distribution:
 ```bash
-$ ./gen_rand.py distr=uniform
+$ ./gen_rand.py  distr=uniform
 ```
 
 To use a uniform distribution between -1 and 0:
 ```bash
-$ ./gen_rand.py distr=uniform distr.a=-1.0 distr.b=0.0
+$ ./gen_rand.py  distr=uniform  distr.a=-1.0  distr.b=0.0
+```
+
+To use a different configuration file:
+```bash
+$ ./gen_rand.py  -cn file_config.yaml
+``` 
+or
+```bash
+$ ./gen_rand.py  --config-name=file_config.yaml
+```
+
+To perform multiple runs with different parameter values:
+```bash
+$ ./gen_rand.py -m distr=uniform,gauss
+``` 
+Note that if multiple parameters are to be varied, the Cartesian product of
+the values will be used.
+
+To view the configuration settings, e.g., for debugging:
+```bash 
+$ ./gen_rand.py  --cfg=job  distr=uniform
 ```
diff --git a/source-code/hydra/conf/config.yaml b/source-code/hydra/conf/config.yaml
@@ -1,4 +1,5 @@
-verbose: False
 n: 1
+file: false
 defaults:
-    - distr: gauss
+  - distr: gauss
+  - _self_
diff --git a/source-code/hydra/conf/distr/gauss.yaml b/source-code/hydra/conf/distr/gauss.yaml
@@ -1,4 +1,3 @@
-distr:
-    name: gauss
-    mu: 0.0
-    sigma: 1.0
+name: gauss
+mu: 0.0
+sigma: 1.0
diff --git a/source-code/hydra/conf/distr/uniform.yaml b/source-code/hydra/conf/distr/uniform.yaml
@@ -1,4 +1,3 @@
-distr:
-    name: uniform
-    a: 0.0
-    b: 1.0
+name: uniform
+a: 0.0
+b: 1.0
diff --git a/source-code/hydra/conf/file_config.yaml b/source-code/hydra/conf/file_config.yaml
@@ -0,0 +1,8 @@
+hydra:
+  job:
+    chdir: true
+n: 1
+file: output.txt
+defaults:
+  - distr: gauss
+  - _self_
diff --git a/source-code/hydra/debug.py b/source-code/hydra/debug.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+from omegaconf import DictConfig, OmegaConf
+import hydra
+
+@hydra.main(version_base=None, config_path='conf/', config_name='config')
+def my_app(cfg):
+    print(OmegaConf.to_yaml(cfg))
+
+if __name__ == '__main__':
+    my_app()
diff --git a/source-code/hydra/gen_rand.py b/source-code/hydra/gen_rand.py
@@ -5,36 +5,34 @@
 import numpy as np
 import sys
 
-LOG = logging.getLogger(sys.argv[0])
 
-@hydra.main(config_path='conf/config.yaml')
+@hydra.main(version_base=None, config_path='conf/', config_name='config')
 def gen_rand(cfg):
-    if cfg.verbose:
-        print(cfg.pretty(), file=sys.stderr)
+    logger = logging.getLogger(__name__)
     if cfg.n <= 0:
-        LOG.error(f'negative number to generate {cfg.n}')
-        return 1
-    LOG.info(f'generating {cfg.n} random numbers')
-    LOG.info(f'using {cfg.distr.name} distribution')
+        logger.error(f'negative number to generate {cfg.n}')
+        sys.exit(1)
+    logger.info(f'generating {cfg.n} random numbers')
+    logger.info(f'using {cfg.distr.name} distribution')
     if cfg.distr.name == 'gauss':
-        LOG.info(f'mu={cfg.distr.mu}, sigma={cfg.distr.sigma}')
+        logger.info(f'mu={cfg.distr.mu}, sigma={cfg.distr.sigma}')
         if cfg.distr.sigma <= 0:
-            LOG.error(f'negative standard deviation {cfg.distr.sigma}')
-            return 1
+            logger.error(f'negative standard deviation {cfg.distr.sigma}')
+            sys.exit(1)
         numbers = np.random.normal(loc=cfg.distr.mu, scale=cfg.distr.sigma,
                                    size=(cfg.n,))
     elif cfg.distr.name == 'uniform':
-        LOG.info(f'a={cfg.distr.a}, b={cfg.distr.b}')
+        logger.info(f'a={cfg.distr.a}, b={cfg.distr.b}')
         if cfg.distr.a >= cfg.distr.b:
-            LOG.warning(f'lower bound exceed upper bound, '
+            logger.warning(f'lower bound exceed upper bound, '
                         f'{cfg.distr.a} >= {cfg.distr.b}')
         numbers = np.random.uniform(cfg.distr.a, cfg.distr.b, size=(cfg.n,))
-    LOG.info('starting output')
+    logger.info('starting output')
+    out = open(cfg.file, 'w') if cfg.file else sys.stdout
     for number in numbers:
-        print(number)
-    LOG.info('output done')
+        print(number, file=out)
+    logger.info('output done')
     return 0
 
 if __name__ == '__main__':
-    status = gen_rand()
-    sys.exit(status)
+    gen_rand()
diff --git a/source-code/processes/monitor.py b/source-code/processes/monitor.py

Original file line number	Diff line number	Diff line change
`@@ -51,20 +51,13 @@`
`51`	`51`	`"sh.ls('-l')"`
`52`	`52`	`]`
`53`	`53`	`},`
`54`		`- {`
`55`		`- "cell_type": "markdown",`
`56`		`- "metadata": {},`
`57`		`- "source": [`
`58`		- "The output can be used by assigning the command to a variable, and using the result's `stdout` attribute. Note that the latter is a sequence of bytes, so it has to be decoded into a UTF-8 string for further processing."
`59`		`- ]`
`60`		`- },`
`61`	`54`	`{`
`62`	`55`	`"cell_type": "code",`
`63`	`56`	`"execution_count": null,`
`64`	`57`	`"metadata": {},`
`65`	`58`	`"outputs": [],`
`66`	`59`	`"source": [`
`67`		`- "cmd = sh.ls('-l', '-a', _encoding='UTF-8')"`
	`60`	`+ "result = sh.ls('-l', '-a')"`
`68`	`61`	`]`
`69`	`62`	`},`
`70`	`63`	`{`
`@@ -73,7 +66,7 @@`
`73`	`66`	`"metadata": {},`
`74`	`67`	`"outputs": [],`
`75`	`68`	`"source": [`
`76`		`- "lines = cmd.stdout.decode(encoding='utf8').split('\\n')"`
	`69`	`+ "lines = result.split('\\n')"`
`77`	`70`	`]`
`78`	`71`	`},`
`79`	`72`	`{`
`@@ -109,7 +102,8 @@`
`109`	`102`	`"metadata": {},`
`110`	`103`	`"outputs": [],`
`111`	`104`	`"source": [`
`112`		`- "sh.ls()"`
	`105`	`+ "for file in (line.strip() for line in sh.ls().split()):\n",`
	`106`	`+ " print(file)"`
`113`	`107`	`]`
`114`	`108`	`},`
`115`	`109`	`{`
`@@ -179,7 +173,7 @@`
`179`	`173`	`"metadata": {},`
`180`	`174`	`"outputs": [],`
`181`	`175`	`"source": [`
`182`		`- "sh.cat('tmp/date_file.txt')"`
	`176`	`+ "print(sh.cat('tmp/date_file.txt'))"`
`183`	`177`	`]`
`184`	`178`	`},`
`185`	`179`	`{`
`@@ -228,14 +222,14 @@`
`228`	`222`	`},`
`229`	`223`	`"outputs": [],`
`230`	`224`	`"source": [`
`231`		`- "sh.grep(sh.ls('-l'), r'\\.ipynb$')"`
	`225`	`+ "print(sh.grep('-e', r'\\.ipynb$', _in=sh.ls('-l')))"`
`232`	`226`	`]`
`233`	`227`	`},`
`234`	`228`	`{`
`235`	`229`	`"cell_type": "markdown",`
`236`	`230`	`"metadata": {},`
`237`	`231`	`"source": [`
`238`		- "Pipe the output of `cut` into `sort`."
	`232`	+ "Pipe the output of `cut` into `sort`. Also use the `_iter` argument to create a generator over standard output."
`239`	`233`	`]`
`240`	`234`	`},`
`241`	`235`	`{`
`@@ -246,7 +240,8 @@`
`246`	`240`	`},`
`247`	`241`	`"outputs": [],`
`248`	`242`	`"source": [`
`249`		`- "sh.sort(sh.cut('-d', ' ', '-f', '5', 'tmp/date_file.txt'), '-r')"`
	`243`	`+ "for line in sh.sort('-r', _in=sh.cut('-d', ' ', '-f', '5', 'tmp/date_file.txt'), _iter=True):\n",`
	`244`	`+ " print(line.strip())"`
`250`	`245`	`]`
`251`	`246`	`},`
`252`	`247`	`{`
`@@ -315,10 +310,12 @@`
`315`	`310`	`},`
`316`	`311`	`"outputs": [],`
`317`	`312`	`"source": [`
	`313`	`+ "process = sh.sleep(10, _bg=True, _bg_exc=False, _timeout=3)\n",`
`318`	`314`	`"try:\n",`
`319`		`- " process = sh.sleep(10, _bg=True, _timeout=3)\n",`
`320`		`- "except TimeoutError as error:\n",`
`321`		`- " print(error)"`
	`315`	`+ " process.wait()\n",`
	`316`	`+ "except sh.TimeoutException as error:\n",`
	`317`	`+ " print('process timed out')\n",`
	`318`	`+ " print(error.exit_code)"`
`322`	`319`	`]`
`323`	`320`	`},`
`324`	`321`	`{`
`@@ -578,7 +575,7 @@`
`578`	`575`	`"metadata": {},`
`579`	`576`	`"outputs": [],`
`580`	`577`	`"source": [`
`581`		`- "process = subprocess.run('ls *.py', stdout=subprocess.PIPE, encoding='utf8', shell=True)"`
	`578`	`+ "process = subprocess.run('ls *.ipynb', stdout=subprocess.PIPE, encoding='utf8', shell=True)"`
`582`	`579`	`]`
`583`	`580`	`},`
`584`	`581`	`{`
`@@ -653,7 +650,9 @@`
`653`	`650`	`{`
`654`	`651`	`"cell_type": "code",`
`655`	`652`	`"execution_count": null,`
`656`		`- "metadata": {},`
	`653`	`+ "metadata": {`
	`654`	`+ "scrolled": true`
	`655`	`+ },`
`657`	`656`	`"outputs": [],`
`658`	`657`	`"source": [`
`659`	`658`	`"process.returncode"`
`@@ -676,7 +675,7 @@`
`676`	`675`	`"name": "python",`
`677`	`676`	`"nbconvert_exporter": "python",`
`678`	`677`	`"pygments_lexer": "ipython3",`
`679`		`- "version": "3.11.3"`
	`678`	`+ "version": "3.11.6"`
`680`	`679`	`}`
`681`	`680`	`},`
`682`	`681`	`"nbformat": 4,`
-Original file line number
+Diff line change
@@ @@ -1,4 +1,3 @@ @@
 -distr:
 -    name: uniform
 -    a: 0.0
 -    b: 1.0
 +name: uniform
 +a: 0.0
 +b: 1.0