diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 9039337..23afda4 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -47,6 +47,6 @@ jobs: sshpass -e ssh ${opt} ${{secrets.CHARON_USER_FRONTEND}} \ "cd _testing; rm -rf swrap; git clone -b ${BRANCH} https://github.com/flow123d/swrap.git; \ pip install --user .; \ - cd swrap/testing/integrated; python3 run_test.py 01_mpi4py" + cd swrap/integrated; python3 run_test.py 01_mpi4py" diff --git a/testing/integrated/01_mpi4py/config.yaml b/integrated/01_mpi4py/config.yaml similarity index 78% rename from testing/integrated/01_mpi4py/config.yaml rename to integrated/01_mpi4py/config.yaml index f900084..843d427 100644 --- a/testing/integrated/01_mpi4py/config.yaml +++ b/integrated/01_mpi4py/config.yaml @@ -1,12 +1,12 @@ common: # these fields are used in every case image: docker://flow123d/geomop-gnu:2.0.0 # image to run in - wrapper: smpiexec + wrapper: smpiexec.py # wrapper to test, currently just mpiexec ##### # PBS options see: https://wiki.metacentrum.cz/wiki/About_scheduling_system#How_to_set_number_of_nodes_and_processors - pbs_select: 1 + pbs_select: 1:mem=4gb:scratch_local=10gb # PBS select option: "-l select={pbs_select}" # syntax select= [+ ] # = [:ncpu=][:mem=][:scratch_=] @@ -32,9 +32,15 @@ cases: # used number of precesses must fit the scheduled number on PBS; see pbs_* keys # the host file is passed automaticaly # for mpiexec see: https://www.open-mpi.org/doc/v3.0/man1/mpiexec.1.php - pbs_select: 2 + pbs_select: 2:mem=4gb:scratch_local=10gb - - name: script - command: -n 4 python3 script.py - pbs_select: 2:ncpus=2 - + - name: simple + command: -n 4 python3 simple.py + pbs_select: 2:ncpus=2:mem=4gb:scratch_local=10gb + + - name: error_in_process + command: -n 4 python3 error_in_process.py + pbs_select: 2:ncpus=2:mem=4gb:scratch_local=10gb + checkers: + check_return_code: 1 + check_find_stdout_regex: ZeroDivisionError diff --git a/integrated/01_mpi4py/error_in_process.py b/integrated/01_mpi4py/error_in_process.py new file mode 100644 index 0000000..c08909c --- /dev/null +++ b/integrated/01_mpi4py/error_in_process.py @@ -0,0 +1,13 @@ +from mpi4py import MPI + + +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +if rank == 0: + # error + print("P{} zero division.".format(rank)) + a = 2 / 0 + +print("P{} finished".format(rank)) diff --git a/testing/integrated/01_mpi4py/job.sh b/integrated/01_mpi4py/job.sh similarity index 100% rename from testing/integrated/01_mpi4py/job.sh rename to integrated/01_mpi4py/job.sh diff --git a/integrated/01_mpi4py/simple.py b/integrated/01_mpi4py/simple.py new file mode 100644 index 0000000..f83833f --- /dev/null +++ b/integrated/01_mpi4py/simple.py @@ -0,0 +1,37 @@ +from mpi4py import MPI +import hashlib +import random +import numpy as np + +comm = MPI.COMM_WORLD +rank = comm.Get_rank() +size = comm.Get_size() + +seed = hashlib.sha256(str(rank).encode()).digest() +random.seed(seed) + +rand_max = 1000_000 +num_sends = 100_000 +num_iter = 10 + +for i in range(num_iter): + sum = 0 + for j in range(1, num_sends + 1): + data_out = random.randrange(rand_max) + dest = (rank + j) % size + req = comm.isend(data_out, dest=dest) + req.wait() + + req = comm.irecv() + data_in = req.wait() + sum += data_out + sum += data_in + sum %= 275_604_541 + + recvbuf = np.empty(1, dtype=int) + sendbuf = sum * np.ones(size, dtype=int) + comm.Reduce_scatter(sendbuf, recvbuf) + + print("[{}] {}".format(rank, recvbuf[0])) + +print("P{} finished".format(rank)) diff --git a/testing/integrated/TODO b/integrated/TODO similarity index 100% rename from testing/integrated/TODO rename to integrated/TODO diff --git a/testing/integrated/charon_pbs_template.sh b/integrated/charon_pbs_template.sh similarity index 100% rename from testing/integrated/charon_pbs_template.sh rename to integrated/charon_pbs_template.sh diff --git a/testing/integrated/osu-microbenchmarks/osu-microbenchmarks_job_01.sh b/integrated/osu-microbenchmarks/osu-microbenchmarks_job_01.sh similarity index 100% rename from testing/integrated/osu-microbenchmarks/osu-microbenchmarks_job_01.sh rename to integrated/osu-microbenchmarks/osu-microbenchmarks_job_01.sh diff --git a/testing/integrated/osu-microbenchmarks/osu-microbenchmarks_job_02.sh b/integrated/osu-microbenchmarks/osu-microbenchmarks_job_02.sh similarity index 100% rename from testing/integrated/osu-microbenchmarks/osu-microbenchmarks_job_02.sh rename to integrated/osu-microbenchmarks/osu-microbenchmarks_job_02.sh diff --git a/testing/integrated/osu-microbenchmarks/osu-microbenchmarks_job_03.sh b/integrated/osu-microbenchmarks/osu-microbenchmarks_job_03.sh similarity index 100% rename from testing/integrated/osu-microbenchmarks/osu-microbenchmarks_job_03.sh rename to integrated/osu-microbenchmarks/osu-microbenchmarks_job_03.sh diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388425 b/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388425 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388425 rename to integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388425 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388426 b/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388426 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388426 rename to integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388426 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388427 b/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388427 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388427 rename to integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388427 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388428 b/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388428 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388428 rename to integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388428 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388429 b/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388429 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388429 rename to integrated/osu-microbenchmarks/results/mpi_excl/osu_get_bw/sing_mpi_test.o11388429 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388361 b/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388361 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388361 rename to integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388361 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388421 b/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388421 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388421 rename to integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388421 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388422 b/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388422 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388422 rename to integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388422 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388423 b/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388423 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388423 rename to integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388423 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388424 b/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388424 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388424 rename to integrated/osu-microbenchmarks/results/mpi_excl/osu_get_latency/sing_mpi_test.o11388424 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341700 b/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341700 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341700 rename to integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341700 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341701 b/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341701 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341701 rename to integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341701 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341702 b/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341702 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341702 rename to integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341702 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341703 b/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341703 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341703 rename to integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341703 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341704 b/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341704 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341704 rename to integrated/osu-microbenchmarks/results/mpi_sing/osu_get_bw/sing_mpi_test.o11341704 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341688 b/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341688 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341688 rename to integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341688 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341689 b/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341689 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341689 rename to integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341689 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341690 b/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341690 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341690 rename to integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341690 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341691 b/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341691 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341691 rename to integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341691 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341692 b/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341692 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341692 rename to integrated/osu-microbenchmarks/results/mpi_sing/osu_get_latency/sing_mpi_test.o11341692 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341856 b/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341856 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341856 rename to integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341856 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341857 b/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341857 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341857 rename to integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341857 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341858 b/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341858 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341858 rename to integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341858 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341859 b/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341859 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341859 rename to integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341859 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341860 b/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341860 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341860 rename to integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_bw/sing_mpi_test.o11341860 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341851 b/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341851 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341851 rename to integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341851 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341852 b/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341852 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341852 rename to integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341852 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341853 b/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341853 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341853 rename to integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341853 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341854 b/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341854 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341854 rename to integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341854 diff --git a/testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341855 b/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341855 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341855 rename to integrated/osu-microbenchmarks/results/mpi_sing_excl/osu_get_latency/sing_mpi_test.o11341855 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11340183 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11340183 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11340183 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11340183 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341316 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341316 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341316 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341316 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341317 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341317 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341317 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341317 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341343 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341343 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341343 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341343 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341344 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341344 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341344 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_bw/sing_mpi_test.o11341344 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11340008 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11340008 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11340008 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11340008 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341314 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341314 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341314 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341314 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341315 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341315 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341315 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341315 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341345 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341345 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341345 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341345 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341346 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341346 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341346 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341346 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341347 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341347 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341347 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341347 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341743 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341743 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341743 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341743 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341744 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341744 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341744 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341744 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341745 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341745 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341745 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341745 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341746 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341746 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341746 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341746 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341747 b/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341747 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341747 rename to integrated/osu-microbenchmarks/results/sing_mpi/osu_get_latency/sing_mpi_test.o11341747 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341805 b/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341805 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341805 rename to integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341805 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341806 b/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341806 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341806 rename to integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341806 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341807 b/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341807 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341807 rename to integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341807 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341808 b/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341808 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341808 rename to integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341808 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341809 b/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341809 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341809 rename to integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_bw/sing_mpi_test.o11341809 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341782 b/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341782 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341782 rename to integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341782 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341783 b/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341783 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341783 rename to integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341783 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341784 b/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341784 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341784 rename to integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341784 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341785 b/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341785 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341785 rename to integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341785 diff --git a/testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341786 b/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341786 similarity index 100% rename from testing/integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341786 rename to integrated/osu-microbenchmarks/results/sing_mpi_excl/osu_get_latency/sing_mpi_test.o11341786 diff --git a/testing/integrated/osu-microbenchmarks/ulhpc-dockerfile-mpich_psm/Dockerfile b/integrated/osu-microbenchmarks/ulhpc-dockerfile-mpich_psm/Dockerfile similarity index 100% rename from testing/integrated/osu-microbenchmarks/ulhpc-dockerfile-mpich_psm/Dockerfile rename to integrated/osu-microbenchmarks/ulhpc-dockerfile-mpich_psm/Dockerfile diff --git a/testing/integrated/run_test.py b/integrated/run_test.py similarity index 93% rename from testing/integrated/run_test.py rename to integrated/run_test.py index ba340cf..4208f28 100644 --- a/testing/integrated/run_test.py +++ b/integrated/run_test.py @@ -128,6 +128,14 @@ def check_return_code(state: JobState, ref_return_code: int) -> str: return f"Error: return code {state.return_code} != {ref_return_code}" +def check_find_stdout_regex(state: JobState, regex: str) -> str: + """ + Check if output contains regular expression. + """ + if re.search(regex, state.get_output()) is None: + return f"Error: output don't contain regular expression: {regex}" + + # def as_list(x): # if type(x) is list: # return x @@ -190,7 +198,7 @@ def __init__(self, queue: Queue, config: Dict[str, Any]): Run._counter += 1 self.id = Run._counter - config['wrapper'] = os.path.join(test_script_dir, "../../src/swrap/", config['wrapper']) + config['wrapper'] = os.path.join(test_script_dir, "../src/swrap/", config['wrapper']) self.config = dict_merge(Run._default_config, config) self.checkers = [(get_check_fn(fn), normalize_args(args)) for fn, args in config['checkers'].items()] @@ -290,8 +298,8 @@ def pbs_submit(self): time.sleep(10) err_msg = None for check_fn, args in self.checkers: - err_msh = check_fn(self.state, *args) - if err_msh is not None: + err_msg = check_fn(self.state, *args) + if err_msg is not None: break self.summary(err_msg) print(f"[{self.id}] done") @@ -304,27 +312,33 @@ def summary(self, err_msg): if err_msg is None: result = "Succeed:" err_msg = "" + success = True else: if self.state.timeout: result = "Timed out:" else: result = "Failed:" err_msg = f"{err_msg}\n{self.state.get_output()}" + success = False msg = f"[{self.id}] {result} {self.short_id}, {self.pbs_script}: {self.config['command']} \n{err_msg}" - self.queue.put((self.id, msg)) - + self.queue.put((self.id, msg, success)) def report(print_queue): messages = [] + all_success = True while True: try: messages.append(print_queue.get_nowait()) print_queue.task_done() except Empty: break - for id, msg in sorted(messages): + for id, msg, success in sorted(messages): print(msg) + if not success: + all_success = False + return all_success + def parse_arguments(): """ @@ -383,7 +397,10 @@ def main(): for f in futures: print(f.result()) - report(print_queue) + all_success = report(print_queue) + if not all_success: + exit(1) + if __name__ == "__main__": main() diff --git a/testing/integrated/test_02_script_venv/python_script_runner.sh b/integrated/test_02_script_venv/python_script_runner.sh similarity index 100% rename from testing/integrated/test_02_script_venv/python_script_runner.sh rename to integrated/test_02_script_venv/python_script_runner.sh diff --git a/testing/integrated/test_02_script_venv/test_02_create_venv.sh b/integrated/test_02_script_venv/test_02_create_venv.sh similarity index 100% rename from testing/integrated/test_02_script_venv/test_02_create_venv.sh rename to integrated/test_02_script_venv/test_02_create_venv.sh diff --git a/testing/integrated/test_02_script_venv/test_02_job.sh b/integrated/test_02_script_venv/test_02_job.sh similarity index 100% rename from testing/integrated/test_02_script_venv/test_02_job.sh rename to integrated/test_02_script_venv/test_02_job.sh diff --git a/testing/integrated/test_02_script_venv/test_02_script.py b/integrated/test_02_script_venv/test_02_script.py similarity index 100% rename from testing/integrated/test_02_script_venv/test_02_script.py rename to integrated/test_02_script_venv/test_02_script.py diff --git a/testing/integrated/test_flow123d/input/01_dirichlet.yaml b/integrated/test_flow123d/input/01_dirichlet.yaml similarity index 100% rename from testing/integrated/test_flow123d/input/01_dirichlet.yaml rename to integrated/test_flow123d/input/01_dirichlet.yaml diff --git a/testing/integrated/test_flow123d/input/square_1x1_xy.msh b/integrated/test_flow123d/input/square_1x1_xy.msh similarity index 100% rename from testing/integrated/test_flow123d/input/square_1x1_xy.msh rename to integrated/test_flow123d/input/square_1x1_xy.msh diff --git a/testing/integrated/test_flow123d/scratch_files b/integrated/test_flow123d/scratch_files similarity index 100% rename from testing/integrated/test_flow123d/scratch_files rename to integrated/test_flow123d/scratch_files diff --git a/testing/integrated/test_flow123d/test_flow123d_job.sh b/integrated/test_flow123d/test_flow123d_job.sh similarity index 100% rename from testing/integrated/test_flow123d/test_flow123d_job.sh rename to integrated/test_flow123d/test_flow123d_job.sh diff --git a/testing/integrated/testing_hostfile b/integrated/testing_hostfile similarity index 100% rename from testing/integrated/testing_hostfile rename to integrated/testing_hostfile diff --git a/testing/integrated/testing_known_hosts b/integrated/testing_known_hosts similarity index 100% rename from testing/integrated/testing_known_hosts rename to integrated/testing_known_hosts diff --git a/testing/integrated/testing_script/python_script_runner.sh b/integrated/testing_script/python_script_runner.sh similarity index 100% rename from testing/integrated/testing_script/python_script_runner.sh rename to integrated/testing_script/python_script_runner.sh diff --git a/testing/integrated/testing_script/script2.py b/integrated/testing_script/script2.py similarity index 100% rename from testing/integrated/testing_script/script2.py rename to integrated/testing_script/script2.py diff --git a/testing/integrated/testing_script/setup_python_environment.sh b/integrated/testing_script/setup_python_environment.sh similarity index 100% rename from testing/integrated/testing_script/setup_python_environment.sh rename to integrated/testing_script/setup_python_environment.sh diff --git a/testing/integrated/testing_script/test_script_job.sh b/integrated/testing_script/test_script_job.sh similarity index 100% rename from testing/integrated/testing_script/test_script_job.sh rename to integrated/testing_script/test_script_job.sh diff --git a/src/swrap/smpiexec.py b/src/swrap/smpiexec.py index 6567273..502accf 100755 --- a/src/swrap/smpiexec.py +++ b/src/swrap/smpiexec.py @@ -6,39 +6,10 @@ from argparse import RawTextHelpFormatter -def flush_print(*margs, **mkwargs): - print(*margs, file=sys.stdout, flush=True, **mkwargs) - -def oscommand(command_string): - flush_print(command_string) - flush_print(os.popen(command_string).read()) - -def create_ssh_agent(): - """ - Setup ssh agent and set appropriate environment variables. - :return: - """ - flush_print("creating ssh agent...") - p = subprocess.Popen('ssh-agent -s', - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=True, universal_newlines=True) - outinfo, errinfo = p.communicate('ssh-agent cmd\n') - # print(outinfo) - - lines = outinfo.split('\n') - for line in lines: - # trim leading and trailing whitespace - line = line.strip() - # ignore blank/empty lines - if not line: - continue - # break off the part before the semicolon - left, right = line.split(';', 1) - if '=' in left: - # get variable and value, put into os.environ - varname, varvalue = left.split('=', 1) - flush_print("setting variable from ssh-agent:", varname, "=", varvalue) - os.environ[varname] = varvalue +sys.path.append(os.path.dirname(os.path.realpath(__file__))) + +from tools import flush_print, oscommand, create_ssh_agent, create_known_hosts_file + def arguments(): parser = argparse.ArgumentParser( @@ -52,7 +23,11 @@ def arguments(): help='comma separated list of paths to be bind to Singularity container') parser.add_argument('-m', '--mpiexec', type=str, metavar="PATH", default="", required=False, help="path (inside the container) to mpiexec to be run, default is 'mpiexec'") - parser.add_argument('-s', '--scratch_copy', type=str, metavar="PATH", default="", required=False, + parser.add_argument('-s', '--scratch_dir', type=str, metavar="PATH", default="", required=False, + help=''' + directory path, where SCRATCHDIR is, overwrite SCRATCHDIR from environment; + ''') + parser.add_argument('-c', '--scratch_copy', type=str, metavar="PATH", default="", required=False, help=''' directory path, its content will be copied to SCRATCHDIR; ''') @@ -117,50 +92,7 @@ def main(): shutil.copy(orig_node_file, node_file) # mprint(os.popen("ls -l").read()) - # Get ssh keys to nodes and append it to $HOME/.ssh/known_hosts - ssh_known_hosts_to_append = [] - if debug: - # ssh_known_hosts_file = 'testing_known_hosts' - ssh_known_hosts_file = 'xxx/.ssh/testing_known_hosts' - else: - assert 'HOME' in os.environ - ssh_known_hosts_file = os.path.join(os.environ['HOME'], '.ssh/known_hosts') - - flush_print("host file name:", ssh_known_hosts_file) - - ssh_known_hosts = [] - if os.path.exists(ssh_known_hosts_file): - with open(ssh_known_hosts_file, 'r') as fp: - ssh_known_hosts = fp.readlines() - else: - flush_print("creating host file...") - dirname = os.path.dirname(ssh_known_hosts_file) - if not os.path.exists(dirname): - os.makedirs(dirname) - - flush_print("reading host file...") - with open(node_file) as fp: - node_names_read = fp.read().splitlines() - # remove duplicates - node_names = list(dict.fromkeys(node_names_read)) - - flush_print("connecting nodes...") - for node in node_names: - # touch all the nodes, so that they are accessible also through container - os.popen('ssh ' + node + ' exit') - # add the nodes to known_hosts so the fingerprint verification is skipped later - # in shell just append # >> ~ /.ssh / known_hosts - # or sort by 3.column in shell: 'sort -k3 -u ~/.ssh/known_hosts' and rewrite - ssh_keys = os.popen('ssh-keyscan -H ' + node) .readlines() - ssh_keys = list((line for line in ssh_keys if not line.startswith('#'))) - for sk in ssh_keys: - splits = sk.split(" ") - if not splits[2] in ssh_known_hosts: - ssh_known_hosts_to_append.append(sk) - - flush_print("finishing host file...") - with open(ssh_known_hosts_file, 'a') as fp: - fp.writelines(ssh_known_hosts_to_append) + node_names = create_known_hosts_file(current_dir, node_file, debug=debug) # mprint(os.environ) create_agent = 'SSH_AUTH_SOCK' not in os.environ @@ -179,8 +111,11 @@ def main(): flush_print("assembling final command...") scratch_dir_path = None - if 'SCRATCHDIR' in os.environ: + if args.scratch_dir: + scratch_dir_path = args.scratch_dir + elif 'SCRATCHDIR' in os.environ: scratch_dir_path = os.environ['SCRATCHDIR'] + if scratch_dir_path and args.scratch_copy: flush_print("Using SCRATCHDIR:", scratch_dir_path) flush_print("copying to SCRATCHDIR on all nodes...") @@ -210,11 +145,11 @@ def main(): for node in node_names: destination_name = username + "@" + node destination_path = destination_name + ':' + scratch_dir_path - command = ' '.join(['scp', source_tar_filepath, destination_path]) + command = ' '.join(['scp', '-o', 'UserKnownHostsFile=known_hosts', source_tar_filepath, destination_path]) oscommand(command) #command = ' '.join(['ssh', destination_name, 'cd', scratch_dir_path, '&&', 'tar --strip-components 1 -xf', source_tar_filepath, '-C /']) - command = ' '.join(['ssh', destination_name, '"cd', scratch_dir_path, '&&', 'tar -xf', source_tar_filename, + command = ' '.join(['ssh', '-o', 'UserKnownHostsFile=known_hosts', destination_name, '"cd', scratch_dir_path, '&&', 'tar -xf', source_tar_filename, '&&', 'rm ', source_tar_filename, '"']) oscommand(command) @@ -238,7 +173,8 @@ def main(): else: bindings_in_launcher = bindings_in_launcher + "," + scratch_dir_path - sing_command = ' '.join(['singularity', 'exec', bindings, image]) + sing_command_list = ['singularity', 'exec', bindings, image] + sing_command = ' '.join(sing_command_list) sing_command_in_launcher = ' '.join(['singularity', 'exec', bindings_in_launcher, image]) flush_print('sing_command:', sing_command) @@ -286,15 +222,15 @@ def main(): # raise Exception("mpiexec path '" + mpiexec_path + "' not found in container!") # D] join mpiexec arguments - mpiexec_args = " ".join([mpiexec_path, '-f', node_file, '-launcher-exec', launcher_path]) + mpiexec_args = [mpiexec_path, '-f', node_file, '-launcher-exec', launcher_path] # F] join all the arguments into final singularity container command - final_command_list = [sing_command, mpiexec_args, *prog_args] + final_command_list = [*sing_command_list, *mpiexec_args, *prog_args] ################################################################################################################### # Final call. ################################################################################################################### - if scratch_dir_path: + if scratch_dir_path and args.scratch_copy: flush_print("Entering SCRATCHDIR:", scratch_dir_path) os.chdir(scratch_dir_path) @@ -304,7 +240,12 @@ def main(): flush_print("=================== smpiexec.py END ===================") if not debug: flush_print("================== Program output START ==================") - proc = subprocess.run(final_command_list) + if scratch_dir_path: + sing_tmp = os.path.join(scratch_dir_path, "singularity_tmp") + else: + sing_tmp = os.path.join(os.environ['HOME'], "singularity_tmp") + os.makedirs(sing_tmp, exist_ok = True) + proc = subprocess.run(final_command_list, env={**os.environ, "SINGULARITY_TMPDIR": sing_tmp}) flush_print("=================== Program output END ===================") exit(proc.returncode) diff --git a/src/swrap/tools.py b/src/swrap/tools.py new file mode 100644 index 0000000..2be0be4 --- /dev/null +++ b/src/swrap/tools.py @@ -0,0 +1,89 @@ +import os +import sys +import subprocess + + +def flush_print(*margs, **mkwargs): + print(*margs, file=sys.stdout, flush=True, **mkwargs) + + +def oscommand(command_string): + flush_print(command_string) + flush_print(os.popen(command_string).read()) + + +def create_ssh_agent(): + """ + Setup ssh agent and set appropriate environment variables. + :return: + """ + flush_print("creating ssh agent...") + p = subprocess.Popen('ssh-agent -s', + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True, universal_newlines=True) + outinfo, errinfo = p.communicate('ssh-agent cmd\n') + # print(outinfo) + + lines = outinfo.split('\n') + for line in lines: + # trim leading and trailing whitespace + line = line.strip() + # ignore blank/empty lines + if not line: + continue + # break off the part before the semicolon + left, right = line.split(';', 1) + if '=' in left: + # get variable and value, put into os.environ + varname, varvalue = left.split('=', 1) + flush_print("setting variable from ssh-agent:", varname, "=", varvalue) + os.environ[varname] = varvalue + + +def create_known_hosts_file(script_dir, node_file, debug=False): + # Get ssh keys to nodes and append it to known_hosts + ssh_known_hosts_to_append = [] + if debug: + # ssh_known_hosts_file = 'testing_known_hosts' + ssh_known_hosts_file = 'xxx/.ssh/testing_known_hosts' + else: + # assert 'HOME' in os.environ + ssh_known_hosts_file = os.path.join(script_dir, 'known_hosts') + + flush_print("host file name:", ssh_known_hosts_file) + + ssh_known_hosts = [] + if os.path.exists(ssh_known_hosts_file): + with open(ssh_known_hosts_file, 'r') as fp: + ssh_known_hosts = fp.readlines() + else: + flush_print("creating host file...") + dirname = os.path.dirname(ssh_known_hosts_file) + if not os.path.exists(dirname): + os.makedirs(dirname) + + flush_print("reading host file...") + with open(node_file) as fp: + node_names_read = fp.read().splitlines() + # remove duplicates + node_names = list(dict.fromkeys(node_names_read)) + + flush_print("connecting nodes...") + for node in node_names: + # touch all the nodes, so that they are accessible also through container + os.popen('ssh ' + node + ' exit') + # add the nodes to known_hosts so the fingerprint verification is skipped later + # in shell just append # >> ~ /.ssh / known_hosts + # or sort by 3.column in shell: 'sort -k3 -u ~/.ssh/known_hosts' and rewrite + ssh_keys = os.popen('ssh-keyscan -H ' + node).readlines() + ssh_keys = list((line for line in ssh_keys if not line.startswith('#'))) + for sk in ssh_keys: + splits = sk.split(" ") + if not splits[2] in ssh_known_hosts: + ssh_known_hosts_to_append.append(sk) + + flush_print("finishing host file...") + with open(ssh_known_hosts_file, 'a') as fp: + fp.writelines(ssh_known_hosts_to_append) + + return node_names diff --git a/testing/integrated/01_mpi4py/script.py b/testing/integrated/01_mpi4py/script.py deleted file mode 100644 index 2039594..0000000 --- a/testing/integrated/01_mpi4py/script.py +++ /dev/null @@ -1,37 +0,0 @@ -from mpi4py import MPI - -comm = MPI.COMM_WORLD -rank = comm.Get_rank() - -if rank == 0: - data = {'a': 1, 'b': 3.14} - print("P0 sending to P1: {}", data) - req = comm.isend(data, dest=1, tag=11) - req.wait() - - data = {'c': 2, 'd': 6.28} - print("P0 sending to P2: {}", data) - req = comm.isend(data, dest=2, tag=12) - req.wait() - print("P0 finished") -elif rank == 1: - req = comm.irecv(source=0, tag=11) - data = req.wait() - print("P1 received: {}", data) - - data["a"] = 3 - data["b"] = 3*3.14 - print("P1 sending to P3: {}", data) - req = comm.isend(data, dest=3, tag=13) - req.wait() - print("P1 finished") -elif rank == 2: - req = comm.irecv(source=0, tag=12) - data = req.wait() - print("P2 received: {}", data) - print("P2 finished") -elif rank == 3: - req = comm.irecv(source=1, tag=13) - data = req.wait() - print("P3 received: {}", data) - print("P3 finished") diff --git a/tox.ini b/tox.ini index 4830ee0..3ff9ab8 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ deps = pytest commands = - pytest --ignore=testing/integrated testing + pytest testing [testenv:coverage-report] deps = coverage