From 0e6a2b754eee3a890ae9a0c4a415ded5d159cbfa Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Tue, 21 May 2024 20:27:32 +0000 Subject: [PATCH 01/10] resolving gnu parallel tc --- lockfile | 0 sciunit2/command/context.py | 40 +++++++++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 lockfile diff --git a/lockfile b/lockfile new file mode 100644 index 0000000..e69de29 diff --git a/sciunit2/command/context.py b/sciunit2/command/context.py index 5e16891..c795c7d 100644 --- a/sciunit2/command/context.py +++ b/sciunit2/command/context.py @@ -5,18 +5,46 @@ from contextlib import contextmanager import os +import shutil +import time + +from getopt import getopt +import sys +import fcntl + +class FileLock: + def __init__(self, file_path): + self.file_path = file_path + self.file_handle = None + + def acquire(self): + self.file_handle = open(self.file_path, 'w') + fcntl.flock(self.file_handle, fcntl.LOCK_EX) + + def release(self): + if self.file_handle: + fcntl.flock(self.file_handle, fcntl.LOCK_UN) + self.file_handle.close() # returns the pkgdir and original command used # to execute execution 'rev' @contextmanager def CheckoutContext(rev): + lock = FileLock('lockfile') + lock.acquire() emgr, repo = sciunit2.workspace.current() - with emgr.exclusive(): - orig = emgr.get(rev).cmd - pkgdir = os.path.join(repo.location, 'cde-package') - repo.cleanup(pkgdir) - repo.checkout(rev) - yield pkgdir, orig + try: + with emgr.exclusive(): + orig = emgr.get(rev).cmd + pkgdir = os.path.join(repo.location, 'cde-package') + pkgdir_rev = os.path.join(repo.location, rev, 'cde-package') + repo.cleanup(pkgdir) + repo.cleanup(pkgdir_rev) + repo.checkout(rev) + shutil.copytree(pkgdir, pkgdir_rev) + yield pkgdir_rev, orig + finally: + lock.release() @contextmanager From 6238485e6f665da838f87b238a97aacb67f8abf8 Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Tue, 21 May 2024 20:27:43 +0000 Subject: [PATCH 02/10] resolving gnu parallel tc --- lockfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 lockfile diff --git a/lockfile b/lockfile deleted file mode 100644 index e69de29..0000000 From 75b6603c93238e07aaf793e88f878555fde3211a Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Tue, 21 May 2024 22:15:57 +0000 Subject: [PATCH 03/10] temp commit --- lockfile | 0 sciunit2/command/commit.py | 6 ++++++ sciunit2/command/context.py | 10 ++++++++++ sciunit2/command/mixin.py | 3 +++ sciunit2/command/repeat.py | 2 ++ sciunit2/core.py | 4 ++++ 6 files changed, 25 insertions(+) create mode 100644 lockfile diff --git a/lockfile b/lockfile new file mode 100644 index 0000000..e69de29 diff --git a/sciunit2/command/commit.py b/sciunit2/command/commit.py index 3373f0b..51e4d04 100644 --- a/sciunit2/command/commit.py +++ b/sciunit2/command/commit.py @@ -20,14 +20,20 @@ def usage(self): 'command as a new execution')] def run(self, args): + print("inside commit") optlist, args = getopt(args, '') if args: raise CommandLineError emgr, repo = sciunit2.workspace.current() # repo is vvpkg pkgdir = os.path.join(repo.location, 'cde-package') + print(pkgdir) with emgr.exclusive(): for cmd in DetachedExecution(pkgdir).getcmd(): rev = emgr.add(cmd) + print("HERE") + print(cmd) + print(rev) + return self.do_commit(pkgdir, rev, emgr, repo) else: raise CommandError('nothing to commit') diff --git a/sciunit2/command/context.py b/sciunit2/command/context.py index c795c7d..83c2a47 100644 --- a/sciunit2/command/context.py +++ b/sciunit2/command/context.py @@ -46,6 +46,16 @@ def CheckoutContext(rev): finally: lock.release() +# @contextmanager +# def CheckoutContext(rev): +# emgr, repo = sciunit2.workspace.current() +# with emgr.exclusive(): +# orig = emgr.get(rev).cmd +# pkgdir = os.path.join(repo.location, 'cde-package') +# repo.cleanup(pkgdir) +# repo.checkout(rev) +# yield pkgdir, orig + @contextmanager def CheckoutContext_Diff(rev): diff --git a/sciunit2/command/mixin.py b/sciunit2/command/mixin.py index 3951a33..dd717a3 100644 --- a/sciunit2/command/mixin.py +++ b/sciunit2/command/mixin.py @@ -13,6 +13,9 @@ # to the end-user during the entire time. class CommitMixin(object): def do_commit(self, pkgdir, rev, emgr, repo): + print("Inside do commit") + print(pkgdir) + print(rev) with Spinner(label='Committing') as sp: # adds the execution to de-duplication engine sz = repo.checkin(rev, pkgdir, sp) diff --git a/sciunit2/command/repeat.py b/sciunit2/command/repeat.py index f79e04c..cc9c27c 100644 --- a/sciunit2/command/repeat.py +++ b/sciunit2/command/repeat.py @@ -19,6 +19,8 @@ def usage(self): def run(self, args): optlist, args = getopt(args, '') + print(optlist) + print(args) if not args: raise CommandLineError with CheckoutContext(args[0]) as (pkgdir, orig): diff --git a/sciunit2/core.py b/sciunit2/core.py index 15a05ce..c54fab6 100644 --- a/sciunit2/core.py +++ b/sciunit2/core.py @@ -35,6 +35,10 @@ def shell(env=None): def repeat(pkgdir, orig, newargs): + print("inside repeat") + print(pkgdir) + print(orig) + print(newargs) if newargs: if not orig: raise CommandError( From 1913666a95b0a1366cc2bdea835d242a22de118a Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Tue, 21 May 2024 22:16:15 +0000 Subject: [PATCH 04/10] temp commit --- lockfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 lockfile diff --git a/lockfile b/lockfile deleted file mode 100644 index e69de29..0000000 From 424000ff8733da9f37f451e303839ac6d09c3a11 Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Wed, 22 May 2024 16:19:35 +0000 Subject: [PATCH 05/10] fixed test cases for parallel repeat --- sciunit2/command/commit.py | 5 ----- sciunit2/command/context.py | 15 ++------------- sciunit2/command/mixin.py | 3 --- sciunit2/command/repeat.py | 2 -- sciunit2/core.py | 12 ++++++++---- 5 files changed, 10 insertions(+), 27 deletions(-) diff --git a/sciunit2/command/commit.py b/sciunit2/command/commit.py index 51e4d04..ac8387e 100644 --- a/sciunit2/command/commit.py +++ b/sciunit2/command/commit.py @@ -20,19 +20,14 @@ def usage(self): 'command as a new execution')] def run(self, args): - print("inside commit") optlist, args = getopt(args, '') if args: raise CommandLineError emgr, repo = sciunit2.workspace.current() # repo is vvpkg pkgdir = os.path.join(repo.location, 'cde-package') - print(pkgdir) with emgr.exclusive(): for cmd in DetachedExecution(pkgdir).getcmd(): rev = emgr.add(cmd) - print("HERE") - print(cmd) - print(rev) return self.do_commit(pkgdir, rev, emgr, repo) else: diff --git a/sciunit2/command/context.py b/sciunit2/command/context.py index 83c2a47..e125d46 100644 --- a/sciunit2/command/context.py +++ b/sciunit2/command/context.py @@ -30,9 +30,9 @@ def release(self): # to execute execution 'rev' @contextmanager def CheckoutContext(rev): - lock = FileLock('lockfile') - lock.acquire() emgr, repo = sciunit2.workspace.current() + lock = FileLock(os.path.join(repo.location ,'lockfile')) + lock.acquire() try: with emgr.exclusive(): orig = emgr.get(rev).cmd @@ -46,17 +46,6 @@ def CheckoutContext(rev): finally: lock.release() -# @contextmanager -# def CheckoutContext(rev): -# emgr, repo = sciunit2.workspace.current() -# with emgr.exclusive(): -# orig = emgr.get(rev).cmd -# pkgdir = os.path.join(repo.location, 'cde-package') -# repo.cleanup(pkgdir) -# repo.checkout(rev) -# yield pkgdir, orig - - @contextmanager def CheckoutContext_Diff(rev): emgr, repo = sciunit2.workspace.current() diff --git a/sciunit2/command/mixin.py b/sciunit2/command/mixin.py index dd717a3..3951a33 100644 --- a/sciunit2/command/mixin.py +++ b/sciunit2/command/mixin.py @@ -13,9 +13,6 @@ # to the end-user during the entire time. class CommitMixin(object): def do_commit(self, pkgdir, rev, emgr, repo): - print("Inside do commit") - print(pkgdir) - print(rev) with Spinner(label='Committing') as sp: # adds the execution to de-duplication engine sz = repo.checkin(rev, pkgdir, sp) diff --git a/sciunit2/command/repeat.py b/sciunit2/command/repeat.py index cc9c27c..f79e04c 100644 --- a/sciunit2/command/repeat.py +++ b/sciunit2/command/repeat.py @@ -19,8 +19,6 @@ def usage(self): def run(self, args): optlist, args = getopt(args, '') - print(optlist) - print(args) if not args: raise CommandLineError with CheckoutContext(args[0]) as (pkgdir, orig): diff --git a/sciunit2/core.py b/sciunit2/core.py index c54fab6..0a75e6f 100644 --- a/sciunit2/core.py +++ b/sciunit2/core.py @@ -35,10 +35,6 @@ def shell(env=None): def repeat(pkgdir, orig, newargs): - print("inside repeat") - print(pkgdir) - print(orig) - print(newargs) if newargs: if not orig: raise CommandError( @@ -47,6 +43,10 @@ def repeat(pkgdir, orig, newargs): with open('cde.log') as f: cd, ls = f os.rename('cde.log', 'cde.log.1') + + parent_pkgdir = os.path.join(os.path.dirname(os.path.dirname(pkgdir)), 'cde-package') + os.makedirs(os.path.join(parent_pkgdir, 'cde.log.1'), exist_ok=True) + with open('cde.log', 'w') as f: # adds the command in a comment f.write_cmd(orig[:1] + newargs) @@ -54,6 +54,10 @@ def repeat(pkgdir, orig, newargs): f.insert(cd) # commands to execute with new arguments f.insert(ls[:1] + newargs) + shutil.copy(os.path.join(pkgdir,'cde.log.1'), os.path.join(parent_pkgdir, 'cde.log.1')) + shutil.copy(os.path.join(pkgdir,'cde.log'), os.path.join(parent_pkgdir, 'cde.log')) + + try: output = subprocess.check_output(['/bin/sh', 'cde.log'], cwd=pkgdir) except subprocess.CalledProcessError as exc: From 6748e613885065f5d7d0a47d54a571118db787da Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Wed, 22 May 2024 16:56:38 +0000 Subject: [PATCH 06/10] improved code readability --- sciunit2/command/context.py | 19 ++----------------- sciunit2/filelock.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 17 deletions(-) create mode 100644 sciunit2/filelock.py diff --git a/sciunit2/command/context.py b/sciunit2/command/context.py index e125d46..c0710cb 100644 --- a/sciunit2/command/context.py +++ b/sciunit2/command/context.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import sciunit2.workspace +import sciunit2.filelock from contextlib import contextmanager import os @@ -9,29 +10,13 @@ import time from getopt import getopt -import sys -import fcntl - -class FileLock: - def __init__(self, file_path): - self.file_path = file_path - self.file_handle = None - - def acquire(self): - self.file_handle = open(self.file_path, 'w') - fcntl.flock(self.file_handle, fcntl.LOCK_EX) - - def release(self): - if self.file_handle: - fcntl.flock(self.file_handle, fcntl.LOCK_UN) - self.file_handle.close() # returns the pkgdir and original command used # to execute execution 'rev' @contextmanager def CheckoutContext(rev): emgr, repo = sciunit2.workspace.current() - lock = FileLock(os.path.join(repo.location ,'lockfile')) + lock = sciunit2.filelock.FileLock(os.path.join(repo.location ,'lockfile')) lock.acquire() try: with emgr.exclusive(): diff --git a/sciunit2/filelock.py b/sciunit2/filelock.py new file mode 100644 index 0000000..a5f01e3 --- /dev/null +++ b/sciunit2/filelock.py @@ -0,0 +1,15 @@ +import fcntl + +class FileLock: + def __init__(self, file_path): + self.file_path = file_path + self.file_handle = None + + def acquire(self): + self.file_handle = open(self.file_path, 'w') + fcntl.flock(self.file_handle, fcntl.LOCK_EX) + + def release(self): + if self.file_handle: + fcntl.flock(self.file_handle, fcntl.LOCK_UN) + self.file_handle.close() \ No newline at end of file From f9a16b276071a4eb8268b9928d2b2b9b685fc7bf Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Wed, 22 May 2024 17:03:03 +0000 Subject: [PATCH 07/10] improved code readability --- sciunit2/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sciunit2/core.py b/sciunit2/core.py index 0a75e6f..dbb0f56 100644 --- a/sciunit2/core.py +++ b/sciunit2/core.py @@ -4,6 +4,7 @@ from sciunit2.util import Chdir from sciunit2.cdelog import open import sciunit2.libexec +import sciunit2.filelock import os import shutil @@ -56,7 +57,6 @@ def repeat(pkgdir, orig, newargs): f.insert(ls[:1] + newargs) shutil.copy(os.path.join(pkgdir,'cde.log.1'), os.path.join(parent_pkgdir, 'cde.log.1')) shutil.copy(os.path.join(pkgdir,'cde.log'), os.path.join(parent_pkgdir, 'cde.log')) - try: output = subprocess.check_output(['/bin/sh', 'cde.log'], cwd=pkgdir) From 240f02cf2b6c7f5e232ae9e7b436b8574af1cb3e Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Wed, 22 May 2024 17:28:35 +0000 Subject: [PATCH 08/10] update test cases --- tests/test_repeat.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_repeat.py b/tests/test_repeat.py index cc49bac..64b3648 100644 --- a/tests/test_repeat.py +++ b/tests/test_repeat.py @@ -62,19 +62,29 @@ def test_all(self): with testpath.modified_env({'SHELL': '/bin/true'}): assert_is_none(testit.sciunit('exec', '-i')) + assert_true(not os.path.exists('tmp/ok/e2')) + assert_true(os.path.exists('tmp/ok/e1')) + with assert_raises(SystemExit) as r: testit.sciunit('repeat', 'e2') assert_equal(r.exception.code, 0) + assert_true(os.path.exists('tmp/ok/e2')) + with assert_raises(SystemExit) as r: testit.sciunit('repeat', 'e2', '-l') assert_equal(r.exception.code, 1) + assert_true(not os.path.exists('tmp/ok/e3')) + assert_is_none(testit.sciunit('commit')) + + assert_true(not os.path.exists('tmp/ok/e3')) with assert_raises(SystemExit) as r: testit.sciunit('repeat', 'e3') assert_equal(r.exception.code, 0) + assert_true(os.path.exists('tmp/ok/e3')) with assert_raises(SystemExit) as r: testit.sciunit('repeat', 'e1', '-L') From 1f3205d178c3fcd449006edde3daeb7c16fd7740 Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Wed, 22 May 2024 17:30:37 +0000 Subject: [PATCH 09/10] update test cases --- sciunit2/command/commit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sciunit2/command/commit.py b/sciunit2/command/commit.py index ac8387e..3373f0b 100644 --- a/sciunit2/command/commit.py +++ b/sciunit2/command/commit.py @@ -28,7 +28,6 @@ def run(self, args): with emgr.exclusive(): for cmd in DetachedExecution(pkgdir).getcmd(): rev = emgr.add(cmd) - return self.do_commit(pkgdir, rev, emgr, repo) else: raise CommandError('nothing to commit') From 4bbecd740147a1f3361c94e9a6b62f51e80aeb6e Mon Sep 17 00:00:00 2001 From: Krishna Kamath Date: Wed, 22 May 2024 17:36:18 +0000 Subject: [PATCH 10/10] added comments --- sciunit2/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sciunit2/core.py b/sciunit2/core.py index dbb0f56..14ee765 100644 --- a/sciunit2/core.py +++ b/sciunit2/core.py @@ -55,6 +55,9 @@ def repeat(pkgdir, orig, newargs): f.insert(cd) # commands to execute with new arguments f.insert(ls[:1] + newargs) + + # this will cause issues if parallel repeat is run. + # Question: Which execution-id to commit in case of sciunit commit after parallel repeat shutil.copy(os.path.join(pkgdir,'cde.log.1'), os.path.join(parent_pkgdir, 'cde.log.1')) shutil.copy(os.path.join(pkgdir,'cde.log'), os.path.join(parent_pkgdir, 'cde.log'))