added parallel execution for repeat

sciunit2/command/context.py

@@ -1,23 +1,35 @@
 from __future__ import absolute_import
 
 import sciunit2.workspace
+import sciunit2.filelock
 
 from contextlib import contextmanager
 import os
 
+import shutil
+import time
+
+from getopt import getopt
 
 # returns the pkgdir and original command used
 # to execute execution 'rev'
 @contextmanager
 def CheckoutContext(rev):
     emgr, repo = sciunit2.workspace.current()
-    with emgr.exclusive():
-        orig = emgr.get(rev).cmd
-        pkgdir = os.path.join(repo.location, 'cde-package')
-        repo.cleanup(pkgdir)
-        repo.checkout(rev)
-        yield pkgdir, orig
-
+    lock = sciunit2.filelock.FileLock(os.path.join(repo.location ,'lockfile'))
+    lock.acquire()
+    try:
+        with emgr.exclusive():
+            orig = emgr.get(rev).cmd
+            pkgdir = os.path.join(repo.location, 'cde-package')
+            pkgdir_rev = os.path.join(repo.location, rev, 'cde-package')
+            repo.cleanup(pkgdir)
+            repo.cleanup(pkgdir_rev)
+            repo.checkout(rev)
+            shutil.copytree(pkgdir, pkgdir_rev)
+            yield pkgdir_rev, orig
+    finally:
+        lock.release()
 
 @contextmanager
 def CheckoutContext_Diff(rev):

sciunit2/core.py

@@ -4,6 +4,7 @@
 from sciunit2.util import Chdir
 from sciunit2.cdelog import open
 import sciunit2.libexec
+import sciunit2.filelock
 
 import os
 import shutil
@@ -43,13 +44,23 @@ def repeat(pkgdir, orig, newargs):
             with open('cde.log') as f:
                 cd, ls = f
             os.rename('cde.log', 'cde.log.1')
+
+            parent_pkgdir = os.path.join(os.path.dirname(os.path.dirname(pkgdir)), 'cde-package')
+            os.makedirs(os.path.join(parent_pkgdir, 'cde.log.1'), exist_ok=True)
+
             with open('cde.log', 'w') as f:
                 # adds the command in a comment
                 f.write_cmd(orig[:1] + newargs)
                 # dir to cd into for executing the commands
                 f.insert(cd)
                 # commands to execute with new arguments
                 f.insert(ls[:1] + newargs)
+
+        # this will cause issues if parallel repeat is run. 
+        # Question: Which execution-id to commit in case of sciunit commit after parallel repeat
+        shutil.copy(os.path.join(pkgdir,'cde.log.1'), os.path.join(parent_pkgdir, 'cde.log.1'))
+        shutil.copy(os.path.join(pkgdir,'cde.log'), os.path.join(parent_pkgdir, 'cde.log'))
+
     try:
         output = subprocess.check_output(['/bin/sh', 'cde.log'], cwd=pkgdir)
     except subprocess.CalledProcessError as exc:

sciunit2/filelock.py

@@ -0,0 +1,15 @@
+import fcntl
+
+class FileLock:
+    def __init__(self, file_path):
+        self.file_path = file_path
+        self.file_handle = None
+
+    def acquire(self):
+        self.file_handle = open(self.file_path, 'w')
+        fcntl.flock(self.file_handle, fcntl.LOCK_EX)
+
+    def release(self):
+        if self.file_handle:
+            fcntl.flock(self.file_handle, fcntl.LOCK_UN)
+            self.file_handle.close()

tests/test_repeat.py

@@ -62,19 +62,29 @@ def test_all(self):
         with testpath.modified_env({'SHELL': '/bin/true'}):
             assert_is_none(testit.sciunit('exec', '-i'))
 
+        assert_true(not os.path.exists('tmp/ok/e2'))
+        assert_true(os.path.exists('tmp/ok/e1'))
+
         with assert_raises(SystemExit) as r:
             testit.sciunit('repeat', 'e2')
         assert_equal(r.exception.code, 0)
 
+        assert_true(os.path.exists('tmp/ok/e2'))
+
         with assert_raises(SystemExit) as r:
             testit.sciunit('repeat', 'e2', '-l')
         assert_equal(r.exception.code, 1)
 
+        assert_true(not os.path.exists('tmp/ok/e3'))
+
         assert_is_none(testit.sciunit('commit'))
+
+        assert_true(not os.path.exists('tmp/ok/e3'))
 
         with assert_raises(SystemExit) as r:
             testit.sciunit('repeat', 'e3')
         assert_equal(r.exception.code, 0)
+        assert_true(os.path.exists('tmp/ok/e3'))
 
         with assert_raises(SystemExit) as r:
             testit.sciunit('repeat', 'e1', '-L')