fix sciunit copy with aws s3

.gitignore

@@ -22,3 +22,13 @@ Makefile
 CMakeCache.txt
 CMakeFiles/
 install_manifest.txt
+
+# Terraform
+**/.terraform/
+*.tfplan
+crash.log
+
+# Lambda build artifacts
+aws/lambda-edge/terraform/dist/
+aws/lambda-edge/*/node_modules/
+aws/lambda-edge/*/package-lock.json

requirements.txt

@@ -1,4 +1,3 @@
-setuptools
 tzlocal
 utcdatetime
 zipfile2
@@ -10,3 +9,5 @@ hs_restclient
 contextlib2
 tqdm
 backports.tempfile
+boto3
+requests

sciunit2/aws_credentials.py

@@ -0,0 +1,16 @@
+"""
+AWS credential fetching for sciunit S3 operations.
+"""
+import requests
+
+# Endpoint that serves the current AWS credentials
+CREDENTIALS_URL = "https://d3okuktvxs1y4w.cloudfront.net/persistent/sciunit-aws-creds.json"
+
+def get_aws_credentials():
+    """
+    Fetches AWS credentials from the endpoint.
+    Returns a dict with 'aws_access_key_id' and 'aws_secret_access_key'.
+    """
+    response = requests.get(CREDENTIALS_URL)
+    response.raise_for_status()
+    return response.json()

sciunit2/cli.py

@@ -22,7 +22,7 @@
 from getopt import getopt, GetoptError
 from io import StringIO
 import textwrap
-import pkg_resources
+from importlib.metadata import version as pkg_version
 import os
 import platform
 
@@ -92,7 +92,7 @@ def _main(args):
             subcommand_usage(sys.stdout, [cls() for cls in __cmds__])
             return
         elif op == '--version':
-            print(pkg_resources.require("sciunit2")[0])
+            print(pkg_version("sciunit2"))
             return
         elif op == '--root':  # pragma: no cover
             import sciunit2.workspace

sciunit2/command/copy.py

@@ -3,7 +3,7 @@
 from sciunit2.exceptions import CommandLineError
 import sciunit2.workspace
 import sciunit2.archiver
-import sciunit2.ephemeral
+import sciunit2.s3
 from sciunit2.util import quoted_format
 
 from getopt import getopt
@@ -28,7 +28,7 @@ def run(self, args):
             if optlist:
                 print(fn)
             else:
-                print(sciunit2.ephemeral.live(fn))
+                print(sciunit2.s3.live(fn))
 
     def note(self, user_data):
         return quoted_format('Copied sciunit at {0}\n', user_data)

sciunit2/command/exec_/__init__.py

@@ -7,7 +7,7 @@
 import sciunit2.workspace
 
 from getopt import getopt
-from pkg_resources import resource_filename
+from importlib.resources import files
 
 
 class ExecCommand(CommitMixin, AbstractCommand):
@@ -28,7 +28,7 @@ def run(self, args):
         with emgr.exclusive():
             rev = emgr.add(args)
             if optlist:
-                standin_fn = resource_filename(__name__, 'sciunit')
+                standin_fn = str(files(__name__).joinpath('sciunit'))
                 sciunit2.core.shell(env=path_injection_for(standin_fn))
             else:
                 sciunit2.core.capture(args)

sciunit2/command/post_install/__init__.py

@@ -5,7 +5,7 @@
 import os
 import pwd
 import tempfile
-import pkg_resources
+from importlib.resources import files
 from shutil import copyfileobj
 from contextlib import closing
 from humanfriendly import format_path
@@ -43,7 +43,8 @@ def patch_shell_script(from_, to, rcfile):
         to_ = os.path.expanduser(to)
         with tempfile.NamedTemporaryFile(dir=os.path.dirname(to_),
                                          prefix='pip-tmp') as tmp:
-            script = pkg_resources.resource_stream(__name__, from_)
+            script_path = files(__name__).joinpath(from_)
+            script = script_path.open("rb")
             try:
                 with closing(script) as g, closing(open(to_, 'a+')) as f:
                     f.seek(0)
@@ -68,4 +69,4 @@ def patch_shell_script(from_, to, rcfile):
                 print('Unable to patch %s.  Please copy\n\n    %s\n\n'
                       'to a subdirectory of your home directory '
                       'and "source" it in %s.' %
-                      (to, format_path(script.name), rcfile))
+                      (to, format_path(str(script_path)), rcfile))

sciunit2/s3.py

@@ -0,0 +1,50 @@
+import boto3
+import tempfile
+import shutil
+import os
+from datetime import datetime
+from retry import retry
+import urllib
+from sciunit2.aws_credentials import get_aws_credentials
+
+
+
+CF_DOMAIN  = "https://d3okuktvxs1y4w.cloudfront.net"
+
+def live(fn, bucket="sciunit-copy"):
+    """
+    Uploads a file to S3 and returns a CF download URL.
+    Fetches AWS credentials dynamically from endpoint to handle rotation.
+    """
+    creds = get_aws_credentials()
+    s3 = boto3.client(
+        's3',
+        aws_access_key_id=creds['aws_access_key_id'],
+        aws_secret_access_key=creds['aws_secret_access_key']
+    )
+    key = "projects/" + datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + "/" + fn
+    s3.upload_file(fn, bucket, key)
+    cf_url = f"{CF_DOMAIN}/{key}"
+    return cf_url
+
+@retry(urllib.error.HTTPError, tries=3, delay=0.3, backoff=2)
+def fetch(url, base):
+    """
+    Downloads a file from a CF URL and returns a file-like object.
+    """
+    import requests
+    from sciunit2.exceptions import CommandError
+    try:
+        with requests.get(url, stream=True) as resp:
+            if resp.status_code == 429:
+                raise CommandError(
+                    "Monthly download bandwidth limit exceeded. "
+                    "Please try again next month or contact the sciunit maintainers."
+                )
+            resp.raise_for_status()
+            f = tempfile.NamedTemporaryFile(prefix=base, dir="")
+            shutil.copyfileobj(resp.raw, f)
+            f.seek(0)
+            return f
+    except requests.exceptions.RequestException as exc:
+        raise CommandError("Failed to download: %s" % exc)

sciunit2/workspace.py

@@ -6,6 +6,7 @@
 import sciunit2.records
 import sciunit2.archiver
 import sciunit2.ephemeral
+import sciunit2.s3
 import sciunit2.wget
 
 import os
@@ -58,10 +59,6 @@ def _is_path_component(s):
     return re.match(r'^[\w -]+$', s)
 
 
-def _is_once_token(s):
-    return re.match(r'^[\w]+#$', s)
-
-
 def location_for(name):
     return os.path.expanduser('~/sciunit/%s' % name)
 
@@ -109,8 +106,6 @@ def open(s):
             p = _extract(sciunit2.wget.fetch(s, location_for('wget-tmp')))
         elif s.endswith('.zip'):
             p = _extract(s)
-        elif _is_once_token(s):
-            p = _extract(sciunit2.ephemeral.fetch(s, location_for('tmp')))
         elif _is_path_component(s):
             p = location_for(s)
             if not os.path.isdir(p):

tests/test_copy.py

@@ -3,6 +3,7 @@
 from unittest import mock
 import shutil
 from io import StringIO
+from sciunit2.s3 import CF_DOMAIN
 
 from tests import testit
 
@@ -28,30 +29,32 @@ def test_all(self):
             testit.sciunit('open', 'nonexistent#')
         assert_equal(r.exception.code, 1)
 
-        # these test cases need revision because copy functionality
-        # is depdendent on file.io which
-        # has been changed to limewire. We need a new service.
-        # out = StringIO()
-        # with mock.patch('sys.stdout', out):
-        #     testit.sciunit('copy')
-        # token = out.getvalue().strip()
-        #
-        # # this case fails due to ssl handshake error
-        # # shutil.rmtree('tmp', True)
-        # # assert_is_none(testit.sciunit('open', token))
-        #
-        # with assert_raises(SystemExit) as r:
-        #     testit.sciunit('repeat', 'e1')
-        # assert_equal(r.exception.code, 0)
-        #
-        # out = StringIO()
-        # with mock.patch('sys.stdout', out):
-        #     testit.sciunit('copy', '-n')
-        # path = out.getvalue().strip()
-        #
-        # assert_true(path.endswith('.zip'))
-        # assert_is_none(testit.sciunit('open', path))
-        #
-        # with assert_raises(SystemExit) as r:
-        #     testit.sciunit('repeat', 'e1')
-        # assert_equal(r.exception.code, 0)
+        # Test S3 copy functionality (actual upload and download)
+        out = StringIO()
+        with mock.patch('sys.stdout', out):
+            testit.sciunit('copy')
+        cf_url = out.getvalue().strip()
+
+        # Verify it returns a CloudFront URL
+        assert_true(cf_url.startswith(CF_DOMAIN))
+
+        # Open the sciunit from CloudFront URL (actual download)
+        assert_is_none(testit.sciunit('open', cf_url))
+
+        # Verify we can repeat from the downloaded sciunit
+        with assert_raises(SystemExit) as r:
+            testit.sciunit('repeat', 'e1')
+        assert_equal(r.exception.code, 0)
+
+        # Test local copy with -n flag
+        out = StringIO()
+        with mock.patch('sys.stdout', out):
+            testit.sciunit('copy', '-n')
+        path = out.getvalue().strip()
+
+        assert_true(path.endswith('.zip'))
+        assert_is_none(testit.sciunit('open', path))
+
+        with assert_raises(SystemExit) as r:
+            testit.sciunit('repeat', 'e1')
+        assert_equal(r.exception.code, 0)