diff --git a/.gitignore b/.gitignore index 977e42d..9ea0f6f 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,13 @@ Makefile CMakeCache.txt CMakeFiles/ install_manifest.txt + +# Terraform +**/.terraform/ +*.tfplan +crash.log + +# Lambda build artifacts +aws/lambda-edge/terraform/dist/ +aws/lambda-edge/*/node_modules/ +aws/lambda-edge/*/package-lock.json \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 21c2402..96a0fda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -setuptools tzlocal utcdatetime zipfile2 @@ -10,3 +9,5 @@ hs_restclient contextlib2 tqdm backports.tempfile +boto3 +requests \ No newline at end of file diff --git a/sciunit2/aws_credentials.py b/sciunit2/aws_credentials.py new file mode 100644 index 0000000..e6e36ca --- /dev/null +++ b/sciunit2/aws_credentials.py @@ -0,0 +1,16 @@ +""" +AWS credential fetching for sciunit S3 operations. +""" +import requests + +# Endpoint that serves the current AWS credentials +CREDENTIALS_URL = "https://d3okuktvxs1y4w.cloudfront.net/persistent/sciunit-aws-creds.json" + +def get_aws_credentials(): + """ + Fetches AWS credentials from the endpoint. + Returns a dict with 'aws_access_key_id' and 'aws_secret_access_key'. + """ + response = requests.get(CREDENTIALS_URL) + response.raise_for_status() + return response.json() diff --git a/sciunit2/cli.py b/sciunit2/cli.py index 3bca3d3..1e14df3 100644 --- a/sciunit2/cli.py +++ b/sciunit2/cli.py @@ -22,7 +22,7 @@ from getopt import getopt, GetoptError from io import StringIO import textwrap -import pkg_resources +from importlib.metadata import version as pkg_version import os import platform @@ -92,7 +92,7 @@ def _main(args): subcommand_usage(sys.stdout, [cls() for cls in __cmds__]) return elif op == '--version': - print(pkg_resources.require("sciunit2")[0]) + print(pkg_version("sciunit2")) return elif op == '--root': # pragma: no cover import sciunit2.workspace diff --git a/sciunit2/command/copy.py b/sciunit2/command/copy.py index 15672cf..6351e94 100644 --- a/sciunit2/command/copy.py +++ b/sciunit2/command/copy.py @@ -3,7 +3,7 @@ from sciunit2.exceptions import CommandLineError import sciunit2.workspace import sciunit2.archiver -import sciunit2.ephemeral +import sciunit2.s3 from sciunit2.util import quoted_format from getopt import getopt @@ -28,7 +28,7 @@ def run(self, args): if optlist: print(fn) else: - print(sciunit2.ephemeral.live(fn)) + print(sciunit2.s3.live(fn)) def note(self, user_data): return quoted_format('Copied sciunit at {0}\n', user_data) diff --git a/sciunit2/command/exec_/__init__.py b/sciunit2/command/exec_/__init__.py index 62cd41d..eb5e1e7 100644 --- a/sciunit2/command/exec_/__init__.py +++ b/sciunit2/command/exec_/__init__.py @@ -7,7 +7,7 @@ import sciunit2.workspace from getopt import getopt -from pkg_resources import resource_filename +from importlib.resources import files class ExecCommand(CommitMixin, AbstractCommand): @@ -28,7 +28,7 @@ def run(self, args): with emgr.exclusive(): rev = emgr.add(args) if optlist: - standin_fn = resource_filename(__name__, 'sciunit') + standin_fn = str(files(__name__).joinpath('sciunit')) sciunit2.core.shell(env=path_injection_for(standin_fn)) else: sciunit2.core.capture(args) diff --git a/sciunit2/command/post_install/__init__.py b/sciunit2/command/post_install/__init__.py index 68daee7..8fcbf29 100644 --- a/sciunit2/command/post_install/__init__.py +++ b/sciunit2/command/post_install/__init__.py @@ -5,7 +5,7 @@ import os import pwd import tempfile -import pkg_resources +from importlib.resources import files from shutil import copyfileobj from contextlib import closing from humanfriendly import format_path @@ -43,7 +43,8 @@ def patch_shell_script(from_, to, rcfile): to_ = os.path.expanduser(to) with tempfile.NamedTemporaryFile(dir=os.path.dirname(to_), prefix='pip-tmp') as tmp: - script = pkg_resources.resource_stream(__name__, from_) + script_path = files(__name__).joinpath(from_) + script = script_path.open("rb") try: with closing(script) as g, closing(open(to_, 'a+')) as f: f.seek(0) @@ -68,4 +69,4 @@ def patch_shell_script(from_, to, rcfile): print('Unable to patch %s. Please copy\n\n %s\n\n' 'to a subdirectory of your home directory ' 'and "source" it in %s.' % - (to, format_path(script.name), rcfile)) + (to, format_path(str(script_path)), rcfile)) diff --git a/sciunit2/s3.py b/sciunit2/s3.py new file mode 100644 index 0000000..5a897a3 --- /dev/null +++ b/sciunit2/s3.py @@ -0,0 +1,50 @@ +import boto3 +import tempfile +import shutil +import os +from datetime import datetime +from retry import retry +import urllib +from sciunit2.aws_credentials import get_aws_credentials + + + +CF_DOMAIN = "https://d3okuktvxs1y4w.cloudfront.net" + +def live(fn, bucket="sciunit-copy"): + """ + Uploads a file to S3 and returns a CF download URL. + Fetches AWS credentials dynamically from endpoint to handle rotation. + """ + creds = get_aws_credentials() + s3 = boto3.client( + 's3', + aws_access_key_id=creds['aws_access_key_id'], + aws_secret_access_key=creds['aws_secret_access_key'] + ) + key = "projects/" + datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + "/" + fn + s3.upload_file(fn, bucket, key) + cf_url = f"{CF_DOMAIN}/{key}" + return cf_url + +@retry(urllib.error.HTTPError, tries=3, delay=0.3, backoff=2) +def fetch(url, base): + """ + Downloads a file from a CF URL and returns a file-like object. + """ + import requests + from sciunit2.exceptions import CommandError + try: + with requests.get(url, stream=True) as resp: + if resp.status_code == 429: + raise CommandError( + "Monthly download bandwidth limit exceeded. " + "Please try again next month or contact the sciunit maintainers." + ) + resp.raise_for_status() + f = tempfile.NamedTemporaryFile(prefix=base, dir="") + shutil.copyfileobj(resp.raw, f) + f.seek(0) + return f + except requests.exceptions.RequestException as exc: + raise CommandError("Failed to download: %s" % exc) \ No newline at end of file diff --git a/sciunit2/workspace.py b/sciunit2/workspace.py index a658467..7808847 100644 --- a/sciunit2/workspace.py +++ b/sciunit2/workspace.py @@ -6,6 +6,7 @@ import sciunit2.records import sciunit2.archiver import sciunit2.ephemeral +import sciunit2.s3 import sciunit2.wget import os @@ -58,10 +59,6 @@ def _is_path_component(s): return re.match(r'^[\w -]+$', s) -def _is_once_token(s): - return re.match(r'^[\w]+#$', s) - - def location_for(name): return os.path.expanduser('~/sciunit/%s' % name) @@ -109,8 +106,6 @@ def open(s): p = _extract(sciunit2.wget.fetch(s, location_for('wget-tmp'))) elif s.endswith('.zip'): p = _extract(s) - elif _is_once_token(s): - p = _extract(sciunit2.ephemeral.fetch(s, location_for('tmp'))) elif _is_path_component(s): p = location_for(s) if not os.path.isdir(p): diff --git a/tests/test_copy.py b/tests/test_copy.py index 83eb907..38a1d2b 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -3,6 +3,7 @@ from unittest import mock import shutil from io import StringIO +from sciunit2.s3 import CF_DOMAIN from tests import testit @@ -28,30 +29,32 @@ def test_all(self): testit.sciunit('open', 'nonexistent#') assert_equal(r.exception.code, 1) - # these test cases need revision because copy functionality - # is depdendent on file.io which - # has been changed to limewire. We need a new service. - # out = StringIO() - # with mock.patch('sys.stdout', out): - # testit.sciunit('copy') - # token = out.getvalue().strip() - # - # # this case fails due to ssl handshake error - # # shutil.rmtree('tmp', True) - # # assert_is_none(testit.sciunit('open', token)) - # - # with assert_raises(SystemExit) as r: - # testit.sciunit('repeat', 'e1') - # assert_equal(r.exception.code, 0) - # - # out = StringIO() - # with mock.patch('sys.stdout', out): - # testit.sciunit('copy', '-n') - # path = out.getvalue().strip() - # - # assert_true(path.endswith('.zip')) - # assert_is_none(testit.sciunit('open', path)) - # - # with assert_raises(SystemExit) as r: - # testit.sciunit('repeat', 'e1') - # assert_equal(r.exception.code, 0) + # Test S3 copy functionality (actual upload and download) + out = StringIO() + with mock.patch('sys.stdout', out): + testit.sciunit('copy') + cf_url = out.getvalue().strip() + + # Verify it returns a CloudFront URL + assert_true(cf_url.startswith(CF_DOMAIN)) + + # Open the sciunit from CloudFront URL (actual download) + assert_is_none(testit.sciunit('open', cf_url)) + + # Verify we can repeat from the downloaded sciunit + with assert_raises(SystemExit) as r: + testit.sciunit('repeat', 'e1') + assert_equal(r.exception.code, 0) + + # Test local copy with -n flag + out = StringIO() + with mock.patch('sys.stdout', out): + testit.sciunit('copy', '-n') + path = out.getvalue().strip() + + assert_true(path.endswith('.zip')) + assert_is_none(testit.sciunit('open', path)) + + with assert_raises(SystemExit) as r: + testit.sciunit('repeat', 'e1') + assert_equal(r.exception.code, 0)