Add LocalStack integration for local S3 testing

.env.example

@@ -0,0 +1,15 @@
+# Production AWS Configuration Template
+# Copy this file to ~/.aws/credentials or set as environment variables
+#
+# For production use:
+# 1. Run: aws configure
+# 2. Enter your real AWS credentials
+# 3. Do NOT set AWS_ENDPOINT_URL
+#
+# This file is for documentation purposes only.
+# Production environments use ~/.aws/credentials or IAM roles.
+
+# AWS_ACCESS_KEY_ID=your-real-access-key-here
+# AWS_SECRET_ACCESS_KEY=your-real-secret-key-here
+# AWS_REGION=us-east-1
+# CETI_BUCKET=ceti-data

.env.localstack

@@ -0,0 +1,9 @@
+# LocalStack Testing Environment
+# This file configures boto3 to use LocalStack instead of real AWS
+# Used for local development and testing only
+
+AWS_ENDPOINT_URL=http://localhost:4566
+AWS_ACCESS_KEY_ID=cetitest
+AWS_SECRET_ACCESS_KEY=cetitest
+AWS_REGION=us-east-1
+CETI_BUCKET=ceti-data-test

Makefile

@@ -24,3 +24,34 @@ release: bumpversion
 
 publish: build_tools build login_twine
 	@python -m twine upload --repository codeartifact dist/ceti-*
+
+# LocalStack testing targets
+localstack-up:
+	@echo "Starting LocalStack..."
+	@docker-compose -f docker-compose.localstack.yml up -d 2>&1
+	@echo "Waiting for LocalStack to be ready..."
+	@sleep 5
+	@echo "LocalStack is ready at http://localhost:4566"
+
+localstack-down:
+	@echo "Stopping LocalStack..."
+	@docker-compose -f docker-compose.localstack.yml down
+	@echo "LocalStack stopped"
+
+localstack-clean:
+	@echo "Cleaning LocalStack data..."
+	@docker-compose -f docker-compose.localstack.yml down -v
+	@echo "LocalStack data cleaned"
+
+localstack-logs:
+	@docker-compose -f docker-compose.localstack.yml logs -f
+
+test-local: localstack-up
+	@echo "Running tests with LocalStack..."
+	@set -a && . $(CURDIR)/.env.localstack && set +a && pytest -v
+	@echo "Stopping LocalStack..."
+	@docker-compose -f docker-compose.localstack.yml down 2>&1
+	@echo "LocalStack stopped"
+
+.PHONY: login login_twine clean build_tools build bumpversion release publish \
+        localstack-up localstack-down localstack-clean localstack-logs test-local

README.md

@@ -185,6 +185,61 @@ You can build a wheel file for binary distribution of the package. The wheel fil
 make build_tools && make build
 ```
 
+### Testing
+
+The project uses LocalStack to emulate AWS S3 for local testing without requiring AWS credentials.
+
+**Requirements:**
+- Docker (for LocalStack)
+- Python 3.7+ (tested with Python 3.13)
+
+**Resource Requirements:**
+- **Disk Space**: ~1.1 GB (Docker image) + ~50 MB (data volume during tests)
+- **Memory**: ~86 MB RAM (idle), up to ~256 MB during active testing
+- **Network**: Initial download of 1.1 GB Docker image (one-time)
+
+#### Quick Start
+
+```console
+# Run all tests with LocalStack (starts/stops automatically)
+make test-local
+```
+
+This command will:
+1. Start LocalStack (pulls Docker image if needed)
+2. Run all 9 tests against local S3 emulator
+3. Stop and clean up LocalStack
+
+#### Manual Testing
+
+For development and debugging, you can manually control LocalStack and run tests:
+
+```console
+# 1. Start LocalStack container
+make localstack-up
+
+# 2. Load environment variables and run tests
+set -a && source .env.localstack && set +a && pytest
+
+# Or run specific test files
+set -a && source .env.localstack && set +a && pytest tests/test_s3upload.py -v
+
+# 3. Stop LocalStack when done
+make localstack-down
+
+# Optional: Clean LocalStack data (removes all buckets/objects)
+make localstack-clean
+```
+
+**Why manual testing?**
+- Keep LocalStack running between test runs (faster iteration)
+- Run specific test files or functions
+- Debug test failures without restarting container
+
+**Note:** `make test-local` is recommended for CI/CD and final validation - it handles all setup/teardown automatically.
+
+For detailed testing documentation, see [docs/TESTING.md](docs/TESTING.md).
+
 ### Releasing a new version
 
 This package follows [semantic versioning](https://semver.org/) approach and [PEP440](https://www.python.org/dev/peps/pep-0440). In order to release a new version run the following steps:

ceti/general_offload.py

@@ -89,9 +89,9 @@ def get_registered_devices(s3client):
 
 
 def cli(args: Namespace):
-    
+
     print()
-    s3client = boto3.client('s3')
+    s3client = boto3.client('s3', endpoint_url=os.getenv('AWS_ENDPOINT_URL'))
     registered_device_ids = get_registered_devices(s3client)
 
     if not os.path.exists(args.data_dir):

ceti/s3upload.py

@@ -83,7 +83,17 @@ def cli(args: Namespace):
 
     files = get_filelist(args.data_directory)
     botocore_config = botocore.config.Config(max_pool_connections=MAX_CONCURRENCY)
-    s3client = boto3.client('s3', config=botocore_config)
+    s3client = boto3.client(
+        's3',
+        config=botocore_config,
+        endpoint_url=os.getenv('AWS_ENDPOINT_URL')
+    )
+
+    # Warn if using LocalStack
+    if os.getenv('AWS_ENDPOINT_URL'):
+        print(f"WARNING: Using LocalStack at {os.getenv('AWS_ENDPOINT_URL')} (not production AWS)")
+        print(f"         Uploading to bucket: {BUCKET_NAME}")
+        print()
 
     if args.debug:
         boto3.set_stream_logger('')

ceti/spark/utils.py

@@ -21,7 +21,7 @@ def get_s3_emr_dir(job_name: str) -> Path:
 
 def upload_files(path_specs: Sequence[Tuple[str, str]]) -> None:
     """Upload files to S3 given src / dst tuples"""
-    s3 = boto3.client('s3')
+    s3 = boto3.client('s3', endpoint_url=os.getenv('AWS_ENDPOINT_URL'))
 
     for src, dst in path_specs:
         uri = urlparse(dst)

docker-compose.localstack.yml

@@ -0,0 +1,24 @@
+services:
+  localstack:
+    image: localstack/localstack:latest
+    container_name: ceti-localstack
+    ports:
+      - "4566:4566"
+    environment:
+      - SERVICES=s3
+      - DEBUG=0
+      - AWS_DEFAULT_REGION=us-east-1
+      - AWS_ACCESS_KEY_ID=cetitest
+      - AWS_SECRET_ACCESS_KEY=cetitest
+    volumes:
+      - localstack-data:/var/lib/localstack
+      - ./scripts/init-localstack.sh:/etc/localstack/init/ready.d/init-buckets.sh
+    networks:
+      - ceti-test
+
+volumes:
+  localstack-data:
+
+networks:
+  ceti-test:
+    driver: bridge