chore: refactor tests

.github/workflows/tests.yml

@@ -24,14 +24,5 @@ jobs:
       - name: Install the project
         run: uv sync --locked --all-extras --dev
 
-      - name: Start MinIO server
-        run: |
-          docker compose -f tests/docker/docker-compose.yml up -d
-          # Wait for MinIO to be healthy
-          sleep 10
-
       - name: Run tests
-        env:
-          MINIO_ROOT_USER: minioadmin
-          MINIO_ROOT_PASSWORD: minioadmin
-        run: uv run pytest -v --cov=opendalfs --cov-report=xml
+        run: make test PYTEST_ARGS="-v --cov=opendalfs --cov-report=xml"

CONTRIBUTING.md

@@ -18,15 +18,10 @@ uv sync --locked --all-extras --dev
 
 ### Dependency Groups
 
-The project uses several dependency groups:
-- `dev`: Development tools (ruff)
-- `test`: Testing tools (pytest, pytest-asyncio, pytest-cov, fsspec, s3fs)
-- `all`: All dependencies combined
-
 Install specific groups as needed:
 ```shell
-pip install -e ".[dev,test]"  # For development and testing
-pip install -e ".[s3]"        # For S3 backend development
+pip install -e ".[dev]"  # For development and testing
+
 ```
 
 ## Testing

Makefile

@@ -0,0 +1,17 @@
+PYTEST_ARGS ?= -v  # Override with e.g. PYTEST_ARGS="-vv --tb=short"
+
+test: test-setup test-exec
+
+test-setup:
+	docker compose -f tests/docker/docker-compose.yml kill
+	docker compose -f tests/docker/docker-compose.yml rm -f
+	docker compose -f tests/docker/docker-compose.yml up -d
+	sleep 5
+
+test-exec:
+	uv run pytest $(PYTEST_ARGS)
+
+test-rebuild:
+	docker compose -f tests/docker/docker-compose.yml kill
+	docker compose -f tests/docker/docker-compose.yml rm -f
+	docker compose -f tests/docker/docker-compose.yml build --no-cache

opendalfs/fs.py

@@ -1,5 +1,5 @@
 from typing import Any
-from fsspec.asyn import AsyncFileSystem, sync
+from fsspec import AbstractFileSystem
 import logging
 from opendal import Operator, AsyncOperator
 from .file import OpendalBufferedFile
@@ -8,8 +8,8 @@
 logger = logging.getLogger("opendalfs")
 
 @generate_blocking_methods
-class OpendalFileSystem(AsyncFileSystem):
-    """OpenDAL implementation of fsspec AsyncFileSystem.
+class OpendalFileSystem(AbstractFileSystem):
+    """OpenDAL implementation for fsspec.
 
     This implementation provides both synchronous and asynchronous access to
     various storage backends supported by OpenDAL.

pyproject.toml

@@ -19,27 +19,24 @@ dependencies = [
 # Development dependencies
 dev = [
     "ruff",              # Code formatting and linting
-]
-
-# Test dependencies
-test = [
     "pytest",           # Test framework
     "pytest-asyncio",   # Async test support
     "pytest-cov",       # Coverage reporting
-    "s3fs",             # S3 filesystem support
-    "boto3",            # AWS S3 client for environment provision
+    "pytest-lazy-fixtures", # Lazy fixture loading
 ]
 
-# All development dependencies
-all = ["opendalfs[dev,test]"]
-
 [tool.setuptools]
 packages = ["opendalfs"]
-package-dir = {"" = "."}
 
 [build-system]
 requires = ["setuptools>=61.0"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools.package-data]
 opendalfs = ["py.typed"]
+
+
+filterwarnings = [
+    "error",
+    "ignore:A plugin raised an exception during an old-style hookwrapper teardown.",
+]

tests/conftest.py

@@ -1,65 +1,29 @@
 import pytest
 from opendalfs import OpendalFileSystem
-import boto3
 import asyncio
+from pytest_lazy_fixtures import lf
+from typing import Generator
 
-
-@pytest.fixture(scope="session")
-def minio_server():
-    """Ensure MinIO server is available for testing."""
-    import socket
-    import time
-
-    # Check if MinIO is accessible
-    retries = 3
-    while retries > 0:
-        try:
-            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            result = sock.connect_ex(("localhost", 9000))
-            if result == 0:
-                sock.close()
-                return  # MinIO is available
-        except Exception as e:
-            print(f"DEBUG: Error connecting to MinIO: {e}, retrying...")
-            pass
-        retries -= 1
-        time.sleep(1)
-
-    pytest.skip(
-        "MinIO is not available. Please start it with: docker-compose -f tests/docker/docker-compose.yml up -d"
-    )
-
-
-@pytest.fixture
-def s3_fs(minio_server):
+@pytest.fixture(scope="function")
+def s3() -> Generator[OpendalFileSystem, None, None]:
     """Create an S3 filesystem for testing sync operations."""
-    from .utils.s3 import create_test_bucket, cleanup_bucket, verify_bucket
-
     fs = OpendalFileSystem(
         scheme="s3",
-        bucket="test-bucket",
+        bucket="test",
         endpoint="http://localhost:9000",
         region="us-east-1",
         access_key_id="minioadmin",
         secret_access_key="minioadmin",
         asynchronous=False,
     )
 
-    create_test_bucket()
-    verify_bucket()
+    yield fs
+    fs.rmdir("/")
 
-    # Verify we can write directly with boto3
-    s3 = boto3.client(
-        "s3",
-        endpoint_url="http://localhost:9000",
-        aws_access_key_id="minioadmin",
-        aws_secret_access_key="minioadmin",
-    )
-    s3.put_object(Bucket="test-bucket", Key="test.txt", Body=b"test")
-    print("DEBUG: Wrote test file directly with boto3")
 
-    yield fs
-    cleanup_bucket()
+FILE_SYSTEMS = [
+    lf("s3"),
+]
 
 
 @pytest.fixture(scope="function")
@@ -68,13 +32,3 @@ async def event_loop():
     loop = asyncio.get_event_loop_policy().new_event_loop()
     yield loop
     loop.close()
-
-
-@pytest.mark.asyncio
-async def test_write_read(s3_fs):
-    """Test basic write and read operations."""
-    for fs in [s3_fs]:
-        content = b"test content"
-        await fs._write("test.txt", content)
-        result = await fs._read("test.txt")
-        assert result == content

tests/docker/docker-compose.yml

@@ -2,15 +2,29 @@ version: '3.7'
 services:
   minio:
     image: minio/minio
-    ports:
-      - "9000:9000"
-      - "9001:9001"
+    container_name: opendalfs-minio
     environment:
       MINIO_ROOT_USER: minioadmin
       MINIO_ROOT_PASSWORD: minioadmin
-    command: server /data --console-address ":9001"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-      interval: 30s
-      timeout: 20s
-      retries: 3 
+    networks:
+      opendalfs_net:
+        aliases:
+          - warehouse.minio
+    ports:
+      - 9001:9001
+      - 9000:9000
+    command: ["server", "/data", "--console-address", ":9001"]
+  mc:
+    depends_on:
+      - minio
+    image: minio/mc
+    container_name: opendalfs-mc
+    entrypoint: >
+      /bin/sh -c "
+      until (/usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin) do echo '...waiting...' && sleep 1; done;
+      /usr/bin/mc mb minio/test;
+      /usr/bin/mc policy set public minio/test;
+      "
+
+networks:
+  opendalfs_net:

tests/test_filesystem.py

@@ -0,0 +1,15 @@
+"""Core functionality tests for filesystem creation and basic operations."""
+
+import pytest
+from tests.conftest import FILE_SYSTEMS
+from opendalfs import OpendalFileSystem
+
+
+@pytest.mark.parametrize("fs", FILE_SYSTEMS)
+def test_write_read(fs: OpendalFileSystem) -> None:
+    """Test basic write and read operations."""
+    with fs.open("foo.bar", "wb") as f:
+        f.write("baz")
+
+    with fs.open("foo.bar", "rb") as f:
+        assert f.read() == "baz"