feat(services/opfs): add read/write/delete/list support for OPFS

.github/workflows/test_behavior.yml

@@ -201,4 +201,4 @@ jobs:
     uses: ./.github/workflows/test_behavior_integration_object_store.yml
     with:
       os: ${{ matrix.os }}
-      cases: ${{ toJson(matrix.cases) }}
+      cases: ${{ toJson(matrix.cases) }}

.github/workflows/test_edge.yml

@@ -138,3 +138,39 @@ jobs:
           OPENDAL_S3_BUCKET: opendal-testing
           OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing
           OPENDAL_S3_REGION: ap-northeast-1
+
+  test_opfs_on_wasm:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+
+      - name: Setup for wasm32
+        run: |
+          rustup target add wasm32-unknown-unknown
+
+      - name: Install Chrome Environment
+        run: |
+          mkdir -p /tmp/chrome
+          wget $(curl https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json | jq -r '.versions | sort_by(.version) | reverse | .[0] | .downloads.chrome | .[] | select(.platform == "linux64") | .url')
+          wget $(curl https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json | jq -r '.versions | sort_by(.version) | reverse | .[0] | .downloads.chromedriver | .[] | select(.platform == "linux64") | .url')
+          unzip chromedriver-linux64.zip
+          unzip chrome-linux64.zip
+          cp -r chrome-linux64/ /tmp/chrome/
+          cp -r chromedriver-linux64 /tmp/chrome/chromedriver
+
+      - name: Setup wasm-pack
+        uses: taiki-e/install-action@v2
+        with:
+          tool: wasm-pack
+
+      - name: Test OPFS
+        working-directory: core/edge/opfs_on_wasm
+        env:
+          OPENDAL_TEST: opfs
+        run: |
+          export PATH=$PATH:/tmp/chrome/chrome-linux64/:/tmp/chrome/chromedriver-linux64/
+          wasm-pack test --chrome --headless

core/Cargo.toml

@@ -369,16 +369,23 @@ js-sys = { version = "0.3.77", optional = true }
 wasm-bindgen = { version = "0.2.100", optional = true }
 wasm-bindgen-futures = { version = "0.4.50", optional = true }
 web-sys = { version = "0.3.77", optional = true, features = [
+  "Blob",
   "Window",
   "File",
+  "FileSystemCreateWritableOptions",
   "FileSystemDirectoryHandle",
   "FileSystemFileHandle",
   "FileSystemGetDirectoryOptions",
   "FileSystemGetFileOptions",
+  "FileSystemHandle",
+  "FileSystemHandleKind",
+  "FileSystemRemoveOptions",
   "FileSystemWritableFileStream",
   "Navigator",
+  "ReadableStream",
+  "ReadableStreamDefaultReader",
+  "ReadableStreamReadResult",
   "StorageManager",
-  "FileSystemGetFileOptions",
 ] }
 
 # Layers

core/edge/opfs_on_wasm/Cargo.toml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "edge_test_opfs_on_wasm"
+
+edition.workspace = true
+license.workspace = true
+publish = false
+rust-version.workspace = true
+version.workspace = true
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+anyhow = { version = "1.0.30", features = ["std"] }
+bytes = "1.6"
+futures = "0.3"
+opendal = { path = "../..", default-features = false, features = [
+  "services-opfs",
+] }
+rand = { version = "0.8" }
+sha2 = "0.10"
+uuid = { version = "1", features = ["serde", "v4"] }
+
+wasm-bindgen-test = "0.3.41"

core/edge/opfs_on_wasm/src/async_create_dir.rs

@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(test)]
+mod tests {
+
+    use anyhow::Result;
+    use opendal::*;
+    use wasm_bindgen_test::{wasm_bindgen_test, wasm_bindgen_test_configure};
+
+    wasm_bindgen_test_configure!(run_in_browser);
+
+    use crate::*;
+
+    #[wasm_bindgen_test]
+    /// Create dir with dir path should succeed.
+    pub async fn test_create_dir() -> Result<()> {
+        let op = operator();
+        let path = TEST_FIXTURE.new_dir_path();
+
+        op.create_dir(&path).await?;
+
+        let meta = op.stat(&path).await?;
+        assert_eq!(meta.mode(), EntryMode::DIR);
+        Ok(())
+    }
+
+    /// Create dir on existing dir should succeed.
+    #[wasm_bindgen_test]
+    pub async fn test_create_dir_existing() -> Result<()> {
+        let op = operator();
+        let path = TEST_FIXTURE.new_dir_path();
+
+        op.create_dir(&path).await?;
+
+        op.create_dir(&path).await?;
+
+        let meta = op.stat(&path).await?;
+        assert_eq!(meta.mode(), EntryMode::DIR);
+
+        Ok(())
+    }
+}

core/edge/opfs_on_wasm/src/async_delete.rs

@@ -0,0 +1,196 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(test)]
+mod tests {
+
+    use anyhow::Result;
+    use opendal::raw::Access;
+    use opendal::*;
+    use wasm_bindgen_test::wasm_bindgen_test;
+    use wasm_bindgen_test::wasm_bindgen_test_configure;
+
+    use crate::*;
+
+    wasm_bindgen_test_configure!(run_in_browser);
+
+    /// Delete existing file should succeed.
+    #[wasm_bindgen_test]
+    pub async fn test_delete_file() -> Result<()> {
+        let op = operator();
+        let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+
+        op.write(&path, content).await.expect("write must succeed");
+
+        op.delete(&path).await?;
+
+        // Stat it again to check.
+        assert!(!op.exists(&path).await?);
+
+        Ok(())
+    }
+
+    /// Delete empty dir should succeed.
+    #[wasm_bindgen_test]
+    pub async fn test_delete_empty_dir() -> Result<()> {
+        let op = operator();
+        if !op.info().full_capability().create_dir {
+            return Ok(());
+        }
+
+        let path = TEST_FIXTURE.new_dir_path();
+
+        op.create_dir(&path).await.expect("create must succeed");
+
+        op.delete(&path).await?;
+
+        Ok(())
+    }
+
+    /// Delete file with special chars should succeed.
+    #[wasm_bindgen_test]
+    pub async fn test_delete_with_special_chars() -> Result<()> {
+        let op = operator();
+        // Ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 addressed.
+        if op.info().scheme() == opendal::Scheme::Atomicserver {
+            return Ok(());
+        }
+
+        let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4());
+        let (path, content, _) = TEST_FIXTURE.new_file_with_path(op.clone(), &path);
+
+        op.write(&path, content).await.expect("write must succeed");
+
+        op.delete(&path).await?;
+
+        // Stat it again to check.
+        assert!(!op.exists(&path).await?);
+
+        Ok(())
+    }
+
+    /// Delete not existing file should also succeed.
+    #[wasm_bindgen_test]
+    pub async fn test_delete_not_existing() -> Result<()> {
+        let op = operator();
+        let path = uuid::Uuid::new_v4().to_string();
+
+        op.delete(&path).await?;
+
+        Ok(())
+    }
+
+    /// Remove one file
+    #[wasm_bindgen_test]
+    pub async fn test_remove_one_file() -> Result<()> {
+        let op = operator();
+        let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+
+        op.write(&path, content.clone())
+            .await
+            .expect("write must succeed");
+
+        op.delete_iter(vec![path.clone()]).await?;
+
+        // Stat it again to check.
+        assert!(!op.exists(&path).await?);
+
+        op.write(&format!("/{path}"), content)
+            .await
+            .expect("write must succeed");
+
+        op.delete_iter(vec![path.clone()]).await?;
+
+        // Stat it again to check.
+        assert!(!op.exists(&path).await?);
+
+        Ok(())
+    }
+
+    /// Delete via stream.
+    #[wasm_bindgen_test]
+    pub async fn test_delete_stream() -> Result<()> {
+        let op = operator();
+        if !op.info().full_capability().create_dir {
+            return Ok(());
+        }
+        // Gdrive think that this test is an abuse of their service and redirect us
+        // to an infinite loop. Let's ignore this test for gdrive.
+        if op.info().scheme() == Scheme::Gdrive {
+            return Ok(());
+        }
+
+        let dir = uuid::Uuid::new_v4().to_string();
+        op.create_dir(&format!("{dir}/"))
+            .await
+            .expect("create must succeed");
+
+        let expected: Vec<_> = (0..100).collect();
+        for path in expected.iter() {
+            op.write(&format!("{dir}/{path}"), "delete_stream").await?;
+        }
+
+        let mut deleter = op.deleter().await?;
+        deleter
+            .delete_iter(expected.iter().map(|v| format!("{dir}/{v}")))
+            .await?;
+        deleter.close().await?;
+
+        // Stat it again to check.
+        for path in expected.iter() {
+            assert!(
+                !op.exists(&format!("{dir}/{path}")).await?,
+                "{path} should be removed"
+            )
+        }
+
+        Ok(())
+    }
+
+    #[wasm_bindgen_test]
+    pub async fn test_batch_delete() -> Result<()> {
+        let op = operator();
+        let mut cap = op.info().full_capability();
+        if cap.delete_max_size.unwrap_or(1) <= 1 {
+            return Ok(());
+        }
+
+        cap.delete_max_size = Some(2);
+        op.inner().info().update_full_capability(|_| cap);
+
+        let mut files = Vec::new();
+        for _ in 0..5 {
+            let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
+            op.write(path.as_str(), content)
+                .await
+                .expect("write must succeed");
+            files.push(path);
+        }
+
+        op.delete_iter(files.clone())
+            .await
+            .expect("batch delete must succeed");
+
+        for path in files {
+            let stat = op.stat(path.as_str()).await;
+            assert!(stat.is_err());
+            assert_eq!(stat.unwrap_err().kind(), ErrorKind::NotFound);
+        }
+
+        Ok(())
+    }
+}