Skip to content

Commit 4b35bab

Browse files
martindurantmartindurant
authored
Merge pull request #8 from martindurant/async
Async
2 parents 80a51b1 + 0a79d62 commit 4b35bab

File tree

11 files changed

+117
-37
lines changed

11 files changed

+117
-37
lines changed

‎.github/workflows/main.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,5 +30,5 @@ jobs:
3030
run: |
3131
pip install -e ./fsspec-proxy
3232
pip install -e ./pyscript-fsspec-client[test]
33-
- name: test
34-
run: pytest -v -s
33+
# - name: test
34+
# run: pytest -v -s

‎example/index.html‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<!DOCTYPE html>
2+
<html lang="en">
3+
<head>
4+
<title>example</title>
5+
6+
<!-- Recommended meta tags -->
7+
<meta charset="UTF-8">
8+
<meta name="viewport" content="width=device-width,initial-scale=1.0">
9+
10+
<link rel="stylesheet" href="https://pyscript.net/releases/2025.8.1/core.css">
11+
<script type="module" src="https://pyscript.net/releases/2025.8.1/core.js"></script>
12+
</head>
13+
<body>
14+
<script type="py" src="./main.py" config="./pyscript.toml" terminal></script>
15+
</body>
16+
</html>

‎example/main.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
from pyscript_fsspec_client import io
2+
from pyscript import PyWorker
3+
4+
config = {
5+
"packages": ["fsspec", "fastparquet"],
6+
"files": {
7+
"./pyscript_fsspec_client/__init__.py": "./pyscript_fsspec_client/__init__.py",
8+
"./pyscript_fsspec_client/client.py": "./pyscript_fsspec_client/client.py",
9+
"./pyscript_fsspec_client/io.py": "./pyscript_fsspec_client/io.py"
10+
}
11+
}
12+
pw = PyWorker("./worker.py", type="pyodide", config=config)
13+
14+
pw.sync.session = io.request

‎example/pyscript.toml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
name = "example"
2+
description = "Usage for pyscript-fsspec-client"
3+
type = "app"
4+
author_name = "Martin Durant"
5+
author_email = "martin.durant@alumni.utoronto.ca"
6+
version = "latest"
7+
packages = [] # only the worker needs installs
8+
9+
[files]
10+
"./pyscript_fsspec_client/__init__.py" = "./pyscript_fsspec_client/__init__.py"
11+
"./pyscript_fsspec_client/client.py" = "./pyscript_fsspec_client/client.py"
12+
"./pyscript_fsspec_client/io.py" = "./pyscript_fsspec_client/io.py"

‎example/pyscript_fsspec_client‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../pyscript-fsspec-client/pyscript_fsspec_client

‎example/worker.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
from pyscript import sync, ffi
2+
3+
import fsspec
4+
import pyscript_fsspec_client.client
5+
6+
fs = fsspec.filesystem("pyscript")
7+
print(fs.ls("local"))
8+
9+
out = fs.cat("local/mdurant/code/fsspec-proxy/pyproject.toml")
10+
print("binary:", type(out), out)
11+
12+
out = fs.cat("local/mdurant/code/fsspec-proxy/pyproject.toml", start=0, end=10)
13+
print("binary:", type(out), out)
14+
15+
fs.pipe_file("local/mdurant/code/fsspec-proxy/OUTPUT", b"hello world")

‎fsspec-proxy/fsspec_proxy/bytes_server.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ async def lifespan(app: fastapi.FastAPI):
2020
app = fastapi.FastAPI(lifespan=lifespan)
2121
app.add_middleware(
2222
CORSMiddleware,
23-
allow_origins=['https://martindurant.pyscriptapps.com'],
23+
allow_origins=['*'],
2424
allow_methods=["GET", "POST", "DELETE", "OPTION", "PUT"],
2525
allow_credentials=True,
2626
allow_headers=["*"]

‎pyscript-fsspec-client/pyscript_fsspec_client/__init__.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-
from .client import PyscriptFileSystem

‎pyscript-fsspec-client/pyscript_fsspec_client/client.py‎

Lines changed: 25 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
import os
1+
from json import dumps, loads
22
import logging
3+
import os
34

4-
import fsspec.utils
5+
from pyscript import sync, ffi
56
from fsspec.spec import AbstractFileSystem, AbstractBufferedFile
6-
from fsspec.implementations.http_sync import RequestsSessionShim
7+
import fsspec.utils
78

89
logger = logging.getLogger("pyscript_fsspec_client")
910
fsspec.utils.setup_logging(logger=logger)
@@ -16,41 +17,35 @@ class PyscriptFileSystem(AbstractFileSystem):
1617
def __init__(self, base_url=default_endpoint):
1718
super().__init__()
1819
self.base_url = base_url
19-
self._session = None
2020

2121
def _split_path(self, path):
2222
key, *relpath = path.split("/", 1)
2323
return key, relpath[0] if relpath else ""
2424

25-
@property
26-
def session(self):
27-
if self._session is None:
28-
try:
29-
import js # noqa: F401
30-
self._session = RequestsSessionShim()
31-
except (ImportError, ModuleNotFoundError):
32-
import requests
33-
self._session = requests.Session()
34-
return self._session
35-
36-
def _call(self, path, method="GET", range=None, binary=False, data=None, json=None, **kw):
37-
logger.debug("request: %s %s %s %s", path, method, kw, range)
25+
def _call(self, path, method="GET", range=None, binary=False, data=0, json=0):
26+
logger.debug("request: %s %s %s", path, method, range)
3827
headers = {}
28+
if binary:
29+
outmode = "bytes"
30+
elif json:
31+
outmode = "json"
32+
else:
33+
outmode = "text"
3934
if range:
4035
headers["Range"] = f"bytes={range[0]}-{range[1]}"
41-
r = self.session.request(
42-
method, f"{self.base_url}/{path}", params=kw, headers=headers,
43-
data=data, json=json
36+
if data:
37+
data = memoryview(data)
38+
outmode = None
39+
out = sync.session(
40+
method, f"{self.base_url}/{path}", ffi.to_js(data),
41+
ffi.to_js(headers), outmode
4442
)
45-
if r.status_code == 404:
46-
raise FileNotFoundError(path)
47-
if r.status_code == 403:
48-
raise PermissionError
49-
r.raise_for_status()
50-
if binary:
51-
return r.content
52-
j = r.json() if callable(r.json) else r.json # inconsistency in shim - to fix!
53-
return j["contents"]
43+
if isinstance(out, str) and out == "ISawAnError":
44+
raise OSError(0, out)
45+
if out is not None and not isinstance(out, str):
46+
# may need a different conversion
47+
out = bytes(out.to_py())
48+
return out
5449

5550
def ls(self, path, detail=True, **kwargs):
5651
path = self._strip_protocol(path)
@@ -104,4 +99,4 @@ def _upload_chunk(self, final=False):
10499
return True
105100
return False
106101

107-
fsspec.register_implementation("pyscript", PyscriptFileSystem)
102+
fsspec.register_implementation("pyscript", PyscriptFileSystem)

‎pyscript-fsspec-client/pyscript_fsspec_client/io.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import json
2+
import pyscript
3+
import js
4+
from pyodide import ffi, console
5+
6+
7+
async def request(method, path, data=None, headers=None,
8+
outmode="text", **kwargs):
9+
if data:
10+
resp = await js.fetch(path, method=method, body=data.buffer, headers=headers or {},
11+
**kwargs)
12+
else:
13+
resp = await js.fetch(path, method=method, headers=headers or {},
14+
**kwargs)
15+
if not resp.ok:
16+
return "ISawAnError"
17+
if resp.status >= 400:
18+
return "ISawAnError"
19+
if outmode == "text":
20+
return await resp.text()
21+
if outmode == "bytes":
22+
return await resp.arrayBuffer()
23+
if outmode is None:
24+
return
25+
return "ISawAnError"

0 commit comments

Comments
 (0)