From d1a4b61649489fc1dbcee5bda5de3dee72551014 Mon Sep 17 00:00:00 2001 From: Devin Smith Date: Wed, 30 Nov 2022 13:21:03 -0800 Subject: [PATCH 01/20] Bump base images to pick up jedi dependency --- docker/registry/all-ai-base/gradle.properties | 2 +- docker/registry/go/gradle.properties | 2 +- docker/registry/nltk-base/gradle.properties | 2 +- docker/registry/node/gradle.properties | 2 +- docker/registry/python/gradle.properties | 2 +- docker/registry/pytorch-base/gradle.properties | 2 +- docker/registry/server-base/gradle.properties | 2 +- docker/registry/sklearn-base/gradle.properties | 2 +- docker/registry/slim-base/gradle.properties | 2 +- .../registry/tensorflow-base/gradle.properties | 2 +- .../src/main/server-jetty/requirements.txt | 6 ++++-- .../main/server-all-ai-netty/requirements.txt | 18 ++++++++++-------- .../src/main/server-netty/requirements.txt | 6 ++++-- .../main/server-nltk-netty/requirements.txt | 6 ++++-- .../main/server-pytorch-netty/requirements.txt | 16 +++++++++------- .../main/server-sklearn-netty/requirements.txt | 6 ++++-- .../server-tensorflow-netty/requirements.txt | 18 ++++++++++-------- 17 files changed, 55 insertions(+), 41 deletions(-) diff --git a/docker/registry/all-ai-base/gradle.properties b/docker/registry/all-ai-base/gradle.properties index b6b28a6077c..881b50890dd 100644 --- a/docker/registry/all-ai-base/gradle.properties +++ b/docker/registry/all-ai-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/all-ai-base:latest -deephaven.registry.imageId=ghcr.io/deephaven/all-ai-base@sha256:0df3f8cda97ee78703d139ae44983837f2a3316aeea5d414826efbd7ccc452f0 +deephaven.registry.imageId=ghcr.io/deephaven/all-ai-base@sha256:7f04d00cfddc493241a024cb42ddc073fb66e405a3b18fe5199445dc616f0bfc diff --git a/docker/registry/go/gradle.properties b/docker/registry/go/gradle.properties index b5aebe516f7..6edff9e10bf 100644 --- a/docker/registry/go/gradle.properties +++ b/docker/registry/go/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=golang:1 -deephaven.registry.imageId=golang@sha256:bf4b15c14a715b9c2d278809254268d57dd74bbee69489e781e9303c814160d5 +deephaven.registry.imageId=golang@sha256:dc76ef03e54c34a00dcdca81e55c242d24b34d231637776c4bb5c1a8e8514253 diff --git a/docker/registry/nltk-base/gradle.properties b/docker/registry/nltk-base/gradle.properties index 0dadc1946b3..6c7ef230fce 100644 --- a/docker/registry/nltk-base/gradle.properties +++ b/docker/registry/nltk-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/nltk-base:latest -deephaven.registry.imageId=ghcr.io/deephaven/nltk-base@sha256:1df4fda4a1f25be302c9d71149b3cf8b42551829ec0c5af43e1a164cce3c2ee9 +deephaven.registry.imageId=ghcr.io/deephaven/nltk-base@sha256:8f569f155d4e8c69c2cc51c0a7f92b59a0eba5974c09969123499a31d1b91b8e diff --git a/docker/registry/node/gradle.properties b/docker/registry/node/gradle.properties index a16a10616eb..cb6b15b0295 100644 --- a/docker/registry/node/gradle.properties +++ b/docker/registry/node/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=node:14 -deephaven.registry.imageId=node@sha256:1172bd6da670f35dca9b2f43eb27c1444c6b3350f7b9619e6a9fdef6cff88d47 +deephaven.registry.imageId=node@sha256:d82d512aec5de4fac53b92b2aa148948c2e72264d650de9e1570283d4f503dbe diff --git a/docker/registry/python/gradle.properties b/docker/registry/python/gradle.properties index a61b838fefc..4735d5fe892 100644 --- a/docker/registry/python/gradle.properties +++ b/docker/registry/python/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=python:3.7 -deephaven.registry.imageId=python@sha256:53e59319730a41a52e8b3c43bd114131c91ebc8689dcece363b796cb4e623cc1 +deephaven.registry.imageId=python@sha256:cb158b66aaaf68b642eea2246a9a4dcc6bc39bc634dccd43812ae44807c70765 diff --git a/docker/registry/pytorch-base/gradle.properties b/docker/registry/pytorch-base/gradle.properties index 3524a23e97c..9a5f007af8b 100644 --- a/docker/registry/pytorch-base/gradle.properties +++ b/docker/registry/pytorch-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/pytorch-base:latest -deephaven.registry.imageId=ghcr.io/deephaven/pytorch-base@sha256:60ca5835d227832a3a0e8938a9c24ed43072713a7dc0e864b3dca6f1bdbbcade +deephaven.registry.imageId=ghcr.io/deephaven/pytorch-base@sha256:bb4e57e18f3b8129c5c8203816658c82745d20ee84b0cf46bf2cb92b86539452 diff --git a/docker/registry/server-base/gradle.properties b/docker/registry/server-base/gradle.properties index c555a370b3f..2a9b9f9fdb2 100644 --- a/docker/registry/server-base/gradle.properties +++ b/docker/registry/server-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/server-base:latest -deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:0684e7e9a2961e8f7b9a964011451206775961d1fc257d4fe8700983d0a7efd2 +deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:25caee1a3a60b25edaeef70613ae1824d79bfd73d4b6507fbaf3b2c35f03c929 diff --git a/docker/registry/sklearn-base/gradle.properties b/docker/registry/sklearn-base/gradle.properties index 397fd80332b..4230a5d0a07 100644 --- a/docker/registry/sklearn-base/gradle.properties +++ b/docker/registry/sklearn-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/sklearn-base:latest -deephaven.registry.imageId=ghcr.io/deephaven/sklearn-base@sha256:721efedc30a28c6ff3da6ea0148a02bbe16e748f684be8de782e147700bfda5b +deephaven.registry.imageId=ghcr.io/deephaven/sklearn-base@sha256:00b170cbcf4e5ca8dfdc25cfdf5e0e20c52cba863e06d60b130b141a902c09a5 diff --git a/docker/registry/slim-base/gradle.properties b/docker/registry/slim-base/gradle.properties index 60bc608989d..37e2a162e95 100644 --- a/docker/registry/slim-base/gradle.properties +++ b/docker/registry/slim-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/slim-base:latest -deephaven.registry.imageId=ghcr.io/deephaven/slim-base@sha256:110c60d37bd188b5c767d210beeed9e8da4eca4bc01dbf0977461188fb884a8a +deephaven.registry.imageId=ghcr.io/deephaven/slim-base@sha256:b99e67e795c46f66894b1d1c54d33b0bbf5831333b6df0a6ce78d537b9c5b1b4 diff --git a/docker/registry/tensorflow-base/gradle.properties b/docker/registry/tensorflow-base/gradle.properties index 473e7b3154b..e9f41004f91 100644 --- a/docker/registry/tensorflow-base/gradle.properties +++ b/docker/registry/tensorflow-base/gradle.properties @@ -1,3 +1,3 @@ io.deephaven.project.ProjectType=DOCKER_REGISTRY deephaven.registry.imageName=ghcr.io/deephaven/tensorflow-base:latest -deephaven.registry.imageId=ghcr.io/deephaven/tensorflow-base@sha256:febafa37c70b569d5485cb042f538ad1adbeb0fd107c5e2c288f1903ae022961 +deephaven.registry.imageId=ghcr.io/deephaven/tensorflow-base@sha256:ace29d216b7deb268a0dc26a6957e2e3dd5ede58d09511ec85c931c21ae94e99 diff --git a/docker/server-jetty/src/main/server-jetty/requirements.txt b/docker/server-jetty/src/main/server-jetty/requirements.txt index 893a167f953..1fbc075f427 100644 --- a/docker/server-jetty/src/main/server-jetty/requirements.txt +++ b/docker/server-jetty/src/main/server-jetty/requirements.txt @@ -1,13 +1,15 @@ deephaven-plugin==0.3.0 -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 java-utilities==0.2.0 +jedi==0.18.2 jpy==0.13.0 llvmlite==0.39.1 numba==0.56.4 numpy==1.21.6 pandas==1.3.5 +parso==0.8.3 python-dateutil==2.8.2 pytz==2022.6 six==1.16.0 typing_extensions==4.4.0 -zipp==3.10.0 +zipp==3.11.0 diff --git a/docker/server/src/main/server-all-ai-netty/requirements.txt b/docker/server/src/main/server-all-ai-netty/requirements.txt index 0df6c12e393..ab7e368e5e6 100644 --- a/docker/server/src/main/server-all-ai-netty/requirements.txt +++ b/docker/server/src/main/server-all-ai-netty/requirements.txt @@ -7,14 +7,15 @@ click==8.1.3 deephaven-plugin==0.3.0 flatbuffers==2.0.7 gast==0.4.0 -google-auth==2.14.0 +google-auth==2.14.1 google-auth-oauthlib==0.4.6 google-pasta==0.2.0 -grpcio==1.50.0 +grpcio==1.51.1 h5py==3.7.0 idna==3.4 -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 java-utilities==0.2.0 +jedi==0.18.2 joblib==1.2.0 jpy==0.13.0 keras==2.7.0 @@ -33,6 +34,7 @@ nvidia-cudnn-cu11==8.5.0.96 oauthlib==3.2.2 opt-einsum==3.3.0 pandas==1.3.5 +parso==0.8.3 protobuf==3.19.6 pyasn1==0.4.8 pyasn1-modules==0.2.8 @@ -45,18 +47,18 @@ rsa==4.9 scikit-learn==1.0.2 scipy==1.7.3 six==1.16.0 -tensorboard==2.10.1 +tensorboard==2.11.0 tensorboard-data-server==0.6.1 tensorboard-plugin-wit==1.8.1 tensorflow==2.7.4 tensorflow-estimator==2.7.0 -tensorflow-io-gcs-filesystem==0.27.0 -termcolor==2.1.0 +tensorflow-io-gcs-filesystem==0.28.0 +termcolor==2.1.1 threadpoolctl==3.1.0 torch==1.13.0 tqdm==4.64.1 typing_extensions==4.4.0 -urllib3==1.26.12 +urllib3==1.26.13 Werkzeug==2.2.2 wrapt==1.14.1 -zipp==3.10.0 +zipp==3.11.0 diff --git a/docker/server/src/main/server-netty/requirements.txt b/docker/server/src/main/server-netty/requirements.txt index 893a167f953..1fbc075f427 100644 --- a/docker/server/src/main/server-netty/requirements.txt +++ b/docker/server/src/main/server-netty/requirements.txt @@ -1,13 +1,15 @@ deephaven-plugin==0.3.0 -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 java-utilities==0.2.0 +jedi==0.18.2 jpy==0.13.0 llvmlite==0.39.1 numba==0.56.4 numpy==1.21.6 pandas==1.3.5 +parso==0.8.3 python-dateutil==2.8.2 pytz==2022.6 six==1.16.0 typing_extensions==4.4.0 -zipp==3.10.0 +zipp==3.11.0 diff --git a/docker/server/src/main/server-nltk-netty/requirements.txt b/docker/server/src/main/server-nltk-netty/requirements.txt index 2935f2e10fe..4fd4499b349 100644 --- a/docker/server/src/main/server-nltk-netty/requirements.txt +++ b/docker/server/src/main/server-nltk-netty/requirements.txt @@ -1,7 +1,8 @@ click==8.1.3 deephaven-plugin==0.3.0 -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 java-utilities==0.2.0 +jedi==0.18.2 joblib==1.2.0 jpy==0.13.0 llvmlite==0.39.1 @@ -9,10 +10,11 @@ nltk==3.6.7 numba==0.56.4 numpy==1.21.6 pandas==1.3.5 +parso==0.8.3 python-dateutil==2.8.2 pytz==2022.6 regex==2022.10.31 six==1.16.0 tqdm==4.64.1 typing_extensions==4.4.0 -zipp==3.10.0 +zipp==3.11.0 diff --git a/docker/server/src/main/server-pytorch-netty/requirements.txt b/docker/server/src/main/server-pytorch-netty/requirements.txt index 665177f5f8c..4936b228eba 100644 --- a/docker/server/src/main/server-pytorch-netty/requirements.txt +++ b/docker/server/src/main/server-pytorch-netty/requirements.txt @@ -3,12 +3,13 @@ cachetools==5.2.0 certifi==2022.9.24 charset-normalizer==2.1.1 deephaven-plugin==0.3.0 -google-auth==2.14.0 +google-auth==2.14.1 google-auth-oauthlib==0.4.6 -grpcio==1.50.0 +grpcio==1.51.1 idna==3.4 -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 java-utilities==0.2.0 +jedi==0.18.2 jpy==0.13.0 llvmlite==0.39.1 Markdown==3.4.1 @@ -17,7 +18,8 @@ numba==0.56.4 numpy==1.21.6 oauthlib==3.2.2 pandas==1.3.5 -protobuf==3.19.6 +parso==0.8.3 +protobuf==3.20.3 pyasn1==0.4.8 pyasn1-modules==0.2.8 python-dateutil==2.8.2 @@ -26,11 +28,11 @@ requests==2.28.1 requests-oauthlib==1.3.1 rsa==4.9 six==1.16.0 -tensorboard==2.10.1 +tensorboard==2.11.0 tensorboard-data-server==0.6.1 tensorboard-plugin-wit==1.8.1 torch==1.10.2 typing_extensions==4.4.0 -urllib3==1.26.12 +urllib3==1.26.13 Werkzeug==2.2.2 -zipp==3.10.0 +zipp==3.11.0 diff --git a/docker/server/src/main/server-sklearn-netty/requirements.txt b/docker/server/src/main/server-sklearn-netty/requirements.txt index f64aeedb9f3..4955820d37c 100644 --- a/docker/server/src/main/server-sklearn-netty/requirements.txt +++ b/docker/server/src/main/server-sklearn-netty/requirements.txt @@ -1,12 +1,14 @@ deephaven-plugin==0.3.0 -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 java-utilities==0.2.0 +jedi==0.18.2 joblib==1.2.0 jpy==0.13.0 llvmlite==0.39.1 numba==0.56.4 numpy==1.21.6 pandas==1.3.5 +parso==0.8.3 python-dateutil==2.8.2 pytz==2022.6 scikit-learn==0.24.2 @@ -14,4 +16,4 @@ scipy==1.7.3 six==1.16.0 threadpoolctl==3.1.0 typing_extensions==4.4.0 -zipp==3.10.0 +zipp==3.11.0 diff --git a/docker/server/src/main/server-tensorflow-netty/requirements.txt b/docker/server/src/main/server-tensorflow-netty/requirements.txt index f43fbd40c84..d8838eb318a 100644 --- a/docker/server/src/main/server-tensorflow-netty/requirements.txt +++ b/docker/server/src/main/server-tensorflow-netty/requirements.txt @@ -6,14 +6,15 @@ charset-normalizer==2.1.1 deephaven-plugin==0.3.0 flatbuffers==2.0.7 gast==0.4.0 -google-auth==2.14.0 +google-auth==2.14.1 google-auth-oauthlib==0.4.6 google-pasta==0.2.0 -grpcio==1.50.0 +grpcio==1.51.1 h5py==3.7.0 idna==3.4 -importlib-metadata==5.0.0 +importlib-metadata==5.1.0 java-utilities==0.2.0 +jedi==0.18.2 jpy==0.13.0 keras==2.7.0 Keras-Preprocessing==1.1.2 @@ -26,6 +27,7 @@ numpy==1.21.6 oauthlib==3.2.2 opt-einsum==3.3.0 pandas==1.3.5 +parso==0.8.3 protobuf==3.19.6 pyasn1==0.4.8 pyasn1-modules==0.2.8 @@ -35,15 +37,15 @@ requests==2.28.1 requests-oauthlib==1.3.1 rsa==4.9 six==1.16.0 -tensorboard==2.10.1 +tensorboard==2.11.0 tensorboard-data-server==0.6.1 tensorboard-plugin-wit==1.8.1 tensorflow==2.7.4 tensorflow-estimator==2.7.0 -tensorflow-io-gcs-filesystem==0.27.0 -termcolor==2.1.0 +tensorflow-io-gcs-filesystem==0.28.0 +termcolor==2.1.1 typing_extensions==4.4.0 -urllib3==1.26.12 +urllib3==1.26.13 Werkzeug==2.2.2 wrapt==1.14.1 -zipp==3.10.0 +zipp==3.11.0 From cb6c66d15756920949c6341eade36d14318c8d6c Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Fri, 7 Oct 2022 10:57:13 -0600 Subject: [PATCH 02/20] Add jetty all-ai image for demo --- docker/server-jetty/build.gradle | 1 + .../main/server-all-ai-jetty/requirements.txt | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt diff --git a/docker/server-jetty/build.gradle b/docker/server-jetty/build.gradle index a66fe027c33..6f1588034eb 100644 --- a/docker/server-jetty/build.gradle +++ b/docker/server-jetty/build.gradle @@ -10,6 +10,7 @@ def targetArch = Architecture.targetArchitecture(project) def baseMapAmd64 = [ 'server-base': 'server-jetty', + 'all-ai-base': 'server-all-ai-jetty', ] // Only the server image is supported on arm64 diff --git a/docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt b/docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt new file mode 100644 index 00000000000..afb60eac87d --- /dev/null +++ b/docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt @@ -0,0 +1,59 @@ +absl-py==1.2.0 +astunparse==1.6.3 +cachetools==5.2.0 +certifi==2022.9.24 +charset-normalizer==2.1.1 +click==8.1.3 +deephaven-plugin==0.2.0 +flatbuffers==2.0.7 +gast==0.4.0 +google-auth==2.12.0 +google-auth-oauthlib==0.4.6 +google-pasta==0.2.0 +grpcio==1.49.1 +h5py==3.7.0 +idna==3.4 +importlib-metadata==4.12.0 +java-utilities==0.2.0 +joblib==1.2.0 +jpy==0.12.0 +keras==2.7.0 +Keras-Preprocessing==1.1.2 +libclang==14.0.6 +llvmlite==0.39.1 +Markdown==3.4.1 +MarkupSafe==2.1.1 +nltk==3.7 +numba==0.56.2 +numpy==1.21.6 +oauthlib==3.2.1 +opt-einsum==3.3.0 +pandas==1.3.5 +pkg_resources==0.0.0 +protobuf==3.19.5 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +python-dateutil==2.8.2 +pytz==2022.2.1 +regex==2022.9.13 +requests==2.28.1 +requests-oauthlib==1.3.1 +rsa==4.9 +scikit-learn==1.0.2 +scipy==1.7.3 +six==1.16.0 +tensorboard==2.10.1 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorflow==2.7.4 +tensorflow-estimator==2.7.0 +tensorflow-io-gcs-filesystem==0.27.0 +termcolor==2.0.1 +threadpoolctl==3.1.0 +torch==1.12.1 +tqdm==4.64.1 +typing_extensions==4.3.0 +urllib3==1.26.12 +Werkzeug==2.2.2 +wrapt==1.14.1 +zipp==3.8.1 From ae37b6224e4192464343c082c6dc49efb9ca7375 Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Fri, 7 Oct 2022 17:15:47 -0600 Subject: [PATCH 03/20] WIP: let our jetty build download dependencies --- docker/server-jetty/src/main/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/server-jetty/src/main/docker/Dockerfile b/docker/server-jetty/src/main/docker/Dockerfile index 5f017364154..a357fad1c67 100644 --- a/docker/server-jetty/src/main/docker/Dockerfile +++ b/docker/server-jetty/src/main/docker/Dockerfile @@ -10,7 +10,7 @@ COPY licenses/ / ARG SERVER COPY $SERVER/requirements.txt requirements.txt RUN set -eux; \ - python -m pip install -q --no-index --no-cache-dir -r requirements.txt; \ + python -m pip install -q --no-cache-dir -r requirements.txt; \ rm requirements.txt COPY wheels/ /wheels From 3587dfe805f77557dcfca610d789610f0fece5a2 Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Tue, 15 Nov 2022 12:23:32 -0600 Subject: [PATCH 04/20] hack in installation of jedi --- docker/server-jetty/src/main/docker/Dockerfile | 2 +- .../src/main/server-all-ai-jetty/requirements.txt | 1 + docker/server-jetty/src/main/server-jetty/requirements.txt | 2 ++ docker/server/src/main/docker/Dockerfile | 4 ++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/server-jetty/src/main/docker/Dockerfile b/docker/server-jetty/src/main/docker/Dockerfile index a357fad1c67..5c5589219b5 100644 --- a/docker/server-jetty/src/main/docker/Dockerfile +++ b/docker/server-jetty/src/main/docker/Dockerfile @@ -15,7 +15,7 @@ RUN set -eux; \ COPY wheels/ /wheels RUN set -eux; \ - python -m pip install -q --no-index --no-cache-dir /wheels/*.whl; \ + python -m pip install -q --no-cache-dir /wheels/*.whl; \ rm -r /wheels ARG DEEPHAVEN_VERSION diff --git a/docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt b/docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt index afb60eac87d..cdd22a54d30 100644 --- a/docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt +++ b/docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt @@ -15,6 +15,7 @@ h5py==3.7.0 idna==3.4 importlib-metadata==4.12.0 java-utilities==0.2.0 +jedi==0.18.1 joblib==1.2.0 jpy==0.12.0 keras==2.7.0 diff --git a/docker/server-jetty/src/main/server-jetty/requirements.txt b/docker/server-jetty/src/main/server-jetty/requirements.txt index 893a167f953..5b220bc7737 100644 --- a/docker/server-jetty/src/main/server-jetty/requirements.txt +++ b/docker/server-jetty/src/main/server-jetty/requirements.txt @@ -1,6 +1,7 @@ deephaven-plugin==0.3.0 importlib-metadata==5.0.0 java-utilities==0.2.0 +jedi==0.18.1 jpy==0.13.0 llvmlite==0.39.1 numba==0.56.4 @@ -11,3 +12,4 @@ pytz==2022.6 six==1.16.0 typing_extensions==4.4.0 zipp==3.10.0 + diff --git a/docker/server/src/main/docker/Dockerfile b/docker/server/src/main/docker/Dockerfile index 01ffcd7e60a..c4aa69c309b 100644 --- a/docker/server/src/main/docker/Dockerfile +++ b/docker/server/src/main/docker/Dockerfile @@ -10,12 +10,12 @@ COPY licenses/ / ARG SERVER COPY $SERVER/requirements.txt requirements.txt RUN set -eux; \ - python -m pip install -q --no-index --no-cache-dir -r requirements.txt; \ + python -m pip install -q --no-cache-dir -r requirements.txt; \ rm requirements.txt COPY wheels/ /wheels RUN set -eux; \ - python -m pip install -q --no-index --no-cache-dir /wheels/*.whl; \ + python -m pip install -q --no-cache-dir /wheels/*.whl; \ rm -r /wheels ARG DEEPHAVEN_VERSION From 7e4fa95ef7bd1d061d960934052c230112e4e04d Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Tue, 15 Nov 2022 12:23:47 -0600 Subject: [PATCH 05/20] Hack in testable version of jedi --- .../lang/parse/CompletionParser.java | 7 ++ .../lang/completion/ChunkerCompleter.java | 2 +- .../console/ConsoleServiceGrpcImpl.java | 112 +++++++++++++++++- 3 files changed, 114 insertions(+), 7 deletions(-) diff --git a/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java b/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java index 2ab10b13cf5..b0a6dee9651 100644 --- a/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java +++ b/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java @@ -118,6 +118,13 @@ public void remove(String uri) { } } + public String getText(String uri) { + final PendingParse doc = docs.get(uri); + if (doc == null) { + throw new IllegalStateException("Unable to find parsed document " + uri); + } + return doc.getText(); + } public ParsedDocument finish(String uri) { final PendingParse doc = docs.get(uri); if (doc == null) { diff --git a/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/ChunkerCompleter.java b/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/ChunkerCompleter.java index f30fdc3a04d..43acc2e3438 100644 --- a/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/ChunkerCompleter.java +++ b/open-api/lang-tools/src/main/java/io/deephaven/lang/completion/ChunkerCompleter.java @@ -354,7 +354,7 @@ private TextEdit.Builder extendEnd(final CompletionItem.Builder item, final Posi } - private String sortable(int i) { + public static String sortable(int i) { StringBuilder res = new StringBuilder(Integer.toString(i, 36)); while (res.length() < 5) { res.insert(0, "0"); diff --git a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java index e77423dac8e..2625af6e5c7 100644 --- a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java +++ b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java @@ -15,6 +15,7 @@ import io.deephaven.extensions.barrage.util.GrpcUtil; import io.deephaven.integrations.python.PythonDeephavenSession; import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.log.LogEntry; import io.deephaven.io.logger.LogBuffer; import io.deephaven.io.logger.LogBufferRecord; import io.deephaven.io.logger.LogBufferRecordListener; @@ -37,11 +38,15 @@ import io.deephaven.server.session.TicketRouter; import io.deephaven.util.SafeCloseable; import io.grpc.stub.StreamObserver; +import org.jpy.PyListWrapper; +import org.jpy.PyObject; import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -266,9 +271,9 @@ public StreamObserver autoCompleteStream( StreamObserver responseObserver) { return GrpcUtil.rpcWrapper(log, responseObserver, () -> { final SessionState session = sessionService.getCurrentSession(); - if (PythonDeephavenSession.SCRIPT_TYPE.equals(scriptSessionProvider.get().scriptType())) { - return new NoopAutoCompleteObserver(responseObserver); - } +// if (PythonDeephavenSession.SCRIPT_TYPE.equals(scriptSessionProvider.get().scriptType())) { +// return new NoopAutoCompleteObserver(responseObserver); +// } return new JavaAutoCompleteObserver(session, responseObserver); }); @@ -279,6 +284,8 @@ public StreamObserver autoCompleteStream( * mangling, and are able to use our flexible parser. */ private static class JavaAutoCompleteObserver implements StreamObserver { + + private boolean canJedi, checkedJedi; private final CompletionParser parser; private final SessionState session; private final StreamObserver responseObserver; @@ -348,8 +355,102 @@ private void getCompletionItems(GetCompletionItemsRequest request, StreamObserver responseObserver) { final ScriptSession scriptSession = exportedConsole.get(); try (final SafeCloseable ignored = scriptSession.getExecutionContext().open()) { - final VersionedTextDocumentIdentifier doc = request.getTextDocument(); final VariableProvider vars = scriptSession.getVariableProvider(); + if (!checkedJedi) { + checkedJedi = true; + long checkStart = System.currentTimeMillis(); + final ScriptSession.Changes result = scriptSession.evaluateScript("try:\n" + + " import jedi\n" + + " jedi.preload_module('deephaven')\n" + + " no_jedi=False\n" + + "except:\n" + + " no_jedi=True"); + canJedi = !vars.getVariable("no_jedi", true); + log.info().append("Prepared for jedi in ").append(System.currentTimeMillis() - checkStart).append("ms").endl(); + log.info().append("Can Jedi? ").append(canJedi).endl(); + } + final VersionedTextDocumentIdentifier doc = request.getTextDocument(); + if (canJedi) { + final long startNano = System.nanoTime(); + String text = parser.getText(doc.getUri()); + try { + final Position pos = request.getPosition(); + String completionVar = "completions"; + scriptSession.setVariable("__jedi_source__", text); + final ScriptSession.Changes changes = scriptSession.evaluateScript( + completionVar + " = jedi.Interpreter(__jedi_source__, [globals()]).complete(" + + (pos.getLine() + 1) + "," + pos.getCharacter() + ")" + ); + final PyListWrapper completes = vars.getVariable(completionVar, null); + List completionResults = new ArrayList<>(); + List completionResults_ = new ArrayList<>(); + List completionResults__ = new ArrayList<>(); + for (PyObject completion : completes) { + String completionName = completion.getAttribute("name").getStringValue(); + int completionPrefix = completion.call("get_completion_prefix_length").getIntValue(); + final CompletionItem.Builder item = CompletionItem.newBuilder(); + final TextEdit.Builder textEdit = item.getTextEditBuilder(); + textEdit.setText(completionName); + final DocumentRange.Builder range = textEdit.getRangeBuilder(); + int start = pos.getCharacter() - completionPrefix; + item.setStart(start); + item.setLabel(completionName); + item.setLength(completionName.length()); + range.getStartBuilder().setLine(pos.getLine()).setCharacter(start); + range.getEndBuilder().setLine(pos.getLine()).setCharacter(start + completionName.length()); + item.setInsertTextFormat(2); + if (completionName.startsWith("__")) { + completionResults__.add(item); + } else if (completionName.startsWith("_")) { + completionResults_.add(item); + } else { + completionResults.add(item); + } + } + int sortPos = 0; + List finalItems = new ArrayList<>(); + for (CompletionItem.Builder res : completionResults) { + res.setSortText(ChunkerCompleter.sortable(sortPos++)); + finalItems.add(res.build()); + } + for (CompletionItem.Builder res : completionResults_) { + res.setSortText(ChunkerCompleter.sortable(sortPos++)); + finalItems.add(res.build()); + } + for (CompletionItem.Builder res : completionResults__) { + res.setSortText(ChunkerCompleter.sortable(sortPos++)); + finalItems.add(res.build()); + } + + final GetCompletionItemsResponse builtItems = GetCompletionItemsResponse.newBuilder() + .setSuccess(true) + .setRequestId(request.getRequestId()) + .addAllItems(finalItems) + .build(); + + safelyExecuteLocked(responseObserver, + () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() + .setCompletionItems(builtItems) + .build())); + String totalNano = Long.toString(System.nanoTime() - startNano ); + // lets track how long completions take, as it's known that some + // modules like numpy can cause slow completion, and we'll want to know + // + log.info().append("Jedi completions took ") + .append( + totalNano.length() > 6 ? + totalNano.substring(0, totalNano.length() - 6) : + "0" + ) + .append('.') + .append(totalNano.substring(6, Math.min(8, totalNano.length()))) + .append("ms").endl(); + // for now, we'll just return right away so we only see jedi results + return; + } catch (Throwable e) { + log.error().append("Jedi completion failure ").append(e).endl(); + } + } final CompletionLookups h = CompletionLookups.preload(scriptSession); // The only stateful part of a completer is the CompletionLookups, which are already // once-per-session-cached @@ -382,8 +483,7 @@ private void getCompletionItems(GetCompletionItemsRequest request, .setRequestId(request.getRequestId()) .addAllItems(results.stream().map( // insertTextFormat is a default we used to set in constructor; for now, we'll - // just - // process the objects before sending back to client + // just process the objects before sending back to client item -> item.setInsertTextFormat(2).build()) .collect(Collectors.toSet())) .build(); From 888e8ff207ffbfd2941241faaf94449cb3a39280 Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Mon, 28 Nov 2022 09:55:53 -0600 Subject: [PATCH 06/20] Wire jedi into python autocomplete "proper"ly --- .../lang/parse/CompletionParser.java | 60 ++-- py/server/deephaven/completer/__init__.py | 95 ++++++ py/server/deephaven/config/__init__.py | 2 +- .../console/ConsoleServiceGrpcImpl.java | 275 +----------------- .../completer/JavaAutoCompleteObserver.java | 176 +++++++++++ .../completer/PythonAutoCompleteObserver.java | 222 ++++++++++++++ 6 files changed, 541 insertions(+), 289 deletions(-) create mode 100644 py/server/deephaven/completer/__init__.py create mode 100644 server/src/main/java/io/deephaven/server/console/completer/JavaAutoCompleteObserver.java create mode 100644 server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java diff --git a/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java b/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java index b0a6dee9651..ec42ac474d6 100644 --- a/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java +++ b/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java @@ -24,6 +24,35 @@ public class CompletionParser implements Closeable { private static final Logger LOGGER = LoggerFactory.getLogger(CompletionParser.class); private final Map docs = new ConcurrentHashMap<>(); + public static String updateDocumentChanges(final String uri, final int version, String document, final List changes) { + for (ChangeDocumentRequest.TextDocumentContentChangeEventOrBuilder change : changes) { + DocumentRange range = change.getRange(); + int length = change.getRangeLength(); + + int offset = LspTools.getOffsetFromPosition(document, range.getStart()); + if (offset < 0) { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn() + .append("Invalid change in document ") + .append(uri) + .append("[") + .append(version) + .append("] @") + .append(range.getStart().getLine()) + .append(":") + .append(range.getStart().getCharacter()) + .endl(); + } + return null; + } + + String prefix = offset > 0 && offset <= document.length() ? document.substring(0, offset) : ""; + String suffix = offset + length < document.length() ? document.substring(offset + length) : ""; + document = prefix + change.getText() + suffix; + } + return document; + } + public ParsedDocument parse(String document) throws ParseException { Chunker chunker = new Chunker(document); final ChunkerDocument doc = chunker.Document(); @@ -49,7 +78,7 @@ private PendingParse startParse(String uri) { return docs.computeIfAbsent(uri, k -> new PendingParse(uri)); } - public void update(final String uri, final String version, + public void update(final String uri, final int version, final List changes) { if (LOGGER.isTraceEnabled()) { LOGGER.trace() @@ -74,32 +103,11 @@ public void update(final String uri, final String version, forceParse = true; } String document = doc.getText(); - for (ChangeDocumentRequest.TextDocumentContentChangeEventOrBuilder change : changes) { - DocumentRange range = change.getRange(); - int length = change.getRangeLength(); - - int offset = LspTools.getOffsetFromPosition(document, range.getStart()); - if (offset < 0) { - if (LOGGER.isWarnEnabled()) { - LOGGER.warn() - .append("Invalid change in document ") - .append(uri) - .append("[") - .append(version) - .append("] @") - .append(range.getStart().getLine()) - .append(":") - .append(range.getStart().getCharacter()) - .endl(); - } - return; - } - - String prefix = offset > 0 && offset <= document.length() ? document.substring(0, offset) : ""; - String suffix = offset + length < document.length() ? document.substring(offset + length) : ""; - document = prefix + change.getText() + suffix; + document = updateDocumentChanges(uri, version, document, changes); + if (document == null) { + return; } - doc.requestParse(version, document, forceParse); + doc.requestParse(Integer.toString(version), document, forceParse); if (LOGGER.isTraceEnabled()) { LOGGER.trace() .append("Finished updating ") diff --git a/py/server/deephaven/completer/__init__.py b/py/server/deephaven/completer/__init__.py new file mode 100644 index 00000000000..35dbb86d8b4 --- /dev/null +++ b/py/server/deephaven/completer/__init__.py @@ -0,0 +1,95 @@ +# +# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending +# + +""" This module allows the user to configure if and how we use jedi to perform autocompletion. +See https://github.com/davidhalter/jedi for information on jedi. + +# To disable autocompletion +from deephaven.completer import CompleterSettings, CompleterMode +CompletionSettings.mode = 'off' + +Valid options for completer_mode are one of: [off, safe, strong]. +off: do not use any autocomplete +safe mode: uses static analysis of source files. Can't execute any code. +strong mode: looks in your globals() for answers to autocomplete and analyzes your runtime python objects +later, we may add slow mode, which uses both static and interpreted completion modes. +""" + +from enum import Enum + + +class CompleterMode(Enum): + off = 'off' + safe = 'safe' + strong = 'strong' + + def __str__(self) -> str: + return self.value + + +class Completer(object): + + def __init__(self): + self._docs = {} + self._versions = {} + # might want to make this a {uri: []} instead of [] + self.pending = [] + try: + import jedi + self.__can_jedi = True + self.mode = CompleterMode.strong + except ImportError: + self.__can_jedi = False + self.mode = CompleterMode.off + + @property + def mode(self) -> CompleterMode: + return self.__mode + + @mode.setter + def mode(self, mode) -> None: + if type(mode) == 'str': + mode = CompleterMode[mode] + self.__mode = mode + + def open_doc(self, text: str, uri: str, version: int) -> None: + self._docs[uri] = text + self._versions[uri] = version + + def get_doc(self, uri: str) -> str: + return self._docs[uri] + + def update_doc(self, text: str, uri: str, version: int) -> None: + self._docs[uri] = text + self._versions[uri] = version + # any pending completions should stop running now. We use a list of Event to signal any running threads to stop + for pending in self.pending: + pending.set() + + def close_doc(self, uri: str) -> None: + del self._docs[uri] + del self._versions[uri] + for pending in self.pending: + pending.set() + + def is_enabled(self) -> bool: + return self.__mode != CompleterMode.off + + def can_jedi(self) -> bool: + return self.__can_jedi + + +jedi_settings = Completer() + +if jedi_settings.mode == CompleterMode.off: + print('No jedi library installed on system path, disabling autocomplete') + +if jedi_settings.mode == CompleterMode.strong: + # start a strong-mode completer thread in bg + pass + + +if jedi_settings.mode == CompleterMode.safe: + # start a safe-mode completer thread in bg + pass \ No newline at end of file diff --git a/py/server/deephaven/config/__init__.py b/py/server/deephaven/config/__init__.py index 589755f54ec..cdc5c971cbd 100644 --- a/py/server/deephaven/config/__init__.py +++ b/py/server/deephaven/config/__init__.py @@ -19,6 +19,6 @@ def get_server_timezone() -> TimeZone: for tz in TimeZone: if j_timezone == tz.value.getTimeZone(): return tz - raise NotImplementedError("can't find the time zone in the TImeZone Enum.") + raise NotImplementedError("can't find the time zone in the TimeZone Enum.") except Exception as e: raise DHError(e, message=f"failed to find a recognized time zone") from e diff --git a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java index 2625af6e5c7..89bcb1a406e 100644 --- a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java +++ b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java @@ -11,45 +11,32 @@ import io.deephaven.engine.updategraph.DynamicNode; import io.deephaven.engine.util.DelegatingScriptSession; import io.deephaven.engine.util.ScriptSession; -import io.deephaven.engine.util.VariableProvider; import io.deephaven.extensions.barrage.util.GrpcUtil; import io.deephaven.integrations.python.PythonDeephavenSession; import io.deephaven.internal.log.LoggerFactory; -import io.deephaven.io.log.LogEntry; import io.deephaven.io.logger.LogBuffer; import io.deephaven.io.logger.LogBufferRecord; import io.deephaven.io.logger.LogBufferRecordListener; import io.deephaven.io.logger.Logger; -import io.deephaven.lang.completion.ChunkerCompleter; -import io.deephaven.lang.completion.CompletionLookups; -import io.deephaven.lang.parse.CompletionParser; -import io.deephaven.lang.parse.LspTools; -import io.deephaven.lang.parse.ParsedDocument; -import io.deephaven.lang.shared.lsp.CompletionCancelled; import io.deephaven.proto.backplane.grpc.FieldInfo; import io.deephaven.proto.backplane.grpc.FieldsChangeUpdate; import io.deephaven.proto.backplane.grpc.Ticket; import io.deephaven.proto.backplane.grpc.TypedTicket; import io.deephaven.proto.backplane.script.grpc.*; +import io.deephaven.server.console.completer.JavaAutoCompleteObserver; +import io.deephaven.server.console.completer.PythonAutoCompleteObserver; import io.deephaven.server.session.SessionCloseableObserver; import io.deephaven.server.session.SessionService; import io.deephaven.server.session.SessionState; import io.deephaven.server.session.SessionState.ExportBuilder; import io.deephaven.server.session.TicketRouter; -import io.deephaven.util.SafeCloseable; import io.grpc.stub.StreamObserver; -import org.jpy.PyListWrapper; import org.jpy.PyObject; import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; import static io.deephaven.extensions.barrage.util.GrpcUtil.safelyExecute; import static io.deephaven.extensions.barrage.util.GrpcUtil.safelyExecuteLocked; @@ -271,256 +258,20 @@ public StreamObserver autoCompleteStream( StreamObserver responseObserver) { return GrpcUtil.rpcWrapper(log, responseObserver, () -> { final SessionState session = sessionService.getCurrentSession(); -// if (PythonDeephavenSession.SCRIPT_TYPE.equals(scriptSessionProvider.get().scriptType())) { -// return new NoopAutoCompleteObserver(responseObserver); -// } - - return new JavaAutoCompleteObserver(session, responseObserver); - }); - } - - /** - * Autocomplete handling for JVM languages, that directly can interact with Java instances without any name - * mangling, and are able to use our flexible parser. - */ - private static class JavaAutoCompleteObserver implements StreamObserver { - - private boolean canJedi, checkedJedi; - private final CompletionParser parser; - private final SessionState session; - private final StreamObserver responseObserver; - - private final Map parsers = new ConcurrentHashMap<>(); - - private CompletionParser ensureParserForSession(SessionState session) { - return parsers.computeIfAbsent(session, s -> { - CompletionParser parser = new CompletionParser(); - s.addOnCloseCallback(() -> { - parsers.remove(s); - parser.close(); + if (PythonDeephavenSession.SCRIPT_TYPE.equals(scriptSessionProvider.get().scriptType())) { + PyObject[] settings = new PyObject[1]; + safelyExecute(()->{ + final ScriptSession scriptSession = scriptSessionProvider.get(); + scriptSession.evaluateScript("from deephaven.completer import jedi_settings\nimport jedi"); + settings[0] = (PyObject) scriptSession.getVariable("jedi_settings"); }); - return parser; - }); - } - - - public JavaAutoCompleteObserver(SessionState session, StreamObserver responseObserver) { - this.session = session; - this.responseObserver = responseObserver; - parser = ensureParserForSession(session); - } - - @Override - public void onNext(AutoCompleteRequest value) { - switch (value.getRequestCase()) { - case OPEN_DOCUMENT: { - final TextDocumentItem doc = value.getOpenDocument().getTextDocument(); - - parser.open(doc.getText(), doc.getUri(), Integer.toString(doc.getVersion())); - break; - } - case CHANGE_DOCUMENT: { - ChangeDocumentRequest request = value.getChangeDocument(); - final VersionedTextDocumentIdentifier text = request.getTextDocument(); - parser.update(text.getUri(), Integer.toString(text.getVersion()), - request.getContentChangesList()); - break; - } - case GET_COMPLETION_ITEMS: { - GetCompletionItemsRequest request = value.getGetCompletionItems(); - SessionState.ExportObject exportedConsole = - session.getExport(request.getConsoleId(), "consoleId"); - session.nonExport() - .require(exportedConsole) - .onError(responseObserver) - .submit(() -> { - getCompletionItems(request, exportedConsole, parser, responseObserver); - }); - break; - } - case CLOSE_DOCUMENT: { - CloseDocumentRequest request = value.getCloseDocument(); - parser.remove(request.getTextDocument().getUri()); - break; - } - case REQUEST_NOT_SET: { - throw GrpcUtil.statusRuntimeException(Code.INVALID_ARGUMENT, - "Autocomplete command missing request"); - } + boolean canJedi = settings[0] != null && settings[0].call("can_jedi").getBooleanValue(); + log.info().append("can jedi? ").append(canJedi).endl(); + return canJedi ? new PythonAutoCompleteObserver(responseObserver, scriptSessionProvider, session) : new NoopAutoCompleteObserver(responseObserver); } - } - - private void getCompletionItems(GetCompletionItemsRequest request, - SessionState.ExportObject exportedConsole, CompletionParser parser, - StreamObserver responseObserver) { - final ScriptSession scriptSession = exportedConsole.get(); - try (final SafeCloseable ignored = scriptSession.getExecutionContext().open()) { - final VariableProvider vars = scriptSession.getVariableProvider(); - if (!checkedJedi) { - checkedJedi = true; - long checkStart = System.currentTimeMillis(); - final ScriptSession.Changes result = scriptSession.evaluateScript("try:\n" + - " import jedi\n" + - " jedi.preload_module('deephaven')\n" + - " no_jedi=False\n" + - "except:\n" + - " no_jedi=True"); - canJedi = !vars.getVariable("no_jedi", true); - log.info().append("Prepared for jedi in ").append(System.currentTimeMillis() - checkStart).append("ms").endl(); - log.info().append("Can Jedi? ").append(canJedi).endl(); - } - final VersionedTextDocumentIdentifier doc = request.getTextDocument(); - if (canJedi) { - final long startNano = System.nanoTime(); - String text = parser.getText(doc.getUri()); - try { - final Position pos = request.getPosition(); - String completionVar = "completions"; - scriptSession.setVariable("__jedi_source__", text); - final ScriptSession.Changes changes = scriptSession.evaluateScript( - completionVar + " = jedi.Interpreter(__jedi_source__, [globals()]).complete(" + - (pos.getLine() + 1) + "," + pos.getCharacter() + ")" - ); - final PyListWrapper completes = vars.getVariable(completionVar, null); - List completionResults = new ArrayList<>(); - List completionResults_ = new ArrayList<>(); - List completionResults__ = new ArrayList<>(); - for (PyObject completion : completes) { - String completionName = completion.getAttribute("name").getStringValue(); - int completionPrefix = completion.call("get_completion_prefix_length").getIntValue(); - final CompletionItem.Builder item = CompletionItem.newBuilder(); - final TextEdit.Builder textEdit = item.getTextEditBuilder(); - textEdit.setText(completionName); - final DocumentRange.Builder range = textEdit.getRangeBuilder(); - int start = pos.getCharacter() - completionPrefix; - item.setStart(start); - item.setLabel(completionName); - item.setLength(completionName.length()); - range.getStartBuilder().setLine(pos.getLine()).setCharacter(start); - range.getEndBuilder().setLine(pos.getLine()).setCharacter(start + completionName.length()); - item.setInsertTextFormat(2); - if (completionName.startsWith("__")) { - completionResults__.add(item); - } else if (completionName.startsWith("_")) { - completionResults_.add(item); - } else { - completionResults.add(item); - } - } - int sortPos = 0; - List finalItems = new ArrayList<>(); - for (CompletionItem.Builder res : completionResults) { - res.setSortText(ChunkerCompleter.sortable(sortPos++)); - finalItems.add(res.build()); - } - for (CompletionItem.Builder res : completionResults_) { - res.setSortText(ChunkerCompleter.sortable(sortPos++)); - finalItems.add(res.build()); - } - for (CompletionItem.Builder res : completionResults__) { - res.setSortText(ChunkerCompleter.sortable(sortPos++)); - finalItems.add(res.build()); - } - - final GetCompletionItemsResponse builtItems = GetCompletionItemsResponse.newBuilder() - .setSuccess(true) - .setRequestId(request.getRequestId()) - .addAllItems(finalItems) - .build(); - - safelyExecuteLocked(responseObserver, - () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() - .setCompletionItems(builtItems) - .build())); - String totalNano = Long.toString(System.nanoTime() - startNano ); - // lets track how long completions take, as it's known that some - // modules like numpy can cause slow completion, and we'll want to know - // - log.info().append("Jedi completions took ") - .append( - totalNano.length() > 6 ? - totalNano.substring(0, totalNano.length() - 6) : - "0" - ) - .append('.') - .append(totalNano.substring(6, Math.min(8, totalNano.length()))) - .append("ms").endl(); - // for now, we'll just return right away so we only see jedi results - return; - } catch (Throwable e) { - log.error().append("Jedi completion failure ").append(e).endl(); - } - } - final CompletionLookups h = CompletionLookups.preload(scriptSession); - // The only stateful part of a completer is the CompletionLookups, which are already - // once-per-session-cached - // so, we'll just create a new completer for each request. No need to hang onto these guys. - final ChunkerCompleter completer = new ChunkerCompleter(log, vars, h); - - final ParsedDocument parsed; - try { - parsed = parser.finish(doc.getUri()); - } catch (CompletionCancelled exception) { - if (log.isTraceEnabled()) { - log.trace().append("Completion canceled").append(exception).endl(); - } - safelyExecuteLocked(responseObserver, - () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() - .setCompletionItems(GetCompletionItemsResponse.newBuilder() - .setSuccess(false) - .setRequestId(request.getRequestId())) - .build())); - return; - } - - int offset = LspTools.getOffsetFromPosition(parsed.getSource(), - request.getPosition()); - final Collection results = - completer.runCompletion(parsed, request.getPosition(), offset); - final GetCompletionItemsResponse mangledResults = - GetCompletionItemsResponse.newBuilder() - .setSuccess(true) - .setRequestId(request.getRequestId()) - .addAllItems(results.stream().map( - // insertTextFormat is a default we used to set in constructor; for now, we'll - // just process the objects before sending back to client - item -> item.setInsertTextFormat(2).build()) - .collect(Collectors.toSet())) - .build(); - - safelyExecuteLocked(responseObserver, - () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() - .setCompletionItems(mangledResults) - .build())); - } catch (Exception exception) { - if (QUIET_AUTOCOMPLETE_ERRORS) { - if (log.isTraceEnabled()) { - log.trace().append("Exception occurred during autocomplete").append(exception).endl(); - } - } else { - log.error().append("Exception occurred during autocomplete").append(exception).endl(); - } - safelyExecuteLocked(responseObserver, - () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() - .setCompletionItems(GetCompletionItemsResponse.newBuilder() - .setSuccess(false) - .setRequestId(request.getRequestId())) - .build())); - } - } - @Override - public void onError(Throwable t) { - // ignore, client doesn't need us, will be cleaned up later - } - - @Override - public void onCompleted() { - // just hang up too, browser will reconnect if interested - synchronized (responseObserver) { - responseObserver.onCompleted(); - } - } + return new JavaAutoCompleteObserver(session, responseObserver); + }); } private static class NoopAutoCompleteObserver implements StreamObserver { diff --git a/server/src/main/java/io/deephaven/server/console/completer/JavaAutoCompleteObserver.java b/server/src/main/java/io/deephaven/server/console/completer/JavaAutoCompleteObserver.java new file mode 100644 index 00000000000..3a3f10f5b70 --- /dev/null +++ b/server/src/main/java/io/deephaven/server/console/completer/JavaAutoCompleteObserver.java @@ -0,0 +1,176 @@ +package io.deephaven.server.console.completer; + +import com.google.rpc.Code; +import io.deephaven.engine.util.ScriptSession; +import io.deephaven.engine.util.VariableProvider; +import io.deephaven.extensions.barrage.util.GrpcUtil; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; +import io.deephaven.lang.completion.ChunkerCompleter; +import io.deephaven.lang.completion.CompletionLookups; +import io.deephaven.lang.parse.CompletionParser; +import io.deephaven.lang.parse.LspTools; +import io.deephaven.lang.parse.ParsedDocument; +import io.deephaven.lang.shared.lsp.CompletionCancelled; +import io.deephaven.proto.backplane.script.grpc.*; +import io.deephaven.server.console.ConsoleServiceGrpcImpl; +import io.deephaven.server.session.SessionState; +import io.deephaven.util.SafeCloseable; +import io.grpc.stub.StreamObserver; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import static io.deephaven.extensions.barrage.util.GrpcUtil.safelyExecuteLocked; + +/** + * Autocomplete handling for JVM languages, that directly can interact with Java instances without any name + * mangling, and are able to use our flexible parser. + */ +public class JavaAutoCompleteObserver implements StreamObserver { + + private static final Logger log = LoggerFactory.getLogger(JavaAutoCompleteObserver.class); + private final CompletionParser parser; + private final SessionState session; + private final StreamObserver responseObserver; + + private final Map parsers = new ConcurrentHashMap<>(); + + private CompletionParser ensureParserForSession(SessionState session) { + return parsers.computeIfAbsent(session, s -> { + CompletionParser parser = new CompletionParser(); + s.addOnCloseCallback(() -> { + parsers.remove(s); + parser.close(); + }); + return parser; + }); + } + + + public JavaAutoCompleteObserver(SessionState session, StreamObserver responseObserver) { + this.session = session; + this.responseObserver = responseObserver; + parser = ensureParserForSession(session); + } + + @Override + public void onNext(AutoCompleteRequest value) { + switch (value.getRequestCase()) { + case OPEN_DOCUMENT: { + final TextDocumentItem doc = value.getOpenDocument().getTextDocument(); + + parser.open(doc.getText(), doc.getUri(), Integer.toString(doc.getVersion())); + break; + } + case CHANGE_DOCUMENT: { + ChangeDocumentRequest request = value.getChangeDocument(); + final VersionedTextDocumentIdentifier text = request.getTextDocument(); + parser.update(text.getUri(), text.getVersion(), + request.getContentChangesList()); + break; + } + case GET_COMPLETION_ITEMS: { + GetCompletionItemsRequest request = value.getGetCompletionItems(); + SessionState.ExportObject exportedConsole = + session.getExport(request.getConsoleId(), "consoleId"); + session.nonExport() + .require(exportedConsole) + .onError(responseObserver) + .submit(() -> { + getCompletionItems(request, exportedConsole, parser, responseObserver); + }); + break; + } + case CLOSE_DOCUMENT: { + CloseDocumentRequest request = value.getCloseDocument(); + parser.remove(request.getTextDocument().getUri()); + break; + } + case REQUEST_NOT_SET: { + throw GrpcUtil.statusRuntimeException(Code.INVALID_ARGUMENT, + "Autocomplete command missing request"); + } + } + } + + private void getCompletionItems(GetCompletionItemsRequest request, + SessionState.ExportObject exportedConsole, CompletionParser parser, + StreamObserver responseObserver) { + final ScriptSession scriptSession = exportedConsole.get(); + try (final SafeCloseable ignored = scriptSession.getExecutionContext().open()) { + final VariableProvider vars = scriptSession.getVariableProvider(); + final VersionedTextDocumentIdentifier doc = request.getTextDocument(); + final CompletionLookups h = CompletionLookups.preload(scriptSession); + // The only stateful part of a completer is the CompletionLookups, which are already + // once-per-session-cached + // so, we'll just create a new completer for each request. No need to hang onto these guys. + final ChunkerCompleter completer = new ChunkerCompleter(log, vars, h); + + final ParsedDocument parsed; + try { + parsed = parser.finish(doc.getUri()); + } catch (CompletionCancelled exception) { + if (log.isTraceEnabled()) { + log.trace().append("Completion canceled").append(exception).endl(); + } + safelyExecuteLocked(responseObserver, + () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() + .setCompletionItems(GetCompletionItemsResponse.newBuilder() + .setSuccess(false) + .setRequestId(request.getRequestId())) + .build())); + return; + } + + int offset = LspTools.getOffsetFromPosition(parsed.getSource(), + request.getPosition()); + final Collection results = + completer.runCompletion(parsed, request.getPosition(), offset); + final GetCompletionItemsResponse mangledResults = + GetCompletionItemsResponse.newBuilder() + .setSuccess(true) + .setRequestId(request.getRequestId()) + .addAllItems(results.stream().map( + // insertTextFormat is a default we used to set in constructor; for now, we'll + // just process the objects before sending back to client + item -> item.setInsertTextFormat(2).build()) + .collect(Collectors.toSet())) + .build(); + + safelyExecuteLocked(responseObserver, + () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() + .setCompletionItems(mangledResults) + .build())); + } catch (Exception exception) { + if (ConsoleServiceGrpcImpl.QUIET_AUTOCOMPLETE_ERRORS) { + if (log.isTraceEnabled()) { + log.trace().append("Exception occurred during autocomplete").append(exception).endl(); + } + } else { + log.error().append("Exception occurred during autocomplete").append(exception).endl(); + } + safelyExecuteLocked(responseObserver, + () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() + .setCompletionItems(GetCompletionItemsResponse.newBuilder() + .setSuccess(false) + .setRequestId(request.getRequestId())) + .build())); + } + } + + @Override + public void onError(Throwable t) { + // ignore, client doesn't need us, will be cleaned up later + } + + @Override + public void onCompleted() { + // just hang up too, browser will reconnect if interested + synchronized (responseObserver) { + responseObserver.onCompleted(); + } + } +} diff --git a/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java new file mode 100644 index 00000000000..58facca28c3 --- /dev/null +++ b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java @@ -0,0 +1,222 @@ +package io.deephaven.server.console.completer; + +import com.google.rpc.Code; +import io.deephaven.engine.util.ScriptSession; +import io.deephaven.engine.util.VariableProvider; +import io.deephaven.extensions.barrage.util.GrpcUtil; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; +import io.deephaven.lang.completion.ChunkerCompleter; +import io.deephaven.lang.completion.CompletionLookups; +import io.deephaven.lang.parse.CompletionParser; +import io.deephaven.lang.parse.LspTools; +import io.deephaven.lang.parse.ParsedDocument; +import io.deephaven.lang.shared.lsp.CompletionCancelled; +import io.deephaven.proto.backplane.script.grpc.*; +import io.deephaven.server.console.ConsoleServiceGrpcImpl; +import io.deephaven.server.session.SessionState; +import io.deephaven.util.SafeCloseable; +import io.grpc.stub.StreamObserver; +import org.jpy.PyListWrapper; +import org.jpy.PyObject; + +import javax.inject.Provider; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static io.deephaven.extensions.barrage.util.GrpcUtil.safelyExecuteLocked; + +/** + * Autocomplete handling for python that will use the jedi library, if it is installed. + */ +public class PythonAutoCompleteObserver implements StreamObserver { + + private static final Logger log = LoggerFactory.getLogger(PythonAutoCompleteObserver.class); + private final Provider scriptSession; + private final SessionState session; + private boolean canJedi, checkedJedi; + private final StreamObserver responseObserver; + + public PythonAutoCompleteObserver(StreamObserver responseObserver, Provider scriptSession, final SessionState session) { + this.scriptSession = scriptSession; + this.session = session; + this.responseObserver = responseObserver; + } + + @Override + @SuppressWarnings("DuplicatedCode") + public void onNext(AutoCompleteRequest value) { + switch (value.getRequestCase()) { + case OPEN_DOCUMENT: { + final OpenDocumentRequest openDoc = value.getOpenDocument(); + final TextDocumentItem doc = openDoc.getTextDocument(); + PyObject completer = (PyObject) scriptSession.get().getVariable("jedi_settings"); + completer.callMethod("open_doc", doc.getText(), doc.getUri(), doc.getVersion()); + break; + } + case CHANGE_DOCUMENT: { + ChangeDocumentRequest request = value.getChangeDocument(); + final VersionedTextDocumentIdentifier text = request.getTextDocument(); + + PyObject completer = (PyObject) scriptSession.get().getVariable("jedi_settings"); + String uri = text.getUri(); + int version = text.getVersion(); + String document = completer.callMethod("get_doc", text.getUri()).getStringValue(); + + final List changes = request.getContentChangesList(); + document = CompletionParser.updateDocumentChanges(uri, version, document, changes); + if (document == null) { + return; + } + + completer.callMethod("update_doc", document, uri, version); + break; + } + case GET_COMPLETION_ITEMS: { + GetCompletionItemsRequest request = value.getGetCompletionItems(); + SessionState.ExportObject exportedConsole = + session.getExport(request.getConsoleId(), "consoleId"); + session.nonExport() + .require(exportedConsole) + .onError(responseObserver) + .submit(() -> { + getCompletionItems(request, exportedConsole, responseObserver); + }); + break; + } + case CLOSE_DOCUMENT: { + CloseDocumentRequest request = value.getCloseDocument(); + PyObject completer = (PyObject) scriptSession.get().getVariable("jedi_settings"); + completer.callMethod("close_doc", request.getTextDocument().getUri()); + break; + } + case REQUEST_NOT_SET: { + throw GrpcUtil.statusRuntimeException(Code.INVALID_ARGUMENT, + "Autocomplete command missing request"); + } + } + } + + private void getCompletionItems(GetCompletionItemsRequest request, + SessionState.ExportObject exportedConsole, + StreamObserver responseObserver) { + final ScriptSession scriptSession = exportedConsole.get(); + try (final SafeCloseable ignored = scriptSession.getExecutionContext().open()) { + final VariableProvider vars = scriptSession.getVariableProvider(); + + PyObject completer = (PyObject) scriptSession.getVariable("jedi_settings"); + boolean canJedi = completer.callMethod("is_enabled").getBooleanValue(); + if (!canJedi) { + log.trace().append("Ignoring completion request because jedi is disabled").endl(); + return; + } + final VersionedTextDocumentIdentifier doc = request.getTextDocument(); + final long startNano = System.nanoTime(); + String text = completer.call("get_doc", doc.getUri()).getStringValue(); + try { + final Position pos = request.getPosition(); + String completionVar = "completions"; + scriptSession.setVariable("__jedi_source__", text); + final ScriptSession.Changes changes = scriptSession.evaluateScript( + completionVar + " = jedi.Interpreter(jedi_settings.get_doc('" + doc.getUri() + "'), [globals()]).complete(" + + (pos.getLine() + 1) + "," + pos.getCharacter() + ")" + ); + final PyListWrapper completes = vars.getVariable(completionVar, null); + List completionResults = new ArrayList<>(); + List completionResults_ = new ArrayList<>(); + List completionResults__ = new ArrayList<>(); + for (PyObject completion : completes) { + String completionName = completion.getAttribute("name").getStringValue(); + int completionPrefix = completion.call("get_completion_prefix_length").getIntValue(); + final CompletionItem.Builder item = CompletionItem.newBuilder(); + final TextEdit.Builder textEdit = item.getTextEditBuilder(); + textEdit.setText(completionName); + final DocumentRange.Builder range = textEdit.getRangeBuilder(); + int start = pos.getCharacter() - completionPrefix; + item.setStart(start); + item.setLabel(completionName); + item.setLength(completionName.length()); + range.getStartBuilder().setLine(pos.getLine()).setCharacter(start); + range.getEndBuilder().setLine(pos.getLine()).setCharacter(start + completionName.length()); + item.setInsertTextFormat(2); + if (completionName.startsWith("__")) { + completionResults__.add(item); + } else if (completionName.startsWith("_")) { + completionResults_.add(item); + } else { + completionResults.add(item); + } + } + int sortPos = 0; + List finalItems = new ArrayList<>(); + for (CompletionItem.Builder res : completionResults) { + res.setSortText(ChunkerCompleter.sortable(sortPos++)); + finalItems.add(res.build()); + } + for (CompletionItem.Builder res : completionResults_) { + res.setSortText(ChunkerCompleter.sortable(sortPos++)); + finalItems.add(res.build()); + } + for (CompletionItem.Builder res : completionResults__) { + res.setSortText(ChunkerCompleter.sortable(sortPos++)); + finalItems.add(res.build()); + } + + final GetCompletionItemsResponse builtItems = GetCompletionItemsResponse.newBuilder() + .setSuccess(true) + .setRequestId(request.getRequestId()) + .addAllItems(finalItems) + .build(); + + safelyExecuteLocked(responseObserver, + () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() + .setCompletionItems(builtItems) + .build())); + String totalNano = Long.toString(System.nanoTime() - startNano); + // lets track how long completions take, as it's known that some + // modules like numpy can cause slow completion, and we'll want to know what was causing them + log.info().append("Jedi completions took ") + .append( + totalNano.length() > 6 ? + totalNano.substring(0, totalNano.length() - 6) : + "0" + ) + .append('.') + .append(totalNano.substring(6, Math.min(8, totalNano.length()))) + .append("ms").endl(); + + } catch (Throwable e) { + log.error().append("Jedi completion failure ").append(e).endl(); + } + } catch (Exception exception) { + if (ConsoleServiceGrpcImpl.QUIET_AUTOCOMPLETE_ERRORS) { + if (log.isTraceEnabled()) { + log.trace().append("Exception occurred during autocomplete").append(exception).endl(); + } + } else { + log.error().append("Exception occurred during autocomplete").append(exception).endl(); + } + safelyExecuteLocked(responseObserver, + () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() + .setCompletionItems(GetCompletionItemsResponse.newBuilder() + .setSuccess(false) + .setRequestId(request.getRequestId())) + .build())); + } + } + + @Override + public void onError(Throwable t) { + // ignore, client doesn't need us, will be cleaned up later + } + + @Override + public void onCompleted() { + // just hang up too, browser will reconnect if interested + synchronized (responseObserver) { + responseObserver.onCompleted(); + } + } +} From 2a335a6ad451ff9513f329717b2b899e118f5d5b Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Thu, 1 Dec 2022 13:00:47 -0600 Subject: [PATCH 07/20] Analyze jedi results in python instead of java --- py/server/deephaven/completer/__init__.py | 47 ++++- server/jetty-app/build.gradle | 4 + .../completer/PythonAutoCompleteObserver.java | 166 ++++++++++-------- 3 files changed, 136 insertions(+), 81 deletions(-) diff --git a/py/server/deephaven/completer/__init__.py b/py/server/deephaven/completer/__init__.py index 35dbb86d8b4..0baf2a80df2 100644 --- a/py/server/deephaven/completer/__init__.py +++ b/py/server/deephaven/completer/__init__.py @@ -6,8 +6,8 @@ See https://github.com/davidhalter/jedi for information on jedi. # To disable autocompletion -from deephaven.completer import CompleterSettings, CompleterMode -CompletionSettings.mode = 'off' +from deephaven.completer import jedi_settings +jedi_settings.mode = 'off' Valid options for completer_mode are one of: [off, safe, strong]. off: do not use any autocomplete @@ -16,8 +16,11 @@ later, we may add slow mode, which uses both static and interpreted completion modes. """ +# TODO: make this python <= 3.8 +from __future__ import annotations from enum import Enum - +from typing import Any +from jedi import Interpreter class CompleterMode(Enum): off = 'off' @@ -79,6 +82,43 @@ def is_enabled(self) -> bool: def can_jedi(self) -> bool: return self.__can_jedi + def do_completion(self, uri: str, version: int, line: int, col: int) -> list[list[Any]]: + if not self._versions[uri] == version: + # if you aren't the newest completion, you get nothing, quickly + print("No text for v{}".format(version)) + return [] + + # run jedi + txt = self.get_doc(uri) + completions = Interpreter(txt, [globals()]).complete(line, col) + # for now, a simple sorting based on number of preceding _ + # we may want to apply additional sorting to each list before combining + results: list = [] + results_: list = [] + results__: list = [] + for complete in completions: + # keep checking the latest version as we run, so updated doc can cancel us + if not self._versions[uri] == version: + return [] + result: list = self.to_result(complete, col) + if result[0].startswith('__'): + results__.append(result) + elif result[0].startswith('_'): + results_.append(result) + else: + results.append(result) + + # put the results together in a better-than-nothing sorting + return results + results_ + results__ + + @staticmethod + def to_result(complete: Any, col: int) -> list[Any]: + name: str = complete.name + prefix_length: int = complete.get_completion_prefix_length() + start: int = col - prefix_length + # all java needs to build a grpc response is completion text (name) and where the completion should start + return [name, start] + jedi_settings = Completer() @@ -89,7 +129,6 @@ def can_jedi(self) -> bool: # start a strong-mode completer thread in bg pass - if jedi_settings.mode == CompleterMode.safe: # start a safe-mode completer thread in bg pass \ No newline at end of file diff --git a/server/jetty-app/build.gradle b/server/jetty-app/build.gradle index 1db697f6c04..fb9b38c8da1 100644 --- a/server/jetty-app/build.gradle +++ b/server/jetty-app/build.gradle @@ -65,6 +65,10 @@ if (hasProperty('debug')) { extraJvmArgs += ['-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005'] } +if (hasProperty('debugAutocomplete')) { + extraJvmArgs += ['-Ddeephaven.console.autocomplete.quiet=false'] +} + if (hasProperty('gcApplication')) { extraJvmArgs += ['-Dio.deephaven.app.GcApplication.enabled=true'] } diff --git a/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java index 58facca28c3..2b63e4c0255 100644 --- a/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java +++ b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java @@ -7,24 +7,17 @@ import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.lang.completion.ChunkerCompleter; -import io.deephaven.lang.completion.CompletionLookups; import io.deephaven.lang.parse.CompletionParser; -import io.deephaven.lang.parse.LspTools; -import io.deephaven.lang.parse.ParsedDocument; -import io.deephaven.lang.shared.lsp.CompletionCancelled; import io.deephaven.proto.backplane.script.grpc.*; import io.deephaven.server.console.ConsoleServiceGrpcImpl; import io.deephaven.server.session.SessionState; import io.deephaven.util.SafeCloseable; import io.grpc.stub.StreamObserver; -import org.jpy.PyListWrapper; import org.jpy.PyObject; import javax.inject.Provider; import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; import static io.deephaven.extensions.barrage.util.GrpcUtil.safelyExecuteLocked; @@ -34,9 +27,13 @@ public class PythonAutoCompleteObserver implements StreamObserver { private static final Logger log = LoggerFactory.getLogger(PythonAutoCompleteObserver.class); + + /** + * We only log timing for completions that take longer than, currently, 100ms + */ + private static final long HUNDRED_MS_IN_NS = 100_000_000; private final Provider scriptSession; private final SessionState session; - private boolean canJedi, checkedJedi; private final StreamObserver responseObserver; public PythonAutoCompleteObserver(StreamObserver responseObserver, Provider scriptSession, final SessionState session) { @@ -104,94 +101,97 @@ private void getCompletionItems(GetCompletionItemsRequest request, StreamObserver responseObserver) { final ScriptSession scriptSession = exportedConsole.get(); try (final SafeCloseable ignored = scriptSession.getExecutionContext().open()) { - final VariableProvider vars = scriptSession.getVariableProvider(); PyObject completer = (PyObject) scriptSession.getVariable("jedi_settings"); boolean canJedi = completer.callMethod("is_enabled").getBooleanValue(); if (!canJedi) { log.trace().append("Ignoring completion request because jedi is disabled").endl(); + // send back an empty response. We may want to include a "don't ask again" flag in the response... + safelyExecuteLocked(responseObserver, + () -> responseObserver.onNext(AutoCompleteResponse.newBuilder().build())); return; } final VersionedTextDocumentIdentifier doc = request.getTextDocument(); + final Position pos = request.getPosition(); final long startNano = System.nanoTime(); - String text = completer.call("get_doc", doc.getUri()).getStringValue(); - try { - final Position pos = request.getPosition(); - String completionVar = "completions"; - scriptSession.setVariable("__jedi_source__", text); - final ScriptSession.Changes changes = scriptSession.evaluateScript( - completionVar + " = jedi.Interpreter(jedi_settings.get_doc('" + doc.getUri() + "'), [globals()]).complete(" + - (pos.getLine() + 1) + "," + pos.getCharacter() + ")" - ); - final PyListWrapper completes = vars.getVariable(completionVar, null); - List completionResults = new ArrayList<>(); - List completionResults_ = new ArrayList<>(); - List completionResults__ = new ArrayList<>(); - for (PyObject completion : completes) { - String completionName = completion.getAttribute("name").getStringValue(); - int completionPrefix = completion.call("get_completion_prefix_length").getIntValue(); - final CompletionItem.Builder item = CompletionItem.newBuilder(); - final TextEdit.Builder textEdit = item.getTextEditBuilder(); - textEdit.setText(completionName); - final DocumentRange.Builder range = textEdit.getRangeBuilder(); - int start = pos.getCharacter() - completionPrefix; - item.setStart(start); - item.setLabel(completionName); - item.setLength(completionName.length()); - range.getStartBuilder().setLine(pos.getLine()).setCharacter(start); - range.getEndBuilder().setLine(pos.getLine()).setCharacter(start + completionName.length()); - item.setInsertTextFormat(2); - if (completionName.startsWith("__")) { - completionResults__.add(item); - } else if (completionName.startsWith("_")) { - completionResults_.add(item); - } else { - completionResults.add(item); - } - } - int sortPos = 0; - List finalItems = new ArrayList<>(); - for (CompletionItem.Builder res : completionResults) { - res.setSortText(ChunkerCompleter.sortable(sortPos++)); - finalItems.add(res.build()); - } - for (CompletionItem.Builder res : completionResults_) { - res.setSortText(ChunkerCompleter.sortable(sortPos++)); - finalItems.add(res.build()); - } - for (CompletionItem.Builder res : completionResults__) { - res.setSortText(ChunkerCompleter.sortable(sortPos++)); - finalItems.add(res.build()); + + if (log.isTraceEnabled()) { + String text = completer.call("get_doc", doc.getUri()).getStringValue(); + log.trace().append("Completion version ").append(doc.getVersion()) + .append(" has source code:").append(text).endl(); + } + final PyObject results = completer.callMethod("do_completion", doc.getUri(), doc.getVersion(), + // our java is 0-indexed lines, 1-indexed chars. jedi is 1-indexed-both. + // we'll keep that translation ugliness to the in-java result-processing. + pos.getLine() + 1, pos.getCharacter()); + if (!results.isList()) { + throw new UnsupportedOperationException("Expected list from jedi_settings.do_completion, got " + results.call("repr")); + } + log.info().append("Got ").append(results.asList().size()).append(" completions from jedi at ") + .append(pos.getLine()).append(", ").append(pos.getCharacter()).endl(); + final long nanosJedi = System.nanoTime(); + // translate from-python list of completion results. For now, each item in the outer list is a [str, int] + // which contains the text of the replacement, and the column where is should be inserted. + List finalItems = new ArrayList<>(); + + for (PyObject result : results.asList()) { + if (!result.isList()) { + throw new UnsupportedOperationException("Expected list-of-lists from jedi_settings.do_completion, " + + "got bad result " + result.call("repr") + " from full results: " + results.call("repr")); } + // we expect [ "completion text", start_column ] as our result. + // in the future we may want to get more interesting info from jedi to pass back to client + final List items = result.asList(); + String completionName = items.get(0).getStringValue(); + int start = items.get(1).getIntValue(); + final CompletionItem.Builder item = CompletionItem.newBuilder(); + final TextEdit.Builder textEdit = item.getTextEditBuilder(); + textEdit.setText(completionName); + final DocumentRange.Builder range = textEdit.getRangeBuilder(); + item.setStart(start); + item.setLabel(completionName); + item.setLength(completionName.length()); + range.getStartBuilder().setLine(pos.getLine()).setCharacter(start); + range.getEndBuilder().setLine(pos.getLine()).setCharacter(start + completionName.length()); + item.setInsertTextFormat(2); + item.setSortText(ChunkerCompleter.sortable(finalItems.size())); + finalItems.add(item.build()); + } + final long nanosBuiltResponse = System.nanoTime(); - final GetCompletionItemsResponse builtItems = GetCompletionItemsResponse.newBuilder() - .setSuccess(true) - .setRequestId(request.getRequestId()) - .addAllItems(finalItems) - .build(); + final GetCompletionItemsResponse builtItems = GetCompletionItemsResponse.newBuilder() + .setSuccess(true) + .setRequestId(request.getRequestId()) + .addAllItems(finalItems) + .build(); + try { safelyExecuteLocked(responseObserver, () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() .setCompletionItems(builtItems) .build())); - String totalNano = Long.toString(System.nanoTime() - startNano); - // lets track how long completions take, as it's known that some + } finally { + // let's track how long completions take, as it's known that some // modules like numpy can cause slow completion, and we'll want to know what was causing them - log.info().append("Jedi completions took ") - .append( - totalNano.length() > 6 ? - totalNano.substring(0, totalNano.length() - 6) : - "0" - ) - .append('.') - .append(totalNano.substring(6, Math.min(8, totalNano.length()))) - .append("ms").endl(); - - } catch (Throwable e) { - log.error().append("Jedi completion failure ").append(e).endl(); + final long totalCompletionNanos = nanosBuiltResponse - startNano; + final long totalJediNanos = nanosJedi - startNano; + final long totalResponseBuildNanos = nanosBuiltResponse - nanosJedi; + // only log completions taking more than 100ms + if (totalCompletionNanos > HUNDRED_MS_IN_NS && log.isTraceEnabled()) { + log.trace().append("Jedi completions timing for ") + .append(finalItems.size()) + .append(" results from doc ") + .append(doc.getUri()) + .append(" (").append(doc.getVersion()).append(")") + .endl(); + log.trace().append("Python time (jedi): ").append(toMillis(totalJediNanos)).endl(); + log.trace().append("Build response time (java): ").append(toMillis(totalResponseBuildNanos)).endl(); + log.trace().append("Total server time: ").append(toMillis(totalCompletionNanos)).endl(); + } } - } catch (Exception exception) { + } catch (Throwable exception) { if (ConsoleServiceGrpcImpl.QUIET_AUTOCOMPLETE_ERRORS) { + exception.printStackTrace(); if (log.isTraceEnabled()) { log.trace().append("Exception occurred during autocomplete").append(exception).endl(); } @@ -204,9 +204,21 @@ private void getCompletionItems(GetCompletionItemsRequest request, .setSuccess(false) .setRequestId(request.getRequestId())) .build())); + if (exception instanceof Error) { + throw exception; + } } } + private String toMillis(final long totalNanos) { + String totalNano = Long.toString(totalNanos); + return (totalNano.length() > 6 ? + totalNano.substring(0, totalNano.length() - 6) : + "0") + "." + ( + totalNano.substring(6, Math.min(8, totalNano.length())) + ) + "ms"; + } + @Override public void onError(Throwable t) { // ignore, client doesn't need us, will be cleaned up later From f6a6ead0bf5e36be240a2a8aff8ce6fb357d074e Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Thu, 1 Dec 2022 13:54:04 -0600 Subject: [PATCH 08/20] Use web version w/ document version in completion requests Also tells docker it's not allowed to download jedi from pypi, as it is now included in base images. --- docker/server-jetty/src/main/docker/Dockerfile | 4 ++-- docker/server/src/main/docker/Dockerfile | 4 ++-- web/client-ui/Dockerfile | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/server-jetty/src/main/docker/Dockerfile b/docker/server-jetty/src/main/docker/Dockerfile index 5c5589219b5..5f017364154 100644 --- a/docker/server-jetty/src/main/docker/Dockerfile +++ b/docker/server-jetty/src/main/docker/Dockerfile @@ -10,12 +10,12 @@ COPY licenses/ / ARG SERVER COPY $SERVER/requirements.txt requirements.txt RUN set -eux; \ - python -m pip install -q --no-cache-dir -r requirements.txt; \ + python -m pip install -q --no-index --no-cache-dir -r requirements.txt; \ rm requirements.txt COPY wheels/ /wheels RUN set -eux; \ - python -m pip install -q --no-cache-dir /wheels/*.whl; \ + python -m pip install -q --no-index --no-cache-dir /wheels/*.whl; \ rm -r /wheels ARG DEEPHAVEN_VERSION diff --git a/docker/server/src/main/docker/Dockerfile b/docker/server/src/main/docker/Dockerfile index c4aa69c309b..01ffcd7e60a 100644 --- a/docker/server/src/main/docker/Dockerfile +++ b/docker/server/src/main/docker/Dockerfile @@ -10,12 +10,12 @@ COPY licenses/ / ARG SERVER COPY $SERVER/requirements.txt requirements.txt RUN set -eux; \ - python -m pip install -q --no-cache-dir -r requirements.txt; \ + python -m pip install -q --no-index --no-cache-dir -r requirements.txt; \ rm requirements.txt COPY wheels/ /wheels RUN set -eux; \ - python -m pip install -q --no-cache-dir /wheels/*.whl; \ + python -m pip install -q --no-index --no-cache-dir /wheels/*.whl; \ rm -r /wheels ARG DEEPHAVEN_VERSION diff --git a/web/client-ui/Dockerfile b/web/client-ui/Dockerfile index fbbb436e115..7c2014e42a7 100644 --- a/web/client-ui/Dockerfile +++ b/web/client-ui/Dockerfile @@ -2,9 +2,9 @@ FROM deephaven/node:local-build WORKDIR /usr/src/app # Most of the time, these versions are the same, except in cases where a patch only affects one of the packages -ARG WEB_VERSION=0.21.1 -ARG GRID_VERSION=0.21.1 -ARG CHART_VERSION=0.21.1 +ARG WEB_VERSION=0.22.2-autocomplete.3 +ARG GRID_VERSION=0.22.2-autocomplete.3 +ARG CHART_VERSION=0.22.2-autocomplete.3 # Pull in the published code-studio package from npmjs and extract is RUN set -eux; \ From 38c4e2648588336b2ad2fbba534947834b56194a Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Thu, 1 Dec 2022 14:06:08 -0600 Subject: [PATCH 09/20] Minor logging cleanup --- py/server/deephaven/completer/__init__.py | 1 - .../io/deephaven/server/console/ConsoleServiceGrpcImpl.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/py/server/deephaven/completer/__init__.py b/py/server/deephaven/completer/__init__.py index 0baf2a80df2..d766db49cd6 100644 --- a/py/server/deephaven/completer/__init__.py +++ b/py/server/deephaven/completer/__init__.py @@ -85,7 +85,6 @@ def can_jedi(self) -> bool: def do_completion(self, uri: str, version: int, line: int, col: int) -> list[list[Any]]: if not self._versions[uri] == version: # if you aren't the newest completion, you get nothing, quickly - print("No text for v{}".format(version)) return [] # run jedi diff --git a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java index 89bcb1a406e..1b0ca47bf92 100644 --- a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java +++ b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java @@ -266,7 +266,7 @@ public StreamObserver autoCompleteStream( settings[0] = (PyObject) scriptSession.getVariable("jedi_settings"); }); boolean canJedi = settings[0] != null && settings[0].call("can_jedi").getBooleanValue(); - log.info().append("can jedi? ").append(canJedi).endl(); + log.info().append(canJedi ? "Using jedi for python autocomplete" : "No jedi dependency available in python environment; disabling autocomplete.").endl(); return canJedi ? new PythonAutoCompleteObserver(responseObserver, scriptSessionProvider, session) : new NoopAutoCompleteObserver(responseObserver); } From fdfaedcef37e1114e1c9fe7416658c113c7d5e38 Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Thu, 1 Dec 2022 14:08:47 -0600 Subject: [PATCH 10/20] Update jetty-all-ai requirements.txt --- .../main/server-all-ai-jetty/requirements.txt | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt b/docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt index cdd22a54d30..ab7e368e5e6 100644 --- a/docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt +++ b/docker/server-jetty/src/main/server-all-ai-jetty/requirements.txt @@ -1,23 +1,23 @@ -absl-py==1.2.0 +absl-py==1.3.0 astunparse==1.6.3 cachetools==5.2.0 certifi==2022.9.24 charset-normalizer==2.1.1 click==8.1.3 -deephaven-plugin==0.2.0 +deephaven-plugin==0.3.0 flatbuffers==2.0.7 gast==0.4.0 -google-auth==2.12.0 +google-auth==2.14.1 google-auth-oauthlib==0.4.6 google-pasta==0.2.0 -grpcio==1.49.1 +grpcio==1.51.1 h5py==3.7.0 idna==3.4 -importlib-metadata==4.12.0 +importlib-metadata==5.1.0 java-utilities==0.2.0 -jedi==0.18.1 +jedi==0.18.2 joblib==1.2.0 -jpy==0.12.0 +jpy==0.13.0 keras==2.7.0 Keras-Preprocessing==1.1.2 libclang==14.0.6 @@ -25,36 +25,40 @@ llvmlite==0.39.1 Markdown==3.4.1 MarkupSafe==2.1.1 nltk==3.7 -numba==0.56.2 +numba==0.56.4 numpy==1.21.6 -oauthlib==3.2.1 +nvidia-cublas-cu11==11.10.3.66 +nvidia-cuda-nvrtc-cu11==11.7.99 +nvidia-cuda-runtime-cu11==11.7.99 +nvidia-cudnn-cu11==8.5.0.96 +oauthlib==3.2.2 opt-einsum==3.3.0 pandas==1.3.5 -pkg_resources==0.0.0 -protobuf==3.19.5 +parso==0.8.3 +protobuf==3.19.6 pyasn1==0.4.8 pyasn1-modules==0.2.8 python-dateutil==2.8.2 -pytz==2022.2.1 -regex==2022.9.13 +pytz==2022.6 +regex==2022.10.31 requests==2.28.1 requests-oauthlib==1.3.1 rsa==4.9 scikit-learn==1.0.2 scipy==1.7.3 six==1.16.0 -tensorboard==2.10.1 +tensorboard==2.11.0 tensorboard-data-server==0.6.1 tensorboard-plugin-wit==1.8.1 tensorflow==2.7.4 tensorflow-estimator==2.7.0 -tensorflow-io-gcs-filesystem==0.27.0 -termcolor==2.0.1 +tensorflow-io-gcs-filesystem==0.28.0 +termcolor==2.1.1 threadpoolctl==3.1.0 -torch==1.12.1 +torch==1.13.0 tqdm==4.64.1 -typing_extensions==4.3.0 -urllib3==1.26.12 +typing_extensions==4.4.0 +urllib3==1.26.13 Werkzeug==2.2.2 wrapt==1.14.1 -zipp==3.8.1 +zipp==3.11.0 From 96a57a5a605459d80974d02bfdafb682d8ca6fb2 Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Thu, 1 Dec 2022 14:10:36 -0600 Subject: [PATCH 11/20] Use official web docker images --- web/client-ui/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/client-ui/Dockerfile b/web/client-ui/Dockerfile index 7c2014e42a7..840d90313aa 100644 --- a/web/client-ui/Dockerfile +++ b/web/client-ui/Dockerfile @@ -2,9 +2,9 @@ FROM deephaven/node:local-build WORKDIR /usr/src/app # Most of the time, these versions are the same, except in cases where a patch only affects one of the packages -ARG WEB_VERSION=0.22.2-autocomplete.3 -ARG GRID_VERSION=0.22.2-autocomplete.3 -ARG CHART_VERSION=0.22.2-autocomplete.3 +ARG WEB_VERSION=0.22.2 +ARG GRID_VERSION=0.22.2 +ARG CHART_VERSION=0.22.2 # Pull in the published code-studio package from npmjs and extract is RUN set -eux; \ From 5ba99bbb5ddc70ef5baf432b6c69d7c8abc902c9 Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Thu, 1 Dec 2022 17:13:38 -0600 Subject: [PATCH 12/20] code review comments --- py/server/deephaven/completer/__init__.py | 115 +----------------- py/server/deephaven/completer/_completer.py | 105 ++++++++++++++++ .../completer/PythonAutoCompleteObserver.java | 8 +- 3 files changed, 112 insertions(+), 116 deletions(-) create mode 100644 py/server/deephaven/completer/_completer.py diff --git a/py/server/deephaven/completer/__init__.py b/py/server/deephaven/completer/__init__.py index d766db49cd6..56c6eb359df 100644 --- a/py/server/deephaven/completer/__init__.py +++ b/py/server/deephaven/completer/__init__.py @@ -16,118 +16,5 @@ later, we may add slow mode, which uses both static and interpreted completion modes. """ -# TODO: make this python <= 3.8 -from __future__ import annotations -from enum import Enum -from typing import Any -from jedi import Interpreter - -class CompleterMode(Enum): - off = 'off' - safe = 'safe' - strong = 'strong' - - def __str__(self) -> str: - return self.value - - -class Completer(object): - - def __init__(self): - self._docs = {} - self._versions = {} - # might want to make this a {uri: []} instead of [] - self.pending = [] - try: - import jedi - self.__can_jedi = True - self.mode = CompleterMode.strong - except ImportError: - self.__can_jedi = False - self.mode = CompleterMode.off - - @property - def mode(self) -> CompleterMode: - return self.__mode - - @mode.setter - def mode(self, mode) -> None: - if type(mode) == 'str': - mode = CompleterMode[mode] - self.__mode = mode - - def open_doc(self, text: str, uri: str, version: int) -> None: - self._docs[uri] = text - self._versions[uri] = version - - def get_doc(self, uri: str) -> str: - return self._docs[uri] - - def update_doc(self, text: str, uri: str, version: int) -> None: - self._docs[uri] = text - self._versions[uri] = version - # any pending completions should stop running now. We use a list of Event to signal any running threads to stop - for pending in self.pending: - pending.set() - - def close_doc(self, uri: str) -> None: - del self._docs[uri] - del self._versions[uri] - for pending in self.pending: - pending.set() - - def is_enabled(self) -> bool: - return self.__mode != CompleterMode.off - - def can_jedi(self) -> bool: - return self.__can_jedi - - def do_completion(self, uri: str, version: int, line: int, col: int) -> list[list[Any]]: - if not self._versions[uri] == version: - # if you aren't the newest completion, you get nothing, quickly - return [] - - # run jedi - txt = self.get_doc(uri) - completions = Interpreter(txt, [globals()]).complete(line, col) - # for now, a simple sorting based on number of preceding _ - # we may want to apply additional sorting to each list before combining - results: list = [] - results_: list = [] - results__: list = [] - for complete in completions: - # keep checking the latest version as we run, so updated doc can cancel us - if not self._versions[uri] == version: - return [] - result: list = self.to_result(complete, col) - if result[0].startswith('__'): - results__.append(result) - elif result[0].startswith('_'): - results_.append(result) - else: - results.append(result) - - # put the results together in a better-than-nothing sorting - return results + results_ + results__ - - @staticmethod - def to_result(complete: Any, col: int) -> list[Any]: - name: str = complete.name - prefix_length: int = complete.get_completion_prefix_length() - start: int = col - prefix_length - # all java needs to build a grpc response is completion text (name) and where the completion should start - return [name, start] - - +from deephaven.completer._completer import Completer jedi_settings = Completer() - -if jedi_settings.mode == CompleterMode.off: - print('No jedi library installed on system path, disabling autocomplete') - -if jedi_settings.mode == CompleterMode.strong: - # start a strong-mode completer thread in bg - pass - -if jedi_settings.mode == CompleterMode.safe: - # start a safe-mode completer thread in bg - pass \ No newline at end of file diff --git a/py/server/deephaven/completer/_completer.py b/py/server/deephaven/completer/_completer.py new file mode 100644 index 00000000000..07cf8d741c7 --- /dev/null +++ b/py/server/deephaven/completer/_completer.py @@ -0,0 +1,105 @@ +# only python 3.8 needs this, but it must be the first expression in the file, so we can't predicate it +from __future__ import annotations +from enum import Enum +from typing import Any +from jedi import Interpreter, Script + + +class CompleterMode(Enum): + off = 'off' + safe = 'safe' + strong = 'strong' + + def __str__(self) -> str: + return self.value + + +class Completer(object): + + def __init__(self): + self._docs = {} + self._versions = {} + # might want to make this a {uri: []} instead of [] + self.pending = [] + try: + import jedi + self.__can_jedi = True + self.mode = CompleterMode.strong + except ImportError: + self.__can_jedi = False + self.mode = CompleterMode.off + + @property + def mode(self) -> CompleterMode: + return self.__mode + + @mode.setter + def mode(self, mode) -> None: + if type(mode) == 'str': + mode = CompleterMode[mode] + self.__mode = mode + + def open_doc(self, text: str, uri: str, version: int) -> None: + self._docs[uri] = text + self._versions[uri] = version + + def get_doc(self, uri: str) -> str: + return self._docs[uri] + + def update_doc(self, text: str, uri: str, version: int) -> None: + self._docs[uri] = text + self._versions[uri] = version + # any pending completions should stop running now. We use a list of Event to signal any running threads to stop + for pending in self.pending: + pending.set() + + def close_doc(self, uri: str) -> None: + del self._docs[uri] + del self._versions[uri] + for pending in self.pending: + pending.set() + + def is_enabled(self) -> bool: + return self.__mode != CompleterMode.off + + def can_jedi(self) -> bool: + return self.__can_jedi + + def do_completion(self, uri: str, version: int, line: int, col: int) -> list[list[Any]]: + if not self._versions[uri] == version: + # if you aren't the newest completion, you get nothing, quickly + return [] + + # run jedi + txt = self.get_doc(uri) + # The Script completer is static analysis only, so we should actually be feeding it a whole document at once. + + completer = Script if self.__mode == CompleterMode.safe else Interpreter + completions = completer(txt, [globals()]).complete(line, col) + # for now, a simple sorting based on number of preceding _ + # we may want to apply additional sorting to each list before combining + results: list = [] + results_: list = [] + results__: list = [] + for complete in completions: + # keep checking the latest version as we run, so updated doc can cancel us + if not self._versions[uri] == version: + return [] + result: list = self.to_result(complete, col) + if result[0].startswith('__'): + results__.append(result) + elif result[0].startswith('_'): + results_.append(result) + else: + results.append(result) + + # put the results together in a better-than-nothing sorting + return results + results_ + results__ + + @staticmethod + def to_result(complete: Any, col: int) -> list[Any]: + name: str = complete.name + prefix_length: int = complete.get_completion_prefix_length() + start: int = col - prefix_length + # all java needs to build a grpc response is completion text (name) and where the completion should start + return [name, start] diff --git a/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java index 2b63e4c0255..e4f4ed4d9c6 100644 --- a/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java +++ b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java @@ -106,9 +106,13 @@ private void getCompletionItems(GetCompletionItemsRequest request, boolean canJedi = completer.callMethod("is_enabled").getBooleanValue(); if (!canJedi) { log.trace().append("Ignoring completion request because jedi is disabled").endl(); - // send back an empty response. We may want to include a "don't ask again" flag in the response... + // send back an empty, failed response... safelyExecuteLocked(responseObserver, - () -> responseObserver.onNext(AutoCompleteResponse.newBuilder().build())); + () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() + .setCompletionItems(GetCompletionItemsResponse.newBuilder() + .setSuccess(false) + .setRequestId(request.getRequestId())) + .build())); return; } final VersionedTextDocumentIdentifier doc = request.getTextDocument(); From a5124244e1a4c80456964f70b290678ff695ee47 Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Thu, 1 Dec 2022 17:56:12 -0600 Subject: [PATCH 13/20] Cleanup logging and warm jedi up a little --- py/server/deephaven/completer/__init__.py | 5 ++++ .../console/ConsoleServiceGrpcImpl.java | 2 +- .../completer/PythonAutoCompleteObserver.java | 29 +++++++++---------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/py/server/deephaven/completer/__init__.py b/py/server/deephaven/completer/__init__.py index 56c6eb359df..6219e30e507 100644 --- a/py/server/deephaven/completer/__init__.py +++ b/py/server/deephaven/completer/__init__.py @@ -17,4 +17,9 @@ """ from deephaven.completer._completer import Completer +from jedi import preload_module, Interpreter + jedi_settings = Completer() +# warm jedi up a little. We could probably off-thread this. +preload_module('deephaven') +Interpreter('', []).complete(1, 0) diff --git a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java index 1b0ca47bf92..240b69b1aef 100644 --- a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java +++ b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java @@ -262,7 +262,7 @@ public StreamObserver autoCompleteStream( PyObject[] settings = new PyObject[1]; safelyExecute(()->{ final ScriptSession scriptSession = scriptSessionProvider.get(); - scriptSession.evaluateScript("from deephaven.completer import jedi_settings\nimport jedi"); + scriptSession.evaluateScript("from deephaven.completer import jedi_settings"); settings[0] = (PyObject) scriptSession.getVariable("jedi_settings"); }); boolean canJedi = settings[0] != null && settings[0].call("can_jedi").getBooleanValue(); diff --git a/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java index e4f4ed4d9c6..e82e0d9d6ae 100644 --- a/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java +++ b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java @@ -2,7 +2,6 @@ import com.google.rpc.Code; import io.deephaven.engine.util.ScriptSession; -import io.deephaven.engine.util.VariableProvider; import io.deephaven.extensions.barrage.util.GrpcUtil; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; @@ -131,8 +130,6 @@ private void getCompletionItems(GetCompletionItemsRequest request, if (!results.isList()) { throw new UnsupportedOperationException("Expected list from jedi_settings.do_completion, got " + results.call("repr")); } - log.info().append("Got ").append(results.asList().size()).append(" completions from jedi at ") - .append(pos.getLine()).append(", ").append(pos.getCharacter()).endl(); final long nanosJedi = System.nanoTime(); // translate from-python list of completion results. For now, each item in the outer list is a [str, int] // which contains the text of the replacement, and the column where is should be inserted. @@ -161,6 +158,7 @@ private void getCompletionItems(GetCompletionItemsRequest request, item.setSortText(ChunkerCompleter.sortable(finalItems.size())); finalItems.add(item.build()); } + final long nanosBuiltResponse = System.nanoTime(); final GetCompletionItemsResponse builtItems = GetCompletionItemsResponse.newBuilder() @@ -182,15 +180,14 @@ private void getCompletionItems(GetCompletionItemsRequest request, final long totalResponseBuildNanos = nanosBuiltResponse - nanosJedi; // only log completions taking more than 100ms if (totalCompletionNanos > HUNDRED_MS_IN_NS && log.isTraceEnabled()) { - log.trace().append("Jedi completions timing for ") + log.trace().append("Found ") .append(finalItems.size()) - .append(" results from doc ") - .append(doc.getUri()) - .append(" (").append(doc.getVersion()).append(")") + .append(" jedi completions from doc ") + .append(doc.getVersion()) + .append("\tjedi_time=").append(toMillis(totalJediNanos)) + .append("\tbuild_response_time=").append(toMillis(totalResponseBuildNanos)) + .append("\ttotal_complete_time=").append(toMillis(totalCompletionNanos)) .endl(); - log.trace().append("Python time (jedi): ").append(toMillis(totalJediNanos)).endl(); - log.trace().append("Build response time (java): ").append(toMillis(totalResponseBuildNanos)).endl(); - log.trace().append("Total server time: ").append(toMillis(totalCompletionNanos)).endl(); } } } catch (Throwable exception) { @@ -215,11 +212,13 @@ private void getCompletionItems(GetCompletionItemsRequest request, } private String toMillis(final long totalNanos) { - String totalNano = Long.toString(totalNanos); - return (totalNano.length() > 6 ? - totalNano.substring(0, totalNano.length() - 6) : - "0") + "." + ( - totalNano.substring(6, Math.min(8, totalNano.length())) + StringBuilder totalNano = new StringBuilder(Long.toString(totalNanos)); + while (totalNano.length() < 7) { + totalNano.insert(0, "0"); + } + int milliCutoff = totalNano.length() - 6; + return totalNano.substring(0, milliCutoff) + "." + ( + totalNano.substring(milliCutoff, Math.min(milliCutoff + 2, totalNano.length())) ) + "ms"; } From c30e795735383ba4b17a07e064c3d95125a78c4c Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Thu, 1 Dec 2022 18:01:40 -0600 Subject: [PATCH 14/20] Ditch whitespace change --- docker/server-jetty/src/main/server-jetty/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/server-jetty/src/main/server-jetty/requirements.txt b/docker/server-jetty/src/main/server-jetty/requirements.txt index 8483bc479f1..1fbc075f427 100644 --- a/docker/server-jetty/src/main/server-jetty/requirements.txt +++ b/docker/server-jetty/src/main/server-jetty/requirements.txt @@ -13,4 +13,3 @@ pytz==2022.6 six==1.16.0 typing_extensions==4.4.0 zipp==3.11.0 - From f9abec8897df67a797596be65c702449c664d794 Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Thu, 1 Dec 2022 18:19:06 -0600 Subject: [PATCH 15/20] spotless fixups --- .../lang/parse/CompletionParser.java | 4 ++- .../console/ConsoleServiceGrpcImpl.java | 8 +++-- .../completer/JavaAutoCompleteObserver.java | 14 ++++----- .../completer/PythonAutoCompleteObserver.java | 29 ++++++++++--------- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java b/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java index ec42ac474d6..31c49c8a570 100644 --- a/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java +++ b/open-api/lang-parser/src/main/java/io/deephaven/lang/parse/CompletionParser.java @@ -24,7 +24,8 @@ public class CompletionParser implements Closeable { private static final Logger LOGGER = LoggerFactory.getLogger(CompletionParser.class); private final Map docs = new ConcurrentHashMap<>(); - public static String updateDocumentChanges(final String uri, final int version, String document, final List changes) { + public static String updateDocumentChanges(final String uri, final int version, String document, + final List changes) { for (ChangeDocumentRequest.TextDocumentContentChangeEventOrBuilder change : changes) { DocumentRange range = change.getRange(); int length = change.getRangeLength(); @@ -133,6 +134,7 @@ public String getText(String uri) { } return doc.getText(); } + public ParsedDocument finish(String uri) { final PendingParse doc = docs.get(uri); if (doc == null) { diff --git a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java index 240b69b1aef..4f076686b1c 100644 --- a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java +++ b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java @@ -260,14 +260,16 @@ public StreamObserver autoCompleteStream( final SessionState session = sessionService.getCurrentSession(); if (PythonDeephavenSession.SCRIPT_TYPE.equals(scriptSessionProvider.get().scriptType())) { PyObject[] settings = new PyObject[1]; - safelyExecute(()->{ + safelyExecute(() -> { final ScriptSession scriptSession = scriptSessionProvider.get(); scriptSession.evaluateScript("from deephaven.completer import jedi_settings"); settings[0] = (PyObject) scriptSession.getVariable("jedi_settings"); }); boolean canJedi = settings[0] != null && settings[0].call("can_jedi").getBooleanValue(); - log.info().append(canJedi ? "Using jedi for python autocomplete" : "No jedi dependency available in python environment; disabling autocomplete.").endl(); - return canJedi ? new PythonAutoCompleteObserver(responseObserver, scriptSessionProvider, session) : new NoopAutoCompleteObserver(responseObserver); + log.info().append(canJedi ? "Using jedi for python autocomplete" + : "No jedi dependency available in python environment; disabling autocomplete.").endl(); + return canJedi ? new PythonAutoCompleteObserver(responseObserver, scriptSessionProvider, session) + : new NoopAutoCompleteObserver(responseObserver); } return new JavaAutoCompleteObserver(session, responseObserver); diff --git a/server/src/main/java/io/deephaven/server/console/completer/JavaAutoCompleteObserver.java b/server/src/main/java/io/deephaven/server/console/completer/JavaAutoCompleteObserver.java index 3a3f10f5b70..5513a1c2692 100644 --- a/server/src/main/java/io/deephaven/server/console/completer/JavaAutoCompleteObserver.java +++ b/server/src/main/java/io/deephaven/server/console/completer/JavaAutoCompleteObserver.java @@ -26,8 +26,8 @@ import static io.deephaven.extensions.barrage.util.GrpcUtil.safelyExecuteLocked; /** - * Autocomplete handling for JVM languages, that directly can interact with Java instances without any name - * mangling, and are able to use our flexible parser. + * Autocomplete handling for JVM languages, that directly can interact with Java instances without any name mangling, + * and are able to use our flexible parser. */ public class JavaAutoCompleteObserver implements StreamObserver { @@ -97,8 +97,8 @@ public void onNext(AutoCompleteRequest value) { } private void getCompletionItems(GetCompletionItemsRequest request, - SessionState.ExportObject exportedConsole, CompletionParser parser, - StreamObserver responseObserver) { + SessionState.ExportObject exportedConsole, CompletionParser parser, + StreamObserver responseObserver) { final ScriptSession scriptSession = exportedConsole.get(); try (final SafeCloseable ignored = scriptSession.getExecutionContext().open()) { final VariableProvider vars = scriptSession.getVariableProvider(); @@ -134,9 +134,9 @@ private void getCompletionItems(GetCompletionItemsRequest request, .setSuccess(true) .setRequestId(request.getRequestId()) .addAllItems(results.stream().map( - // insertTextFormat is a default we used to set in constructor; for now, we'll - // just process the objects before sending back to client - item -> item.setInsertTextFormat(2).build()) + // insertTextFormat is a default we used to set in constructor; for now, we'll + // just process the objects before sending back to client + item -> item.setInsertTextFormat(2).build()) .collect(Collectors.toSet())) .build(); diff --git a/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java index e82e0d9d6ae..e64d1295a3c 100644 --- a/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java +++ b/server/src/main/java/io/deephaven/server/console/completer/PythonAutoCompleteObserver.java @@ -35,7 +35,8 @@ public class PythonAutoCompleteObserver implements StreamObserver responseObserver; - public PythonAutoCompleteObserver(StreamObserver responseObserver, Provider scriptSession, final SessionState session) { + public PythonAutoCompleteObserver(StreamObserver responseObserver, + Provider scriptSession, final SessionState session) { this.scriptSession = scriptSession; this.session = session; this.responseObserver = responseObserver; @@ -61,7 +62,8 @@ public void onNext(AutoCompleteRequest value) { int version = text.getVersion(); String document = completer.callMethod("get_doc", text.getUri()).getStringValue(); - final List changes = request.getContentChangesList(); + final List changes = + request.getContentChangesList(); document = CompletionParser.updateDocumentChanges(uri, version, document, changes); if (document == null) { return; @@ -96,8 +98,8 @@ public void onNext(AutoCompleteRequest value) { } private void getCompletionItems(GetCompletionItemsRequest request, - SessionState.ExportObject exportedConsole, - StreamObserver responseObserver) { + SessionState.ExportObject exportedConsole, + StreamObserver responseObserver) { final ScriptSession scriptSession = exportedConsole.get(); try (final SafeCloseable ignored = scriptSession.getExecutionContext().open()) { @@ -108,10 +110,10 @@ private void getCompletionItems(GetCompletionItemsRequest request, // send back an empty, failed response... safelyExecuteLocked(responseObserver, () -> responseObserver.onNext(AutoCompleteResponse.newBuilder() - .setCompletionItems(GetCompletionItemsResponse.newBuilder() - .setSuccess(false) - .setRequestId(request.getRequestId())) - .build())); + .setCompletionItems(GetCompletionItemsResponse.newBuilder() + .setSuccess(false) + .setRequestId(request.getRequestId())) + .build())); return; } final VersionedTextDocumentIdentifier doc = request.getTextDocument(); @@ -128,7 +130,8 @@ private void getCompletionItems(GetCompletionItemsRequest request, // we'll keep that translation ugliness to the in-java result-processing. pos.getLine() + 1, pos.getCharacter()); if (!results.isList()) { - throw new UnsupportedOperationException("Expected list from jedi_settings.do_completion, got " + results.call("repr")); + throw new UnsupportedOperationException( + "Expected list from jedi_settings.do_completion, got " + results.call("repr")); } final long nanosJedi = System.nanoTime(); // translate from-python list of completion results. For now, each item in the outer list is a [str, int] @@ -137,7 +140,8 @@ private void getCompletionItems(GetCompletionItemsRequest request, for (PyObject result : results.asList()) { if (!result.isList()) { - throw new UnsupportedOperationException("Expected list-of-lists from jedi_settings.do_completion, " + + throw new UnsupportedOperationException("Expected list-of-lists from jedi_settings.do_completion, " + + "got bad result " + result.call("repr") + " from full results: " + results.call("repr")); } // we expect [ "completion text", start_column ] as our result. @@ -217,9 +221,8 @@ private String toMillis(final long totalNanos) { totalNano.insert(0, "0"); } int milliCutoff = totalNano.length() - 6; - return totalNano.substring(0, milliCutoff) + "." + ( - totalNano.substring(milliCutoff, Math.min(milliCutoff + 2, totalNano.length())) - ) + "ms"; + return totalNano.substring(0, milliCutoff) + "." + + (totalNano.substring(milliCutoff, Math.min(milliCutoff + 2, totalNano.length()))) + "ms"; } @Override From 97b4b0c9214e7c07d8ce486e8e9b265aa779a05c Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Fri, 2 Dec 2022 12:22:10 -0600 Subject: [PATCH 16/20] save a copy of globals() while we have it in scope --- py/server/deephaven/completer/_completer.py | 8 +++++++- .../deephaven/server/console/ConsoleServiceGrpcImpl.java | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/py/server/deephaven/completer/_completer.py b/py/server/deephaven/completer/_completer.py index 07cf8d741c7..333c3b4d916 100644 --- a/py/server/deephaven/completer/_completer.py +++ b/py/server/deephaven/completer/_completer.py @@ -19,6 +19,8 @@ class Completer(object): def __init__(self): self._docs = {} self._versions = {} + # we will replace this w/ top-level globals() when we open the document + self.__scope = globals() # might want to make this a {uri: []} instead of [] self.pending = [] try: @@ -65,6 +67,9 @@ def is_enabled(self) -> bool: def can_jedi(self) -> bool: return self.__can_jedi + def set_scope(self, scope: dict) -> None: + self.__scope = scope + def do_completion(self, uri: str, version: int, line: int, col: int) -> list[list[Any]]: if not self._versions[uri] == version: # if you aren't the newest completion, you get nothing, quickly @@ -75,7 +80,8 @@ def do_completion(self, uri: str, version: int, line: int, col: int) -> list[lis # The Script completer is static analysis only, so we should actually be feeding it a whole document at once. completer = Script if self.__mode == CompleterMode.safe else Interpreter - completions = completer(txt, [globals()]).complete(line, col) + + completions = completer(txt, [this.__scope]).complete(line, col) # for now, a simple sorting based on number of preceding _ # we may want to apply additional sorting to each list before combining results: list = [] diff --git a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java index 4f076686b1c..a04a12ff2d7 100644 --- a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java +++ b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java @@ -262,7 +262,7 @@ public StreamObserver autoCompleteStream( PyObject[] settings = new PyObject[1]; safelyExecute(() -> { final ScriptSession scriptSession = scriptSessionProvider.get(); - scriptSession.evaluateScript("from deephaven.completer import jedi_settings"); + scriptSession.evaluateScript("from deephaven.completer import jedi_settings ; jedi_settings.set_scope(globals())"); settings[0] = (PyObject) scriptSession.getVariable("jedi_settings"); }); boolean canJedi = settings[0] != null && settings[0].call("can_jedi").getBooleanValue(); From 1a24e893701ba0cf2ddd32b4c388eace1e5c344c Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Fri, 2 Dec 2022 12:26:25 -0600 Subject: [PATCH 17/20] woops, fix typo --- py/server/deephaven/completer/_completer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/server/deephaven/completer/_completer.py b/py/server/deephaven/completer/_completer.py index 333c3b4d916..652d8e605b9 100644 --- a/py/server/deephaven/completer/_completer.py +++ b/py/server/deephaven/completer/_completer.py @@ -81,7 +81,7 @@ def do_completion(self, uri: str, version: int, line: int, col: int) -> list[lis completer = Script if self.__mode == CompleterMode.safe else Interpreter - completions = completer(txt, [this.__scope]).complete(line, col) + completions = completer(txt, [self.__scope]).complete(line, col) # for now, a simple sorting based on number of preceding _ # we may want to apply additional sorting to each list before combining results: list = [] From ed99c5d985c199f4967bbfff1db05a5dccf2977a Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Fri, 2 Dec 2022 12:27:48 -0600 Subject: [PATCH 18/20] add optional dependency on jedi from deephaven server --- py/server/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/py/server/setup.py b/py/server/setup.py index 027a0b53d6b..c11ddfeea65 100644 --- a/py/server/setup.py +++ b/py/server/setup.py @@ -62,6 +62,9 @@ def normalize_version(version): # TODO(deephaven-core#3082): Remove numba dependency workarounds 'numba; python_version < "3.11"', ], + extras_require={ + "autocomplete": ["jedi==0.18.2"], + }, entry_points={ 'deephaven.plugin': ['registration_cls = deephaven.pandasplugin:PandasPluginRegistration'] } From 733d8bed98d3e2756d6613e2905c11b39b2e971c Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Fri, 2 Dec 2022 12:34:12 -0600 Subject: [PATCH 19/20] spotless fixups --- .../io/deephaven/server/console/ConsoleServiceGrpcImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java index a04a12ff2d7..e6edf6181a0 100644 --- a/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java +++ b/server/src/main/java/io/deephaven/server/console/ConsoleServiceGrpcImpl.java @@ -262,7 +262,8 @@ public StreamObserver autoCompleteStream( PyObject[] settings = new PyObject[1]; safelyExecute(() -> { final ScriptSession scriptSession = scriptSessionProvider.get(); - scriptSession.evaluateScript("from deephaven.completer import jedi_settings ; jedi_settings.set_scope(globals())"); + scriptSession.evaluateScript( + "from deephaven.completer import jedi_settings ; jedi_settings.set_scope(globals())"); settings[0] = (PyObject) scriptSession.getVariable("jedi_settings"); }); boolean canJedi = settings[0] != null && settings[0].call("can_jedi").getBooleanValue(); From 0ad8d98aea8643eba17ca74bd7ceb60fc4ce8f3a Mon Sep 17 00:00:00 2001 From: James X Nelson Date: Fri, 2 Dec 2022 12:38:09 -0600 Subject: [PATCH 20/20] fix failing test --- .../lang/completion/ChunkerCompletionHandlerTest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompletionHandlerTest.groovy b/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompletionHandlerTest.groovy index 8b0f647087b..e852bf6cd5d 100644 --- a/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompletionHandlerTest.groovy +++ b/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompletionHandlerTest.groovy @@ -160,8 +160,8 @@ b = 2 c = 3 """ String src2 = "t = " - p.update(uri, "0", [ makeChange(0, 0, src1) ]) - p.update(uri, "1", [ makeChange(3, 0, src2) ]) + p.update(uri, 0, [ makeChange(0, 0, src1) ]) + p.update(uri, 1, [ makeChange(3, 0, src2) ]) doc = p.finish(uri) VariableProvider variables = Mock(VariableProvider) {