From 601dd251517d0d62562c67054186fe6cdccd7a14 Mon Sep 17 00:00:00 2001 From: william-lukusa Date: Fri, 5 Feb 2021 10:03:37 +0100 Subject: [PATCH 1/8] implementations of api client for jupyter notebooks --- dataikuapi/dss/notebook.py | 38 ++++++++++++++++++++++++++---- dataikuapi/dss/project.py | 48 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/dataikuapi/dss/notebook.py b/dataikuapi/dss/notebook.py index b6e8c1c3..92915fe7 100644 --- a/dataikuapi/dss/notebook.py +++ b/dataikuapi/dss/notebook.py @@ -4,11 +4,12 @@ class DSSNotebook(object): """ A Python/R/Scala notebook on the DSS instance """ - def __init__(self, client, project_key, notebook_name, state=None): + def __init__(self, client, project_key, notebook_name, state=None, content=None): self.client = client self.project_key = project_key self.notebook_name = notebook_name self.state = state + self.content = content self.state_is_peek = True def unload(self, session_id=None): @@ -27,14 +28,19 @@ def unload(self, session_id=None): raise Exception("Several sessions of the notebook are running, choose one") else: session_id = state['activeSessions'][0].get('sessionId', None) - return self.client._perform_json("DELETE", "/projects/%s/notebooks/" % self.project_key, params={'notebookName' : self.notebook_name, 'sessionId' : session_id}) + return self.client._perform_json("DELETE", + "/projects/%s/jupyter-notebooks/%s/sessions/%s" % (self.project_key, self.notebook_name, session_id)) def get_state(self): """ Get the status of the notebook """ - if self.state is None: - self.state = self.client._perform_json("GET", "/projects/%s/notebooks/" % self.project_key, params={'notebookName' : self.notebook_name}) + notebook_list = self.client._perform_json("GET", + "/projects/%s/jupyter-notebooks/" % self.project_key, + params={"active": False}) + for notebook in notebook_list: + if notebook.get("name") == self.notebook_name: + self.state = notebook return self.state def get_sessions(self): @@ -48,6 +54,30 @@ def get_sessions(self): raise Exception("Notebook isn't running") return state['activeSessions'] + def get_content(self): + """ + Get the content of this notebook (metadata, cells, nbformat) + """ + if self.content is None: + self.content = self.client._perform_json("GET", + "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name)) + return self.content + + def save(self): + """ + Save the content of this notebook + """ + return self.client._perform_json("PUT", + "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name), + body=self.content) + + def delete(self): + """ + Delete this jupyter notebook and stop all of its active sessions. + """ + return self.client._perform_json("DELETE", + "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name)) + ######################################################## # Discussions ######################################################## diff --git a/dataikuapi/dss/project.py b/dataikuapi/dss/project.py index 6557e8b6..3531dc3d 100644 --- a/dataikuapi/dss/project.py +++ b/dataikuapi/dss/project.py @@ -825,6 +825,54 @@ def new_job_definition_builder(self, job_type='NON_RECURSIVE_FORCED_BUILD'): warnings.warn("new_job_definition_builder is deprecated, please use new_job", DeprecationWarning) return JobDefinitionBuilder(self, job_type) + ######################################################## + # Jupyter Notebooks + ######################################################## + + def list_jupyter_notebooks(self, as_objects=True, active=False): + """ + List the jupyter notebooks of a project. + + :param bool as_objects: if True, return the jupyter notebooks as a :class:`dataikuapi.dss.notebook.DSSNotebook` + notebook handles instead of raw JSON + :param bool active: if True, only return currently running jupyter notebooks. + + + :returns: The list of the notebooks - see as_objects for more information + :rtype: list + """ + notebooks = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/" % self.project_key, + params={"active": active}) + if as_objects: + return [DSSNotebook(self.client, notebook['projectKey'], notebook['name'], notebook) for notebook in notebooks] + else: + return notebooks + + def get_jupyter_notebook(self, notebook_name): + """ + Get a handle to interact with a specific jupyter notebook + + :param str notebook_name: The name of the jupyter notebook to retrieve + :returns: A handle to interact with this jupyter notebook + :rtype: :class:`~dataikuapi.dss.notebook.DSSNotebook` jupyter notebook handle + """ + notebook_content = self.client._perform_json("GET", + "/projects/%s/jupyter-notebooks/%s" % (self.project_key, notebook_name)) + return DSSNotebook(self.client, self.project_key, notebook_name, None, notebook_content) + + def create_jupyter_notebook(self, notebook_name, notebook_content): + """ + Create a new jupyter notebook. + + :param str notebook_name: the name of the notebook to create + :param dict notebook_content: the data of the notebook to create, as a dict. + The data will be converted to a JSON string internally. + Use ``get_state()`` on a similar existing ``DSSNotebook`` object in order to get a sample definition object + """ + return self.client._perform_json("POST", + "/projects/%s/jupyter-notebooks/%s" % (self.project_key, notebook_name), + body=notebook_content) + ######################################################## # Continuous activities ######################################################## From c5fbe909d7ae43af2670a7e81ca1fed3d0bee612 Mon Sep 17 00:00:00 2001 From: william-lukusa Date: Fri, 5 Feb 2021 11:12:41 +0100 Subject: [PATCH 2/8] fix arguments --- dataikuapi/dss/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataikuapi/dss/project.py b/dataikuapi/dss/project.py index 3531dc3d..62087643 100644 --- a/dataikuapi/dss/project.py +++ b/dataikuapi/dss/project.py @@ -844,7 +844,7 @@ def list_jupyter_notebooks(self, as_objects=True, active=False): notebooks = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/" % self.project_key, params={"active": active}) if as_objects: - return [DSSNotebook(self.client, notebook['projectKey'], notebook['name'], notebook) for notebook in notebooks] + return [DSSNotebook(self.client, notebook['projectKey'], notebook['name'], content=notebook) for notebook in notebooks] else: return notebooks @@ -858,7 +858,7 @@ def get_jupyter_notebook(self, notebook_name): """ notebook_content = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, notebook_name)) - return DSSNotebook(self.client, self.project_key, notebook_name, None, notebook_content) + return DSSNotebook(self.client, self.project_key, notebook_name, content=notebook_content) def create_jupyter_notebook(self, notebook_name, notebook_content): """ From d4f2f70a3dddd07300d0d7753980d27d0ae0e69c Mon Sep 17 00:00:00 2001 From: william-lukusa Date: Fri, 5 Feb 2021 11:51:01 +0100 Subject: [PATCH 3/8] initialize state instead of content in list --- dataikuapi/dss/notebook.py | 1 + dataikuapi/dss/project.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dataikuapi/dss/notebook.py b/dataikuapi/dss/notebook.py index 92915fe7..568d9324 100644 --- a/dataikuapi/dss/notebook.py +++ b/dataikuapi/dss/notebook.py @@ -41,6 +41,7 @@ def get_state(self): for notebook in notebook_list: if notebook.get("name") == self.notebook_name: self.state = notebook + return self.state return self.state def get_sessions(self): diff --git a/dataikuapi/dss/project.py b/dataikuapi/dss/project.py index 62087643..dc5eba20 100644 --- a/dataikuapi/dss/project.py +++ b/dataikuapi/dss/project.py @@ -844,7 +844,7 @@ def list_jupyter_notebooks(self, as_objects=True, active=False): notebooks = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/" % self.project_key, params={"active": active}) if as_objects: - return [DSSNotebook(self.client, notebook['projectKey'], notebook['name'], content=notebook) for notebook in notebooks] + return [DSSNotebook(self.client, notebook_state['projectKey'], notebook_state['name'], state=notebook_state) for notebook_state in notebooks] else: return notebooks From 08301dc8dc4d311657d13c6c09457fa775ec9c33 Mon Sep 17 00:00:00 2001 From: william-lukusa Date: Mon, 8 Feb 2021 15:34:52 +0100 Subject: [PATCH 4/8] Cosmetics --- dataikuapi/dss/notebook.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dataikuapi/dss/notebook.py b/dataikuapi/dss/notebook.py index 568d9324..e630f348 100644 --- a/dataikuapi/dss/notebook.py +++ b/dataikuapi/dss/notebook.py @@ -14,7 +14,7 @@ def __init__(self, client, project_key, notebook_name, state=None, content=None) def unload(self, session_id=None): """ - Stop the notebook and release its resources + Stop this Jupyter notebook and release its resources """ state = self.get_state() if state is None: @@ -33,12 +33,12 @@ def unload(self, session_id=None): def get_state(self): """ - Get the status of the notebook + Get the status of this Jupyter notebook """ - notebook_list = self.client._perform_json("GET", + notebook_states = self.client._perform_json("GET", "/projects/%s/jupyter-notebooks/" % self.project_key, params={"active": False}) - for notebook in notebook_list: + for notebook in notebook_states: if notebook.get("name") == self.notebook_name: self.state = notebook return self.state @@ -46,7 +46,7 @@ def get_state(self): def get_sessions(self): """ - Get the list of the running sessions of this notebook + Get the list of the running sessions of this Jupyter notebook """ state = self.get_state() if state is None: @@ -57,7 +57,7 @@ def get_sessions(self): def get_content(self): """ - Get the content of this notebook (metadata, cells, nbformat) + Get the content of this Jupyter notebook (metadata, cells, nbformat) """ if self.content is None: self.content = self.client._perform_json("GET", @@ -66,7 +66,7 @@ def get_content(self): def save(self): """ - Save the content of this notebook + Save the content of this Jupyter notebook """ return self.client._perform_json("PUT", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name), @@ -74,7 +74,7 @@ def save(self): def delete(self): """ - Delete this jupyter notebook and stop all of its active sessions. + Delete this Jupyter notebook and stop all of its active sessions. """ return self.client._perform_json("DELETE", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, self.notebook_name)) From a23d027b374512e48a0c1dc2eedab52fad45903e Mon Sep 17 00:00:00 2001 From: william-lukusa Date: Tue, 9 Feb 2021 10:00:32 +0100 Subject: [PATCH 5/8] Use changes from PR#11966 : * Use new sessions endpoint in get_sessions call * Put back cache for state with a refresh parameter to get the state directly from backend * Return created notebook as object --- dataikuapi/dss/notebook.py | 42 ++++++++++++++++++++------------------ dataikuapi/dss/project.py | 11 ++++++---- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/dataikuapi/dss/notebook.py b/dataikuapi/dss/notebook.py index e630f348..848a752b 100644 --- a/dataikuapi/dss/notebook.py +++ b/dataikuapi/dss/notebook.py @@ -16,44 +16,46 @@ def unload(self, session_id=None): """ Stop this Jupyter notebook and release its resources """ - state = self.get_state() - if state is None: + sessions = self.get_sessions() + if sessions is None: raise Exception("Notebook isn't running") - if state.get('activeSessions', None) is None: - raise Exception("Notebook isn't running") - if len(state['activeSessions']) == 0: + if len(sessions) == 0: raise Exception("Notebook isn't running") if session_id is None: - if len(state['activeSessions']) > 1: + if len(sessions) > 1: raise Exception("Several sessions of the notebook are running, choose one") else: - session_id = state['activeSessions'][0].get('sessionId', None) + session_id = sessions[0].get('sessionId', None) return self.client._perform_json("DELETE", "/projects/%s/jupyter-notebooks/%s/sessions/%s" % (self.project_key, self.notebook_name, session_id)) - def get_state(self): + def get_state(self, refresh=False): """ Get the status of this Jupyter notebook + + :param bool refresh: if True, get the status of the notebook from the backend """ notebook_states = self.client._perform_json("GET", - "/projects/%s/jupyter-notebooks/" % self.project_key, - params={"active": False}) - for notebook in notebook_states: - if notebook.get("name") == self.notebook_name: - self.state = notebook + "/projects/%s/jupyter-notebooks/" % self.project_key, + params={"active": False}) + if self.state is None or refresh: + for state in notebook_states: + if state.get("name") == self.notebook_name: + self.state = state return self.state return self.state def get_sessions(self): """ - Get the list of the running sessions of this Jupyter notebook + Get the list of running sessions of this Jupyter notebook """ - state = self.get_state() - if state is None: - raise Exception("Notebook isn't running") - if state.get('activeSessions', None) is None: - raise Exception("Notebook isn't running") - return state['activeSessions'] + + if self.state is None: + self.state = {} + sessions = self.client._perform_json("GET", + "/projects/%s/jupyter-notebooks/%s/sessions" % (self.project_key, self.notebook_name)) + self.state["activeSessions"] = sessions + return sessions def get_content(self): """ diff --git a/dataikuapi/dss/project.py b/dataikuapi/dss/project.py index dc5eba20..72174422 100644 --- a/dataikuapi/dss/project.py +++ b/dataikuapi/dss/project.py @@ -862,16 +862,19 @@ def get_jupyter_notebook(self, notebook_name): def create_jupyter_notebook(self, notebook_name, notebook_content): """ - Create a new jupyter notebook. + Create a new jupyter notebook and get a handle to interact with it :param str notebook_name: the name of the notebook to create :param dict notebook_content: the data of the notebook to create, as a dict. The data will be converted to a JSON string internally. Use ``get_state()`` on a similar existing ``DSSNotebook`` object in order to get a sample definition object + :returns: A handle to interact with the newly created jupyter notebook + :rtype: :class:`~dataikuapi.dss.notebook.DSSNotebook` jupyter notebook handle """ - return self.client._perform_json("POST", - "/projects/%s/jupyter-notebooks/%s" % (self.project_key, notebook_name), - body=notebook_content) + self.client._perform_json("POST", + "/projects/%s/jupyter-notebooks/%s" % (self.project_key, notebook_name), + body=notebook_content) + return DSSNotebook(self.client, self.project_key, notebook_name, content=notebook_content) ######################################################## # Continuous activities From c9a9a0658d3e5103f137be2f5744b6cbd195779e Mon Sep 17 00:00:00 2001 From: william-lukusa Date: Tue, 9 Feb 2021 11:48:36 +0100 Subject: [PATCH 6/8] return content from api call rather than sent for create --- dataikuapi/dss/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataikuapi/dss/project.py b/dataikuapi/dss/project.py index 72174422..91ceec64 100644 --- a/dataikuapi/dss/project.py +++ b/dataikuapi/dss/project.py @@ -871,10 +871,10 @@ def create_jupyter_notebook(self, notebook_name, notebook_content): :returns: A handle to interact with the newly created jupyter notebook :rtype: :class:`~dataikuapi.dss.notebook.DSSNotebook` jupyter notebook handle """ - self.client._perform_json("POST", + created_notebook_content = self.client._perform_json("POST", "/projects/%s/jupyter-notebooks/%s" % (self.project_key, notebook_name), body=notebook_content) - return DSSNotebook(self.client, self.project_key, notebook_name, content=notebook_content) + return DSSNotebook(self.client, self.project_key, notebook_name, content=created_notebook_content) ######################################################## # Continuous activities From 54cba7621c2d44b05cff3124aa9c017e0f23b3bc Mon Sep 17 00:00:00 2001 From: william-lukusa Date: Mon, 15 Feb 2021 14:59:39 +0100 Subject: [PATCH 7/8] Rename state to notebook_state; break instead of return --- dataikuapi/dss/notebook.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataikuapi/dss/notebook.py b/dataikuapi/dss/notebook.py index 848a752b..317531ba 100644 --- a/dataikuapi/dss/notebook.py +++ b/dataikuapi/dss/notebook.py @@ -39,10 +39,10 @@ def get_state(self, refresh=False): "/projects/%s/jupyter-notebooks/" % self.project_key, params={"active": False}) if self.state is None or refresh: - for state in notebook_states: - if state.get("name") == self.notebook_name: - self.state = state - return self.state + for notebook_state in notebook_states: + if notebook_state.get("name") == self.notebook_name: + self.state = notebook_state + break return self.state def get_sessions(self): From 745fb80c9cfe01275ef6ea77d9a9a74cc4437733 Mon Sep 17 00:00:00 2001 From: william-lukusa Date: Mon, 15 Feb 2021 15:33:48 +0100 Subject: [PATCH 8/8] "use get_content" instead of get_state in 'create_jupyter_notebook' method documentation --- dataikuapi/dss/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/project.py b/dataikuapi/dss/project.py index 91ceec64..59783445 100644 --- a/dataikuapi/dss/project.py +++ b/dataikuapi/dss/project.py @@ -867,7 +867,7 @@ def create_jupyter_notebook(self, notebook_name, notebook_content): :param str notebook_name: the name of the notebook to create :param dict notebook_content: the data of the notebook to create, as a dict. The data will be converted to a JSON string internally. - Use ``get_state()`` on a similar existing ``DSSNotebook`` object in order to get a sample definition object + Use ``get_content()`` on a similar existing ``DSSNotebook`` object in order to get a sample definition object :returns: A handle to interact with the newly created jupyter notebook :rtype: :class:`~dataikuapi.dss.notebook.DSSNotebook` jupyter notebook handle """