Fix BLOB store/get, Update API.py GET

.gitignore

@@ -1,6 +1,7 @@
 **/target
 tmp
 **/\.~lock*
+scripts
 
 # Byte-compiled / optimized / DLL files
 **/__pycache__/

cwms/api.py

@@ -26,6 +26,7 @@
 the error.
 """
 
+import base64
 import json
 import logging
 from json import JSONDecodeError
@@ -188,28 +189,16 @@ def get_xml(
     Raises:
         ApiError: If an error response is return by the API.
     """
-
-    headers = {"Accept": api_version_text(api_version)}
-    response = SESSION.get(endpoint, params=params, headers=headers)
-    response.close()
-
-    if response.status_code < 200 or response.status_code >= 300:
-        logging.error(f"CDA Error: response={response}")
-        raise ApiError(response)
-
-    try:
-        return response.content.decode("utf-8")
-    except JSONDecodeError as error:
-        logging.error(f"Error decoding CDA response as xml: {error}")
-        return {}
+    # Wrap the primary get for backwards compatibility
+    return get(endpoint=endpoint, params=params, api_version=api_version)
 
 
 def get(
     endpoint: str,
     params: Optional[RequestParams] = None,
     *,
     api_version: int = API_VERSION,
-) -> JSON:
+) -> Any:
     """Make a GET request to the CWMS Data API.
 
     Args:
@@ -228,17 +217,28 @@ def get(
     """
 
     headers = {"Accept": api_version_text(api_version)}
-    response = SESSION.get(endpoint, params=params, headers=headers)
-    response.close()
-    if response.status_code < 200 or response.status_code >= 300:
-        logging.error(f"CDA Error: response={response}")
-        raise ApiError(response)
-
-    try:
-        return cast(JSON, response.json())
-    except JSONDecodeError as error:
-        logging.error(f"Error decoding CDA response as json: {error}")
-        return {}
+    with SESSION.get(endpoint, params=params, headers=headers) as response:
+        if not response.ok:
+            logging.error(f"CDA Error: response={response}")
+            raise ApiError(response)
+        try:
+            # Avoid case sensitivity issues with the content type header
+            content_type = response.headers.get("Content-Type", "").lower()
+            # Most CDA content is JSON
+            if "application/json" in content_type or not content_type:
+                return cast(JSON, response.json())
+            # Use automatic charset detection with .text
+            if "text/plain" in content_type or "text/" in content_type:
+                return response.text
+            if content_type.startswith("image/"):
+                return base64.b64encode(response.content).decode("utf-8")
+            # Fallback for remaining content types
+            return response.content.decode("utf-8")
+        except JSONDecodeError as error:
+            logging.error(
+                f"Error decoding CDA response as JSON: {error} on line {error.lineno}\n\tFalling back to text"
+            )
+            return response.text
 
 
 def get_with_paging(
@@ -247,7 +247,7 @@ def get_with_paging(
     params: RequestParams,
     *,
     api_version: int = API_VERSION,
-) -> JSON:
+) -> Any:
     """Make a GET request to the CWMS Data API with paging.
 
     Args:
@@ -312,12 +312,10 @@ def post(
     if isinstance(data, dict) or isinstance(data, list):
         data = json.dumps(data)
 
-    response = SESSION.post(endpoint, params=params, headers=headers, data=data)
-    response.close()
-
-    if response.status_code < 200 or response.status_code >= 300:
-        logging.error(f"CDA Error: response={response}")
-        raise ApiError(response)
+    with SESSION.post(endpoint, params=params, headers=headers, data=data) as response:
+        if not response.ok:
+            logging.error(f"CDA Error: response={response}")
+            raise ApiError(response)
 
 
 def patch(
@@ -346,16 +344,13 @@ def patch(
     """
 
     headers = {"accept": "*/*", "Content-Type": api_version_text(api_version)}
-    if data is None:
-        response = SESSION.patch(endpoint, params=params, headers=headers)
-    else:
-        if isinstance(data, dict) or isinstance(data, list):
-            data = json.dumps(data)
-        response = SESSION.patch(endpoint, params=params, headers=headers, data=data)
-    response.close()
-    if response.status_code < 200 or response.status_code >= 300:
-        logging.error(f"CDA Error: response={response}")
-        raise ApiError(response)
+
+    if data and isinstance(data, dict) or isinstance(data, list):
+        data = json.dumps(data)
+    with SESSION.patch(endpoint, params=params, headers=headers, data=data) as response:
+        if not response.ok:
+            logging.error(f"CDA Error: response={response}")
+            raise ApiError(response)
 
 
 def delete(
@@ -379,8 +374,7 @@ def delete(
     """
 
     headers = {"Accept": api_version_text(api_version)}
-    response = SESSION.delete(endpoint, params=params, headers=headers)
-    response.close()
-    if response.status_code < 200 or response.status_code >= 300:
-        logging.error(f"CDA Error: response={response}")
-        raise ApiError(response)
+    with SESSION.delete(endpoint, params=params, headers=headers) as response:
+        if not response.ok:
+            logging.error(f"CDA Error: response={response}")
+            raise ApiError(response)

cwms/catalog/blobs.py

@@ -1,29 +1,40 @@
+import base64
 from typing import Optional
 
 import cwms.api as api
 from cwms.cwms_types import JSON, Data
+from cwms.utils.checks import is_base64
 
+STORE_DICT = """data = {
+    "office-id": "SWT",
+    "id": "MYFILE_OR_BLOB_ID.TXT",
+    "description": "Your description here",
+    "media-type-id": "application/octet-stream",
+    "value": "STRING of content or BASE64_ENCODED_STRING"
+}
+"""
 
-def get_blob(blob_id: str, office_id: str) -> Data:
-    """Get a single clob.
+
+def get_blob(blob_id: str, office_id: str) -> str:
+    """Get a single BLOB (Binary Large Object).
 
     Parameters
         ----------
             blob_id: string
-                Specifies the id of the blob
+                Specifies the id of the blob. ALL blob ids are UPPERCASE.
             office_id: string
                 Specifies the office of the blob.
 
 
         Returns
         -------
-            cwms data type.  data.json will return the JSON output and data.df will return a dataframe
+            str: the value returned based on the content-type it was stored with as a string
     """
 
     endpoint = f"blobs/{blob_id}"
     params = {"office": office_id}
     response = api.get(endpoint, params, api_version=1)
-    return Data(response)
+    return str(response)
 
 
 def get_blobs(
@@ -50,36 +61,39 @@ def get_blobs(
     endpoint = "blobs"
     params = {"office": office_id, "page-size": page_size, "like": blob_id_like}
 
-    response = api.get(endpoint, params, api_version=1)
+    response = api.get(endpoint, params, api_version=2)
     return Data(response, selector="blobs")
 
 
 def store_blobs(data: JSON, fail_if_exists: Optional[bool] = True) -> None:
-    """Create New Blob
+    f"""Create New Blob
 
     Parameters
-        ----------
-            Data: JSON dictionary
-                JSON containing information of Blob to be updated
-                    {
-                    "office-id": "string",
-                    "id": "string",
-                    "description": "string",
-                    "media-type-id": "string",
-                    "value": "string"
-                    }
-            fail_if_exists: Boolean
-                Create will fail if provided ID already exists. Default: true
+    ----------
+        **Note**: The "id" field is automatically cast to uppercase.
 
-        Returns
-        -------
-            None
+        Data: JSON dictionary
+            JSON containing information of Blob to be updated.
+
+            {STORE_DICT}
+        fail_if_exists: Boolean
+            Create will fail if the provided ID already exists. Default: True
+
+    Returns
+    -------
+        None
     """
 
     if not isinstance(data, dict):
-        raise ValueError("Cannot store a Blob without a JSON data dictionary")
+        raise ValueError(
+            f"Cannot store a Blob without a JSON data dictionary:\n{STORE_DICT}"
+        )
+
+    # Encode value if it's not already Base64-encoded
+    if "value" in data and not is_base64(data["value"]):
+        # Encode to bytes, then Base64, then decode to string for storing
+        data["value"] = base64.b64encode(data["value"].encode("utf-8")).decode("utf-8")
 
     endpoint = "blobs"
     params = {"fail-if-exists": fail_if_exists}
-
     return api.post(endpoint, data, params, api_version=1)

cwms/ratings/ratings.py

@@ -15,7 +15,7 @@ def rating_current_effective_date(rating_id: str, office_id: str) -> Any:
     """Retrieve the most recent effective date for a specific rating id.
 
     Returns
-        datatime
+        Any
             the datetime of the most recent effective date for a rating id. If max effective date is
             not present for rating_id then None will be returned
 
@@ -46,7 +46,7 @@ def get_current_rating(
             The owning office of the rating specifications. If no office is provided information from all offices will
             be returned
         rating_table_in_df: Bool, Optional Default = True
-            define if the independant and dependant variables should be stored as a dataframe
+            define if the independent and dependant variables should be stored as a dataframe
     Returns
     -------
         Data : Data
@@ -112,7 +112,7 @@ def get_ratings_xml(
     timezone: Optional[str] = None,
     method: Optional[str] = "EAGER",
 ) -> Any:
-    """Retrives ratings for a specific rating-id
+    """Retrieves ratings for a specific rating-id
 
     Parameters
     ----------
@@ -124,7 +124,7 @@ def get_ratings_xml(
         begin: datetime, optional
             the start of the time window for data to be included in the response.  This is based on the effective date of the ratings
         end: datetime, optional
-            the end of the time window for data to be included int he reponse. This is based on the effective date of the ratings
+            the end of the time window for data to be included int he response. This is based on the effective date of the ratings
         timezone:
             the time zone of the values in the being and end fields if not specified UTC is used
         method:
@@ -225,13 +225,13 @@ def rating_simple_df_to_json(
     active: Optional[bool] = True,
 ) -> JSON:
     """This function converts a dataframe to a json dictionary in the correct format to be posted using the store_ratings function. Can
-    only be used for simple ratings with a indenpendant and 1 dependant variable.
+    only be used for simple ratings with a independent and 1 dependant variable.
 
     Parameters
     ----------
         data: pd.Dataframe
             Rating Table to be stored to an exiting rating specification and template.  Can only have 2 columns ind and dep. ind
-            contained the indenpendant variable and dep contains the dependent variable.
+            contained the independent variable and dep contains the dependent variable.
                         ind	dep
                     0	9.62	0.01
                     1	9.63	0.01
@@ -249,7 +249,7 @@ def rating_simple_df_to_json(
         office_id: str
             the owning office of the rating
         units: str
-            units for both the independant and dependent variable seperated by ; i.e. ft;cfs or ft;ft.
+            units for both the independent and dependent variable separated by ; i.e. ft;cfs or ft;ft.
         effective_date: datetime,
             The effective date of the rating curve to be stored.
         transition_start_date: datetime Optional = None
@@ -384,7 +384,7 @@ def delete_ratings(
 
 
 def store_rating(data: Any, store_template: Optional[bool] = True) -> None:
-    """Will create a new ratingset including template/spec and rating
+    """Will create a new rating-set including template/spec and rating
 
     Parameters
     ----------
@@ -403,7 +403,7 @@ def store_rating(data: Any, store_template: Optional[bool] = True) -> None:
 
     if not isinstance(data, dict) and xml_heading not in data:
         raise ValueError(
-            "Cannot store a timeseries without a JSON data dictionaryor in XML"
+            "Cannot store a timeseries without a JSON data dictionary or in XML"
         )
 
     if xml_heading in data:

cwms/ratings/ratings_spec.py

@@ -8,7 +8,7 @@
 
 
 def get_rating_spec(rating_id: str, office_id: str) -> Data:
-    """Retrives a single rating spec
+    """Retrieves a single rating spec
 
       Parameters
       ----------
@@ -37,15 +37,15 @@ def get_rating_specs(
     rating_id_mask: Optional[str] = None,
     page_size: int = 500000,
 ) -> Data:
-    """Retrives a list of rating specification
+    """Retrieves a list of rating specification
 
       Parameters
       ----------
           office_id: string, optional
               The owning office of the rating specifications. If no office is provided information from all offices will
               be returned
           rating-id-mask: string, optional
-              Posix regular expression that specifies the rating ids to be included in the reponce.  If not specified all
+              Posix regular expression that specifies the rating ids to be included in the response.  If not specified all
               rating specs shall be returned.
           page-size: int, optional, default is 5000000: Specifies the number of records to obtain in
               a single call.
@@ -111,7 +111,7 @@ def rating_spec_df_to_xml(data: pd.DataFrame) -> str:
     Parameters
     ----------
     data : pd_dataframe
-        pandas dataframe that contrains rating specification paramters
+        pandas dataframe that contains rating specification parameters
         should follow same formate the is returned from get_rating_spec function
     Returns
     -------
@@ -134,10 +134,10 @@ def rating_spec_df_to_xml(data: pd.DataFrame) -> str:
       <auto-migrate-extension>{str(data.loc[0,'auto-migrate-extension']).lower()}</auto-migrate-extension>
       <ind-rounding-specs>"""
 
-    ind_rouding = data.loc[0, "independent-rounding-specs"]
-    if isinstance(ind_rouding, list):
+    ind_rounding = data.loc[0, "independent-rounding-specs"]
+    if isinstance(ind_rounding, list):
         i = 1
-        for rounding in ind_rouding:
+        for rounding in ind_rounding:
             spec_xml = (
                 spec_xml
                 + f"""\n   <ind-rounding-spec position="{i}">{rounding['value']}</ind-rounding-spec>"""