Skip to content

Fix BLOB store/get, Update API.py GET #137

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
krowvin merged 27 commits into from
Jun 3, 2025
Merged

Fix BLOB store/get, Update API.py GET #137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
849914f
Initial setup of location group function defs
krowvin Feb 13, 2025
902c24d
Merge branch 'main' of https://github.com/HydrologicEngineeringCenter…
krowvin Mar 12, 2025
e2f1d01
Bump version, fix typo in project toml description
krowvin Mar 12, 2025
4328f08
Change response close method to context manager (with) to ensure clos…
krowvin Mar 12, 2025
b9319c8
Switch to using response.ok as 300 are usually handled by requests li…
krowvin Mar 12, 2025
a9555b2
Update blob return type, make get requests dynamic on response conten…
krowvin Mar 12, 2025
3fc330c
Fix clob to BLOB in get, add extra notes for storing/gets to pydocs
krowvin Mar 12, 2025
5b89757
Create base64 check utility for blobs
krowvin Mar 12, 2025
a200088
Add blurb to users on how to format the BLOB storage, correct note, e…
krowvin Mar 12, 2025
7c186d8
Not sure how to set the headers in the mock requests response, for no…
krowvin Mar 12, 2025
2f35807
Ensure string or JSON can return from blob
krowvin Mar 12, 2025
e504397
Forgot to install precommit on this box, fix sort order of typing for…
krowvin Mar 12, 2025
ae96f0d
Attempt casting get_blob to string and reverting type union for get
krowvin Mar 12, 2025
fada38a
Wrap all the responses for get that are not json in a dictionary so t…
krowvin Mar 12, 2025
1792120
Remove duplicate group methods
krowvin Mar 13, 2025
2fe3602
Ensure accept mimetype is set for proper file type response;
krowvin Mar 13, 2025
9c0befd
Change all references to 102 (xml, v2) to format="xml" and version 2
krowvin Mar 13, 2025
32719b9
Remove print statements
krowvin Mar 13, 2025
841e0c2
Run spell checker through all files
krowvin Mar 13, 2025
d402002
Correct api_version_text to not set a version for 0 and 1; remove tes…
krowvin Mar 13, 2025
75ac1dc
Revert "Change all references to 102 (xml, v2) to format="xml" and ve…
krowvin May 29, 2025
9308abe
Revert "Ensure accept mimetype is set for proper file type response;"
krowvin May 29, 2025
efeb86d
Change to Any and update other mimetypes to pure output (not JSON wra…
krowvin May 29, 2025
fcaf173
Bump version, handle Any for get_with_paging
krowvin May 29, 2025
3efade4
Handle base64 encoded content (images)
krowvin May 29, 2025
73bdcad
Correct base64 import position
krowvin May 29, 2025
fd6d917
Merge branch 'main' into bug/get-blob-response
Enovotny May 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
**/target
tmp
**/\.~lock*
scripts

# Byte-compiled / optimized / DLL files
**/__pycache__/
Expand Down
90 changes: 42 additions & 48 deletions cwms/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
the error.
"""

import base64
import json
import logging
from json import JSONDecodeError
Expand Down Expand Up @@ -188,28 +189,16 @@ def get_xml(
Raises:
ApiError: If an error response is return by the API.
"""

headers = {"Accept": api_version_text(api_version)}
response = SESSION.get(endpoint, params=params, headers=headers)
response.close()

if response.status_code < 200 or response.status_code >= 300:
logging.error(f"CDA Error: response={response}")
raise ApiError(response)

try:
return response.content.decode("utf-8")
except JSONDecodeError as error:
logging.error(f"Error decoding CDA response as xml: {error}")
return {}
# Wrap the primary get for backwards compatibility
return get(endpoint=endpoint, params=params, api_version=api_version)


def get(
endpoint: str,
params: Optional[RequestParams] = None,
*,
api_version: int = API_VERSION,
) -> JSON:
) -> Any:
"""Make a GET request to the CWMS Data API.

Args:
Expand All @@ -228,17 +217,28 @@ def get(
"""

headers = {"Accept": api_version_text(api_version)}
response = SESSION.get(endpoint, params=params, headers=headers)
response.close()
if response.status_code < 200 or response.status_code >= 300:
logging.error(f"CDA Error: response={response}")
raise ApiError(response)

try:
return cast(JSON, response.json())
except JSONDecodeError as error:
logging.error(f"Error decoding CDA response as json: {error}")
return {}
with SESSION.get(endpoint, params=params, headers=headers) as response:
if not response.ok:
logging.error(f"CDA Error: response={response}")
raise ApiError(response)
try:
# Avoid case sensitivity issues with the content type header
content_type = response.headers.get("Content-Type", "").lower()
# Most CDA content is JSON
if "application/json" in content_type or not content_type:
return cast(JSON, response.json())
# Use automatic charset detection with .text
if "text/plain" in content_type or "text/" in content_type:
return response.text
if content_type.startswith("image/"):
return base64.b64encode(response.content).decode("utf-8")
# Fallback for remaining content types
return response.content.decode("utf-8")
except JSONDecodeError as error:
logging.error(
f"Error decoding CDA response as JSON: {error} on line {error.lineno}\n\tFalling back to text"
)
return response.text


def get_with_paging(
Expand All @@ -247,7 +247,7 @@ def get_with_paging(
params: RequestParams,
*,
api_version: int = API_VERSION,
) -> JSON:
) -> Any:
"""Make a GET request to the CWMS Data API with paging.

Args:
Expand Down Expand Up @@ -312,12 +312,10 @@ def post(
if isinstance(data, dict) or isinstance(data, list):
data = json.dumps(data)

response = SESSION.post(endpoint, params=params, headers=headers, data=data)
response.close()

if response.status_code < 200 or response.status_code >= 300:
logging.error(f"CDA Error: response={response}")
raise ApiError(response)
with SESSION.post(endpoint, params=params, headers=headers, data=data) as response:
if not response.ok:
logging.error(f"CDA Error: response={response}")
raise ApiError(response)


def patch(
Expand Down Expand Up @@ -346,16 +344,13 @@ def patch(
"""

headers = {"accept": "*/*", "Content-Type": api_version_text(api_version)}
if data is None:
response = SESSION.patch(endpoint, params=params, headers=headers)
else:
if isinstance(data, dict) or isinstance(data, list):
data = json.dumps(data)
response = SESSION.patch(endpoint, params=params, headers=headers, data=data)
response.close()
if response.status_code < 200 or response.status_code >= 300:
logging.error(f"CDA Error: response={response}")
raise ApiError(response)

if data and isinstance(data, dict) or isinstance(data, list):
data = json.dumps(data)
with SESSION.patch(endpoint, params=params, headers=headers, data=data) as response:
if not response.ok:
logging.error(f"CDA Error: response={response}")
raise ApiError(response)


def delete(
Expand All @@ -379,8 +374,7 @@ def delete(
"""

headers = {"Accept": api_version_text(api_version)}
response = SESSION.delete(endpoint, params=params, headers=headers)
response.close()
if response.status_code < 200 or response.status_code >= 300:
logging.error(f"CDA Error: response={response}")
raise ApiError(response)
with SESSION.delete(endpoint, params=params, headers=headers) as response:
if not response.ok:
logging.error(f"CDA Error: response={response}")
raise ApiError(response)
62 changes: 38 additions & 24 deletions cwms/catalog/blobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,29 +1,40 @@
import base64
from typing import Optional

import cwms.api as api
from cwms.cwms_types import JSON, Data
from cwms.utils.checks import is_base64

STORE_DICT = """data = {
"office-id": "SWT",
"id": "MYFILE_OR_BLOB_ID.TXT",
"description": "Your description here",
"media-type-id": "application/octet-stream",
"value": "STRING of content or BASE64_ENCODED_STRING"
}
"""

def get_blob(blob_id: str, office_id: str) -> Data:
"""Get a single clob.

def get_blob(blob_id: str, office_id: str) -> str:
"""Get a single BLOB (Binary Large Object).

Parameters
----------
blob_id: string
Specifies the id of the blob
Specifies the id of the blob. ALL blob ids are UPPERCASE.
office_id: string
Specifies the office of the blob.


Returns
-------
cwms data type. data.json will return the JSON output and data.df will return a dataframe
str: the value returned based on the content-type it was stored with as a string
"""

endpoint = f"blobs/{blob_id}"
params = {"office": office_id}
response = api.get(endpoint, params, api_version=1)
return Data(response)
return str(response)


def get_blobs(
Expand All @@ -50,36 +61,39 @@ def get_blobs(
endpoint = "blobs"
params = {"office": office_id, "page-size": page_size, "like": blob_id_like}

response = api.get(endpoint, params, api_version=1)
response = api.get(endpoint, params, api_version=2)
return Data(response, selector="blobs")


def store_blobs(data: JSON, fail_if_exists: Optional[bool] = True) -> None:
"""Create New Blob
f"""Create New Blob

Parameters
----------
Data: JSON dictionary
JSON containing information of Blob to be updated
{
"office-id": "string",
"id": "string",
"description": "string",
"media-type-id": "string",
"value": "string"
}
fail_if_exists: Boolean
Create will fail if provided ID already exists. Default: true
----------
**Note**: The "id" field is automatically cast to uppercase.

Returns
-------
None
Data: JSON dictionary
JSON containing information of Blob to be updated.

{STORE_DICT}
fail_if_exists: Boolean
Create will fail if the provided ID already exists. Default: True

Returns
-------
None
"""

if not isinstance(data, dict):
raise ValueError("Cannot store a Blob without a JSON data dictionary")
raise ValueError(
f"Cannot store a Blob without a JSON data dictionary:\n{STORE_DICT}"
)

# Encode value if it's not already Base64-encoded
if "value" in data and not is_base64(data["value"]):
# Encode to bytes, then Base64, then decode to string for storing
data["value"] = base64.b64encode(data["value"].encode("utf-8")).decode("utf-8")

endpoint = "blobs"
params = {"fail-if-exists": fail_if_exists}

return api.post(endpoint, data, params, api_version=1)
18 changes: 9 additions & 9 deletions cwms/ratings/ratings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def rating_current_effective_date(rating_id: str, office_id: str) -> Any:
"""Retrieve the most recent effective date for a specific rating id.

Returns
datatime
Any
the datetime of the most recent effective date for a rating id. If max effective date is
not present for rating_id then None will be returned

Expand Down Expand Up @@ -46,7 +46,7 @@ def get_current_rating(
The owning office of the rating specifications. If no office is provided information from all offices will
be returned
rating_table_in_df: Bool, Optional Default = True
define if the independant and dependant variables should be stored as a dataframe
define if the independent and dependant variables should be stored as a dataframe
Returns
-------
Data : Data
Expand Down Expand Up @@ -112,7 +112,7 @@ def get_ratings_xml(
timezone: Optional[str] = None,
method: Optional[str] = "EAGER",
) -> Any:
"""Retrives ratings for a specific rating-id
"""Retrieves ratings for a specific rating-id

Parameters
----------
Expand All @@ -124,7 +124,7 @@ def get_ratings_xml(
begin: datetime, optional
the start of the time window for data to be included in the response. This is based on the effective date of the ratings
end: datetime, optional
the end of the time window for data to be included int he reponse. This is based on the effective date of the ratings
the end of the time window for data to be included int he response. This is based on the effective date of the ratings
timezone:
the time zone of the values in the being and end fields if not specified UTC is used
method:
Expand Down Expand Up @@ -225,13 +225,13 @@ def rating_simple_df_to_json(
active: Optional[bool] = True,
) -> JSON:
"""This function converts a dataframe to a json dictionary in the correct format to be posted using the store_ratings function. Can
only be used for simple ratings with a indenpendant and 1 dependant variable.
only be used for simple ratings with a independent and 1 dependant variable.

Parameters
----------
data: pd.Dataframe
Rating Table to be stored to an exiting rating specification and template. Can only have 2 columns ind and dep. ind
contained the indenpendant variable and dep contains the dependent variable.
contained the independent variable and dep contains the dependent variable.
ind dep
0 9.62 0.01
1 9.63 0.01
Expand All @@ -249,7 +249,7 @@ def rating_simple_df_to_json(
office_id: str
the owning office of the rating
units: str
units for both the independant and dependent variable seperated by ; i.e. ft;cfs or ft;ft.
units for both the independent and dependent variable separated by ; i.e. ft;cfs or ft;ft.
effective_date: datetime,
The effective date of the rating curve to be stored.
transition_start_date: datetime Optional = None
Expand Down Expand Up @@ -384,7 +384,7 @@ def delete_ratings(


def store_rating(data: Any, store_template: Optional[bool] = True) -> None:
"""Will create a new ratingset including template/spec and rating
"""Will create a new rating-set including template/spec and rating

Parameters
----------
Expand All @@ -403,7 +403,7 @@ def store_rating(data: Any, store_template: Optional[bool] = True) -> None:

if not isinstance(data, dict) and xml_heading not in data:
raise ValueError(
"Cannot store a timeseries without a JSON data dictionaryor in XML"
"Cannot store a timeseries without a JSON data dictionary or in XML"
)

if xml_heading in data:
Expand Down
14 changes: 7 additions & 7 deletions cwms/ratings/ratings_spec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@


def get_rating_spec(rating_id: str, office_id: str) -> Data:
"""Retrives a single rating spec
"""Retrieves a single rating spec

Parameters
----------
Expand Down Expand Up @@ -37,15 +37,15 @@ def get_rating_specs(
rating_id_mask: Optional[str] = None,
page_size: int = 500000,
) -> Data:
"""Retrives a list of rating specification
"""Retrieves a list of rating specification

Parameters
----------
office_id: string, optional
The owning office of the rating specifications. If no office is provided information from all offices will
be returned
rating-id-mask: string, optional
Posix regular expression that specifies the rating ids to be included in the reponce. If not specified all
Posix regular expression that specifies the rating ids to be included in the response. If not specified all
rating specs shall be returned.
page-size: int, optional, default is 5000000: Specifies the number of records to obtain in
a single call.
Expand Down Expand Up @@ -111,7 +111,7 @@ def rating_spec_df_to_xml(data: pd.DataFrame) -> str:
Parameters
----------
data : pd_dataframe
pandas dataframe that contrains rating specification paramters
pandas dataframe that contains rating specification parameters
should follow same formate the is returned from get_rating_spec function
Returns
-------
Expand All @@ -134,10 +134,10 @@ def rating_spec_df_to_xml(data: pd.DataFrame) -> str:
<auto-migrate-extension>{str(data.loc[0,'auto-migrate-extension']).lower()}</auto-migrate-extension>
<ind-rounding-specs>"""

ind_rouding = data.loc[0, "independent-rounding-specs"]
if isinstance(ind_rouding, list):
ind_rounding = data.loc[0, "independent-rounding-specs"]
if isinstance(ind_rounding, list):
i = 1
for rounding in ind_rouding:
for rounding in ind_rounding:
spec_xml = (
spec_xml
+ f"""\n <ind-rounding-spec position="{i}">{rounding['value']}</ind-rounding-spec>"""
Expand Down
Loading