automated push from c6b2d6be2ed21d94f1b8a868c28719fe8ba196e9

Author: Vic Putz

pyproject.toml

@@ -16,6 +16,7 @@ classifiers = [
 	    "Operating System :: OS Independent",
 	    ]
 requires = [
+	 "aiohttp >= 3.7.3",
 	 "backoff >= 1.10.0",
 	 "requests >= 2.24.0",
 	 "python-decouple >= 3.3",

qcware/api_calls/__init__.py

@@ -1,2 +1,3 @@
-from .api_call import (api_call, post_call, wait_for_call, handle_result,
-                       retrieve_result, async_retrieve_result, status, cancel)
+from .api_call import (api_call, post_call, async_post_call, wait_for_call,
+                       handle_result, retrieve_result, async_retrieve_result,
+                       status, cancel)

qcware/api_calls/api_call.py

@@ -3,6 +3,7 @@
 from urllib.parse import urljoin
 import backoff
 from ..request import post, get
+from ..async_request import post as async_post, get as async_get
 from ..exceptions import ApiCallExecutionError, ApiTimeoutError
 from ..util.transforms import client_result_from_wire
 from ..config import (client_timeout, do_client_api_compatibility_check_once,
@@ -12,6 +13,17 @@
 
 
 def post_call(endpoint: str, data: dict):
+    api_call_context = data.get('api_call_context', None)
+    if api_call_context is None:
+        api_call_context = current_context()
+    host = api_call_context.qcware_host
+    # replace the ApiCallContext class with a jsonable dict
+    data['api_call_context'] = api_call_context.dict()
+    url = urljoin(host, endpoint)
+    return post(url, data)
+
+
+async def async_post_call(endpoint: str, data: dict):
     """
     Centralizes the post for the API call.  Assumes the data dict
     contains an entry with key 'api_key'; if this is missing or set to
@@ -25,31 +37,37 @@ def post_call(endpoint: str, data: dict):
     # replace the ApiCallContext class with a jsonable dict
     data['api_call_context'] = api_call_context.dict()
     url = urljoin(host, endpoint)
-    return post(url, data)
+    return await async_post(url, data)
 
 
 def api_call(api_call_context: ApiCallContext, call_token: str):
     api_call_context = current_context(
     ) if api_call_context is None else api_call_context
-    api_call_context = api_call_context.dict()
     do_client_api_compatibility_check_once()
-    return post(f'{api_call_context["qcware_host"]}/api_calls', locals())
+    return post(
+        f'{api_call_context.qcware_host}/api_calls',
+        dict(api_call_context=api_call_context.dict(), call_token=call_token))
+
+
+async def async_api_call(api_call_context: ApiCallContext, call_token: str):
+    api_call_context = current_context(
+    ) if api_call_context is None else api_call_context
+    do_client_api_compatibility_check_once()
+    return await async_post(
+        f'{api_call_context.qcware_host}/api_calls',
+        dict(api_call_context=api_call_context.dict(), call_token=call_token))
 
 
 def status(call_token: str):
     api_call_context = current_context()
-    api_call_context = api_call_context.dict()
     do_client_api_compatibility_check_once()
-    return post(f'{api_call_context["qcware_host"]}/api_calls/status',
-                locals())
+    return post(f'{api_call_context.qcware_host}/api_calls/status', locals())
 
 
 def cancel(call_token: str):
     api_call_context = current_context()
-    api_call_context = api_call_context.dict()
     do_client_api_compatibility_check_once()
-    return post(f'{api_call_context["qcware_host"]}/api_calls/cancel',
-                locals())
+    return post(f'{api_call_context.qcware_host}/api_calls/cancel', locals())
 
 
 def _print_waiting_handler(details: Dict):
@@ -70,6 +88,18 @@ def wait_for_call(call_token: str, api_call_context=None):
     return api_call(api_call_context, call_token)
 
 
+@backoff.on_predicate(backoff.constant,
+                      interval=1,
+                      predicate=lambda a: a.get('state') == 'open',
+                      max_time=client_timeout,
+                      on_backoff=_print_waiting_handler)
+async def async_wait_for_call(call_token: str, api_call_context=None):
+    api_call_context = current_context(
+    ) if api_call_context is None else api_call_context
+    # backoff.on_predicate is mildly problematic.
+    return await async_api_call(api_call_context, call_token)
+
+
 def handle_result(api_call):
     if api_call['state'] == 'error':
         if 'result_url' in api_call:
@@ -145,6 +175,6 @@ async def async_retrieve_result(call_token: str,
     """
     while True:
         try:
-            return handle_result(wait_for_call(call_token))
+            return handle_result(await async_wait_for_call(call_token))
         except ApiTimeoutError as e:
             await asyncio.sleep(async_interval_between_tries())

qcware/async_request.py

@@ -0,0 +1,63 @@
+import backoff
+import requests
+import asyncio
+import aiohttp
+
+from .exceptions import ApiCallFailedError, ApiCallResultUnavailableError
+
+_client_session = None
+
+
+def client_session() -> aiohttp.ClientSession:
+    """
+    Singleton guardian for client session.  This may need to be moved
+    to being a contextvar, and it could be that the whole python Client
+    needs to be made instantiable (for sessions).  But since aiohttp is
+    single-threaded this should be OK for now.
+    """
+    global _client_session
+    if _client_session is None:
+        _client_session = aiohttp.ClientSession()
+    return _client_session
+
+
+def _fatal_code(e):
+    return 400 <= e.response.status_code < 500
+
+
+# By default, evidently aiohttp doesn't raise exceptions for non-200
+# statuses, so backoff has trouble unless you specifically ask
+# using raise_for_status = True; see
+# https://stackoverflow.com/questions/56152651/how-to-retry-async-aiohttp-requests-depending-on-the-status-code
+
+
+@backoff.on_exception(backoff.expo,
+                      requests.exceptions.RequestException,
+                      max_tries=3,
+                      giveup=_fatal_code)
+def post_request(url, data):
+    return client_session().post(url, json=data, raise_for_status=True)
+
+
+@backoff.on_exception(backoff.expo,
+                      requests.exceptions.RequestException,
+                      max_tries=3,
+                      giveup=_fatal_code)
+def get_request(url):
+    return client_session().get(url, raise_for_status=True)
+
+
+async def post(url, data):
+    async with post_request(url, data) as response:
+        if response.status >= 400:
+            raise ApiCallFailedError(response.json()['message'])
+        return await response.json()
+
+
+async def get(url):
+    async with get_request(url) as response:
+        if response.status >= 400:
+            raise ApiCallResultUnavailableError(
+                'Unable to retrieve result, please try again later or contact support'
+            )
+        return await response.text()

qcware/config/config.py

@@ -1,10 +1,10 @@
-from decouple import config, UndefinedValueError
+from decouple import config, UndefinedValueError  # type: ignore
 from urllib.parse import urlparse, urljoin
 from typing import Optional
 from functools import reduce
 from packaging import version
 import requests
-import colorama
+import colorama  # type: ignore
 from .api_semver import api_semver
 import os
 from pydantic import BaseModel
@@ -53,13 +53,17 @@ def qcware_api_key(override: Optional[str] = None) -> str:
     return result
 
 
-def is_valid_host_url(url: str) -> bool:
+def is_valid_host_url(url: Optional[str]) -> bool:
     """
     Checks if a host url is valid.  A valid host url is just a scheme
     (http/https), a net location, and no path.
     """
-    result = urlparse(url)
-    return all([result.scheme, result.netloc]) and not result.path
+    if url is None:
+        result = False
+    else:
+        parse_result = urlparse(url)
+        result = all([parse_result.scheme, parse_result.netloc]) and not parse_result.path
+    return result
 
 
 def qcware_host(override: Optional[str] = None) -> str:
@@ -80,7 +84,8 @@ def qcware_host(override: Optional[str] = None) -> str:
     # check to make sure the host is a valid url
 
     if is_valid_host_url(result):
-        return result
+        # type ignored below because if we get here, it's a valid string
+        return result  # type:ignore
     else:
         raise ConfigurationError(
             f"Configured QCWARE_HOST ({result}): does not seem to be"
@@ -328,7 +333,7 @@ class ApiCredentials(BaseModel):
 
 class ApiCallContext(BaseModel):
     qcware_host: Optional[str] = None
-    credentials: Optional[ApiCredentials] = None
+    credentials = ApiCredentials()
     server_timeout: Optional[int] = None
     client_timeout: Optional[int] = None
     async_interval_between_tries: Optional[float] = None
@@ -355,7 +360,7 @@ def root_context() -> ApiCallContext:
                                default=SchedulingMode.immediate))
 
 
-_contexts = contextvars.ContextVar('contexts', default=[])
+_contexts: contextvars.ContextVar = contextvars.ContextVar('contexts', default=[])
 
 
 def push_context(**kwargs):

qcware/optimization/api/solve_binary.py

@@ -4,7 +4,7 @@
 
 import asyncio
 from ... import logger
-from ...api_calls import post_call, wait_for_call, handle_result, async_retrieve_result
+from ...api_calls import post_call, async_post_call, wait_for_call, handle_result, async_retrieve_result
 from ...util.transforms import client_args_to_wire
 from ...exceptions import ApiTimeoutError
 from ...config import (ApiCallContext, client_timeout,
@@ -991,7 +991,7 @@ async def async_solve_binary(
 :rtype: dict
     """
     data = client_args_to_wire('optimization.solve_binary', **locals())
-    api_call = post_call('optimization/solve_binary', data)
+    api_call = await async_post_call('optimization/solve_binary', data)
     logger.info(
         f'API call to optimization.solve_binary successful. Your API token is {api_call["uid"]}'
     )

qcware/request.py

@@ -3,6 +3,19 @@
 
 from .exceptions import ApiCallFailedError, ApiCallResultUnavailableError
 
+_client_session = None
+
+
+def client_session() -> requests.Session:
+    """
+    Singleton guardian for client session
+    """
+    global _client_session
+    if _client_session is None:
+        _client_session = requests.Session()
+    return _client_session
+    
+
 
 def _fatal_code(e):
     return 400 <= e.response.status_code < 500
@@ -13,15 +26,15 @@ def _fatal_code(e):
                       max_tries=3,
                       giveup=_fatal_code)
 def post_request(url, data):
-    return requests.post(url, json=data)
+    return client_session().post(url, json=data)
 
 
 @backoff.on_exception(backoff.expo,
                       requests.exceptions.RequestException,
                       max_tries=3,
                       giveup=_fatal_code)
 def get_request(url):
-    return requests.get(url)
+    return client_session().get(url)
 
 
 def post(url, data):

requirements.txt

@@ -1,3 +1,4 @@
+aiohttp==3.7.3
 pydantic==1.7.3
 wheel==0.36.2
 backoff==1.10.0
@@ -11,4 +12,4 @@ lz4==3.1.3
 # git+ssh://git@github.com:/qcware/quasar-dev.git@4dd310ed594f829e85f7ff49139289119b049e6b
 qubovert==1.2.3
 tabulate==0.8.8
-qcware-quasar==1.0.2
+qcware-quasar==1.0.2