Skip to content

Commit 899df69

Browse files
authored
VER: Release 0.40.0
See release notes.
2 parents 39a9e36 + 9439845 commit 899df69

22 files changed

+889
-22
lines changed

CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
# Changelog
22

3+
## 0.40.0 - 2024-08-27
4+
5+
#### Enhancements
6+
- Added `adjustment_factors.get_range(...)` method for `Reference` client
7+
- Added `security_master.get_range(...)` method for `Reference` client
8+
- Added `security_master.get_last(...)` method for `Reference` client
9+
- Upgraded `databento-dbn` to 0.20.1
10+
311
## 0.39.3 - 2024-08-20
412

513
#### Enhancements

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ The library is fully compatible with the latest distribution of Anaconda 3.8 and
3232
The minimum dependencies as found in the `pyproject.toml` are also listed below:
3333
- python = "^3.8"
3434
- aiohttp = "^3.8.3"
35-
- databento-dbn = "0.20.0"
35+
- databento-dbn = "0.20.1"
3636
- numpy= ">=1.23.5"
3737
- pandas = ">=1.5.3"
3838
- pip-system-certs = ">=4.0" (Windows only)

databento/common/constants.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,3 +106,24 @@
106106
"financial_year_end_date",
107107
"exp_completion_date",
108108
]
109+
110+
ADJUSTMENT_FACTORS_DATETIME_COLUMNS: Final[list[str]] = [
111+
"ts_created",
112+
]
113+
114+
ADJUSTMENT_FACTORS_DATE_COLUMNS: Final[list[str]] = [
115+
"ex_date",
116+
]
117+
118+
SECURITY_MASTER_DATETIME_COLUMNS: Final[list[str]] = [
119+
"ts_record",
120+
"ts_effective",
121+
"ts_created",
122+
]
123+
124+
SECURITY_MASTER_DATE_COLUMNS: Final[list[str]] = [
125+
"listing_created_date",
126+
"listing_date",
127+
"delisting_date",
128+
"shares_outstanding_date",
129+
]

databento/common/parsing.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,14 @@
44
from datetime import date
55
from functools import partial
66
from functools import singledispatch
7+
from io import BytesIO
8+
from io import TextIOWrapper
79
from numbers import Integral
10+
from typing import IO
811
from typing import Any
912

1013
import pandas as pd
14+
import zstandard
1115
from databento_dbn import SType
1216

1317
from databento.common.constants import ALL_SYMBOLS
@@ -418,3 +422,28 @@ def convert_datetime_columns(df: pd.DataFrame, columns: list[str]) -> None:
418422
if column not in df:
419423
continue
420424
df[column] = df[column].apply(convert_to_datetime)
425+
426+
427+
def convert_ndjson_to_df(data: bytes, compressed: bool) -> pd.DataFrame:
428+
"""
429+
Convert the given NDJSON bytes `data` to a pandas DataFrame.
430+
431+
Parameters
432+
----------
433+
data : bytes
434+
The NDJSON data as bytes to be converted.
435+
compressed : bool
436+
If the content is zstd compressed.
437+
438+
Returns
439+
-------
440+
pandas.DataFrame
441+
442+
"""
443+
if compressed:
444+
decompressor = zstandard.ZstdDecompressor()
445+
reader: IO[bytes] = decompressor.stream_reader(data)
446+
else:
447+
reader = BytesIO(data)
448+
449+
return pd.read_json(TextIOWrapper(reader), lines=True)

databento/historical/api/batch.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ def submit_job(
9292
symbols : Iterable[str | int] or str or int
9393
The instrument symbols to filter for. Takes up to 2,000 symbols per request.
9494
If more than 1 symbol is specified, the data is merged and sorted by time.
95-
If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
95+
If 'ALL_SYMBOLS' or `None` then will select **all** symbols.
9696
schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades' # noqa
9797
The data record schema for the request.
9898
start : pd.Timestamp or date or str or int

databento/historical/api/metadata.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ def get_record_count(
288288
Defaults to the same value as `start`.
289289
symbols : Iterable[str | int] or str or int, optional
290290
The instrument symbols to filter for. Takes up to 2,000 symbols per request.
291-
If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
291+
If 'ALL_SYMBOLS' or `None` then will select **all** symbols.
292292
schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades' # noqa
293293
The data record schema for the request.
294294
stype_in : SType or str, default 'raw_symbol'
@@ -357,7 +357,7 @@ def get_billable_size(
357357
Defaults to the same value as `start`.
358358
symbols : Iterable[str | int] or str or int, optional
359359
The instrument symbols to filter for. Takes up to 2,000 symbols per request.
360-
If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
360+
If 'ALL_SYMBOLS' or `None` then will select **all** symbols.
361361
schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades' # noqa
362362
The data record schema for the request.
363363
stype_in : SType or str, default 'raw_symbol'
@@ -429,7 +429,7 @@ def get_cost(
429429
The data feed mode for the request.
430430
symbols : Iterable[str | int] or str or int, optional
431431
The instrument symbols to filter for. Takes up to 2,000 symbols per request.
432-
If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
432+
If 'ALL_SYMBOLS' or `None` then will select **all** symbols.
433433
schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades' # noqa
434434
The data record schema for the request.
435435
stype_in : SType or str, default 'raw_symbol'

databento/historical/api/timeseries.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ def get_range(
7171
symbols : Iterable[str | int], or str, or int, optional
7272
The instrument symbols to filter for. Takes up to 2,000 symbols per request.
7373
If more than 1 symbol is specified, the data is merged and sorted by time.
74-
If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
74+
If 'ALL_SYMBOLS' or `None` then will select **all** symbols.
7575
schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades'
7676
The data record schema for the request.
7777
stype_in : SType or str, default 'raw_symbol'
@@ -168,7 +168,7 @@ async def get_range_async(
168168
symbols : Iterable[str | int] or str or int, optional
169169
The instrument symbols to filter for. Takes up to 2,000 symbols per request.
170170
If more than 1 symbol is specified, the data is merged and sorted by time.
171-
If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
171+
If 'ALL_SYMBOLS' or `None` then will select **all** symbols.
172172
schema : Schema or str {'mbo', 'mbp-1', 'mbp-10', 'trades', 'tbbo', 'ohlcv-1s', 'ohlcv-1m', 'ohlcv-1h', 'ohlcv-1d', 'definition', 'statistics', 'status'}, default 'trades' # noqa
173173
The data record schema for the request.
174174
stype_in : SType or str, default 'raw_symbol'
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
from __future__ import annotations
2+
3+
from collections.abc import Iterable
4+
from datetime import date
5+
6+
import pandas as pd
7+
from databento_dbn import Compression
8+
from databento_dbn import SType
9+
10+
from databento.common import API_VERSION
11+
from databento.common.constants import ADJUSTMENT_FACTORS_DATE_COLUMNS
12+
from databento.common.constants import ADJUSTMENT_FACTORS_DATETIME_COLUMNS
13+
from databento.common.http import BentoHttpAPI
14+
from databento.common.parsing import convert_date_columns
15+
from databento.common.parsing import convert_datetime_columns
16+
from databento.common.parsing import convert_ndjson_to_df
17+
from databento.common.parsing import datetime_to_string
18+
from databento.common.parsing import optional_datetime_to_string
19+
from databento.common.parsing import optional_string_to_list
20+
from databento.common.parsing import optional_symbols_list_to_list
21+
22+
23+
class AdjustmentFactorsHttpAPI(BentoHttpAPI):
24+
"""
25+
Provides request methods for the adjustment factors HTTP API endpoints.
26+
"""
27+
28+
def __init__(self, key: str, gateway: str) -> None:
29+
super().__init__(key=key, gateway=gateway)
30+
self._base_url = gateway + f"/v{API_VERSION}/adjustment_factors"
31+
32+
def get_range(
33+
self,
34+
start: pd.Timestamp | date | str | int,
35+
end: pd.Timestamp | date | str | int | None = None,
36+
symbols: Iterable[str] | str | None = None,
37+
stype_in: SType | str = "raw_symbol",
38+
countries: Iterable[str] | str | None = None,
39+
security_types: Iterable[str] | str | None = None,
40+
) -> pd.DataFrame:
41+
"""
42+
Request a new adjustment factors time series from Databento.
43+
44+
Makes a `POST /adjustment_factors.get_range` HTTP request.
45+
46+
The `ex_date` column will be used to filter the time range and order the records.
47+
It will also be set as the index of the resulting data frame.
48+
49+
Parameters
50+
----------
51+
start : pd.Timestamp or date or str or int
52+
The start datetime of the request time range (inclusive) based on `ex_date`.
53+
Assumes UTC as timezone unless passed a tz-aware object.
54+
If an integer is passed, then this represents nanoseconds since the UNIX epoch.
55+
end : pd.Timestamp or date or str or int, optional
56+
The end datetime of the request time range (exclusive) based on `ex_date`.
57+
Assumes UTC as timezone unless passed a tz-aware object.
58+
If an integer is passed, then this represents nanoseconds since the UNIX epoch.
59+
symbols : Iterable[str] or str, optional
60+
The symbols to filter for. Takes up to 2,000 symbols per request.
61+
If more than 1 symbol is specified, the data is merged and sorted by time.
62+
If 'ALL_SYMBOLS' or `None` then will select **all** symbols.
63+
stype_in : SType or str, default 'raw_symbol'
64+
The input symbology type to resolve from.
65+
Use any of 'raw_symbol', 'nasdaq_symbol', 'isin', 'us_code'.
66+
countries : Iterable[str] or str, optional
67+
The listing countries to filter for.
68+
Takes any number of two letter ISO 3166-1 alpha-2 country codes per request.
69+
If not specified then will select **all** listing countries by default.
70+
See [CNTRY](https://databento.com/docs/standards-and-conventions/reference-data-enums#cntry) enum.
71+
security_types : Iterable[str] or str, optional
72+
The security types to filter for.
73+
Takes any number of security types per request.
74+
If not specified then will select **all** security types by default.
75+
See [SECTYPE](https://databento.com/docs/standards-and-conventions/reference-data-enums#sectype) enum.
76+
77+
Returns
78+
-------
79+
pandas.DataFrame
80+
The data converted into a data frame.
81+
82+
"""
83+
symbols_list = optional_symbols_list_to_list(symbols, SType.RAW_SYMBOL)
84+
countries = optional_string_to_list(countries)
85+
security_types = optional_string_to_list(security_types)
86+
87+
data: dict[str, object | None] = {
88+
"start": datetime_to_string(start),
89+
"end": optional_datetime_to_string(end),
90+
"symbols": ",".join(symbols_list),
91+
"stype_in": stype_in,
92+
"countries": ",".join(countries) if countries else None,
93+
"security_types": ",".join(security_types) if security_types else None,
94+
"compression": str(Compression.ZSTD), # Always request zstd
95+
}
96+
97+
response = self._post(
98+
url=self._base_url + ".get_range",
99+
data=data,
100+
basic_auth=True,
101+
)
102+
103+
df = convert_ndjson_to_df(response.content, compressed=True)
104+
if df.empty:
105+
return df
106+
107+
convert_datetime_columns(df, ADJUSTMENT_FACTORS_DATETIME_COLUMNS)
108+
convert_date_columns(df, ADJUSTMENT_FACTORS_DATE_COLUMNS)
109+
110+
df.set_index("ex_date", inplace=True)
111+
df.sort_index(inplace=True)
112+
113+
return df

databento/reference/api/corporate.py

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,8 @@
22

33
from collections.abc import Iterable
44
from datetime import date
5-
from io import BytesIO
6-
from io import StringIO
75

86
import pandas as pd
9-
import zstandard
107
from databento_dbn import Compression
118
from databento_dbn import SType
129

@@ -16,6 +13,7 @@
1613
from databento.common.http import BentoHttpAPI
1714
from databento.common.parsing import convert_date_columns
1815
from databento.common.parsing import convert_datetime_columns
16+
from databento.common.parsing import convert_ndjson_to_df
1917
from databento.common.parsing import datetime_to_string
2018
from databento.common.parsing import optional_datetime_to_string
2119
from databento.common.parsing import optional_string_to_list
@@ -49,23 +47,27 @@ def get_range(
4947
5048
Makes a `POST /corporate_actions.get_range` HTTP request.
5149
50+
The specified `index` will be used to filter the time range and order the records.
51+
It will also be set as the index of the resulting data frame.
52+
5253
Parameters
5354
----------
5455
start : pd.Timestamp or date or str or int
55-
The start datetime of the request time range (inclusive).
56+
The start datetime of the request time range (inclusive) based on `index`.
5657
Assumes UTC as timezone unless passed a tz-aware object.
5758
If an integer is passed, then this represents nanoseconds since the UNIX epoch.
5859
end : pd.Timestamp or date or str or int, optional
59-
The end datetime of the request time range (exclusive).
60+
The end datetime of the request time range (exclusive) based on `index`.
6061
Assumes UTC as timezone unless passed a tz-aware object.
6162
If an integer is passed, then this represents nanoseconds since the UNIX epoch.
6263
index : str, default 'event_date'
63-
The index column to filter the `start` and `end` time range on.
64+
The index column used for filtering the `start` and `end` time range
65+
and for record ordering.
6466
Use any of 'event_date', 'ex_date' or 'ts_record'.
6567
symbols : Iterable[str] or str, optional
6668
The symbols to filter for. Takes up to 2,000 symbols per request.
6769
If more than 1 symbol is specified, the data is merged and sorted by time.
68-
If 'ALL_SYMBOLS' or `None` then will be for **all** symbols.
70+
If 'ALL_SYMBOLS' or `None` then will select **all** symbols.
6971
stype_in : SType or str, default 'raw_symbol'
7072
The input symbology type to resolve from.
7173
Use any of 'raw_symbol', 'nasdaq_symbol', 'isin', 'us_code',
@@ -124,10 +126,7 @@ def get_range(
124126
basic_auth=True,
125127
)
126128

127-
decompressor = zstandard.ZstdDecompressor()
128-
decompressed_content = decompressor.stream_reader(BytesIO(response.content)).read()
129-
130-
df = pd.read_json(StringIO(decompressed_content.decode()), lines=True)
129+
df = convert_ndjson_to_df(response.content, compressed=True)
131130
if df.empty:
132131
return df
133132

0 commit comments

Comments
 (0)