From 49c51ca8da1221873efb6ff72f7dcb69f9b62ced Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 11 Feb 2025 16:56:45 +0100 Subject: [PATCH 1/7] Support Gift transaction event, map timeline events using customerSupportChat payload when available --- pyproject.toml | 1 + pytr/_types.py | 42 ++++++++++++++++++++++++++++++++++++++++++ pytr/event.py | 2 ++ pytr/timeline.py | 32 +++++++++++++++++++++++++++++--- tests/test_timeline.py | 21 +++++++++++++++++++++ 5 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 pytr/_types.py create mode 100644 tests/test_timeline.py diff --git a/pyproject.toml b/pyproject.toml index 140b62b1..4d27f1d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "shtab", "websockets>=14", "babel", + "python-jsonpath>=1.3.0", ] [project.scripts] diff --git a/pytr/_types.py b/pytr/_types.py new file mode 100644 index 00000000..89f9856c --- /dev/null +++ b/pytr/_types.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import Any, Literal, NotRequired, TypedDict + + +class TimelineDetailV2(TypedDict): + """ + Incomplete typed representation of the TR `timelineDetailV2` object. + """ + + id: str + sections: list[TimelineDetailV2_Section] + + +class TimelineDetailV2_Section(TypedDict): + title: str + type: Literal["header", "table", "steps"] + data: dict[str, Any] | list[dict[str, Any]] + + +class TimelineDetailV2_CustomerSupportChatAction(TypedDict): + type: Literal["customerSupportChat"] + payload: TimelineDetailV2_CustomerSupportChatAction_Payload + style: str + type: str + + +class TimelineDetailV2_CustomerSupportChatAction_Payload(TypedDict): + contextParams: TimelineDetailV2_CustomerSupportChatAction_ContextParamms + contextCategory: str + + +class TimelineDetailV2_CustomerSupportChatAction_ContextParamms(TypedDict): + chat_flow_key: str + timelineEventId: str + savingsPlanId: NotRequired[str] + primId: NotRequired[str] + groupId: NotRequired[str] + createdAt: NotRequired[str] + amount: NotRequired[str] + iban: NotRequired[str] + interestPayoutId: NotRequired[str] diff --git a/pytr/event.py b/pytr/event.py index 7cccbf07..54b81236 100644 --- a/pytr/event.py +++ b/pytr/event.py @@ -73,6 +73,8 @@ class EventType(Enum): "SAVINGS_PLAN_INVOICE_CREATED": ConditionalEventType.TRADE_INVOICE, "benefits_spare_change_execution": ConditionalEventType.TRADE_INVOICE, "TRADE_INVOICE": ConditionalEventType.TRADE_INVOICE, + # Gifting + "GIFTER_TRANSACTION": PPEventType.TRANSFER_OUT, } diff --git a/pytr/timeline.py b/pytr/timeline.py index f1ce2b5f..e5cf312c 100644 --- a/pytr/timeline.py +++ b/pytr/timeline.py @@ -1,5 +1,10 @@ import json from datetime import datetime +from typing import cast + +import jsonpath + +from pytr._types import TimelineDetailV2, TimelineDetailV2_CustomerSupportChatAction from .transactions import export_transactions from .utils import get_logger @@ -125,9 +130,15 @@ def process_timelineDetail(self, response, dl): create other_events.json, events_with_documents.json and account_transactions.csv """ - event = self.timeline_events.get(response["id"], None) - if event is None: - raise UnsupportedEventError(response["id"]) + # Find the ID of the corresponding timeline event. This is burried deep in the last section of the + # response that contains the customer support information. + support_action = get_customer_support_chat_action(response) + if support_action: + timeline_event_id = support_action["payload"]["contextParams"]["timelineEventId"] + else: + timeline_event_id = response["id"] + + event = self.timeline_events.get(timeline_event_id, None) self.received_detail += 1 event["details"] = response @@ -205,3 +216,18 @@ def finish_timeline_details(self, dl): ) dl.work_responses() + + +def get_customer_support_chat_action( + timeline_detail: TimelineDetailV2, +) -> TimelineDetailV2_CustomerSupportChatAction | None: + """ + From a `timelineDetailV2` object, find the `customerSupportChat` object. + """ + + JSONPATH = '$.sections[*].data[?(@.detail.action.type == "customerSupportChat")].detail.action' + + for action in jsonpath.finditer(JSONPATH, timeline_detail): + return cast(TimelineDetailV2_CustomerSupportChatAction, action.obj) + + return None diff --git a/tests/test_timeline.py b/tests/test_timeline.py new file mode 100644 index 00000000..c3fa576f --- /dev/null +++ b/tests/test_timeline.py @@ -0,0 +1,21 @@ +import json + +from pytr.timeline import get_customer_support_chat_action + + +def test__get_customer_support_chat_action() -> None: + with open("tests/sample_event.json", "r") as file: + sample_data = json.load(file) + + data = get_customer_support_chat_action(sample_data["details"]) + assert data is not None + assert data == { + "payload": { + "contextParams": { + "timelineEventId": "d8a5aa3d-12a4-465a-90ad-3fca36eff19a", + "chat_flow_key": "NHC_0024_deposit_report_an_issue", + }, + "contextCategory": "NHC", + }, + "type": "customerSupportChat", + } From 84684e30137fd331b8d8543ab30ef9c0061a8414 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 11 Feb 2025 17:01:18 +0100 Subject: [PATCH 2/7] still raise UnsupportedEventError --- pytr/timeline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytr/timeline.py b/pytr/timeline.py index e5cf312c..c76709f5 100644 --- a/pytr/timeline.py +++ b/pytr/timeline.py @@ -139,6 +139,8 @@ def process_timelineDetail(self, response, dl): timeline_event_id = response["id"] event = self.timeline_events.get(timeline_event_id, None) + if event is None: + raise UnsupportedEventError(response["id"]) self.received_detail += 1 event["details"] = response From 69c5397b4c5989db6ab6876c6fda41e726bd24e9 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Tue, 11 Feb 2025 17:04:29 +0100 Subject: [PATCH 3/7] move logging of events that have no matching timeline event into Timeline class --- pytr/dl.py | 9 ++------- pytr/timeline.py | 8 +++----- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/pytr/dl.py b/pytr/dl.py index 295cff02..f09f2aae 100644 --- a/pytr/dl.py +++ b/pytr/dl.py @@ -6,7 +6,7 @@ from requests_futures.sessions import FuturesSession from pytr.api import TradeRepublicError -from pytr.timeline import Timeline, UnsupportedEventError +from pytr.timeline import Timeline from pytr.utils import get_logger, preview @@ -77,12 +77,7 @@ async def dl_loop(self): elif subscription.get("type", "") == "timelineActivityLog": await self.tl.get_next_timeline_activity_log(response) elif subscription.get("type", "") == "timelineDetailV2": - try: - self.tl.process_timelineDetail(response, self) - except UnsupportedEventError: - self.log.warning("Ignoring unsupported event %s", response) - self.tl.skipped_detail += 1 - self.tl.check_if_done(self) + self.tl.process_timelineDetail(response, self) else: self.log.warning(f"unmatched subscription of type '{subscription['type']}':\n{preview(response)}") diff --git a/pytr/timeline.py b/pytr/timeline.py index c76709f5..361c3b18 100644 --- a/pytr/timeline.py +++ b/pytr/timeline.py @@ -10,10 +10,6 @@ from .utils import get_logger -class UnsupportedEventError(Exception): - pass - - class Timeline: def __init__(self, tr, max_age_timestamp): self.tr = tr @@ -140,7 +136,9 @@ def process_timelineDetail(self, response, dl): event = self.timeline_events.get(timeline_event_id, None) if event is None: - raise UnsupportedEventError(response["id"]) + self.log.warning("Missing timeline event %r for detail: %s", timeline_event_id, json.dumps(response)) + self.skipped_detail += 1 + return self.received_detail += 1 event["details"] = response From f55538108389f7c3d7369563f2c0955382bb3f9c Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 14 Feb 2025 12:02:15 +0100 Subject: [PATCH 4/7] satisfy mypy --- pyproject.toml | 1 + pytr/_types.py | 5 +++-- pytr/timeline.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5e4c8b82..4b39c90a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "websockets>=14", "babel", "python-jsonpath>=1.3.0", + "typing-extensions>=4.12.2", ] [project.scripts] diff --git a/pytr/_types.py b/pytr/_types.py index 89f9856c..28b57923 100644 --- a/pytr/_types.py +++ b/pytr/_types.py @@ -1,6 +1,8 @@ from __future__ import annotations -from typing import Any, Literal, NotRequired, TypedDict +from typing import Any, Literal, TypedDict + +from typing_extensions import NotRequired class TimelineDetailV2(TypedDict): @@ -22,7 +24,6 @@ class TimelineDetailV2_CustomerSupportChatAction(TypedDict): type: Literal["customerSupportChat"] payload: TimelineDetailV2_CustomerSupportChatAction_Payload style: str - type: str class TimelineDetailV2_CustomerSupportChatAction_Payload(TypedDict): diff --git a/pytr/timeline.py b/pytr/timeline.py index 361c3b18..dc5aed1e 100644 --- a/pytr/timeline.py +++ b/pytr/timeline.py @@ -1,6 +1,6 @@ import json from datetime import datetime -from typing import cast +from typing import Optional, cast import jsonpath @@ -220,7 +220,7 @@ def finish_timeline_details(self, dl): def get_customer_support_chat_action( timeline_detail: TimelineDetailV2, -) -> TimelineDetailV2_CustomerSupportChatAction | None: +) -> Optional[TimelineDetailV2_CustomerSupportChatAction]: """ From a `timelineDetailV2` object, find the `customerSupportChat` object. """ From 69d819b1770032c1aeb11d94217e354b90ea4e42 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 14 Feb 2025 12:23:25 +0100 Subject: [PATCH 5/7] add warning log for debugging #173 --- pytr/_types.py | 2 +- pytr/timeline.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pytr/_types.py b/pytr/_types.py index 28b57923..fa3b874a 100644 --- a/pytr/_types.py +++ b/pytr/_types.py @@ -33,7 +33,7 @@ class TimelineDetailV2_CustomerSupportChatAction_Payload(TypedDict): class TimelineDetailV2_CustomerSupportChatAction_ContextParamms(TypedDict): chat_flow_key: str - timelineEventId: str + timelineEventId: NotRequired[str] savingsPlanId: NotRequired[str] primId: NotRequired[str] groupId: NotRequired[str] diff --git a/pytr/timeline.py b/pytr/timeline.py index dc5aed1e..2641052d 100644 --- a/pytr/timeline.py +++ b/pytr/timeline.py @@ -40,7 +40,7 @@ async def get_next_timeline_transactions(self, response=None): for event in response["items"]: if ( self.max_age_timestamp == 0 - or datetime.fromisoformat(event["timestamp"][:19]).timestamp() >= self.max_age_timestamp + or datetime.fromisoformat(event["timestamp"]).timestamp() >= self.max_age_timestamp ): event["source"] = "timelineTransaction" self.timeline_events[event["id"]] = event @@ -129,9 +129,14 @@ def process_timelineDetail(self, response, dl): # Find the ID of the corresponding timeline event. This is burried deep in the last section of the # response that contains the customer support information. support_action = get_customer_support_chat_action(response) - if support_action: - timeline_event_id = support_action["payload"]["contextParams"]["timelineEventId"] + if support_action and (timeline_event_id := support_action["payload"]["contextParams"].get("timelineEventId")): + pass else: + if support_action: + # DEBUG + self.log.warning( + "This event has no timelineEventId in the customerSupportChat action: %s", json.dumps(response) + ) timeline_event_id = response["id"] event = self.timeline_events.get(timeline_event_id, None) From 66a75af734b46237138f28f500e0b0a74357848a Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Fri, 14 Feb 2025 15:09:50 +0100 Subject: [PATCH 6/7] map `card_successful_verification` to new `PPEventType.OTHER` --- pytr/event.py | 3 +++ pytr/timeline.py | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pytr/event.py b/pytr/event.py index 4564730f..885bcc8f 100644 --- a/pytr/event.py +++ b/pytr/event.py @@ -34,6 +34,7 @@ class PPEventType(EventType): TAX_REFUND = "TAX_REFUND" TRANSFER_IN = "TRANSFER_IN" # Currently not mapped to TRANSFER_OUT = "TRANSFER_OUT" # Currently not mapped to + OTHER = "OTHER" # Events that have no impact on the account balance. tr_event_type_mapping = { @@ -76,6 +77,8 @@ class PPEventType(EventType): "TRADE_INVOICE": ConditionalEventType.TRADE_INVOICE, # Gifting "GIFTER_TRANSACTION": PPEventType.TRANSFER_OUT, + # Other + "card_successful_verification": PPEventType.OTHER, } diff --git a/pytr/timeline.py b/pytr/timeline.py index 2641052d..77fd39e5 100644 --- a/pytr/timeline.py +++ b/pytr/timeline.py @@ -132,11 +132,6 @@ def process_timelineDetail(self, response, dl): if support_action and (timeline_event_id := support_action["payload"]["contextParams"].get("timelineEventId")): pass else: - if support_action: - # DEBUG - self.log.warning( - "This event has no timelineEventId in the customerSupportChat action: %s", json.dumps(response) - ) timeline_event_id = response["id"] event = self.timeline_events.get(timeline_event_id, None) From 0783fed87899cb45bc707463ff84c2c49f20c040 Mon Sep 17 00:00:00 2001 From: Niklas Rosenstein Date: Sat, 15 Feb 2025 01:35:05 +0100 Subject: [PATCH 7/7] rename _types to types after uv 0.5.31 --- pytr/timeline.py | 2 +- pytr/{_types.py => types.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename pytr/{_types.py => types.py} (100%) diff --git a/pytr/timeline.py b/pytr/timeline.py index 77fd39e5..b1fb0404 100644 --- a/pytr/timeline.py +++ b/pytr/timeline.py @@ -4,7 +4,7 @@ import jsonpath -from pytr._types import TimelineDetailV2, TimelineDetailV2_CustomerSupportChatAction +from pytr.types import TimelineDetailV2, TimelineDetailV2_CustomerSupportChatAction from .transactions import export_transactions from .utils import get_logger diff --git a/pytr/_types.py b/pytr/types.py similarity index 100% rename from pytr/_types.py rename to pytr/types.py