From 5095f561d88982b5b6847560bd7c1faffa06c1e2 Mon Sep 17 00:00:00 2001 From: MisterSeajay Date: Mon, 1 Dec 2025 21:32:48 +0000 Subject: [PATCH 01/11] fix Issue-148 and apply black formatting --- src/secops/chronicle/parser.py | 67 ++++++++++++++-------------------- 1 file changed, 28 insertions(+), 39 deletions(-) diff --git a/src/secops/chronicle/parser.py b/src/secops/chronicle/parser.py index 082cfd40..ec05bac2 100644 --- a/src/secops/chronicle/parser.py +++ b/src/secops/chronicle/parser.py @@ -14,10 +14,10 @@ # """Parser management functionality for Chronicle.""" -from typing import Dict, Any, List, Optional -from secops.exceptions import APIError import base64 +from typing import Any, Dict, List, Optional +from secops.exceptions import APIError # Constants for size limits MAX_LOG_SIZE = 10 * 1024 * 1024 # 10MB per log @@ -26,7 +26,9 @@ def activate_parser( - client, log_type: str, id: str # pylint: disable=redefined-builtin + client, + log_type: str, + id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: """Activate a custom parser. @@ -55,7 +57,9 @@ def activate_parser( def activate_release_candidate_parser( - client, log_type: str, id: str # pylint: disable=redefined-builtin + client, + log_type: str, + id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: """Activate the release candidate parser making it live for that customer. @@ -84,7 +88,9 @@ def activate_release_candidate_parser( def copy_parser( - client, log_type: str, id: str # pylint: disable=redefined-builtin + client, + log_type: str, + id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: """Makes a copy of a prebuilt parser. @@ -100,8 +106,7 @@ def copy_parser( APIError: If the API request fails """ url = ( - f"{client.base_url}/{client.instance_id}/logTypes/{log_type}" - f"/parsers/{id}:copy" + f"{client.base_url}/{client.instance_id}/logTypes/{log_type}/parsers/{id}:copy" ) body = {} response = client.session.post(url, json=body) @@ -148,7 +153,9 @@ def create_parser( def deactivate_parser( - client, log_type: str, id: str # pylint: disable=redefined-builtin + client, + log_type: str, + id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: """Deactivate a custom parser. @@ -196,10 +203,7 @@ def delete_parser( Raises: APIError: If the API request fails """ - url = ( - f"{client.base_url}/{client.instance_id}/logTypes/{log_type}" - f"/parsers/{id}" - ) + url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}/parsers/{id}" params = {"force": force} response = client.session.delete(url, params=params) @@ -210,7 +214,9 @@ def delete_parser( def get_parser( - client, log_type: str, id: str # pylint: disable=redefined-builtin + client, + log_type: str, + id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: """Get a Parser by ID. @@ -225,10 +231,7 @@ def get_parser( Raises: APIError: If the API request fails """ - url = ( - f"{client.base_url}/{client.instance_id}/logTypes/{log_type}" - f"/parsers/{id}" - ) + url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}/parsers/{id}" response = client.session.get(url) if response.status_code != 200: @@ -263,10 +266,7 @@ def list_parsers( parsers = [] while more: - url = ( - f"{client.base_url}/{client.instance_id}" - f"/logTypes/{log_type}/parsers" - ) + url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}/parsers" params = { "pageSize": page_size, @@ -284,8 +284,8 @@ def list_parsers( if "parsers" in data: parsers.extend(data["parsers"]) - if "next_page_token" in data: - params["pageToken"] = data["next_page_token"] + if "nextPageToken" in data: + page_token = data["nextPageToken"] else: more = False @@ -364,27 +364,19 @@ def run_parser( # Check number of logs if len(logs) > MAX_LOGS: - raise ValueError( - f"Number of logs ({len(logs)}) exceeds maximum of {MAX_LOGS}" - ) + raise ValueError(f"Number of logs ({len(logs)}) exceeds maximum of {MAX_LOGS}") # Validate parser_extension_code type if provided - if parser_extension_code is not None and not isinstance( - parser_extension_code, str - ): + if parser_extension_code is not None and not isinstance(parser_extension_code, str): raise TypeError( "parser_extension_code must be a string or None, got " f"{type(parser_extension_code).__name__}" ) # Build request - url = ( - f"{client.base_url}/{client.instance_id}/logTypes/{log_type}:runParser" - ) + url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}:runParser" - parser = { - "cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8") - } + parser = {"cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8")} parser_extension = None if parser_extension_code: @@ -397,10 +389,7 @@ def run_parser( body = { "parser": parser, "parser_extension": parser_extension, - "log": [ - base64.b64encode(log.encode("utf-8")).decode("utf-8") - for log in logs - ], + "log": [base64.b64encode(log.encode("utf-8")).decode("utf-8") for log in logs], "statedump_allowed": statedump_allowed, } From 2b5a3be6f73f8211a6f71277e28d78e19d8db3f8 Mon Sep 17 00:00:00 2001 From: MisterSeajay Date: Tue, 2 Dec 2025 15:45:53 +0000 Subject: [PATCH 02/11] run black formatter --- src/secops/chronicle/parser.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/secops/chronicle/parser.py b/src/secops/chronicle/parser.py index ec05bac2..e28444af 100644 --- a/src/secops/chronicle/parser.py +++ b/src/secops/chronicle/parser.py @@ -105,9 +105,7 @@ def copy_parser( Raises: APIError: If the API request fails """ - url = ( - f"{client.base_url}/{client.instance_id}/logTypes/{log_type}/parsers/{id}:copy" - ) + url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}/parsers/{id}:copy" body = {} response = client.session.post(url, json=body) @@ -364,19 +362,27 @@ def run_parser( # Check number of logs if len(logs) > MAX_LOGS: - raise ValueError(f"Number of logs ({len(logs)}) exceeds maximum of {MAX_LOGS}") + raise ValueError( + f"Number of logs ({len(logs)}) exceeds maximum of {MAX_LOGS}" + ) # Validate parser_extension_code type if provided - if parser_extension_code is not None and not isinstance(parser_extension_code, str): + if parser_extension_code is not None and not isinstance( + parser_extension_code, str + ): raise TypeError( "parser_extension_code must be a string or None, got " f"{type(parser_extension_code).__name__}" ) # Build request - url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}:runParser" + url = ( + f"{client.base_url}/{client.instance_id}/logTypes/{log_type}:runParser" + ) - parser = {"cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8")} + parser = { + "cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8") + } parser_extension = None if parser_extension_code: @@ -389,7 +395,10 @@ def run_parser( body = { "parser": parser, "parser_extension": parser_extension, - "log": [base64.b64encode(log.encode("utf-8")).decode("utf-8") for log in logs], + "log": [ + base64.b64encode(log.encode("utf-8")).decode("utf-8") + for log in logs + ], "statedump_allowed": statedump_allowed, } From 17737d7894a005cb5632c4bfe0b39c552cbbe993 Mon Sep 17 00:00:00 2001 From: MisterSeajay Date: Tue, 2 Dec 2025 16:37:41 +0000 Subject: [PATCH 03/11] break long urls below 80 char line limit --- src/secops/chronicle/parser.py | 46 +++++++++++++++------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/src/secops/chronicle/parser.py b/src/secops/chronicle/parser.py index e28444af..7fba12b3 100644 --- a/src/secops/chronicle/parser.py +++ b/src/secops/chronicle/parser.py @@ -44,8 +44,8 @@ def activate_parser( APIError: If the API request fails """ url = ( - f"{client.base_url}/{client.instance_id}/logTypes/{log_type}" - f"/parsers/{id}:activate" + f"{client.base_url}/{client.instance_id}" + f"/logTypes/{log_type}/parsers/{id}:activate" ) body = {} response = client.session.post(url, json=body) @@ -75,8 +75,8 @@ def activate_release_candidate_parser( APIError: If the API request fails """ url = ( - f"{client.base_url}/{client.instance_id}/logTypes/{log_type}" - f"/parsers/{id}:activateReleaseCandidateParser" + f"{client.base_url}/{client.instance_id}" + f"/logTypes/{log_type}/parsers/{id}:activateReleaseCandidateParser" ) body = {} response = client.session.post(url, json=body) @@ -105,7 +105,10 @@ def copy_parser( Raises: APIError: If the API request fails """ - url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}/parsers/{id}:copy" + url = ( + f"{client.base_url}/{client.instance_id}" + f"/logTypes/{log_type}/parsers/{id}:copy" + ) body = {} response = client.session.post(url, json=body) @@ -169,8 +172,8 @@ def deactivate_parser( APIError: If the API request fails """ url = ( - f"{client.base_url}/{client.instance_id}/logTypes/{log_type}" - f"/parsers/{id}:deactivate" + f"{client.base_url}/{client.instance_id}" + f"/logTypes/{log_type}/parsers/{id}:deactivate" ) body = {} response = client.session.post(url, json=body) @@ -201,7 +204,7 @@ def delete_parser( Raises: APIError: If the API request fails """ - url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}/parsers/{id}" + url = f"{client.base_url}/{client.instance_id}" f"/logTypes/{log_type}/parsers/{id}" params = {"force": force} response = client.session.delete(url, params=params) @@ -229,7 +232,8 @@ def get_parser( Raises: APIError: If the API request fails """ - url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}/parsers/{id}" + url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}" + url += f"/parsers/{id}" response = client.session.get(url) if response.status_code != 200: @@ -264,7 +268,8 @@ def list_parsers( parsers = [] while more: - url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}/parsers" + url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}" + url += "/parsers" params = { "pageSize": page_size, @@ -362,27 +367,19 @@ def run_parser( # Check number of logs if len(logs) > MAX_LOGS: - raise ValueError( - f"Number of logs ({len(logs)}) exceeds maximum of {MAX_LOGS}" - ) + raise ValueError(f"Number of logs ({len(logs)}) exceeds maximum of {MAX_LOGS}") # Validate parser_extension_code type if provided - if parser_extension_code is not None and not isinstance( - parser_extension_code, str - ): + if parser_extension_code is not None and not isinstance(parser_extension_code, str): raise TypeError( "parser_extension_code must be a string or None, got " f"{type(parser_extension_code).__name__}" ) # Build request - url = ( - f"{client.base_url}/{client.instance_id}/logTypes/{log_type}:runParser" - ) + url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}:runParser" - parser = { - "cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8") - } + parser = {"cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8")} parser_extension = None if parser_extension_code: @@ -395,10 +392,7 @@ def run_parser( body = { "parser": parser, "parser_extension": parser_extension, - "log": [ - base64.b64encode(log.encode("utf-8")).decode("utf-8") - for log in logs - ], + "log": [base64.b64encode(log.encode("utf-8")).decode("utf-8") for log in logs], "statedump_allowed": statedump_allowed, } From 00c7ea52a69c3dae450d9796ca3fcf4ccd420eeb Mon Sep 17 00:00:00 2001 From: MisterSeajay Date: Tue, 2 Dec 2025 16:37:53 +0000 Subject: [PATCH 04/11] break long urls below 80 char line limit --- src/secops/chronicle/parser.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/secops/chronicle/parser.py b/src/secops/chronicle/parser.py index 7fba12b3..253c6ae0 100644 --- a/src/secops/chronicle/parser.py +++ b/src/secops/chronicle/parser.py @@ -204,7 +204,8 @@ def delete_parser( Raises: APIError: If the API request fails """ - url = f"{client.base_url}/{client.instance_id}" f"/logTypes/{log_type}/parsers/{id}" + url = f"{client.base_url}/{client.instance_id}" + url += f"/logTypes/{log_type}/parsers/{id}" params = {"force": force} response = client.session.delete(url, params=params) @@ -232,8 +233,8 @@ def get_parser( Raises: APIError: If the API request fails """ - url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}" - url += f"/parsers/{id}" + url = f"{client.base_url}/{client.instance_id}" + url += f"/logTypes/{log_type}/parsers/{id}" response = client.session.get(url) if response.status_code != 200: @@ -268,8 +269,8 @@ def list_parsers( parsers = [] while more: - url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}" - url += "/parsers" + url = f"{client.base_url}/{client.instance_id}" + url += f"/logTypes/{log_type}/parsers" params = { "pageSize": page_size, @@ -377,7 +378,8 @@ def run_parser( ) # Build request - url = f"{client.base_url}/{client.instance_id}/logTypes/{log_type}:runParser" + url = f"{client.base_url}/{client.instance_id}" + url += f"/logTypes/{log_type}:runParser" parser = {"cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8")} From 6f3b3da6633312323e71159f83dbc8f6e5566436 Mon Sep 17 00:00:00 2001 From: MisterSeajay Date: Tue, 2 Dec 2025 17:00:39 +0000 Subject: [PATCH 05/11] Use ChronicleClient type hint --- src/secops/chronicle/parser.py | 40 +++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/secops/chronicle/parser.py b/src/secops/chronicle/parser.py index 253c6ae0..d9419742 100644 --- a/src/secops/chronicle/parser.py +++ b/src/secops/chronicle/parser.py @@ -15,8 +15,9 @@ """Parser management functionality for Chronicle.""" import base64 -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union +from secops.chronicle.client import ChronicleClient from secops.exceptions import APIError # Constants for size limits @@ -26,7 +27,7 @@ def activate_parser( - client, + client: ChronicleClient, log_type: str, id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: @@ -57,7 +58,7 @@ def activate_parser( def activate_release_candidate_parser( - client, + client: ChronicleClient, log_type: str, id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: @@ -88,7 +89,7 @@ def activate_release_candidate_parser( def copy_parser( - client, + client: ChronicleClient, log_type: str, id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: @@ -119,7 +120,7 @@ def copy_parser( def create_parser( - client, + client: ChronicleClient, log_type: str, parser_code: str, validated_on_empty_logs: bool = True, @@ -154,7 +155,7 @@ def create_parser( def deactivate_parser( - client, + client: ChronicleClient, log_type: str, id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: @@ -185,7 +186,7 @@ def deactivate_parser( def delete_parser( - client, + client: ChronicleClient, log_type: str, id: str, # pylint: disable=redefined-builtin force: bool = False, @@ -216,7 +217,7 @@ def delete_parser( def get_parser( - client, + client: ChronicleClient, log_type: str, id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: @@ -244,10 +245,10 @@ def get_parser( def list_parsers( - client, + client: ChronicleClient, log_type: str = "-", page_size: int = 100, - page_token: str = None, + page_token: Optional[Union[str, None]] = None, filter: str = None, # pylint: disable=redefined-builtin ) -> List[Any]: """List parsers. @@ -297,7 +298,7 @@ def list_parsers( def run_parser( - client: "ChronicleClient", + client: ChronicleClient, log_type: str, parser_code: str, parser_extension_code: Optional[str], @@ -368,10 +369,14 @@ def run_parser( # Check number of logs if len(logs) > MAX_LOGS: - raise ValueError(f"Number of logs ({len(logs)}) exceeds maximum of {MAX_LOGS}") + raise ValueError( + f"Number of logs ({len(logs)}) exceeds maximum of {MAX_LOGS}" + ) # Validate parser_extension_code type if provided - if parser_extension_code is not None and not isinstance(parser_extension_code, str): + if parser_extension_code is not None and not isinstance( + parser_extension_code, str + ): raise TypeError( "parser_extension_code must be a string or None, got " f"{type(parser_extension_code).__name__}" @@ -381,7 +386,9 @@ def run_parser( url = f"{client.base_url}/{client.instance_id}" url += f"/logTypes/{log_type}:runParser" - parser = {"cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8")} + parser = { + "cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8") + } parser_extension = None if parser_extension_code: @@ -394,7 +401,10 @@ def run_parser( body = { "parser": parser, "parser_extension": parser_extension, - "log": [base64.b64encode(log.encode("utf-8")).decode("utf-8") for log in logs], + "log": [ + base64.b64encode(log.encode("utf-8")).decode("utf-8") + for log in logs + ], "statedump_allowed": statedump_allowed, } From d88ffae16c18fb541086165658cf9b7e1a2c06a3 Mon Sep 17 00:00:00 2001 From: MisterSeajay Date: Tue, 2 Dec 2025 17:03:01 +0000 Subject: [PATCH 06/11] Revert client: ChronicleClient type hint. :-( --- src/secops/chronicle/parser.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/secops/chronicle/parser.py b/src/secops/chronicle/parser.py index d9419742..77a14040 100644 --- a/src/secops/chronicle/parser.py +++ b/src/secops/chronicle/parser.py @@ -17,7 +17,6 @@ import base64 from typing import Any, Dict, List, Optional, Union -from secops.chronicle.client import ChronicleClient from secops.exceptions import APIError # Constants for size limits @@ -27,7 +26,7 @@ def activate_parser( - client: ChronicleClient, + client: "ChronicleClient", log_type: str, id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: @@ -58,7 +57,7 @@ def activate_parser( def activate_release_candidate_parser( - client: ChronicleClient, + client: "ChronicleClient", log_type: str, id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: @@ -89,7 +88,7 @@ def activate_release_candidate_parser( def copy_parser( - client: ChronicleClient, + client: "ChronicleClient", log_type: str, id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: @@ -120,7 +119,7 @@ def copy_parser( def create_parser( - client: ChronicleClient, + client: "ChronicleClient", log_type: str, parser_code: str, validated_on_empty_logs: bool = True, @@ -155,7 +154,7 @@ def create_parser( def deactivate_parser( - client: ChronicleClient, + client: "ChronicleClient", log_type: str, id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: @@ -186,7 +185,7 @@ def deactivate_parser( def delete_parser( - client: ChronicleClient, + client: "ChronicleClient", log_type: str, id: str, # pylint: disable=redefined-builtin force: bool = False, @@ -217,7 +216,7 @@ def delete_parser( def get_parser( - client: ChronicleClient, + client: "ChronicleClient", log_type: str, id: str, # pylint: disable=redefined-builtin ) -> Dict[str, Any]: @@ -245,7 +244,7 @@ def get_parser( def list_parsers( - client: ChronicleClient, + client: "ChronicleClient", log_type: str = "-", page_size: int = 100, page_token: Optional[Union[str, None]] = None, @@ -298,7 +297,7 @@ def list_parsers( def run_parser( - client: ChronicleClient, + client: "ChronicleClient", log_type: str, parser_code: str, parser_extension_code: Optional[str], From 5c85e831154b8c6a79906ca053454206ffec35f0 Mon Sep 17 00:00:00 2001 From: MisterSeajay Date: Tue, 2 Dec 2025 17:15:17 +0000 Subject: [PATCH 07/11] update unit tests for paginated lists --- tests/chronicle/test_parser.py | 163 +++++++++++++++++++++++++++------ 1 file changed, 136 insertions(+), 27 deletions(-) diff --git a/tests/chronicle/test_parser.py b/tests/chronicle/test_parser.py index 99dc5986..e734972f 100644 --- a/tests/chronicle/test_parser.py +++ b/tests/chronicle/test_parser.py @@ -15,10 +15,15 @@ """Tests for Chronicle parser functions.""" import base64 -import pytest from unittest.mock import Mock, patch + +import pytest + from secops.chronicle.client import ChronicleClient from secops.chronicle.parser import ( + MAX_LOG_SIZE, + MAX_LOGS, + MAX_TOTAL_SIZE, activate_parser, activate_release_candidate_parser, copy_parser, @@ -28,9 +33,6 @@ get_parser, list_parsers, run_parser, - MAX_LOG_SIZE, - MAX_LOGS, - MAX_TOTAL_SIZE, ) from secops.exceptions import APIError @@ -100,7 +102,9 @@ def test_activate_parser_error(chronicle_client, mock_error_response): # --- activate_release_candidate_parser Tests --- -def test_activate_release_candidate_parser_success(chronicle_client, mock_response): +def test_activate_release_candidate_parser_success( + chronicle_client, mock_response +): """Test activate_release_candidate_parser function for success.""" log_type = "SOME_LOG_TYPE" parser_id = "pa_67890" @@ -118,7 +122,9 @@ def test_activate_release_candidate_parser_success(chronicle_client, mock_respon assert result == {} -def test_activate_release_candidate_parser_error(chronicle_client, mock_error_response): +def test_activate_release_candidate_parser_error( + chronicle_client, mock_error_response +): """Test activate_release_candidate_parser function for API error.""" log_type = "SOME_LOG_TYPE" parser_id = "pa_67890" @@ -127,7 +133,9 @@ def test_activate_release_candidate_parser_error(chronicle_client, mock_error_re chronicle_client.session, "post", return_value=mock_error_response ): with pytest.raises(APIError) as exc_info: - activate_release_candidate_parser(chronicle_client, log_type, parser_id) + activate_release_candidate_parser( + chronicle_client, log_type, parser_id + ) assert "Failed to activate parser: Error message" in str(exc_info.value) @@ -166,7 +174,9 @@ def test_copy_parser_error(chronicle_client, mock_error_response): # --- create_parser Tests --- -def test_create_parser_success_default_validation(chronicle_client, mock_response): +def test_create_parser_success_default_validation( + chronicle_client, mock_response +): """Test create_parser function for success with default validated_on_empty_logs.""" log_type = "NIX_SYSTEM" parser_code = "filter {}" @@ -186,14 +196,18 @@ def test_create_parser_success_default_validation(chronicle_client, mock_respons mock_post.assert_called_once_with( expected_url, json={ - "cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8"), + "cbn": base64.b64encode(parser_code.encode("utf-8")).decode( + "utf-8" + ), "validated_on_empty_logs": True, }, ) assert result == expected_parser_info -def test_create_parser_success_with_validation_false(chronicle_client, mock_response): +def test_create_parser_success_with_validation_false( + chronicle_client, mock_response +): """Test create_parser function for success with validated_on_empty_logs=False.""" log_type = "NIX_SYSTEM" parser_code = "filter {}" @@ -208,14 +222,19 @@ def test_create_parser_success_with_validation_false(chronicle_client, mock_resp chronicle_client.session, "post", return_value=mock_response ) as mock_post: result = create_parser( - chronicle_client, log_type, parser_code, validated_on_empty_logs=False + chronicle_client, + log_type, + parser_code, + validated_on_empty_logs=False, ) expected_url = f"{chronicle_client.base_url}/{chronicle_client.instance_id}/logTypes/{log_type}/parsers" mock_post.assert_called_once_with( expected_url, json={ - "cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8"), + "cbn": base64.b64encode(parser_code.encode("utf-8")).decode( + "utf-8" + ), "validated_on_empty_logs": False, }, ) @@ -262,7 +281,9 @@ def test_deactivate_parser_error(chronicle_client, mock_error_response): ): with pytest.raises(APIError) as exc_info: deactivate_parser(chronicle_client, log_type, parser_id) - assert "Failed to deactivate parser: Error message" in str(exc_info.value) + assert "Failed to deactivate parser: Error message" in str( + exc_info.value + ) # --- delete_parser Tests --- @@ -278,7 +299,9 @@ def test_delete_parser_success_no_force(chronicle_client, mock_response): result = delete_parser(chronicle_client, log_type, parser_id) expected_url = f"{chronicle_client.base_url}/{chronicle_client.instance_id}/logTypes/{log_type}/parsers/{parser_id}" - mock_delete.assert_called_once_with(expected_url, params={"force": False}) + mock_delete.assert_called_once_with( + expected_url, params={"force": False} + ) assert result == {} @@ -291,10 +314,14 @@ def test_delete_parser_success_with_force(chronicle_client, mock_response): with patch.object( chronicle_client.session, "delete", return_value=mock_response ) as mock_delete: - result = delete_parser(chronicle_client, log_type, parser_id, force=True) + result = delete_parser( + chronicle_client, log_type, parser_id, force=True + ) expected_url = f"{chronicle_client.base_url}/{chronicle_client.instance_id}/logTypes/{log_type}/parsers/{parser_id}" - mock_delete.assert_called_once_with(expected_url, params={"force": True}) + mock_delete.assert_called_once_with( + expected_url, params={"force": True} + ) assert result == {} @@ -362,7 +389,8 @@ def test_list_parsers_single_page_success(chronicle_client, mock_response): expected_url = f"{chronicle_client.base_url}/{chronicle_client.instance_id}/logTypes/{log_type}/parsers" mock_get.assert_called_once_with( - expected_url, params={"pageSize": 100, "pageToken": None, "filter": None} + expected_url, + params={"pageSize": 100, "pageToken": None, "filter": None}, ) assert result == expected_parsers @@ -381,7 +409,8 @@ def test_list_parsers_no_parsers_success(chronicle_client, mock_response): expected_url = f"{chronicle_client.base_url}/{chronicle_client.instance_id}/logTypes/{log_type}/parsers" mock_get.assert_called_once_with( - expected_url, params={"pageSize": 100, "pageToken": None, "filter": None} + expected_url, + params={"pageSize": 100, "pageToken": None, "filter": None}, ) assert result == [] @@ -430,6 +459,72 @@ def test_list_parsers_with_optional_params(chronicle_client, mock_response): assert result == expected_parsers +def test_list_parsers_multi_page_pagination(chronicle_client, mock_response): + """Test list_parsers function with multi-page pagination (Issue 148). + + This test validates that the pagination fix correctly handles the + 'nextPageToken' field (not 'next_page_token') returned by the API. + """ + log_type = "WINDOWS" + + # First page of parsers with nextPageToken + first_page_parsers = [ + {"name": "pa_windows_1", "id": "pa_windows_1"}, + {"name": "pa_windows_2", "id": "pa_windows_2"}, + ] + + # Second page of parsers without nextPageToken (last page) + second_page_parsers = [ + {"name": "pa_windows_3", "id": "pa_windows_3"}, + ] + + # Mock responses for each page + first_response = Mock() + first_response.status_code = 200 + first_response.json.return_value = { + "parsers": first_page_parsers, + "nextPageToken": "page2_token", + } + + second_response = Mock() + second_response.status_code = 200 + second_response.json.return_value = { + "parsers": second_page_parsers, + # No nextPageToken - this is the last page + } + + with patch.object( + chronicle_client.session, + "get", + side_effect=[first_response, second_response], + ) as mock_get: + result = list_parsers(chronicle_client, log_type=log_type, page_size=2) + + # Verify we made two API calls (one per page) + assert mock_get.call_count == 2 + + # Verify first call + expected_url = ( + f"{chronicle_client.base_url}/{chronicle_client.instance_id}" + f"/logTypes/{log_type}/parsers" + ) + first_call = mock_get.call_args_list[0] + assert first_call[0][0] == expected_url + assert first_call[1]["params"]["pageSize"] == 2 + assert first_call[1]["params"]["pageToken"] is None + + # Verify second call uses the nextPageToken from first response + second_call = mock_get.call_args_list[1] + assert second_call[0][0] == expected_url + assert second_call[1]["params"]["pageSize"] == 2 + assert second_call[1]["params"]["pageToken"] == "page2_token" + + # Verify all parsers from both pages are returned + expected_all_parsers = first_page_parsers + second_page_parsers + assert result == expected_all_parsers + assert len(result) == 3 + + # --- run_parser Tests --- def test_run_parser_success(chronicle_client, mock_response): """Test run_parser function for success.""" @@ -473,9 +568,11 @@ def test_run_parser_success(chronicle_client, mock_response): assert request_body["parser"]["cbn"] == base64.b64encode( parser_code.encode("utf8") ).decode("utf-8") - assert request_body["parser_extension"]["cbn_snippet"] == base64.b64encode( - parser_extension_code.encode("utf8") - ).decode("utf-8") + assert request_body["parser_extension"][ + "cbn_snippet" + ] == base64.b64encode(parser_extension_code.encode("utf8")).decode( + "utf-8" + ) assert len(request_body["log"]) == 2 assert request_body["log"][0] == base64.b64encode( logs[0].encode("utf8") @@ -584,7 +681,9 @@ def test_run_parser_error(chronicle_client, mock_error_response): logs=logs, ) # Check for the new detailed error message format - assert "Failed to evaluate parser for log type 'WINDOWS'" in str(exc_info.value) + assert "Failed to evaluate parser for log type 'WINDOWS'" in str( + exc_info.value + ) assert "Bad request" in str(exc_info.value) @@ -718,7 +817,9 @@ def test_run_parser_validation_invalid_extension_type(chronicle_client): parser_extension_code=123, # type: ignore logs=["test log"], ) - assert "parser_extension_code must be a string or None" in str(exc_info.value) + assert "parser_extension_code must be a string or None" in str( + exc_info.value + ) def test_run_parser_detailed_error_400(chronicle_client, mock_response): @@ -726,7 +827,9 @@ def test_run_parser_detailed_error_400(chronicle_client, mock_response): mock_response.status_code = 400 mock_response.text = "Invalid log type: INVALID_TYPE" - with patch.object(chronicle_client.session, "post", return_value=mock_response): + with patch.object( + chronicle_client.session, "post", return_value=mock_response + ): with pytest.raises(APIError) as exc_info: run_parser( chronicle_client, @@ -736,7 +839,9 @@ def test_run_parser_detailed_error_400(chronicle_client, mock_response): logs=["test log"], ) error_msg = str(exc_info.value) - assert "Failed to evaluate parser for log type 'INVALID_TYPE'" in error_msg + assert ( + "Failed to evaluate parser for log type 'INVALID_TYPE'" in error_msg + ) assert "Bad request" in error_msg assert "Log type 'INVALID_TYPE' may not be valid" in error_msg @@ -746,7 +851,9 @@ def test_run_parser_detailed_error_404(chronicle_client, mock_response): mock_response.status_code = 404 mock_response.text = "Not found" - with patch.object(chronicle_client.session, "post", return_value=mock_response): + with patch.object( + chronicle_client.session, "post", return_value=mock_response + ): with pytest.raises(APIError) as exc_info: run_parser( chronicle_client, @@ -764,7 +871,9 @@ def test_run_parser_detailed_error_413(chronicle_client, mock_response): mock_response.status_code = 413 mock_response.text = "Request entity too large" - with patch.object(chronicle_client.session, "post", return_value=mock_response): + with patch.object( + chronicle_client.session, "post", return_value=mock_response + ): with pytest.raises(APIError) as exc_info: run_parser( chronicle_client, From d0d6fa953c9ff2de45a189f1ed8b80e09a24ee4c Mon Sep 17 00:00:00 2001 From: MisterSeajay Date: Tue, 2 Dec 2025 17:30:35 +0000 Subject: [PATCH 08/11] make long url string construction consistent --- src/secops/chronicle/parser.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/secops/chronicle/parser.py b/src/secops/chronicle/parser.py index 77a14040..3bc6c1b0 100644 --- a/src/secops/chronicle/parser.py +++ b/src/secops/chronicle/parser.py @@ -204,8 +204,10 @@ def delete_parser( Raises: APIError: If the API request fails """ - url = f"{client.base_url}/{client.instance_id}" - url += f"/logTypes/{log_type}/parsers/{id}" + url = ( + f"{client.base_url}/{client.instance_id}" + f"/logTypes/{log_type}/parsers/{id}" + ) params = {"force": force} response = client.session.delete(url, params=params) @@ -233,8 +235,10 @@ def get_parser( Raises: APIError: If the API request fails """ - url = f"{client.base_url}/{client.instance_id}" - url += f"/logTypes/{log_type}/parsers/{id}" + url = ( + f"{client.base_url}/{client.instance_id}" + f"/logTypes/{log_type}/parsers/{id}" + ) response = client.session.get(url) if response.status_code != 200: @@ -269,8 +273,10 @@ def list_parsers( parsers = [] while more: - url = f"{client.base_url}/{client.instance_id}" - url += f"/logTypes/{log_type}/parsers" + url = ( + f"{client.base_url}/{client.instance_id}" + f"/logTypes/{log_type}/parsers" + ) params = { "pageSize": page_size, @@ -382,8 +388,10 @@ def run_parser( ) # Build request - url = f"{client.base_url}/{client.instance_id}" - url += f"/logTypes/{log_type}:runParser" + url = ( + f"{client.base_url}/{client.instance_id}" + f"/logTypes/{log_type}:runParser" + ) parser = { "cbn": base64.b64encode(parser_code.encode("utf-8")).decode("utf-8") From 45c0eb92cc335d731371a5cdac816fb4bf0b82dd Mon Sep 17 00:00:00 2001 From: Mihir Vala <179564180+mihirvala-crestdata@users.noreply.github.com> Date: Thu, 4 Dec 2025 14:02:58 +0530 Subject: [PATCH 09/11] chore: added page_size condition. Added documentation. Fixed tests. --- README.md | 9 +++- src/secops/chronicle/client.py | 18 +++++--- src/secops/chronicle/parser.py | 32 +++++++++----- tests/chronicle/test_parser.py | 77 ++++++++++++++++++++++++++-------- 4 files changed, 101 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 4830fd38..0f755bf3 100644 --- a/README.md +++ b/README.md @@ -1190,13 +1190,20 @@ print(f"Parser ID: {parser_id}") Retrieve, list, copy, activate/deactivate, and delete parsers: ```python -# List all parsers +# List all parsers (returns complete list) parsers = chronicle.list_parsers() for parser in parsers: parser_id = parser.get("name", "").split("/")[-1] state = parser.get("state") print(f"Parser ID: {parser_id}, State: {state}") +# Manual pagination: get raw API response with nextPageToken +response = chronicle.list_parsers(page_size=50) +parsers = response.get("parsers", []) +next_token = response.get("nextPageToken") +# Use next_token for subsequent calls: +# response = chronicle.list_parsers(page_size=50, page_token=next_token) + log_type = "WINDOWS_AD" # Get specific parser diff --git a/src/secops/chronicle/client.py b/src/secops/chronicle/client.py index a6910e21..b4bebdc2 100644 --- a/src/secops/chronicle/client.py +++ b/src/secops/chronicle/client.py @@ -1662,20 +1662,26 @@ def get_parser( def list_parsers( self, log_type: str = "-", - page_size: int = 100, - page_token: str = None, + page_size: Optional[int] = None, + page_token: Optional[str] = None, filter: str = None, # pylint: disable=redefined-builtin - ) -> List[Any]: + ) -> Union[List[Any], Dict[str, Any]]: """List parsers. Args: log_type: Log type to filter by - page_size: The maximum number of parsers to return - page_token: A page token, received from a previous ListParsers call + page_size: The maximum number of parsers to return per page. + If provided, returns raw API response with pagination info. + If None (default), auto-paginates and returns all parsers. + page_token: A page token, received from a previous ListParsers + call. filter: Optional filter expression Returns: - List of parser dictionaries + If page_size is None: List of all parsers + (auto-paginated) + If page_size is provided: List of parsers with next page token if + available. Raises: APIError: If the API request fails diff --git a/src/secops/chronicle/parser.py b/src/secops/chronicle/parser.py index 3bc6c1b0..0401e837 100644 --- a/src/secops/chronicle/parser.py +++ b/src/secops/chronicle/parser.py @@ -250,21 +250,25 @@ def get_parser( def list_parsers( client: "ChronicleClient", log_type: str = "-", - page_size: int = 100, - page_token: Optional[Union[str, None]] = None, + page_size: Optional[int] = None, + page_token: Optional[str] = None, filter: str = None, # pylint: disable=redefined-builtin -) -> List[Any]: +) -> Union[List[Any], Dict[str, Any]]: """List parsers. Args: client: ChronicleClient instance log_type: Log type to filter by - page_size: The maximum number of parsers to return - page_token: A page token, received from a previous ListParsers call + page_size: The maximum number of parsers to return per page. + If provided, returns raw API response with pagination info. + If None (default), auto-paginates and returns all parsers. + page_token: A page token, received from a previous ListParsers call. filter: Optional filter expression Returns: - List of parser dictionaries + If page_size is None: List of all parsers. + If page_size is provided: List of parsers with next page token if + available. Raises: APIError: If the API request fails @@ -278,11 +282,14 @@ def list_parsers( f"/logTypes/{log_type}/parsers" ) - params = { - "pageSize": page_size, - "pageToken": page_token, - "filter": filter, - } + params = {} + + if page_size: + params["pageSize"] = page_size + if page_token: + params["pageToken"] = page_token + if filter: + params["filter"] = filter response = client.session.get(url, params=params) @@ -291,6 +298,9 @@ def list_parsers( data = response.json() + if page_size is not None: + return data + if "parsers" in data: parsers.extend(data["parsers"]) diff --git a/tests/chronicle/test_parser.py b/tests/chronicle/test_parser.py index e734972f..ee7b6307 100644 --- a/tests/chronicle/test_parser.py +++ b/tests/chronicle/test_parser.py @@ -390,7 +390,7 @@ def test_list_parsers_single_page_success(chronicle_client, mock_response): expected_url = f"{chronicle_client.base_url}/{chronicle_client.instance_id}/logTypes/{log_type}/parsers" mock_get.assert_called_once_with( expected_url, - params={"pageSize": 100, "pageToken": None, "filter": None}, + params={}, ) assert result == expected_parsers @@ -410,7 +410,7 @@ def test_list_parsers_no_parsers_success(chronicle_client, mock_response): expected_url = f"{chronicle_client.base_url}/{chronicle_client.instance_id}/logTypes/{log_type}/parsers" mock_get.assert_called_once_with( expected_url, - params={"pageSize": 100, "pageToken": None, "filter": None}, + params={}, ) assert result == [] @@ -427,14 +427,20 @@ def test_list_parsers_error(chronicle_client, mock_error_response): assert "Failed to list parsers: Error message" in str(exc_info.value) -def test_list_parsers_with_optional_params(chronicle_client, mock_response): - """Test list_parsers function with custom page_size, page_token, and filter.""" +def test_list_parsers_with_page_size_returns_raw_response( + chronicle_client, mock_response +): + """Test list_parsers returns raw API response when page_size is provided.""" log_type = "CUSTOM_LOG_TYPE" page_size = 50 page_token = "custom_token_xyz" filter_query = "name=contains('custom')" expected_parsers = [{"name": "pa_custom_1"}] - mock_response.json.return_value = {"parsers": expected_parsers} + expected_response = { + "parsers": expected_parsers, + "nextPageToken": "next_token_abc", + } + mock_response.json.return_value = expected_response with patch.object( chronicle_client.session, "get", return_value=mock_response @@ -447,7 +453,10 @@ def test_list_parsers_with_optional_params(chronicle_client, mock_response): filter=filter_query, ) - expected_url = f"{chronicle_client.base_url}/{chronicle_client.instance_id}/logTypes/{log_type}/parsers" + expected_url = ( + f"{chronicle_client.base_url}/{chronicle_client.instance_id}" + f"/logTypes/{log_type}/parsers" + ) mock_get.assert_called_once_with( expected_url, params={ @@ -456,14 +465,16 @@ def test_list_parsers_with_optional_params(chronicle_client, mock_response): "filter": filter_query, }, ) - assert result == expected_parsers + # With page_size provided, returns raw response dict + assert result == expected_response + assert "nextPageToken" in result -def test_list_parsers_multi_page_pagination(chronicle_client, mock_response): - """Test list_parsers function with multi-page pagination (Issue 148). +def test_list_parsers_auto_pagination(chronicle_client): + """Test list_parsers auto-paginates when page_size is None (default). - This test validates that the pagination fix correctly handles the - 'nextPageToken' field (not 'next_page_token') returned by the API. + This test validates that the pagination correctly handles the + 'nextPageToken' field returned by the API and fetches all pages. """ log_type = "WINDOWS" @@ -498,33 +509,65 @@ def test_list_parsers_multi_page_pagination(chronicle_client, mock_response): "get", side_effect=[first_response, second_response], ) as mock_get: - result = list_parsers(chronicle_client, log_type=log_type, page_size=2) + # No page_size means auto-pagination + result = list_parsers(chronicle_client, log_type=log_type) # Verify we made two API calls (one per page) assert mock_get.call_count == 2 - # Verify first call + # Verify first call uses default page size of 100 expected_url = ( f"{chronicle_client.base_url}/{chronicle_client.instance_id}" f"/logTypes/{log_type}/parsers" ) first_call = mock_get.call_args_list[0] assert first_call[0][0] == expected_url - assert first_call[1]["params"]["pageSize"] == 2 - assert first_call[1]["params"]["pageToken"] is None # Verify second call uses the nextPageToken from first response second_call = mock_get.call_args_list[1] assert second_call[0][0] == expected_url - assert second_call[1]["params"]["pageSize"] == 2 assert second_call[1]["params"]["pageToken"] == "page2_token" - # Verify all parsers from both pages are returned + # Verify all parsers from both pages are returned as a list expected_all_parsers = first_page_parsers + second_page_parsers assert result == expected_all_parsers assert len(result) == 3 +def test_list_parsers_manual_pagination_single_page( + chronicle_client, mock_response +): + """Test list_parsers returns raw response for manual pagination.""" + log_type = "MANUAL_LOG_TYPE" + page_size = 10 + expected_parsers = [{"name": "pa_manual_1"}] + expected_response = { + "parsers": expected_parsers, + "nextPageToken": "next_page_token", + } + mock_response.json.return_value = expected_response + + with patch.object( + chronicle_client.session, "get", return_value=mock_response + ) as mock_get: + result = list_parsers( + chronicle_client, log_type=log_type, page_size=page_size + ) + + expected_url = ( + f"{chronicle_client.base_url}/{chronicle_client.instance_id}" + f"/logTypes/{log_type}/parsers" + ) + mock_get.assert_called_once_with( + expected_url, + params={"pageSize": page_size}, + ) + # Returns raw response dict, not just the parsers list + assert result == expected_response + assert "parsers" in result + assert "nextPageToken" in result + + # --- run_parser Tests --- def test_run_parser_success(chronicle_client, mock_response): """Test run_parser function for success.""" From 928a44056a7b08e5696860ee43851fa55c0af76d Mon Sep 17 00:00:00 2001 From: Mihir Vala <179564180+mihirvala-crestdata@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:24:52 +0530 Subject: [PATCH 10/11] chore: fixed cli for parser list --- src/secops/cli/commands/parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/secops/cli/commands/parser.py b/src/secops/cli/commands/parser.py index ae98771a..0ca058c8 100644 --- a/src/secops/cli/commands/parser.py +++ b/src/secops/cli/commands/parser.py @@ -345,12 +345,12 @@ def handle_parser_run_command(args, chronicle): else: # If no parser code provided, # try to find an active parser for the log type - parsers = chronicle.list_parsers( + parser_list_response = chronicle.list_parsers( args.log_type, page_size=1, - page_token=None, filter="STATE=ACTIVE", ) + parsers = parser_list_response.get("parsers", []) if len(parsers) < 1: raise SecOpsError( "No parser file provided and an active parser could not " From 97146764355c7a9bd3b67f4f60738967f419b457 Mon Sep 17 00:00:00 2001 From: Mihir Vala <179564180+mihirvala-crestdata@users.noreply.github.com> Date: Mon, 8 Dec 2025 14:08:15 +0530 Subject: [PATCH 11/11] chore: added changelog. updated project version --- CHANGELOG.md | 6 ++++++ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c16b02a9..d1b0ae70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.27.2] - 2025-12-08 +### Updated +- Parser list method to handle pagination properly + - Method auto paginates and returns all when no page size is provided. + - When page size is provided, method returns response with next page token. + ## [0.27.1] - 2025-12-05 ### Updated - Updated Chronicle client to expose API version param for following: diff --git a/pyproject.toml b/pyproject.toml index 3266dd1d..20f35768 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "secops" -version = "0.27.1" +version = "0.27.2" description = "Python SDK for wrapping the Google SecOps API for common use cases" readme = "README.md" requires-python = ">=3.7"