From 34127c1c35cfeab81c80ba8e2b1379d2bcf91572 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 10:36:45 -0500 Subject: [PATCH 01/54] support custom TopCP images --- .../TopCP_code_generator/query_translate.py | 11 +- .../tests/test_custom_image_tag.py | 91 +++++++++++ helm/servicex/templates/app/configmap.yaml | 7 + helm/servicex/values.yaml | 15 +- servicex_app/servicex_app/code_gen_adapter.py | 2 +- .../resources/transformation/submit.py | 84 ++++++++++- .../test_custom_image_validation.py | 141 ++++++++++++++++++ 7 files changed, 345 insertions(+), 6 deletions(-) create mode 100644 code_generator_TopCPToolkit/tests/test_custom_image_tag.py create mode 100644 servicex_app/servicex_app_test/test_custom_image_validation.py diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index ac7ff9ac3..773a5ea69 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -36,6 +36,12 @@ "ifTrue": ["--no-filter"], "ifFalse": None, }, + "image_tag": { + "properType": str, + "properTypeString": "string", + "default": "2.17.0-25.2.45", + "optional": True, + }, } @@ -52,9 +58,12 @@ def generate_files_from_query(query, query_file_path): "customConfig", ] - # ensure all keys are specified + # ensure all required keys are specified for key in options: if key not in jquery: + # Skip optional parameters + if options[key].get("optional", False): + continue raise ValueError( key + " must be specified. May be type None or ", options[key]["properTypeString"], diff --git a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py new file mode 100644 index 000000000..18c2cf60e --- /dev/null +++ b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py @@ -0,0 +1,91 @@ +"""Test custom image tag functionality for TopCP code generator.""" + +import json +import os +import tempfile +import unittest + +from servicex.TopCP_code_generator.query_translate import generate_files_from_query + + +class TestCustomImageTag(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.temp_dir) + + def test_custom_image_tag_in_query_validation(self): + """Test that custom image tag is accepted in query validation.""" + query = { + "reco": "reco_config_content", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False, + "image_tag": "v2.20.0_v0.2" + } + + # This should not raise an exception + generate_files_from_query(json.dumps(query), self.temp_dir) + + def test_image_tag_not_required(self): + """Test that image_tag is optional and doesn't break existing queries.""" + query = { + "reco": "reco_config_content", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False + } + + # This should not raise an exception + generate_files_from_query(json.dumps(query), self.temp_dir) + + def test_image_tag_none_allowed(self): + """Test that None image tag is accepted.""" + query = { + "reco": "reco_config_content", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False, + "image_tag": None + } + + # This should not raise an exception + generate_files_from_query(json.dumps(query), self.temp_dir) + + def test_generated_files_created_with_image_tag(self): + """Test that all expected files are generated when image_tag is present.""" + query = { + "reco": "reco_config_content", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False, + "image_tag": "v2.20.0_v0.2" + } + + generate_files_from_query(json.dumps(query), self.temp_dir) + + # Check that reco.yaml is created + reco_file = os.path.join(self.temp_dir, "reco.yaml") + self.assertTrue(os.path.exists(reco_file)) + + with open(reco_file, 'r') as f: + content = f.read() + self.assertEqual(content, "reco_config_content") + + # Check that generated_transformer.py is created + transformer_file = os.path.join(self.temp_dir, "generated_transformer.py") + self.assertTrue(os.path.exists(transformer_file)) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index e543e2608..67067878a 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -178,6 +178,13 @@ data: CODE_GEN_IMAGES = { {{ join "," $code_gen_images }} } {{- end }} + {{- if .Values.codeGen.topcp.enabled }} + # TopCP custom image configuration + TOPCP_ALLOWED_REPOSITORIES = {{ .Values.codeGen.topcp.allowedRepositories | toJson }} + TOPCP_IMAGE_TAG_PATTERN = '{{ .Values.codeGen.topcp.imageTagPattern }}' + TOPCP_DEFAULT_BASE_IMAGE = '{{ .Values.codeGen.topcp.defaultBaseImage }}' + {{- end }} + {{- $didFinders := list }} {{- if .Values.didFinder.CERNOpenData.enabled }} {{- $didFinders = append $didFinders "cernopendata" }} diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index 40de0b959..02d1707fd 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -108,6 +108,12 @@ codeGen: tag: develop defaultScienceContainerImage: sslhep/servicex_science_image_topcp defaultScienceContainerTag: 2.17.0-25.2.45 + # Configuration for custom image tag validation + allowedRepositories: + - "sslhep/servicex_science_image_topcp" + - "registry.gitlab.com/topcp-project/toolkit" + imageTagPattern: "^v?\\d+\\.\\d+\\.\\d+[-_]v?\\d+\\.\\d+$" + defaultBaseImage: "sslhep/servicex_science_image_topcp" didFinder: CERNOpenData: @@ -146,7 +152,8 @@ logging: minio: volumePermissions: image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell auth: rootPassword: leftfoot1 rootUser: miniouser @@ -168,7 +175,8 @@ postgres: postgresql: volumePermissions: image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell global: postgresql: auth: @@ -189,7 +197,8 @@ rabbitmq: volumePermissions: enabled: true image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell extraConfiguration: |- consumer_timeout = 3600000 diff --git a/servicex_app/servicex_app/code_gen_adapter.py b/servicex_app/servicex_app/code_gen_adapter.py index ea88516c5..b4361acd2 100644 --- a/servicex_app/servicex_app/code_gen_adapter.py +++ b/servicex_app/servicex_app/code_gen_adapter.py @@ -53,7 +53,7 @@ def generate_code_for_selection( :param namespace: Namespace in which to place resulting ConfigMap. :param user_codegen_name: Name provided by user for selecting the codegen URL from config dictionary - :returns a tuple of (config map name, default transformer image) + :returns a tuple of (config map name, default transformer image, language, command) """ from io import BytesIO from zipfile import ZipFile diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 51e57668e..3aec30274 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -26,6 +26,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid +import re from datetime import datetime, timezone from typing import Optional, List @@ -39,7 +40,63 @@ from werkzeug.exceptions import BadRequest +def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) -> tuple[str, str]: + """ + Validate custom image tag for TopCP transformations. + + :param image_tag: The custom image tag provided by user + :param config: Flask application configuration + :param codegen_name: The code generator name (e.g., 'topcp') + :returns: tuple of (validated_image_name, validated_tag) + :raises: BadRequest if validation fails + """ + if not image_tag or not image_tag.strip(): + return None, None + + # Only validate for TopCP requests + if codegen_name != 'topcp': + return None, None + + # Get validation configuration (with defaults) + allowed_repos = config.get('TOPCP_ALLOWED_REPOSITORIES', [ + 'sslhep/servicex_science_image_topcp', + 'registry.gitlab.com/topcp-project/toolkit' + ]) + tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') + default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', 'sslhep/servicex_science_image_topcp') + + # Validate tag format + if not re.match(tag_pattern, image_tag): + raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2") + + # For now, use the default base image with the custom tag + # In the future, this could be extended to allow custom repositories + validated_image = default_base_image + validated_tag = image_tag + + return validated_image, validated_tag + + class SubmitTransformationRequest(ServiceXResource): + + def _extract_custom_image_tag(self, selection: str, codegen_name: str) -> Optional[str]: + """ + Extract custom image tag from TopCP selection query. + + :param selection: The query selection string + :param codegen_name: The code generator name + :returns: Custom image tag if present and valid, None otherwise + """ + if codegen_name != 'topcp': + return None + + try: + import json + query = json.loads(selection) + return query.get('image_tag') + except (json.JSONDecodeError, AttributeError): + # Invalid JSON or non-dict selection + return None @classmethod def make_api( cls, @@ -212,6 +269,9 @@ def post(self): files=0, ) + # Extract custom image tag from TopCP queries before code generation + custom_image_tag = self._extract_custom_image_tag(request_rec.selection, user_codegen_name) + # The first thing to do is make sure the requested selection is correct, # and can generate the requested code ( @@ -223,7 +283,29 @@ def post(self): request_rec, namespace, user_codegen_name ) - request_rec.image = codegen_transformer_image + # Handle custom image tag for TopCP + if custom_image_tag: + try: + validated_image, validated_tag = validate_custom_image_tag( + custom_image_tag, config, user_codegen_name + ) + if validated_image and validated_tag: + # Override the default science container image + request_rec.image = f"{validated_image}:{validated_tag}" + current_app.logger.info( + f"Using custom TopCP image: {request_rec.image}", + extra={"requestId": request_id} + ) + else: + request_rec.image = codegen_transformer_image + except BadRequest as e: + current_app.logger.error( + f"Invalid custom image tag: {str(e)}", + extra={"requestId": request_id} + ) + return {"message": str(e)}, 400 + else: + request_rec.image = codegen_transformer_image # Check to make sure the transformer docker image actually exists (if enabled) if config["TRANSFORMER_VALIDATE_DOCKER_IMAGE"]: diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py new file mode 100644 index 000000000..43a40d193 --- /dev/null +++ b/servicex_app/servicex_app_test/test_custom_image_validation.py @@ -0,0 +1,141 @@ +"""Test custom image tag validation for TopCP transformations.""" + +import json +import unittest +from werkzeug.exceptions import BadRequest + +from servicex_app.resources.transformation.submit import validate_custom_image_tag, SubmitTransformationRequest + + +class TestCustomImageValidation(unittest.TestCase): + + def setUp(self): + self.config = { + 'TOPCP_ALLOWED_REPOSITORIES': [ + 'sslhep/servicex_science_image_topcp', + 'registry.gitlab.com/topcp-project/toolkit' + ], + 'TOPCP_IMAGE_TAG_PATTERN': r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$', + 'TOPCP_DEFAULT_BASE_IMAGE': 'sslhep/servicex_science_image_topcp' + } + + def test_valid_image_tag_format(self): + """Test that valid image tag formats are accepted.""" + test_cases = [ + "v2.20.0_v0.2", + "2.20.0_v0.2", + "v1.15.0-v1.0", + "3.0.0_v2.5" + ] + + for tag in test_cases: + with self.subTest(tag=tag): + image, result_tag = validate_custom_image_tag(tag, self.config, 'topcp') + self.assertEqual(image, 'sslhep/servicex_science_image_topcp') + self.assertEqual(result_tag, tag) + + def test_invalid_image_tag_format(self): + """Test that invalid image tag formats are rejected.""" + test_cases = [ + "latest", + "2.20.0", + "v2.20.0", + "invalid-tag", + "v2.20.0_v0.2.1", # Too many version components + "2.20_v0.2", # Missing patch version + ] + + for tag in test_cases: + with self.subTest(tag=tag): + with self.assertRaises(BadRequest): + validate_custom_image_tag(tag, self.config, 'topcp') + + def test_non_topcp_codegen_returns_none(self): + """Test that non-TopCP code generators return None.""" + image, tag = validate_custom_image_tag("v2.20.0_v0.2", self.config, 'uproot') + self.assertIsNone(image) + self.assertIsNone(tag) + + def test_empty_image_tag_returns_none(self): + """Test that empty image tag returns None.""" + test_cases = [None, "", " "] + + for tag in test_cases: + with self.subTest(tag=tag): + image, result_tag = validate_custom_image_tag(tag, self.config, 'topcp') + self.assertIsNone(image) + self.assertIsNone(result_tag) + + def test_missing_config_uses_defaults(self): + """Test that missing configuration uses default values.""" + minimal_config = {} + + image, tag = validate_custom_image_tag("v2.20.0_v0.2", minimal_config, 'topcp') + self.assertEqual(image, 'sslhep/servicex_science_image_topcp') + self.assertEqual(tag, "v2.20.0_v0.2") + + +class TestImageTagExtraction(unittest.TestCase): + + def setUp(self): + self.submit_request = SubmitTransformationRequest() + + def test_extract_image_tag_from_topcp_query(self): + """Test extraction of image tag from TopCP query.""" + query = { + "reco": "reco_config", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False, + "image_tag": "v2.20.0_v0.2" + } + + result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + self.assertEqual(result, "v2.20.0_v0.2") + + def test_extract_image_tag_missing_returns_none(self): + """Test that missing image tag returns None.""" + query = { + "reco": "reco_config", + "parton": None, + "particle": None, + "max_events": -1, + "no_systematics": False, + "no_filter": False + } + + result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + self.assertIsNone(result) + + def test_extract_image_tag_non_topcp_returns_none(self): + """Test that non-TopCP codegen returns None.""" + query = { + "selection": "some_selection", + "image_tag": "v2.20.0_v0.2" + } + + result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'uproot') + self.assertIsNone(result) + + def test_extract_image_tag_invalid_json_returns_none(self): + """Test that invalid JSON returns None.""" + invalid_json = "{ invalid json }" + + result = self.submit_request._extract_custom_image_tag(invalid_json, 'topcp') + self.assertIsNone(result) + + def test_extract_image_tag_none_value(self): + """Test that None image tag value is returned as None.""" + query = { + "reco": "reco_config", + "image_tag": None + } + + result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + self.assertIsNone(result) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From ba7101415838b6c80e8b350d251e0503ce643103 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Sep 2025 15:37:26 +0000 Subject: [PATCH 02/54] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tests/test_custom_image_tag.py | 15 ++-- .../resources/transformation/submit.py | 47 ++++++++---- .../test_custom_image_validation.py | 74 +++++++++---------- 3 files changed, 77 insertions(+), 59 deletions(-) diff --git a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py index 18c2cf60e..19ffd5344 100644 --- a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py +++ b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py @@ -14,6 +14,7 @@ def setUp(self): def tearDown(self): import shutil + shutil.rmtree(self.temp_dir) def test_custom_image_tag_in_query_validation(self): @@ -25,7 +26,7 @@ def test_custom_image_tag_in_query_validation(self): "max_events": -1, "no_systematics": False, "no_filter": False, - "image_tag": "v2.20.0_v0.2" + "image_tag": "v2.20.0_v0.2", } # This should not raise an exception @@ -39,7 +40,7 @@ def test_image_tag_not_required(self): "particle": None, "max_events": -1, "no_systematics": False, - "no_filter": False + "no_filter": False, } # This should not raise an exception @@ -54,7 +55,7 @@ def test_image_tag_none_allowed(self): "max_events": -1, "no_systematics": False, "no_filter": False, - "image_tag": None + "image_tag": None, } # This should not raise an exception @@ -69,7 +70,7 @@ def test_generated_files_created_with_image_tag(self): "max_events": -1, "no_systematics": False, "no_filter": False, - "image_tag": "v2.20.0_v0.2" + "image_tag": "v2.20.0_v0.2", } generate_files_from_query(json.dumps(query), self.temp_dir) @@ -78,7 +79,7 @@ def test_generated_files_created_with_image_tag(self): reco_file = os.path.join(self.temp_dir, "reco.yaml") self.assertTrue(os.path.exists(reco_file)) - with open(reco_file, 'r') as f: + with open(reco_file, "r") as f: content = f.read() self.assertEqual(content, "reco_config_content") @@ -87,5 +88,5 @@ def test_generated_files_created_with_image_tag(self): self.assertTrue(os.path.exists(transformer_file)) -if __name__ == '__main__': - unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 3aec30274..4fe0e789d 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -40,7 +40,9 @@ from werkzeug.exceptions import BadRequest -def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) -> tuple[str, str]: +def validate_custom_image_tag( + image_tag: str, config: dict, codegen_name: str +) -> tuple[str, str]: """ Validate custom image tag for TopCP transformations. @@ -54,20 +56,29 @@ def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) - return None, None # Only validate for TopCP requests - if codegen_name != 'topcp': + if codegen_name != "topcp": return None, None # Get validation configuration (with defaults) - allowed_repos = config.get('TOPCP_ALLOWED_REPOSITORIES', [ - 'sslhep/servicex_science_image_topcp', - 'registry.gitlab.com/topcp-project/toolkit' - ]) - tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') - default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', 'sslhep/servicex_science_image_topcp') + allowed_repos = config.get( + "TOPCP_ALLOWED_REPOSITORIES", + [ + "sslhep/servicex_science_image_topcp", + "registry.gitlab.com/topcp-project/toolkit", + ], + ) + tag_pattern = config.get( + "TOPCP_IMAGE_TAG_PATTERN", r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$" + ) + default_base_image = config.get( + "TOPCP_DEFAULT_BASE_IMAGE", "sslhep/servicex_science_image_topcp" + ) # Validate tag format if not re.match(tag_pattern, image_tag): - raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2") + raise BadRequest( + f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2" + ) # For now, use the default base image with the custom tag # In the future, this could be extended to allow custom repositories @@ -79,7 +90,9 @@ def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) - class SubmitTransformationRequest(ServiceXResource): - def _extract_custom_image_tag(self, selection: str, codegen_name: str) -> Optional[str]: + def _extract_custom_image_tag( + self, selection: str, codegen_name: str + ) -> Optional[str]: """ Extract custom image tag from TopCP selection query. @@ -87,16 +100,18 @@ def _extract_custom_image_tag(self, selection: str, codegen_name: str) -> Option :param codegen_name: The code generator name :returns: Custom image tag if present and valid, None otherwise """ - if codegen_name != 'topcp': + if codegen_name != "topcp": return None try: import json + query = json.loads(selection) - return query.get('image_tag') + return query.get("image_tag") except (json.JSONDecodeError, AttributeError): # Invalid JSON or non-dict selection return None + @classmethod def make_api( cls, @@ -270,7 +285,9 @@ def post(self): ) # Extract custom image tag from TopCP queries before code generation - custom_image_tag = self._extract_custom_image_tag(request_rec.selection, user_codegen_name) + custom_image_tag = self._extract_custom_image_tag( + request_rec.selection, user_codegen_name + ) # The first thing to do is make sure the requested selection is correct, # and can generate the requested code @@ -294,14 +311,14 @@ def post(self): request_rec.image = f"{validated_image}:{validated_tag}" current_app.logger.info( f"Using custom TopCP image: {request_rec.image}", - extra={"requestId": request_id} + extra={"requestId": request_id}, ) else: request_rec.image = codegen_transformer_image except BadRequest as e: current_app.logger.error( f"Invalid custom image tag: {str(e)}", - extra={"requestId": request_id} + extra={"requestId": request_id}, ) return {"message": str(e)}, 400 else: diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py index 43a40d193..c0198f46a 100644 --- a/servicex_app/servicex_app_test/test_custom_image_validation.py +++ b/servicex_app/servicex_app_test/test_custom_image_validation.py @@ -4,34 +4,32 @@ import unittest from werkzeug.exceptions import BadRequest -from servicex_app.resources.transformation.submit import validate_custom_image_tag, SubmitTransformationRequest +from servicex_app.resources.transformation.submit import ( + validate_custom_image_tag, + SubmitTransformationRequest, +) class TestCustomImageValidation(unittest.TestCase): def setUp(self): self.config = { - 'TOPCP_ALLOWED_REPOSITORIES': [ - 'sslhep/servicex_science_image_topcp', - 'registry.gitlab.com/topcp-project/toolkit' + "TOPCP_ALLOWED_REPOSITORIES": [ + "sslhep/servicex_science_image_topcp", + "registry.gitlab.com/topcp-project/toolkit", ], - 'TOPCP_IMAGE_TAG_PATTERN': r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$', - 'TOPCP_DEFAULT_BASE_IMAGE': 'sslhep/servicex_science_image_topcp' + "TOPCP_IMAGE_TAG_PATTERN": r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$", + "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", } def test_valid_image_tag_format(self): """Test that valid image tag formats are accepted.""" - test_cases = [ - "v2.20.0_v0.2", - "2.20.0_v0.2", - "v1.15.0-v1.0", - "3.0.0_v2.5" - ] + test_cases = ["v2.20.0_v0.2", "2.20.0_v0.2", "v1.15.0-v1.0", "3.0.0_v2.5"] for tag in test_cases: with self.subTest(tag=tag): - image, result_tag = validate_custom_image_tag(tag, self.config, 'topcp') - self.assertEqual(image, 'sslhep/servicex_science_image_topcp') + image, result_tag = validate_custom_image_tag(tag, self.config, "topcp") + self.assertEqual(image, "sslhep/servicex_science_image_topcp") self.assertEqual(result_tag, tag) def test_invalid_image_tag_format(self): @@ -42,17 +40,17 @@ def test_invalid_image_tag_format(self): "v2.20.0", "invalid-tag", "v2.20.0_v0.2.1", # Too many version components - "2.20_v0.2", # Missing patch version + "2.20_v0.2", # Missing patch version ] for tag in test_cases: with self.subTest(tag=tag): with self.assertRaises(BadRequest): - validate_custom_image_tag(tag, self.config, 'topcp') + validate_custom_image_tag(tag, self.config, "topcp") def test_non_topcp_codegen_returns_none(self): """Test that non-TopCP code generators return None.""" - image, tag = validate_custom_image_tag("v2.20.0_v0.2", self.config, 'uproot') + image, tag = validate_custom_image_tag("v2.20.0_v0.2", self.config, "uproot") self.assertIsNone(image) self.assertIsNone(tag) @@ -62,7 +60,7 @@ def test_empty_image_tag_returns_none(self): for tag in test_cases: with self.subTest(tag=tag): - image, result_tag = validate_custom_image_tag(tag, self.config, 'topcp') + image, result_tag = validate_custom_image_tag(tag, self.config, "topcp") self.assertIsNone(image) self.assertIsNone(result_tag) @@ -70,8 +68,8 @@ def test_missing_config_uses_defaults(self): """Test that missing configuration uses default values.""" minimal_config = {} - image, tag = validate_custom_image_tag("v2.20.0_v0.2", minimal_config, 'topcp') - self.assertEqual(image, 'sslhep/servicex_science_image_topcp') + image, tag = validate_custom_image_tag("v2.20.0_v0.2", minimal_config, "topcp") + self.assertEqual(image, "sslhep/servicex_science_image_topcp") self.assertEqual(tag, "v2.20.0_v0.2") @@ -89,10 +87,12 @@ def test_extract_image_tag_from_topcp_query(self): "max_events": -1, "no_systematics": False, "no_filter": False, - "image_tag": "v2.20.0_v0.2" + "image_tag": "v2.20.0_v0.2", } - result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + result = self.submit_request._extract_custom_image_tag( + json.dumps(query), "topcp" + ) self.assertEqual(result, "v2.20.0_v0.2") def test_extract_image_tag_missing_returns_none(self): @@ -103,39 +103,39 @@ def test_extract_image_tag_missing_returns_none(self): "particle": None, "max_events": -1, "no_systematics": False, - "no_filter": False + "no_filter": False, } - result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + result = self.submit_request._extract_custom_image_tag( + json.dumps(query), "topcp" + ) self.assertIsNone(result) def test_extract_image_tag_non_topcp_returns_none(self): """Test that non-TopCP codegen returns None.""" - query = { - "selection": "some_selection", - "image_tag": "v2.20.0_v0.2" - } + query = {"selection": "some_selection", "image_tag": "v2.20.0_v0.2"} - result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'uproot') + result = self.submit_request._extract_custom_image_tag( + json.dumps(query), "uproot" + ) self.assertIsNone(result) def test_extract_image_tag_invalid_json_returns_none(self): """Test that invalid JSON returns None.""" invalid_json = "{ invalid json }" - result = self.submit_request._extract_custom_image_tag(invalid_json, 'topcp') + result = self.submit_request._extract_custom_image_tag(invalid_json, "topcp") self.assertIsNone(result) def test_extract_image_tag_none_value(self): """Test that None image tag value is returned as None.""" - query = { - "reco": "reco_config", - "image_tag": None - } + query = {"reco": "reco_config", "image_tag": None} - result = self.submit_request._extract_custom_image_tag(json.dumps(query), 'topcp') + result = self.submit_request._extract_custom_image_tag( + json.dumps(query), "topcp" + ) self.assertIsNone(result) -if __name__ == '__main__': - unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() From 86d536d545a03f2ef3e6fc4605a25cb2983f6c57 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 10:52:48 -0500 Subject: [PATCH 03/54] resolve flake8 issues --- .../tests/test_custom_image_tag.py | 2 +- .../resources/transformation/submit.py | 17 +++++++++-------- .../test_custom_image_validation.py | 6 ++++-- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py index 18c2cf60e..635228a85 100644 --- a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py +++ b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py @@ -88,4 +88,4 @@ def test_generated_files_created_with_image_tag(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 3aec30274..2e16772b3 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -58,16 +58,15 @@ def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) - return None, None # Get validation configuration (with defaults) - allowed_repos = config.get('TOPCP_ALLOWED_REPOSITORIES', [ - 'sslhep/servicex_science_image_topcp', - 'registry.gitlab.com/topcp-project/toolkit' - ]) - tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') - default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', 'sslhep/servicex_science_image_topcp') + tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', + r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') + default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', + 'sslhep/servicex_science_image_topcp') # Validate tag format if not re.match(tag_pattern, image_tag): - raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2") + raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. " + f"Expected format: v2.20.0_v0.2") # For now, use the default base image with the custom tag # In the future, this could be extended to allow custom repositories @@ -97,6 +96,7 @@ def _extract_custom_image_tag(self, selection: str, codegen_name: str) -> Option except (json.JSONDecodeError, AttributeError): # Invalid JSON or non-dict selection return None + @classmethod def make_api( cls, @@ -270,7 +270,8 @@ def post(self): ) # Extract custom image tag from TopCP queries before code generation - custom_image_tag = self._extract_custom_image_tag(request_rec.selection, user_codegen_name) + custom_image_tag = self._extract_custom_image_tag( + request_rec.selection, user_codegen_name) # The first thing to do is make sure the requested selection is correct, # and can generate the requested code diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py index 43a40d193..ce02bf41a 100644 --- a/servicex_app/servicex_app_test/test_custom_image_validation.py +++ b/servicex_app/servicex_app_test/test_custom_image_validation.py @@ -4,7 +4,9 @@ import unittest from werkzeug.exceptions import BadRequest -from servicex_app.resources.transformation.submit import validate_custom_image_tag, SubmitTransformationRequest +from servicex_app.resources.transformation.submit import ( + validate_custom_image_tag, SubmitTransformationRequest +) class TestCustomImageValidation(unittest.TestCase): @@ -138,4 +140,4 @@ def test_extract_image_tag_none_value(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From b518d4086aca9f0f84623e8c0ece934c67bdace7 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 10:55:27 -0500 Subject: [PATCH 04/54] Revert "resolve flake8 issues" This reverts commit 86d536d545a03f2ef3e6fc4605a25cb2983f6c57. --- .../tests/test_custom_image_tag.py | 2 +- .../resources/transformation/submit.py | 17 ++++++++--------- .../test_custom_image_validation.py | 6 ++---- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py index 635228a85..18c2cf60e 100644 --- a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py +++ b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py @@ -88,4 +88,4 @@ def test_generated_files_created_with_image_tag(self): if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 2e16772b3..3aec30274 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -58,15 +58,16 @@ def validate_custom_image_tag(image_tag: str, config: dict, codegen_name: str) - return None, None # Get validation configuration (with defaults) - tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', - r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') - default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', - 'sslhep/servicex_science_image_topcp') + allowed_repos = config.get('TOPCP_ALLOWED_REPOSITORIES', [ + 'sslhep/servicex_science_image_topcp', + 'registry.gitlab.com/topcp-project/toolkit' + ]) + tag_pattern = config.get('TOPCP_IMAGE_TAG_PATTERN', r'^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$') + default_base_image = config.get('TOPCP_DEFAULT_BASE_IMAGE', 'sslhep/servicex_science_image_topcp') # Validate tag format if not re.match(tag_pattern, image_tag): - raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. " - f"Expected format: v2.20.0_v0.2") + raise BadRequest(f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2") # For now, use the default base image with the custom tag # In the future, this could be extended to allow custom repositories @@ -96,7 +97,6 @@ def _extract_custom_image_tag(self, selection: str, codegen_name: str) -> Option except (json.JSONDecodeError, AttributeError): # Invalid JSON or non-dict selection return None - @classmethod def make_api( cls, @@ -270,8 +270,7 @@ def post(self): ) # Extract custom image tag from TopCP queries before code generation - custom_image_tag = self._extract_custom_image_tag( - request_rec.selection, user_codegen_name) + custom_image_tag = self._extract_custom_image_tag(request_rec.selection, user_codegen_name) # The first thing to do is make sure the requested selection is correct, # and can generate the requested code diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py index ce02bf41a..43a40d193 100644 --- a/servicex_app/servicex_app_test/test_custom_image_validation.py +++ b/servicex_app/servicex_app_test/test_custom_image_validation.py @@ -4,9 +4,7 @@ import unittest from werkzeug.exceptions import BadRequest -from servicex_app.resources.transformation.submit import ( - validate_custom_image_tag, SubmitTransformationRequest -) +from servicex_app.resources.transformation.submit import validate_custom_image_tag, SubmitTransformationRequest class TestCustomImageValidation(unittest.TestCase): @@ -140,4 +138,4 @@ def test_extract_image_tag_none_value(self): if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file From 8286d955e87989412e7be2004b07095c30f120db Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 11:05:12 -0500 Subject: [PATCH 05/54] fix broken test --- .../servicex/TopCP_code_generator/query_translate.py | 2 +- .../servicex_app/resources/transformation/submit.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index 773a5ea69..0b5518e82 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -87,7 +87,7 @@ def generate_files_from_query(query, query_file_path): ) # check for reco.yaml, parton.yaml and particle.yaml files - if isinstance(jquery[key], str): + if isinstance(jquery[key], str) and "fileName" in options[key]: with open( os.path.join(query_file_path, options[key]["fileName"]), "w" ) as file: diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 4fe0e789d..9d9469c5d 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -60,13 +60,6 @@ def validate_custom_image_tag( return None, None # Get validation configuration (with defaults) - allowed_repos = config.get( - "TOPCP_ALLOWED_REPOSITORIES", - [ - "sslhep/servicex_science_image_topcp", - "registry.gitlab.com/topcp-project/toolkit", - ], - ) tag_pattern = config.get( "TOPCP_IMAGE_TAG_PATTERN", r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$" ) From 2e706b7643a5e79df206f6489c6aff63070dbc44 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 11:10:05 -0500 Subject: [PATCH 06/54] add back allowed_repos logic --- .../resources/transformation/submit.py | 16 ++++++++++-- .../test_custom_image_validation.py | 26 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 9d9469c5d..07a7dce3b 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -60,6 +60,13 @@ def validate_custom_image_tag( return None, None # Get validation configuration (with defaults) + allowed_repos = config.get( + "TOPCP_ALLOWED_REPOSITORIES", + [ + "sslhep/servicex_science_image_topcp", + "registry.gitlab.com/topcp-project/toolkit", + ], + ) tag_pattern = config.get( "TOPCP_IMAGE_TAG_PATTERN", r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$" ) @@ -73,8 +80,13 @@ def validate_custom_image_tag( f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2" ) - # For now, use the default base image with the custom tag - # In the future, this could be extended to allow custom repositories + # Validate that the default base image is in the allowed repositories + if default_base_image not in allowed_repos: + raise BadRequest( + f"Default base image {default_base_image} is not in allowed repositories: " + f"{allowed_repos}" + ) + validated_image = default_base_image validated_tag = image_tag diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py index c0198f46a..9bfc9f4ec 100644 --- a/servicex_app/servicex_app_test/test_custom_image_validation.py +++ b/servicex_app/servicex_app_test/test_custom_image_validation.py @@ -72,6 +72,32 @@ def test_missing_config_uses_defaults(self): self.assertEqual(image, "sslhep/servicex_science_image_topcp") self.assertEqual(tag, "v2.20.0_v0.2") + def test_repository_validation_success(self): + """Test that repository validation passes when default image is in allowed repos.""" + config = { + "TOPCP_ALLOWED_REPOSITORIES": [ + "sslhep/servicex_science_image_topcp", + "registry.gitlab.com/topcp-project/toolkit", + ], + "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", + } + + image, tag = validate_custom_image_tag("v2.20.0_v0.2", config, "topcp") + self.assertEqual(image, "sslhep/servicex_science_image_topcp") + self.assertEqual(tag, "v2.20.0_v0.2") + + def test_repository_validation_failure(self): + """Test that repository validation fails when default image is not in allowed repos.""" + config = { + "TOPCP_ALLOWED_REPOSITORIES": [ + "registry.gitlab.com/topcp-project/toolkit", + ], + "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", + } + + with self.assertRaises(BadRequest): + validate_custom_image_tag("v2.20.0_v0.2", config, "topcp") + class TestImageTagExtraction(unittest.TestCase): From fe976f3e14ffb4492b05283645b8999efbf38de5 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 23 Sep 2025 11:17:11 -0500 Subject: [PATCH 07/54] remove accidental values changes --- helm/servicex/values.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index 02d1707fd..4c1ad0a58 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -152,8 +152,7 @@ logging: minio: volumePermissions: image: -# repository: bitnami/bitnami-shell-archived - repository: bitnami/os-shell + repository: bitnami/bitnami-shell-archived auth: rootPassword: leftfoot1 rootUser: miniouser @@ -175,8 +174,7 @@ postgres: postgresql: volumePermissions: image: -# repository: bitnami/bitnami-shell-archived - repository: bitnami/os-shell + repository: bitnami/bitnami-shell-archived global: postgresql: auth: @@ -197,8 +195,7 @@ rabbitmq: volumePermissions: enabled: true image: -# repository: bitnami/bitnami-shell-archived - repository: bitnami/os-shell + repository: bitnami/bitnami-shell-archived extraConfiguration: |- consumer_timeout = 3600000 From e12925647fd016a4f8be50c847a0672ba4566281 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Wed, 8 Oct 2025 09:33:47 -0500 Subject: [PATCH 08/54] moving towards MVP (app server is failing due to bad mount) --- Procfile | 3 +- code_generator_TopCPToolkit/Dockerfile | 7 +- .../TopCP_code_generator/query_translate.py | 18 ++- .../request_translator.py | 20 ++- .../{ => servicex}/boot.sh | 2 +- .../templates/transform_single_file.py | 13 +- helm/servicex/templates/app/configmap.yaml | 1 + .../templates/codegen/deployment.yaml | 9 ++ helm/servicex/values.yaml | 22 ++-- servicex_app/pyproject.toml | 2 +- servicex_app/servicex_app/code_gen_adapter.py | 10 +- .../servicex_app/docker_repo_adapter.py | 82 ++++++++++++ .../resources/transformation/submit.py | 120 +++--------------- 13 files changed, 179 insertions(+), 130 deletions(-) rename code_generator_TopCPToolkit/{ => servicex}/boot.sh (54%) diff --git a/Procfile b/Procfile index cfe64a9c2..8ddfaea4b 100644 --- a/Procfile +++ b/Procfile @@ -1,4 +1,5 @@ -minikube-mount: minikube mount $LOCAL_DIR:/mnt/servicex +minikube-mount-app: minikube mount $CHART_DIR:/mnt/servicex +minikube-mount-topcp: minikube mount $LOCAL_DIR/code_generator_TopCPToolkit/servicex:/mnt/topcp helm-install: sleep 5; cd $CHART_DIR && helm install -f $VALUES_FILE servicex . && tail -f /dev/null port-forward-app: sleep 30 && cd $LOCAL_DIR && bash local/port-forward.sh app port-forward-minio: sleep 20 && cd $LOCAL_DIR && bash local/port-forward.sh minio diff --git a/code_generator_TopCPToolkit/Dockerfile b/code_generator_TopCPToolkit/Dockerfile index ec8adf0ce..98136d379 100644 --- a/code_generator_TopCPToolkit/Dockerfile +++ b/code_generator_TopCPToolkit/Dockerfile @@ -17,9 +17,6 @@ RUN poetry config virtualenvs.create false && \ RUN pip install gunicorn -COPY boot.sh ./ -RUN chmod 755 boot.sh - COPY transformer_capabilities.json ./ RUN chmod 644 transformer_capabilities.json @@ -29,9 +26,11 @@ RUN chmod 777 -R servicex COPY app.conf . RUN chmod 755 app.conf +RUN chmod 755 /home/servicex/servicex/boot.sh + USER servicex ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf" EXPOSE 5000 -ENTRYPOINT ["./boot.sh"] +ENTRYPOINT ["/home/servicex/servicex/boot.sh"] diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index 0b5518e82..1644e6fad 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -36,10 +36,10 @@ "ifTrue": ["--no-filter"], "ifFalse": None, }, - "image_tag": { + "docker_image": { "properType": str, "properTypeString": "string", - "default": "2.17.0-25.2.45", + "default": "sslhep/servicex_science_image_topcp:2.17.0-25.2.45", "optional": True, }, } @@ -48,6 +48,15 @@ def generate_files_from_query(query, query_file_path): jquery = json.loads(query) + # transformer_image = jquery.get("docker_image", "sslhep/servicex_science_image_topcp:2.17.0-25.2.45") + # metadata = { + # "transformer_image": transformer_image + # } + # with open( + # os.path.join(query_file_path, "image_metadata.json"), "w" + # ) as metadata_file: + # json.dump(metadata, metadata_file) + runTopCommand = [ "runTop_el.py", "-i", @@ -70,7 +79,10 @@ def generate_files_from_query(query, query_file_path): ) for key in jquery: - # ensure only aviable options are allowed + if key == "docker_image": + continue + + # ensure only available options are allowed if key not in options: raise KeyError( key + " is not implemented. Available keys: " + str(options.keys()) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index 76ebcf0c8..d1ff541cd 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -26,6 +26,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os +import json import shutil from . import query_translate from servicex_codegen.code_generator import ( @@ -62,11 +63,26 @@ def generate_code(self, query, cache_path: str): capabilities_path = os.environ.get( "CAPABILITIES_PATH", "/home/servicex/transformer_capabilities.json" ) + + # Generate query files first to create any metadata + query_translate.generate_files_from_query(query, query_file_path) + + # Copy capabilities file shutil.copyfile( capabilities_path, os.path.join(query_file_path, "transformer_capabilities.json"), ) - query_translate.generate_files_from_query(query, query_file_path) - return GeneratedFileResult(_hash, query_file_path) + # Check if custom transformer image was specified + # image_metadata_path = os.path.join(query_file_path, "image_metadata.json") + # transformer_image = "sslhep/servicex_science_image_topcp:2.17.0-25.2.45" + # if os.path.exists(image_metadata_path): + # with open(image_metadata_path, "r") as f: + # metadata = json.load(f) + # transformer_image = metadata.get("transformer_image", transformer_image) + # + # generated_file = GeneratedFileResult(_hash, query_file_path) + # generated_file.image = transformer_image + # + # return generated_file diff --git a/code_generator_TopCPToolkit/boot.sh b/code_generator_TopCPToolkit/servicex/boot.sh similarity index 54% rename from code_generator_TopCPToolkit/boot.sh rename to code_generator_TopCPToolkit/servicex/boot.sh index 599ab4610..350504fd7 100644 --- a/code_generator_TopCPToolkit/boot.sh +++ b/code_generator_TopCPToolkit/servicex/boot.sh @@ -4,7 +4,7 @@ action=${1:-web_service} if [ "$action" = "web_service" ] ; then mkdir instance - exec gunicorn -b :5000 --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" + exec gunicorn -b :5000 --reload --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" else echo "Unknown action '$action'" fi diff --git a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py index 264b1c51f..92722f3aa 100644 --- a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py +++ b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py @@ -12,24 +12,27 @@ def transform_single_file(file_path: str, output_path: Path, output_format: str) # create input.txt file for event loop and insert file_path as only line with open("input.txt", "w") as f: f.write(file_path) - # move reco.yaml, parton.yaml and particle.yaml if they exit to CONFIG_LOC loacation + + # Get CONFIG_LOC with a sensible default (current working directory) + config_loc = os.environ.get("CONFIG_LOC", os.getcwd()) + + # move reco.yaml, parton.yaml and particle.yaml if they exist to CONFIG_LOC location if os.path.exists("/generated/reco.yaml"): shutil.copyfile( "/generated/reco.yaml", - os.path.join(os.environ.get("CONFIG_LOC"), "reco.yaml"), + os.path.join(config_loc, "reco.yaml"), ) if os.path.exists("/generated/parton.yaml"): shutil.copyfile( "/generated/parton.yaml", - os.path.join(os.environ.get("CONFIG_LOC"), "parton.yaml"), + os.path.join(config_loc, "parton.yaml"), ) if os.path.exists("/generated/particle.yaml"): shutil.copyfile( "/generated/particle.yaml", - os.path.join(os.environ.get("CONFIG_LOC"), "particle.yaml"), + os.path.join(config_loc, "particle.yaml"), ) - generated_transformer.runTop_el() subprocess.run(["mv", "output.root", output_path]) diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index 67067878a..7f0be9c96 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -183,6 +183,7 @@ data: TOPCP_ALLOWED_REPOSITORIES = {{ .Values.codeGen.topcp.allowedRepositories | toJson }} TOPCP_IMAGE_TAG_PATTERN = '{{ .Values.codeGen.topcp.imageTagPattern }}' TOPCP_DEFAULT_BASE_IMAGE = '{{ .Values.codeGen.topcp.defaultBaseImage }}' + TOPCP_DEFAULT_BASE_TAG = '{{ .Values.codeGen.topcp.defaultBaseTag }}' {{- end }} {{- $didFinders := list }} diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index 97d225ae9..02460ca29 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -35,5 +35,14 @@ spec: imagePullPolicy: {{ .pullPolicy }} ports: - containerPort: 5000 + volumeMounts: + - name: host-topcp-volume + mountPath: /home/servicex/servicex + + volumes: + - name: host-topcp-volume + hostPath: + path: /mnt/topcp + type: DirectoryOrCreate {{- end }} {{- end }} diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index 4c1ad0a58..556882012 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -104,15 +104,14 @@ codeGen: topcp: enabled: true image: sslhep/servicex_code_gen_topcp - pullPolicy: Always + pullPolicy: IfNotPresent tag: develop defaultScienceContainerImage: sslhep/servicex_science_image_topcp defaultScienceContainerTag: 2.17.0-25.2.45 - # Configuration for custom image tag validation allowedRepositories: - - "sslhep/servicex_science_image_topcp" - - "registry.gitlab.com/topcp-project/toolkit" - imageTagPattern: "^v?\\d+\\.\\d+\\.\\d+[-_]v?\\d+\\.\\d+$" + - "sslhep" + - "mattshirley" + imageTagPattern: ".*" defaultBaseImage: "sslhep/servicex_science_image_topcp" didFinder: @@ -152,7 +151,8 @@ logging: minio: volumePermissions: image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell auth: rootPassword: leftfoot1 rootUser: miniouser @@ -174,7 +174,8 @@ postgres: postgresql: volumePermissions: image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell global: postgresql: auth: @@ -195,7 +196,8 @@ rabbitmq: volumePermissions: enabled: true image: - repository: bitnami/bitnami-shell-archived +# repository: bitnami/bitnami-shell-archived + repository: bitnami/os-shell extraConfiguration: |- consumer_timeout = 3600000 @@ -218,8 +220,8 @@ transformer: sidecarImage: sslhep/servicex_sidecar_transformer sidecarTag: develop - sidecarPullPolicy: Always - scienceContainerPullPolicy: Always + sidecarPullPolicy: IfNotPresent + scienceContainerPullPolicy: IfNotPresent language: python exec: # replace me diff --git a/servicex_app/pyproject.toml b/servicex_app/pyproject.toml index 7dab5eaa3..7f70bdc26 100644 --- a/servicex_app/pyproject.toml +++ b/servicex_app/pyproject.toml @@ -34,7 +34,7 @@ blinker = "^1.5" pre-commit = "^2.20.0" minio = "^7.1.12" flask-migrate = "^3.1.0" -psycopg2 = "^2.9.5" +psycopg2-binary = "^2.9.5" python-logstash = "^0.4.8" humanize = "^4.4.0" gunicorn = "^23.0.0" diff --git a/servicex_app/servicex_app/code_gen_adapter.py b/servicex_app/servicex_app/code_gen_adapter.py index b4361acd2..7917a0eb7 100644 --- a/servicex_app/servicex_app/code_gen_adapter.py +++ b/servicex_app/servicex_app/code_gen_adapter.py @@ -25,6 +25,8 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from typing import Optional + import requests from requests_toolbelt.multipart import decoder @@ -43,7 +45,11 @@ def post_request(self, post_url, post_obj): return result def generate_code_for_selection( - self, request_record: TransformRequest, namespace: str, user_codegen_name: str + self, + request_record: TransformRequest, + namespace: str, + user_codegen_name: str, + custom_image: Optional[str] = None, ) -> tuple[str, str, str, str]: """ Generates the C++ code for a request's selection string. @@ -68,6 +74,8 @@ def generate_code_for_selection( if not post_url: raise ValueError(f"{user_codegen_name}, code generator unavailable for use") + print("generate_code_for_selection") + print(request_record.selection) result = self.post_request( post_url + "/servicex/generated-code", post_obj={ diff --git a/servicex_app/servicex_app/docker_repo_adapter.py b/servicex_app/servicex_app/docker_repo_adapter.py index 01976ed40..46e8773cd 100644 --- a/servicex_app/servicex_app/docker_repo_adapter.py +++ b/servicex_app/servicex_app/docker_repo_adapter.py @@ -64,3 +64,85 @@ def check_image_exists(self, tagged_image: str) -> bool: f"last updated {r.json()['last_updated']}" ) return True + + @servicex_retry() + def get_image_manifest(self, repo: str, image: str, tag: str) -> requests.Response: + """Get Docker image manifest from registry API v2.""" + # Try Docker Hub v2 API first for manifest + query = f"https://registry-1.docker.io/v2/{repo}/{image}/manifests/{tag}" + headers = {"Accept": "application/vnd.docker.distribution.manifest.v2+json"} + try: + r = requests.get(query, headers=headers, timeout=REQUEST_TIMEOUT) + if r.status_code == 200: + return r + except Exception: + pass + + # Fall back to Docker Hub v1 API for basic info + query = f"{self.registry_endpoint}/v2/repositories/{repo}/{image}/tags/{tag}" + r = requests.get(query, timeout=REQUEST_TIMEOUT) + return r + + def get_image_info(self, tagged_image: str) -> dict: + """ + Get detailed information about a Docker image including layers and configuration. + + :param tagged_image: Full Docker image name, e.g. "sslhep/servicex_app:latest" + :return: Dictionary containing image metadata, layers, config, etc. + """ + search_result = re.search("(.+)/(.+):(.+)", tagged_image) + if not search_result or len(search_result.groups()) != 3: + current_app.logger.warning(f"Invalid image format: {tagged_image}") + return None + + (repo, image, tag) = search_result.groups() + + try: + # Get manifest/detailed info + r = self.get_image_manifest(repo, image, tag) + + if r.status_code != 200: + current_app.logger.warning( + f"Failed to get image info for {tagged_image}: {r.status_code}" + ) + return None + + manifest_data = r.json() + + # Extract relevant information depending on API version + image_info = { + "digest": manifest_data.get("digest"), + "layers": [], + "config": {}, + "history": [], + } + + # Handle Docker Registry v2 manifest format + if "layers" in manifest_data: + image_info["layers"] = [ + layer.get("digest", "") for layer in manifest_data["layers"] + ] + + # Handle config blob reference + if "config" in manifest_data: + config_digest = manifest_data["config"].get("digest") + if config_digest: + # For full implementation, we'd fetch the config blob here + # For now, store the reference + image_info["config"] = {"digest": config_digest} + + # Handle Docker Hub v1 API response format + if "last_updated" in manifest_data: + image_info["last_updated"] = manifest_data["last_updated"] + + # Add some basic metadata + image_info["tag_info"] = manifest_data + + current_app.logger.debug(f"Retrieved image info for {tagged_image}") + return image_info + + except Exception as e: + current_app.logger.warning( + f"Error retrieving image info for {tagged_image}: {str(e)}" + ) + return None diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 07a7dce3b..5d977fc98 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -27,6 +27,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid import re +import json from datetime import datetime, timezone from typing import Optional, List @@ -39,84 +40,7 @@ from servicex_app.resources.servicex_resource import ServiceXResource from werkzeug.exceptions import BadRequest - -def validate_custom_image_tag( - image_tag: str, config: dict, codegen_name: str -) -> tuple[str, str]: - """ - Validate custom image tag for TopCP transformations. - - :param image_tag: The custom image tag provided by user - :param config: Flask application configuration - :param codegen_name: The code generator name (e.g., 'topcp') - :returns: tuple of (validated_image_name, validated_tag) - :raises: BadRequest if validation fails - """ - if not image_tag or not image_tag.strip(): - return None, None - - # Only validate for TopCP requests - if codegen_name != "topcp": - return None, None - - # Get validation configuration (with defaults) - allowed_repos = config.get( - "TOPCP_ALLOWED_REPOSITORIES", - [ - "sslhep/servicex_science_image_topcp", - "registry.gitlab.com/topcp-project/toolkit", - ], - ) - tag_pattern = config.get( - "TOPCP_IMAGE_TAG_PATTERN", r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$" - ) - default_base_image = config.get( - "TOPCP_DEFAULT_BASE_IMAGE", "sslhep/servicex_science_image_topcp" - ) - - # Validate tag format - if not re.match(tag_pattern, image_tag): - raise BadRequest( - f"Invalid TopCP image tag format: {image_tag}. Expected format: v2.20.0_v0.2" - ) - - # Validate that the default base image is in the allowed repositories - if default_base_image not in allowed_repos: - raise BadRequest( - f"Default base image {default_base_image} is not in allowed repositories: " - f"{allowed_repos}" - ) - - validated_image = default_base_image - validated_tag = image_tag - - return validated_image, validated_tag - - class SubmitTransformationRequest(ServiceXResource): - - def _extract_custom_image_tag( - self, selection: str, codegen_name: str - ) -> Optional[str]: - """ - Extract custom image tag from TopCP selection query. - - :param selection: The query selection string - :param codegen_name: The code generator name - :returns: Custom image tag if present and valid, None otherwise - """ - if codegen_name != "topcp": - return None - - try: - import json - - query = json.loads(selection) - return query.get("image_tag") - except (json.JSONDecodeError, AttributeError): - # Invalid JSON or non-dict selection - return None - @classmethod def make_api( cls, @@ -243,6 +167,9 @@ def post(self): file_list = args.get("file-list") user_codegen_name = args.get("codegen") + print("CODEGEN!!!") + print(config["CODE_GEN_IMAGES"]) + print(user_codegen_name) code_gen_image_name = config["CODE_GEN_IMAGES"].get(user_codegen_name, None) namespace = config["TRANSFORMER_NAMESPACE"] @@ -271,6 +198,9 @@ def post(self): # TODO: need to check to make sure bucket was created # WHat happens if object-store and object_store is None? + print("SELECTION!!!") + print(args["selection"]) + request_rec = TransformRequest( request_id=str(request_id), title=args.get("title"), @@ -289,10 +219,12 @@ def post(self): files=0, ) - # Extract custom image tag from TopCP queries before code generation - custom_image_tag = self._extract_custom_image_tag( - request_rec.selection, user_codegen_name - ) + + selection = json.loads(args["selection"]) + custom_docker_image = None + if "docker_image" in selection: + custom_docker_image = selection["docker_image"] + del selection["docker_image"] # The first thing to do is make sure the requested selection is correct, # and can generate the requested code @@ -305,27 +237,11 @@ def post(self): request_rec, namespace, user_codegen_name ) - # Handle custom image tag for TopCP - if custom_image_tag: - try: - validated_image, validated_tag = validate_custom_image_tag( - custom_image_tag, config, user_codegen_name - ) - if validated_image and validated_tag: - # Override the default science container image - request_rec.image = f"{validated_image}:{validated_tag}" - current_app.logger.info( - f"Using custom TopCP image: {request_rec.image}", - extra={"requestId": request_id}, - ) - else: - request_rec.image = codegen_transformer_image - except BadRequest as e: - current_app.logger.error( - f"Invalid custom image tag: {str(e)}", - extra={"requestId": request_id}, - ) - return {"message": str(e)}, 400 + print("TEST!!") + print(codegen_transformer_image) + + if custom_docker_image: + request_rec.image = custom_docker_image else: request_rec.image = codegen_transformer_image From 03864761e6a6182c3fd4ea89edda8be0355eb15b Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 21 Oct 2025 13:13:52 -0500 Subject: [PATCH 09/54] update directory structure --- Procfile | 4 ++-- code_generator_TopCPToolkit/Dockerfile | 4 ++-- code_generator_TopCPToolkit/{servicex => }/boot.sh | 0 helm/servicex/templates/codegen/deployment.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename code_generator_TopCPToolkit/{servicex => }/boot.sh (100%) mode change 100644 => 100755 diff --git a/Procfile b/Procfile index 8ddfaea4b..8b0636d50 100644 --- a/Procfile +++ b/Procfile @@ -1,5 +1,5 @@ -minikube-mount-app: minikube mount $CHART_DIR:/mnt/servicex -minikube-mount-topcp: minikube mount $LOCAL_DIR/code_generator_TopCPToolkit/servicex:/mnt/topcp +minikube-mount-app: minikube mount $LOCAL_DIR:/mnt/servicex +minikube-mount-topcp: minikube mount $LOCAL_DIR/code_generator_TopCPToolkit:/mnt/topcp helm-install: sleep 5; cd $CHART_DIR && helm install -f $VALUES_FILE servicex . && tail -f /dev/null port-forward-app: sleep 30 && cd $LOCAL_DIR && bash local/port-forward.sh app port-forward-minio: sleep 20 && cd $LOCAL_DIR && bash local/port-forward.sh minio diff --git a/code_generator_TopCPToolkit/Dockerfile b/code_generator_TopCPToolkit/Dockerfile index 98136d379..79a3a34ec 100644 --- a/code_generator_TopCPToolkit/Dockerfile +++ b/code_generator_TopCPToolkit/Dockerfile @@ -26,11 +26,11 @@ RUN chmod 777 -R servicex COPY app.conf . RUN chmod 755 app.conf -RUN chmod 755 /home/servicex/servicex/boot.sh +RUN chmod 755 /home/servicex/boot.sh USER servicex ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf" EXPOSE 5000 -ENTRYPOINT ["/home/servicex/servicex/boot.sh"] +ENTRYPOINT ["/home/servicex/boot.sh"] \ No newline at end of file diff --git a/code_generator_TopCPToolkit/servicex/boot.sh b/code_generator_TopCPToolkit/boot.sh old mode 100644 new mode 100755 similarity index 100% rename from code_generator_TopCPToolkit/servicex/boot.sh rename to code_generator_TopCPToolkit/boot.sh diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index 02460ca29..8f029b61f 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -37,7 +37,7 @@ spec: - containerPort: 5000 volumeMounts: - name: host-topcp-volume - mountPath: /home/servicex/servicex + mountPath: /home/servicex volumes: - name: host-topcp-volume From d0776b043d20da1edc0f6b21b726cb3c99305b1d Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 21 Oct 2025 16:13:18 -0500 Subject: [PATCH 10/54] tighten flake8 --- .../TopCP_code_generator/query_translate.py | 9 --------- .../TopCP_code_generator/request_translator.py | 13 ------------- .../servicex/templates/transform_single_file.py | 1 - .../servicex_app/resources/transformation/submit.py | 5 +---- 4 files changed, 1 insertion(+), 27 deletions(-) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index 1644e6fad..76e87e96d 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -48,15 +48,6 @@ def generate_files_from_query(query, query_file_path): jquery = json.loads(query) - # transformer_image = jquery.get("docker_image", "sslhep/servicex_science_image_topcp:2.17.0-25.2.45") - # metadata = { - # "transformer_image": transformer_image - # } - # with open( - # os.path.join(query_file_path, "image_metadata.json"), "w" - # ) as metadata_file: - # json.dump(metadata, metadata_file) - runTopCommand = [ "runTop_el.py", "-i", diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index d1ff541cd..8e8a3839a 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -26,7 +26,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -import json import shutil from . import query_translate from servicex_codegen.code_generator import ( @@ -74,15 +73,3 @@ def generate_code(self, query, cache_path: str): ) return GeneratedFileResult(_hash, query_file_path) - # Check if custom transformer image was specified - # image_metadata_path = os.path.join(query_file_path, "image_metadata.json") - # transformer_image = "sslhep/servicex_science_image_topcp:2.17.0-25.2.45" - # if os.path.exists(image_metadata_path): - # with open(image_metadata_path, "r") as f: - # metadata = json.load(f) - # transformer_image = metadata.get("transformer_image", transformer_image) - # - # generated_file = GeneratedFileResult(_hash, query_file_path) - # generated_file.image = transformer_image - # - # return generated_file diff --git a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py index 92722f3aa..d42db1585 100644 --- a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py +++ b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py @@ -3,7 +3,6 @@ from pathlib import Path import subprocess import shutil -import generated_transformer instance = os.environ.get("INSTANCE_NAME", "Unknown") diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 5d977fc98..6f1bea80b 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -26,7 +26,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid -import re import json from datetime import datetime, timezone from typing import Optional, List @@ -40,6 +39,7 @@ from servicex_app.resources.servicex_resource import ServiceXResource from werkzeug.exceptions import BadRequest + class SubmitTransformationRequest(ServiceXResource): @classmethod def make_api( @@ -100,7 +100,6 @@ def _initialize_dataset_manager( request_id: str, config: dict, ) -> DatasetManager: - # did xor file_list if bool(did) == bool(file_list): raise BadRequest("Must provide did or file-list but not both") @@ -152,7 +151,6 @@ def post(self): uuid.uuid4() ) # make sure we have a request id for all messages try: - try: args = self.parser.parse_args() except BadRequest as bad_request: @@ -219,7 +217,6 @@ def post(self): files=0, ) - selection = json.loads(args["selection"]) custom_docker_image = None if "docker_image" in selection: From f1b7b43dffda323a3c0e8b95f888568b0d9b2d39 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 21 Oct 2025 16:34:53 -0500 Subject: [PATCH 11/54] remove comments and unused env vars --- helm/servicex/templates/app/configmap.yaml | 3 --- servicex_app/servicex_app/code_gen_adapter.py | 2 -- .../servicex_app/resources/transformation/submit.py | 9 --------- 3 files changed, 14 deletions(-) diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index e7615fd91..e35ec9645 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -186,9 +186,6 @@ data: {{- if .Values.codeGen.topcp.enabled }} # TopCP custom image configuration TOPCP_ALLOWED_REPOSITORIES = {{ .Values.codeGen.topcp.allowedRepositories | toJson }} - TOPCP_IMAGE_TAG_PATTERN = '{{ .Values.codeGen.topcp.imageTagPattern }}' - TOPCP_DEFAULT_BASE_IMAGE = '{{ .Values.codeGen.topcp.defaultBaseImage }}' - TOPCP_DEFAULT_BASE_TAG = '{{ .Values.codeGen.topcp.defaultBaseTag }}' {{- end }} {{- $didFinders := list }} diff --git a/servicex_app/servicex_app/code_gen_adapter.py b/servicex_app/servicex_app/code_gen_adapter.py index 7917a0eb7..b6ede3034 100644 --- a/servicex_app/servicex_app/code_gen_adapter.py +++ b/servicex_app/servicex_app/code_gen_adapter.py @@ -74,8 +74,6 @@ def generate_code_for_selection( if not post_url: raise ValueError(f"{user_codegen_name}, code generator unavailable for use") - print("generate_code_for_selection") - print(request_record.selection) result = self.post_request( post_url + "/servicex/generated-code", post_obj={ diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 6f1bea80b..82eaaeb9f 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -165,9 +165,6 @@ def post(self): file_list = args.get("file-list") user_codegen_name = args.get("codegen") - print("CODEGEN!!!") - print(config["CODE_GEN_IMAGES"]) - print(user_codegen_name) code_gen_image_name = config["CODE_GEN_IMAGES"].get(user_codegen_name, None) namespace = config["TRANSFORMER_NAMESPACE"] @@ -196,9 +193,6 @@ def post(self): # TODO: need to check to make sure bucket was created # WHat happens if object-store and object_store is None? - print("SELECTION!!!") - print(args["selection"]) - request_rec = TransformRequest( request_id=str(request_id), title=args.get("title"), @@ -234,9 +228,6 @@ def post(self): request_rec, namespace, user_codegen_name ) - print("TEST!!") - print(codegen_transformer_image) - if custom_docker_image: request_rec.image = custom_docker_image else: From 8a4fbb7fed83cd63b507fac72319a80710d37f04 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 21:37:35 +0000 Subject: [PATCH 12/54] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- code_generator_TopCPToolkit/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_generator_TopCPToolkit/Dockerfile b/code_generator_TopCPToolkit/Dockerfile index 79a3a34ec..6da3bd25b 100644 --- a/code_generator_TopCPToolkit/Dockerfile +++ b/code_generator_TopCPToolkit/Dockerfile @@ -33,4 +33,4 @@ USER servicex ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf" EXPOSE 5000 -ENTRYPOINT ["/home/servicex/boot.sh"] \ No newline at end of file +ENTRYPOINT ["/home/servicex/boot.sh"] From 0fd050d9277f7991a3f45013bf379285b7e0c8e1 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 23 Oct 2025 14:12:34 -0500 Subject: [PATCH 13/54] rewrite custom_docker_image logic --- .../servicex_app/resources/transformation/submit.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 82eaaeb9f..471638b91 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -211,11 +211,14 @@ def post(self): files=0, ) - selection = json.loads(args["selection"]) + print(f"selection: {args.get('selection')}") custom_docker_image = None - if "docker_image" in selection: - custom_docker_image = selection["docker_image"] - del selection["docker_image"] + try: + selection = json.loads(args["selection"]) + if "docker_image" in selection: + custom_docker_image = selection["docker_image"] + except json.decoder.JSONDecodeError: + pass # The first thing to do is make sure the requested selection is correct, # and can generate the requested code From d7bb6cd0a49180b7a5f976aea4e0a47982e2386c Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 28 Oct 2025 15:30:27 -0500 Subject: [PATCH 14/54] add back runTop_el --- .../TopCP_code_generator/query_translate.py | 4 - .../templates/transform_single_file.py | 2 + .../templates/codegen/deployment.yaml | 4 + servicex_app/pyproject.toml | 2 +- servicex_app/servicex_app/code_gen_adapter.py | 2 +- .../servicex_app/docker_repo_adapter.py | 82 ------------------- 6 files changed, 8 insertions(+), 88 deletions(-) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index 76e87e96d..4923b0d6a 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -39,7 +39,6 @@ "docker_image": { "properType": str, "properTypeString": "string", - "default": "sslhep/servicex_science_image_topcp:2.17.0-25.2.45", "optional": True, }, } @@ -70,9 +69,6 @@ def generate_files_from_query(query, query_file_path): ) for key in jquery: - if key == "docker_image": - continue - # ensure only available options are allowed if key not in options: raise KeyError( diff --git a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py index d42db1585..7c2c047b1 100644 --- a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py +++ b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py @@ -3,6 +3,7 @@ from pathlib import Path import subprocess import shutil +import generated_transformer instance = os.environ.get("INSTANCE_NAME", "Unknown") @@ -32,6 +33,7 @@ def transform_single_file(file_path: str, output_path: Path, output_format: str) os.path.join(config_loc, "particle.yaml"), ) + generated_transformer.runTop_el() subprocess.run(["mv", "output.root", output_path]) diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index 8f029b61f..e4b02ffe9 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -35,14 +35,18 @@ spec: imagePullPolicy: {{ .pullPolicy }} ports: - containerPort: 5000 + {{- if eq $codeGenName "topcp" }} volumeMounts: - name: host-topcp-volume mountPath: /home/servicex + {{- end }} + {{- if eq $codeGenName "topcp" }} volumes: - name: host-topcp-volume hostPath: path: /mnt/topcp type: DirectoryOrCreate + {{- end }} {{- end }} {{- end }} diff --git a/servicex_app/pyproject.toml b/servicex_app/pyproject.toml index 1b9b01fa1..5964b38b2 100644 --- a/servicex_app/pyproject.toml +++ b/servicex_app/pyproject.toml @@ -34,7 +34,7 @@ blinker = "^1.5" pre-commit = "^2.20.0" minio = "^7.1.12" flask-migrate = "^3.1.0" -psycopg2-binary = "^2.9.5" +psycopg2 = "^2.9.5" python-logstash = "^0.4.8" humanize = "^4.4.0" gunicorn = "^23.0.0" diff --git a/servicex_app/servicex_app/code_gen_adapter.py b/servicex_app/servicex_app/code_gen_adapter.py index b6ede3034..c088f3c06 100644 --- a/servicex_app/servicex_app/code_gen_adapter.py +++ b/servicex_app/servicex_app/code_gen_adapter.py @@ -59,7 +59,7 @@ def generate_code_for_selection( :param namespace: Namespace in which to place resulting ConfigMap. :param user_codegen_name: Name provided by user for selecting the codegen URL from config dictionary - :returns a tuple of (config map name, default transformer image, language, command) + :returns a tuple of (config map name, default transformer image) """ from io import BytesIO from zipfile import ZipFile diff --git a/servicex_app/servicex_app/docker_repo_adapter.py b/servicex_app/servicex_app/docker_repo_adapter.py index 46e8773cd..01976ed40 100644 --- a/servicex_app/servicex_app/docker_repo_adapter.py +++ b/servicex_app/servicex_app/docker_repo_adapter.py @@ -64,85 +64,3 @@ def check_image_exists(self, tagged_image: str) -> bool: f"last updated {r.json()['last_updated']}" ) return True - - @servicex_retry() - def get_image_manifest(self, repo: str, image: str, tag: str) -> requests.Response: - """Get Docker image manifest from registry API v2.""" - # Try Docker Hub v2 API first for manifest - query = f"https://registry-1.docker.io/v2/{repo}/{image}/manifests/{tag}" - headers = {"Accept": "application/vnd.docker.distribution.manifest.v2+json"} - try: - r = requests.get(query, headers=headers, timeout=REQUEST_TIMEOUT) - if r.status_code == 200: - return r - except Exception: - pass - - # Fall back to Docker Hub v1 API for basic info - query = f"{self.registry_endpoint}/v2/repositories/{repo}/{image}/tags/{tag}" - r = requests.get(query, timeout=REQUEST_TIMEOUT) - return r - - def get_image_info(self, tagged_image: str) -> dict: - """ - Get detailed information about a Docker image including layers and configuration. - - :param tagged_image: Full Docker image name, e.g. "sslhep/servicex_app:latest" - :return: Dictionary containing image metadata, layers, config, etc. - """ - search_result = re.search("(.+)/(.+):(.+)", tagged_image) - if not search_result or len(search_result.groups()) != 3: - current_app.logger.warning(f"Invalid image format: {tagged_image}") - return None - - (repo, image, tag) = search_result.groups() - - try: - # Get manifest/detailed info - r = self.get_image_manifest(repo, image, tag) - - if r.status_code != 200: - current_app.logger.warning( - f"Failed to get image info for {tagged_image}: {r.status_code}" - ) - return None - - manifest_data = r.json() - - # Extract relevant information depending on API version - image_info = { - "digest": manifest_data.get("digest"), - "layers": [], - "config": {}, - "history": [], - } - - # Handle Docker Registry v2 manifest format - if "layers" in manifest_data: - image_info["layers"] = [ - layer.get("digest", "") for layer in manifest_data["layers"] - ] - - # Handle config blob reference - if "config" in manifest_data: - config_digest = manifest_data["config"].get("digest") - if config_digest: - # For full implementation, we'd fetch the config blob here - # For now, store the reference - image_info["config"] = {"digest": config_digest} - - # Handle Docker Hub v1 API response format - if "last_updated" in manifest_data: - image_info["last_updated"] = manifest_data["last_updated"] - - # Add some basic metadata - image_info["tag_info"] = manifest_data - - current_app.logger.debug(f"Retrieved image info for {tagged_image}") - return image_info - - except Exception as e: - current_app.logger.warning( - f"Error retrieving image info for {tagged_image}: {str(e)}" - ) - return None From e5024d01c56addc15cd6e8ac460c6b9cea4854a1 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 28 Oct 2025 15:33:02 -0500 Subject: [PATCH 15/54] remove old test files --- .../tests/test_custom_image_tag.py | 92 ---------- .../test_custom_image_validation.py | 167 ------------------ 2 files changed, 259 deletions(-) delete mode 100644 code_generator_TopCPToolkit/tests/test_custom_image_tag.py delete mode 100644 servicex_app/servicex_app_test/test_custom_image_validation.py diff --git a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py b/code_generator_TopCPToolkit/tests/test_custom_image_tag.py deleted file mode 100644 index 19ffd5344..000000000 --- a/code_generator_TopCPToolkit/tests/test_custom_image_tag.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Test custom image tag functionality for TopCP code generator.""" - -import json -import os -import tempfile -import unittest - -from servicex.TopCP_code_generator.query_translate import generate_files_from_query - - -class TestCustomImageTag(unittest.TestCase): - def setUp(self): - self.temp_dir = tempfile.mkdtemp() - - def tearDown(self): - import shutil - - shutil.rmtree(self.temp_dir) - - def test_custom_image_tag_in_query_validation(self): - """Test that custom image tag is accepted in query validation.""" - query = { - "reco": "reco_config_content", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - "image_tag": "v2.20.0_v0.2", - } - - # This should not raise an exception - generate_files_from_query(json.dumps(query), self.temp_dir) - - def test_image_tag_not_required(self): - """Test that image_tag is optional and doesn't break existing queries.""" - query = { - "reco": "reco_config_content", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - } - - # This should not raise an exception - generate_files_from_query(json.dumps(query), self.temp_dir) - - def test_image_tag_none_allowed(self): - """Test that None image tag is accepted.""" - query = { - "reco": "reco_config_content", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - "image_tag": None, - } - - # This should not raise an exception - generate_files_from_query(json.dumps(query), self.temp_dir) - - def test_generated_files_created_with_image_tag(self): - """Test that all expected files are generated when image_tag is present.""" - query = { - "reco": "reco_config_content", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - "image_tag": "v2.20.0_v0.2", - } - - generate_files_from_query(json.dumps(query), self.temp_dir) - - # Check that reco.yaml is created - reco_file = os.path.join(self.temp_dir, "reco.yaml") - self.assertTrue(os.path.exists(reco_file)) - - with open(reco_file, "r") as f: - content = f.read() - self.assertEqual(content, "reco_config_content") - - # Check that generated_transformer.py is created - transformer_file = os.path.join(self.temp_dir, "generated_transformer.py") - self.assertTrue(os.path.exists(transformer_file)) - - -if __name__ == "__main__": - unittest.main() diff --git a/servicex_app/servicex_app_test/test_custom_image_validation.py b/servicex_app/servicex_app_test/test_custom_image_validation.py deleted file mode 100644 index 9bfc9f4ec..000000000 --- a/servicex_app/servicex_app_test/test_custom_image_validation.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Test custom image tag validation for TopCP transformations.""" - -import json -import unittest -from werkzeug.exceptions import BadRequest - -from servicex_app.resources.transformation.submit import ( - validate_custom_image_tag, - SubmitTransformationRequest, -) - - -class TestCustomImageValidation(unittest.TestCase): - - def setUp(self): - self.config = { - "TOPCP_ALLOWED_REPOSITORIES": [ - "sslhep/servicex_science_image_topcp", - "registry.gitlab.com/topcp-project/toolkit", - ], - "TOPCP_IMAGE_TAG_PATTERN": r"^v?\d+\.\d+\.\d+[-_]v?\d+\.\d+$", - "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", - } - - def test_valid_image_tag_format(self): - """Test that valid image tag formats are accepted.""" - test_cases = ["v2.20.0_v0.2", "2.20.0_v0.2", "v1.15.0-v1.0", "3.0.0_v2.5"] - - for tag in test_cases: - with self.subTest(tag=tag): - image, result_tag = validate_custom_image_tag(tag, self.config, "topcp") - self.assertEqual(image, "sslhep/servicex_science_image_topcp") - self.assertEqual(result_tag, tag) - - def test_invalid_image_tag_format(self): - """Test that invalid image tag formats are rejected.""" - test_cases = [ - "latest", - "2.20.0", - "v2.20.0", - "invalid-tag", - "v2.20.0_v0.2.1", # Too many version components - "2.20_v0.2", # Missing patch version - ] - - for tag in test_cases: - with self.subTest(tag=tag): - with self.assertRaises(BadRequest): - validate_custom_image_tag(tag, self.config, "topcp") - - def test_non_topcp_codegen_returns_none(self): - """Test that non-TopCP code generators return None.""" - image, tag = validate_custom_image_tag("v2.20.0_v0.2", self.config, "uproot") - self.assertIsNone(image) - self.assertIsNone(tag) - - def test_empty_image_tag_returns_none(self): - """Test that empty image tag returns None.""" - test_cases = [None, "", " "] - - for tag in test_cases: - with self.subTest(tag=tag): - image, result_tag = validate_custom_image_tag(tag, self.config, "topcp") - self.assertIsNone(image) - self.assertIsNone(result_tag) - - def test_missing_config_uses_defaults(self): - """Test that missing configuration uses default values.""" - minimal_config = {} - - image, tag = validate_custom_image_tag("v2.20.0_v0.2", minimal_config, "topcp") - self.assertEqual(image, "sslhep/servicex_science_image_topcp") - self.assertEqual(tag, "v2.20.0_v0.2") - - def test_repository_validation_success(self): - """Test that repository validation passes when default image is in allowed repos.""" - config = { - "TOPCP_ALLOWED_REPOSITORIES": [ - "sslhep/servicex_science_image_topcp", - "registry.gitlab.com/topcp-project/toolkit", - ], - "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", - } - - image, tag = validate_custom_image_tag("v2.20.0_v0.2", config, "topcp") - self.assertEqual(image, "sslhep/servicex_science_image_topcp") - self.assertEqual(tag, "v2.20.0_v0.2") - - def test_repository_validation_failure(self): - """Test that repository validation fails when default image is not in allowed repos.""" - config = { - "TOPCP_ALLOWED_REPOSITORIES": [ - "registry.gitlab.com/topcp-project/toolkit", - ], - "TOPCP_DEFAULT_BASE_IMAGE": "sslhep/servicex_science_image_topcp", - } - - with self.assertRaises(BadRequest): - validate_custom_image_tag("v2.20.0_v0.2", config, "topcp") - - -class TestImageTagExtraction(unittest.TestCase): - - def setUp(self): - self.submit_request = SubmitTransformationRequest() - - def test_extract_image_tag_from_topcp_query(self): - """Test extraction of image tag from TopCP query.""" - query = { - "reco": "reco_config", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - "image_tag": "v2.20.0_v0.2", - } - - result = self.submit_request._extract_custom_image_tag( - json.dumps(query), "topcp" - ) - self.assertEqual(result, "v2.20.0_v0.2") - - def test_extract_image_tag_missing_returns_none(self): - """Test that missing image tag returns None.""" - query = { - "reco": "reco_config", - "parton": None, - "particle": None, - "max_events": -1, - "no_systematics": False, - "no_filter": False, - } - - result = self.submit_request._extract_custom_image_tag( - json.dumps(query), "topcp" - ) - self.assertIsNone(result) - - def test_extract_image_tag_non_topcp_returns_none(self): - """Test that non-TopCP codegen returns None.""" - query = {"selection": "some_selection", "image_tag": "v2.20.0_v0.2"} - - result = self.submit_request._extract_custom_image_tag( - json.dumps(query), "uproot" - ) - self.assertIsNone(result) - - def test_extract_image_tag_invalid_json_returns_none(self): - """Test that invalid JSON returns None.""" - invalid_json = "{ invalid json }" - - result = self.submit_request._extract_custom_image_tag(invalid_json, "topcp") - self.assertIsNone(result) - - def test_extract_image_tag_none_value(self): - """Test that None image tag value is returned as None.""" - query = {"reco": "reco_config", "image_tag": None} - - result = self.submit_request._extract_custom_image_tag( - json.dumps(query), "topcp" - ) - self.assertIsNone(result) - - -if __name__ == "__main__": - unittest.main() From 0125f414d6ec7586e94f9ff41105fb4325a54bc5 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Wed, 29 Oct 2025 13:50:07 -0500 Subject: [PATCH 16/54] fix local volume mount --- helm/servicex/templates/codegen/deployment.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index e4b02ffe9..b14e66ed3 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -36,17 +36,25 @@ spec: ports: - containerPort: 5000 {{- if eq $codeGenName "topcp" }} + {{- if eq $.Values.app.environment "dev" }} + {{- if $.Values.app.reload }} volumeMounts: - name: host-topcp-volume mountPath: /home/servicex {{- end }} + {{- end }} + {{- end }} {{- if eq $codeGenName "topcp" }} + {{- if eq $.Values.app.environment "dev" }} + {{- if $.Values.app.reload }} volumes: - name: host-topcp-volume hostPath: path: /mnt/topcp type: DirectoryOrCreate {{- end }} + {{- end }} + {{- end }} {{- end }} {{- end }} From b49b9ee32f39847bbc825087f5357963a515b22f Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Wed, 29 Oct 2025 14:15:21 -0500 Subject: [PATCH 17/54] change reload to mountLocal --- helm/servicex/templates/codegen/deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index b14e66ed3..b5204764e 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -37,7 +37,7 @@ spec: - containerPort: 5000 {{- if eq $codeGenName "topcp" }} {{- if eq $.Values.app.environment "dev" }} - {{- if $.Values.app.reload }} + {{- if $.Values.app.mountLocal }} volumeMounts: - name: host-topcp-volume mountPath: /home/servicex @@ -47,7 +47,7 @@ spec: {{- if eq $codeGenName "topcp" }} {{- if eq $.Values.app.environment "dev" }} - {{- if $.Values.app.reload }} + {{- if $.Values.app.mountLocal }} volumes: - name: host-topcp-volume hostPath: From 668633645997fe1cee4cccf9f7e9b6c836fb503f Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 30 Oct 2025 20:50:35 -0500 Subject: [PATCH 18/54] update helm charts and other config stuff --- code_generator_TopCPToolkit/boot.sh | 14 ++++- .../request_translator.py | 2 - .../templates/transform_single_file.py | 4 +- helm/servicex/templates/app/configmap.yaml | 2 +- .../templates/codegen/deployment.yaml | 5 ++ helm/servicex/values.yaml | 4 +- .../resources/transformation/submit.py | 55 ++++++++++++++++--- 7 files changed, 68 insertions(+), 18 deletions(-) diff --git a/code_generator_TopCPToolkit/boot.sh b/code_generator_TopCPToolkit/boot.sh index 350504fd7..76cd19549 100755 --- a/code_generator_TopCPToolkit/boot.sh +++ b/code_generator_TopCPToolkit/boot.sh @@ -1,10 +1,22 @@ #!/bin/bash +# Initialize reload flag +RELOAD="" + +# Parse command line arguments +for arg in "$@" +do + if [ "$arg" = "--reload" ]; then + RELOAD="--reload" + break + fi +done + # Running the web server? action=${1:-web_service} if [ "$action" = "web_service" ] ; then mkdir instance - exec gunicorn -b :5000 --reload --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" + exec gunicorn -b :5000 $RELOAD --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" else echo "Unknown action '$action'" fi diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index 8e8a3839a..3a7ef9021 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -63,10 +63,8 @@ def generate_code(self, query, cache_path: str): "CAPABILITIES_PATH", "/home/servicex/transformer_capabilities.json" ) - # Generate query files first to create any metadata query_translate.generate_files_from_query(query, query_file_path) - # Copy capabilities file shutil.copyfile( capabilities_path, os.path.join(query_file_path, "transformer_capabilities.json"), diff --git a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py index 7c2c047b1..7efe01a6d 100644 --- a/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py +++ b/code_generator_TopCPToolkit/servicex/templates/transform_single_file.py @@ -13,10 +13,8 @@ def transform_single_file(file_path: str, output_path: Path, output_format: str) with open("input.txt", "w") as f: f.write(file_path) - # Get CONFIG_LOC with a sensible default (current working directory) - config_loc = os.environ.get("CONFIG_LOC", os.getcwd()) - # move reco.yaml, parton.yaml and particle.yaml if they exist to CONFIG_LOC location + config_loc = os.environ.get("CONFIG_LOC", os.getcwd()) if os.path.exists("/generated/reco.yaml"): shutil.copyfile( "/generated/reco.yaml", diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index e35ec9645..b95b0c681 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -185,7 +185,7 @@ data: {{- if .Values.codeGen.topcp.enabled }} # TopCP custom image configuration - TOPCP_ALLOWED_REPOSITORIES = {{ .Values.codeGen.topcp.allowedRepositories | toJson }} + TOPCP_ALLOWED_IMAGES = {{ .Values.codeGen.topcp.allowedImages | toJson }} {{- end }} {{- $didFinders := list }} diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index b5204764e..6330ec560 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -19,6 +19,11 @@ spec: containers: - name: {{ $.Release.Name }}-code-gen-{{ $codeGenName }} image: {{ .image }}:{{ .tag }} + {{- if eq $.Values.app.environment "dev" }} + {{- if $.Values.app.reload }} + command: [ "./boot.sh", "web_service", "--reload" ] + {{- end }} + {{- end }} env: - name: INSTANCE_NAME value: {{ $.Release.Name }} diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index cda785e1c..ff460f2b0 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -108,8 +108,8 @@ codeGen: tag: develop defaultScienceContainerImage: sslhep/servicex_science_image_topcp defaultScienceContainerTag: 2.17.0-25.2.45 - allowedRepositories: - - "sslhep" + allowedImages: + - "sslhep/servicex_science_image_topcp:" defaultBaseImage: "sslhep/servicex_science_image_topcp" didFinder: diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 471638b91..b4404600e 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -27,6 +27,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid import json +import os from datetime import datetime, timezone from typing import Optional, List @@ -40,6 +41,36 @@ from werkzeug.exceptions import BadRequest +def validate_custom_docker_image(image_name: str) -> bool: + allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES") + + if not allowed_images_json: + raise BadRequest( + "Custom Docker images are not allowed." + ) + + try: + allowed_prefixes = json.loads(allowed_images_json) + + if not isinstance(allowed_prefixes, list): + raise BadRequest( + "TopCP allowed images are improperly configured." + ) + + for prefix in allowed_prefixes: + if image_name.startswith(prefix): + return True + + raise BadRequest( + f"Custom Docker image '{image_name}' not allowed." + ) + + except json.JSONDecodeError as e: + raise BadRequest( + "TopCP allowed images are improperly configured." + ) + + class SubmitTransformationRequest(ServiceXResource): @classmethod def make_api( @@ -211,15 +242,6 @@ def post(self): files=0, ) - print(f"selection: {args.get('selection')}") - custom_docker_image = None - try: - selection = json.loads(args["selection"]) - if "docker_image" in selection: - custom_docker_image = selection["docker_image"] - except json.decoder.JSONDecodeError: - pass - # The first thing to do is make sure the requested selection is correct, # and can generate the requested code ( @@ -231,6 +253,21 @@ def post(self): request_rec, namespace, user_codegen_name ) + custom_docker_image = None + if user_codegen_name == "topcp": + try: + selection = json.loads(args["selection"]) + if "docker_image" in selection: + custom_docker_image = selection["docker_image"] + try: + validate_custom_docker_image(custom_docker_image) + except BadRequest as e: + current_app.logger.error( + str(e), extra={"requestId": request_id} + ) + except json.decoder.JSONDecodeError: + pass + if custom_docker_image: request_rec.image = custom_docker_image else: From 0f4f3f44140f3665f771c92f4f60042582de86cf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 01:50:51 +0000 Subject: [PATCH 19/54] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../resources/transformation/submit.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index b4404600e..38411b165 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -45,30 +45,22 @@ def validate_custom_docker_image(image_name: str) -> bool: allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES") if not allowed_images_json: - raise BadRequest( - "Custom Docker images are not allowed." - ) + raise BadRequest("Custom Docker images are not allowed.") try: allowed_prefixes = json.loads(allowed_images_json) if not isinstance(allowed_prefixes, list): - raise BadRequest( - "TopCP allowed images are improperly configured." - ) + raise BadRequest("TopCP allowed images are improperly configured.") for prefix in allowed_prefixes: if image_name.startswith(prefix): return True - raise BadRequest( - f"Custom Docker image '{image_name}' not allowed." - ) + raise BadRequest(f"Custom Docker image '{image_name}' not allowed.") except json.JSONDecodeError as e: - raise BadRequest( - "TopCP allowed images are improperly configured." - ) + raise BadRequest("TopCP allowed images are improperly configured.") class SubmitTransformationRequest(ServiceXResource): From 8aeff9eb38ed20db850efe3f90fc5443630918b4 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 30 Oct 2025 20:57:03 -0500 Subject: [PATCH 20/54] resolve flake8 --- servicex_app/servicex_app/resources/transformation/submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index b4404600e..2b01e9500 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -65,7 +65,7 @@ def validate_custom_docker_image(image_name: str) -> bool: f"Custom Docker image '{image_name}' not allowed." ) - except json.JSONDecodeError as e: + except json.JSONDecodeError: raise BadRequest( "TopCP allowed images are improperly configured." ) From 981504b9964e8acc5c1ea7de4d3fe26108fdc6c2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 01:58:40 +0000 Subject: [PATCH 21/54] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- servicex_app/servicex_app/resources/transformation/submit.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 6e6512f3d..ef1a50485 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -60,9 +60,7 @@ def validate_custom_docker_image(image_name: str) -> bool: raise BadRequest(f"Custom Docker image '{image_name}' not allowed.") except json.JSONDecodeError: - raise BadRequest( - "TopCP allowed images are improperly configured." - ) + raise BadRequest("TopCP allowed images are improperly configured.") class SubmitTransformationRequest(ServiceXResource): From 28523ed9bf1db86217f0223f4292bd6db63ac172 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 30 Oct 2025 21:16:03 -0500 Subject: [PATCH 22/54] move boot.sh --- code_generator_TopCPToolkit/Dockerfile | 4 ++-- code_generator_TopCPToolkit/{ => servicex}/boot.sh | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename code_generator_TopCPToolkit/{ => servicex}/boot.sh (100%) diff --git a/code_generator_TopCPToolkit/Dockerfile b/code_generator_TopCPToolkit/Dockerfile index 6da3bd25b..98136d379 100644 --- a/code_generator_TopCPToolkit/Dockerfile +++ b/code_generator_TopCPToolkit/Dockerfile @@ -26,11 +26,11 @@ RUN chmod 777 -R servicex COPY app.conf . RUN chmod 755 app.conf -RUN chmod 755 /home/servicex/boot.sh +RUN chmod 755 /home/servicex/servicex/boot.sh USER servicex ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf" EXPOSE 5000 -ENTRYPOINT ["/home/servicex/boot.sh"] +ENTRYPOINT ["/home/servicex/servicex/boot.sh"] diff --git a/code_generator_TopCPToolkit/boot.sh b/code_generator_TopCPToolkit/servicex/boot.sh similarity index 100% rename from code_generator_TopCPToolkit/boot.sh rename to code_generator_TopCPToolkit/servicex/boot.sh From ce26f8f31b521b580758bd17043eb00d62cf02b2 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 30 Oct 2025 21:33:05 -0500 Subject: [PATCH 23/54] update boot.sh command --- helm/servicex/templates/codegen/deployment.yaml | 4 +++- helm/servicex/values.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index 6330ec560..b6b4d97e1 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -20,8 +20,10 @@ spec: - name: {{ $.Release.Name }}-code-gen-{{ $codeGenName }} image: {{ .image }}:{{ .tag }} {{- if eq $.Values.app.environment "dev" }} + {{- if eq $codeGenName "topcp" }} {{- if $.Values.app.reload }} - command: [ "./boot.sh", "web_service", "--reload" ] + command: [ "./servicex/boot.sh", "web_service", "--reload" ] + {{- end }} {{- end }} {{- end }} env: diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index ff460f2b0..4333cdf3e 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -104,7 +104,7 @@ codeGen: topcp: enabled: true image: sslhep/servicex_code_gen_topcp - pullPolicy: Always + pullPolicy: IfNotPresent tag: develop defaultScienceContainerImage: sslhep/servicex_science_image_topcp defaultScienceContainerTag: 2.17.0-25.2.45 From 6c2059f86d1b63190b34ae98a453e9a7cff18eb9 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Fri, 31 Oct 2025 16:19:01 -0500 Subject: [PATCH 24/54] update test coverage --- .../resources/transformation/submit.py | 13 +- .../resources/transformation/test_submit.py | 203 +++++++++++++++++- 2 files changed, 207 insertions(+), 9 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index ef1a50485..7c0f3ab35 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -38,7 +38,7 @@ from servicex_app.did_parser import DIDParser from servicex_app.models import TransformRequest, db, TransformStatus from servicex_app.resources.servicex_resource import ServiceXResource -from werkzeug.exceptions import BadRequest +from werkzeug.exceptions import BadRequest, HTTPException def validate_custom_docker_image(image_name: str) -> bool: @@ -251,14 +251,9 @@ def post(self): selection = json.loads(args["selection"]) if "docker_image" in selection: custom_docker_image = selection["docker_image"] - try: - validate_custom_docker_image(custom_docker_image) - except BadRequest as e: - current_app.logger.error( - str(e), extra={"requestId": request_id} - ) + validate_custom_docker_image(custom_docker_image) except json.decoder.JSONDecodeError: - pass + raise BadRequest("Malformed JSON submitted") if custom_docker_image: request_rec.image = custom_docker_image @@ -310,6 +305,8 @@ def post(self): "Transformation request submitted!", extra={"requestId": request_id} ) return {"request_id": str(request_id)} + except HTTPException: + raise except Exception as eek: current_app.logger.exception( "Got exception while submitting transformation request", diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index 4eba86c1b..e13095ffe 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -25,17 +25,22 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import json +import os from datetime import datetime, timezone -from unittest.mock import ANY +from unittest.mock import ANY, patch +import pytest from celery import Celery from pytest import fixture +from werkzeug.exceptions import BadRequest from servicex_app import LookupResultProcessor from servicex_app.code_gen_adapter import CodeGenAdapter from servicex_app.dataset_manager import DatasetManager from servicex_app.models import Dataset from servicex_app.models import TransformRequest, DatasetStatus, TransformStatus +from servicex_app.resources.transformation.submit import validate_custom_docker_image from servicex_app.transformer_manager import TransformerManager from servicex_app_test.resource_test_base import ResourceTestBase @@ -588,3 +593,199 @@ def test_submit_transformation_with_title( saved_obj = TransformRequest.lookup(request_id) assert saved_obj assert saved_obj.title == title + + +class TestValidateCustomDockerImage: + """Tests for the validate_custom_docker_image function""" + + def test_validate_with_matching_prefix(self): + """Test validation succeeds when image matches an allowed prefix""" + with patch.dict( + os.environ, + {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, + ): + result = validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) + assert result is True + + def test_validate_with_multiple_prefixes(self): + """Test validation with multiple allowed prefixes""" + with patch.dict( + os.environ, + { + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + }, + ): + assert ( + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:latest" + ) + is True + ) + assert ( + validate_custom_docker_image("docker.io/ssl-hep/custom:v1") is True + ) + + def test_validate_with_no_matching_prefix(self): + """Test validation fails when image doesn't match any allowed prefix""" + with patch.dict( + os.environ, + {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, + ): + with pytest.raises(BadRequest, match="not allowed"): + validate_custom_docker_image("unauthorized/image:latest") + + def test_validate_with_no_env_variable(self): + """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" + with patch.dict(os.environ, {}, clear=True): + with pytest.raises(BadRequest, match="Custom Docker images are not allowed"): + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + + def test_validate_with_invalid_json(self): + """Test validation fails with invalid JSON in env variable""" + with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "not-valid-json"}): + with pytest.raises(BadRequest, match="improperly configured"): + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + + def test_validate_with_non_list_json(self): + """Test validation fails when JSON is not a list""" + with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": '{"key": "value"}'}): + with pytest.raises(BadRequest, match="improperly configured"): + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + + def test_validate_with_empty_list(self): + """Test validation fails when allowed list is empty""" + with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "[]"}): + with pytest.raises(BadRequest, match="not allowed"): + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + + +class TestSubmitTransformationRequestCustomImage(ResourceTestBase): + """Tests for custom Docker image feature in transformation requests""" + + @staticmethod + def _generate_transformation_request(**kwargs): + request = { + "did": "123-45-678", + "selection": "test-string", + "result-destination": "object-store", + "result-format": "root-file", + "workers": 10, + "codegen": "topcp", + } + request.update(kwargs) + return request + + @fixture + def mock_dataset_manager_from_did(self, mocker): + dm = mocker.Mock() + dm.dataset = Dataset( + name="rucio://123-45-678", + did_finder="rucio", + lookup_status=DatasetStatus.looking, + last_used=datetime.now(tz=timezone.utc), + last_updated=datetime.fromtimestamp(0), + ) + dm.name = "rucio://123-45-678" + dm.id = 42 + mock_from_did = mocker.patch.object(DatasetManager, "from_did", return_value=dm) + return mock_from_did + + @fixture + def mock_codegen(self, mocker): + mock_code_gen = mocker.MagicMock(CodeGenAdapter) + mock_code_gen.generate_code_for_selection.return_value = ( + "my-cm", + "sslhep/servicex_science_image_topcp:2.17.0", + "bash", + "echo", + ) + return mock_code_gen + + def test_submit_topcp_with_custom_docker_image( + self, mock_dataset_manager_from_did, mock_codegen, mock_app_version + ): + """Test submitting a TopCP transformation with a valid custom docker image""" + extra_config = { + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + } + with patch.dict( + os.environ, + {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, + ): + client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + with client.application.app_context(): + selection_dict = {"docker_image": "sslhep/servicex_science_image_topcp:custom"} + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) + ) + + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 200 + request_id = response.json["request_id"] + + saved_obj = TransformRequest.lookup(request_id) + assert saved_obj + assert saved_obj.image == "sslhep/servicex_science_image_topcp:custom" + + def test_submit_topcp_with_invalid_custom_docker_image( + self, mock_dataset_manager_from_did, mock_codegen + ): + """Test submitting a TopCP transformation with an invalid custom docker image returns 400""" + extra_config = { + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + } + with patch.dict( + os.environ, + {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, + ): + client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + with client.application.app_context(): + selection_dict = {"docker_image": "unauthorized/image:latest"} + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) + ) + + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 400 + + def test_submit_topcp_with_non_json_selection( + self, mock_dataset_manager_from_did, mock_codegen, mock_app_version + ): + """Test submitting a TopCP transformation with non-JSON selection string""" + extra_config = { + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + } + client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + with client.application.app_context(): + request = self._generate_transformation_request(selection="not-json-string") + + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 400 + + def test_submit_custom_topcp_without_env_var( + self, mock_dataset_manager_from_did, mock_codegen + ): + """Test submitting TopCP with custom image when TOPCP_ALLOWED_IMAGES is not set""" + extra_config = { + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + } + with patch.dict(os.environ, {}, clear=True): + client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + with client.application.app_context(): + selection_dict = {"docker_image": "sslhep/servicex_science_image_topcp:custom"} + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) + ) + + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 400 From 3dfd2ad8e5dca333025fb304dc678f74f4017e35 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 21:19:24 +0000 Subject: [PATCH 25/54] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../resources/transformation/test_submit.py | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index e13095ffe..c07ff818b 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -623,9 +623,7 @@ def test_validate_with_multiple_prefixes(self): ) is True ) - assert ( - validate_custom_docker_image("docker.io/ssl-hep/custom:v1") is True - ) + assert validate_custom_docker_image("docker.io/ssl-hep/custom:v1") is True def test_validate_with_no_matching_prefix(self): """Test validation fails when image doesn't match any allowed prefix""" @@ -639,26 +637,36 @@ def test_validate_with_no_matching_prefix(self): def test_validate_with_no_env_variable(self): """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" with patch.dict(os.environ, {}, clear=True): - with pytest.raises(BadRequest, match="Custom Docker images are not allowed"): - validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + with pytest.raises( + BadRequest, match="Custom Docker images are not allowed" + ): + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) def test_validate_with_invalid_json(self): """Test validation fails with invalid JSON in env variable""" with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "not-valid-json"}): with pytest.raises(BadRequest, match="improperly configured"): - validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) def test_validate_with_non_list_json(self): """Test validation fails when JSON is not a list""" with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": '{"key": "value"}'}): with pytest.raises(BadRequest, match="improperly configured"): - validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) def test_validate_with_empty_list(self): """Test validation fails when allowed list is empty""" with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "[]"}): with pytest.raises(BadRequest, match="not allowed"): - validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) class TestSubmitTransformationRequestCustomImage(ResourceTestBase): @@ -714,9 +722,13 @@ def test_submit_topcp_with_custom_docker_image( os.environ, {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, ): - client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) with client.application.app_context(): - selection_dict = {"docker_image": "sslhep/servicex_science_image_topcp:custom"} + selection_dict = { + "docker_image": "sslhep/servicex_science_image_topcp:custom" + } request = self._generate_transformation_request( selection=json.dumps(selection_dict) ) @@ -742,7 +754,9 @@ def test_submit_topcp_with_invalid_custom_docker_image( os.environ, {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, ): - client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) with client.application.app_context(): selection_dict = {"docker_image": "unauthorized/image:latest"} request = self._generate_transformation_request( @@ -761,7 +775,9 @@ def test_submit_topcp_with_non_json_selection( extra_config = { "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} } - client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) with client.application.app_context(): request = self._generate_transformation_request(selection="not-json-string") @@ -778,9 +794,13 @@ def test_submit_custom_topcp_without_env_var( "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} } with patch.dict(os.environ, {}, clear=True): - client = self._test_client(code_gen_service=mock_codegen, extra_config=extra_config) + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) with client.application.app_context(): - selection_dict = {"docker_image": "sslhep/servicex_science_image_topcp:custom"} + selection_dict = { + "docker_image": "sslhep/servicex_science_image_topcp:custom" + } request = self._generate_transformation_request( selection=json.dumps(selection_dict) ) From 5aa97d6421723a2382c036b3c5d9b1b85932d7b5 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Fri, 31 Oct 2025 16:25:58 -0500 Subject: [PATCH 26/54] resolve flake8 --- .../resources/transformation/test_submit.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index e13095ffe..f76d308f0 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -614,7 +614,8 @@ def test_validate_with_multiple_prefixes(self): with patch.dict( os.environ, { - "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + "TOPCP_ALLOWED_IMAGES": + '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' }, ): assert ( @@ -734,7 +735,7 @@ def test_submit_topcp_with_custom_docker_image( def test_submit_topcp_with_invalid_custom_docker_image( self, mock_dataset_manager_from_did, mock_codegen ): - """Test submitting a TopCP transformation with an invalid custom docker image returns 400""" + """Submitting a TopCP transformation with an invalid custom docker image fails""" extra_config = { "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} } From 8cce09658c79fec35c5a4a004add086121337cba Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 21:27:16 +0000 Subject: [PATCH 27/54] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../servicex_app_test/resources/transformation/test_submit.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index f66866126..2559d8711 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -614,8 +614,7 @@ def test_validate_with_multiple_prefixes(self): with patch.dict( os.environ, { - "TOPCP_ALLOWED_IMAGES": - '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' }, ): assert ( From 01c453af046d5aaa50d2f5b4aba4f81066719749 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Fri, 31 Oct 2025 16:32:38 -0500 Subject: [PATCH 28/54] resolve flake8 again --- .../resources/transformation/test_submit.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index 2559d8711..b480d7cc3 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -46,7 +46,6 @@ class TestSubmitTransformationRequest(ResourceTestBase): - @staticmethod def _generate_transformation_request(**kwargs): request = { @@ -437,7 +436,6 @@ def test_submit_transformation_request_no_docker_check( code_gen_service=mock_codegen, ) with client.application.app_context(): - request = self._generate_transformation_request() response = client.post( "/servicex/transformation", json=request, headers=self.fake_header() @@ -565,7 +563,6 @@ def test_submit_transformation_auth_enabled( extra_config={"ENABLE_AUTH": True}, code_gen_service=mock_codegen ) with client.application.app_context(): - response = client.post( "/servicex/transformation", json=self._generate_transformation_request(), @@ -614,7 +611,8 @@ def test_validate_with_multiple_prefixes(self): with patch.dict( os.environ, { - "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + "TOPCP_ALLOWED_IMAGES": """["sslhep/servicex_science_image_topcp:", + "docker.io/ssl-hep/"]""" }, ): assert ( From 91343d260d7bb0ef140471678a505b7aba98e4ab Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 4 Nov 2025 17:57:03 -0600 Subject: [PATCH 29/54] remove unneeded defaultBaseImage from values.yaml --- helm/servicex/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index 4333cdf3e..80d99dce7 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -110,7 +110,6 @@ codeGen: defaultScienceContainerTag: 2.17.0-25.2.45 allowedImages: - "sslhep/servicex_science_image_topcp:" - defaultBaseImage: "sslhep/servicex_science_image_topcp" didFinder: CERNOpenData: From a172125fd116cca3f29cb6a2ec57a4a216c40245 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Mon, 10 Nov 2025 16:57:21 -0600 Subject: [PATCH 30/54] properly source app config values --- helm/servicex/templates/app/configmap.yaml | 2 +- .../servicex_app/resources/transformation/submit.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index b95b0c681..59add3396 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -185,7 +185,7 @@ data: {{- if .Values.codeGen.topcp.enabled }} # TopCP custom image configuration - TOPCP_ALLOWED_IMAGES = {{ .Values.codeGen.topcp.allowedImages | toJson }} + TOPCP_ALLOWED_IMAGES = {{ .Values.codeGen.topcp.allowedImages | toJson | quote }} {{- end }} {{- $didFinders := list }} diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 7c0f3ab35..6cbfc7a2a 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -42,17 +42,15 @@ def validate_custom_docker_image(image_name: str) -> bool: - allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES") + allowed_images_json = current_app.config.get("TOPCP_ALLOWED_IMAGES") if not allowed_images_json: raise BadRequest("Custom Docker images are not allowed.") try: allowed_prefixes = json.loads(allowed_images_json) - if not isinstance(allowed_prefixes, list): raise BadRequest("TopCP allowed images are improperly configured.") - for prefix in allowed_prefixes: if image_name.startswith(prefix): return True @@ -252,6 +250,8 @@ def post(self): if "docker_image" in selection: custom_docker_image = selection["docker_image"] validate_custom_docker_image(custom_docker_image) + print("test!!") + print(custom_docker_image) except json.decoder.JSONDecodeError: raise BadRequest("Malformed JSON submitted") From 78ddeac71bb8c04af0d4c62dd03fdab6ec1d68cf Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Mon, 10 Nov 2025 17:00:16 -0600 Subject: [PATCH 31/54] resolve flake8 --- servicex_app/servicex_app/resources/transformation/submit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 6cbfc7a2a..f2b445518 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -27,7 +27,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid import json -import os from datetime import datetime, timezone from typing import Optional, List From 6bbd73b477667485b4edb96002df5c6f8a076147 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Mon, 10 Nov 2025 17:30:32 -0600 Subject: [PATCH 32/54] update tests to use app.config --- .../resources/transformation/submit.py | 2 - .../resources/transformation/test_submit.py | 147 +++++++++--------- 2 files changed, 73 insertions(+), 76 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index f2b445518..6209d8750 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -249,8 +249,6 @@ def post(self): if "docker_image" in selection: custom_docker_image = selection["docker_image"] validate_custom_docker_image(custom_docker_image) - print("test!!") - print(custom_docker_image) except json.decoder.JSONDecodeError: raise BadRequest("Malformed JSON submitted") diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index b480d7cc3..31addbf00 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -26,9 +26,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json -import os from datetime import datetime, timezone -from unittest.mock import ANY, patch +from unittest.mock import ANY import pytest from celery import Celery @@ -592,15 +591,16 @@ def test_submit_transformation_with_title( assert saved_obj.title == title -class TestValidateCustomDockerImage: +class TestValidateCustomDockerImage(ResourceTestBase): """Tests for the validate_custom_docker_image function""" def test_validate_with_matching_prefix(self): """Test validation succeeds when image matches an allowed prefix""" - with patch.dict( - os.environ, - {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, - ): + extra_config = { + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]' + } + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): result = validate_custom_docker_image( "sslhep/servicex_science_image_topcp:2.17.0" ) @@ -608,13 +608,11 @@ def test_validate_with_matching_prefix(self): def test_validate_with_multiple_prefixes(self): """Test validation with multiple allowed prefixes""" - with patch.dict( - os.environ, - { - "TOPCP_ALLOWED_IMAGES": """["sslhep/servicex_science_image_topcp:", - "docker.io/ssl-hep/"]""" - }, - ): + extra_config = { + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + } + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): assert ( validate_custom_docker_image( "sslhep/servicex_science_image_topcp:latest" @@ -625,16 +623,18 @@ def test_validate_with_multiple_prefixes(self): def test_validate_with_no_matching_prefix(self): """Test validation fails when image doesn't match any allowed prefix""" - with patch.dict( - os.environ, - {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, - ): + extra_config = { + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]' + } + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): with pytest.raises(BadRequest, match="not allowed"): validate_custom_docker_image("unauthorized/image:latest") def test_validate_with_no_env_variable(self): """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" - with patch.dict(os.environ, {}, clear=True): + client = self._test_client() + with client.application.app_context(): with pytest.raises( BadRequest, match="Custom Docker images are not allowed" ): @@ -644,7 +644,9 @@ def test_validate_with_no_env_variable(self): def test_validate_with_invalid_json(self): """Test validation fails with invalid JSON in env variable""" - with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "not-valid-json"}): + extra_config = {"TOPCP_ALLOWED_IMAGES": "not-valid-json"} + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): with pytest.raises(BadRequest, match="improperly configured"): validate_custom_docker_image( "sslhep/servicex_science_image_topcp:2.17.0" @@ -652,7 +654,9 @@ def test_validate_with_invalid_json(self): def test_validate_with_non_list_json(self): """Test validation fails when JSON is not a list""" - with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": '{"key": "value"}'}): + extra_config = {"TOPCP_ALLOWED_IMAGES": '{"key": "value"}'} + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): with pytest.raises(BadRequest, match="improperly configured"): validate_custom_docker_image( "sslhep/servicex_science_image_topcp:2.17.0" @@ -660,7 +664,9 @@ def test_validate_with_non_list_json(self): def test_validate_with_empty_list(self): """Test validation fails when allowed list is empty""" - with patch.dict(os.environ, {"TOPCP_ALLOWED_IMAGES": "[]"}): + extra_config = {"TOPCP_ALLOWED_IMAGES": "[]"} + client = self._test_client(extra_config=extra_config) + with client.application.app_context(): with pytest.raises(BadRequest, match="not allowed"): validate_custom_docker_image( "sslhep/servicex_science_image_topcp:2.17.0" @@ -714,57 +720,51 @@ def test_submit_topcp_with_custom_docker_image( ): """Test submitting a TopCP transformation with a valid custom docker image""" extra_config = { - "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"}, + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]', } - with patch.dict( - os.environ, - {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, - ): - client = self._test_client( - code_gen_service=mock_codegen, extra_config=extra_config + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) + with client.application.app_context(): + selection_dict = { + "docker_image": "sslhep/servicex_science_image_topcp:custom" + } + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) ) - with client.application.app_context(): - selection_dict = { - "docker_image": "sslhep/servicex_science_image_topcp:custom" - } - request = self._generate_transformation_request( - selection=json.dumps(selection_dict) - ) - response = client.post( - "/servicex/transformation", json=request, headers=self.fake_header() - ) - assert response.status_code == 200 - request_id = response.json["request_id"] + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 200 + request_id = response.json["request_id"] - saved_obj = TransformRequest.lookup(request_id) - assert saved_obj - assert saved_obj.image == "sslhep/servicex_science_image_topcp:custom" + saved_obj = TransformRequest.lookup(request_id) + assert saved_obj + assert saved_obj.image == "sslhep/servicex_science_image_topcp:custom" def test_submit_topcp_with_invalid_custom_docker_image( self, mock_dataset_manager_from_did, mock_codegen ): """Submitting a TopCP transformation with an invalid custom docker image fails""" extra_config = { - "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} + "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"}, + "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]', } - with patch.dict( - os.environ, - {"TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]'}, - ): - client = self._test_client( - code_gen_service=mock_codegen, extra_config=extra_config + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) + with client.application.app_context(): + selection_dict = {"docker_image": "unauthorized/image:latest"} + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) ) - with client.application.app_context(): - selection_dict = {"docker_image": "unauthorized/image:latest"} - request = self._generate_transformation_request( - selection=json.dumps(selection_dict) - ) - response = client.post( - "/servicex/transformation", json=request, headers=self.fake_header() - ) - assert response.status_code == 400 + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 400 def test_submit_topcp_with_non_json_selection( self, mock_dataset_manager_from_did, mock_codegen, mock_app_version @@ -791,19 +791,18 @@ def test_submit_custom_topcp_without_env_var( extra_config = { "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} } - with patch.dict(os.environ, {}, clear=True): - client = self._test_client( - code_gen_service=mock_codegen, extra_config=extra_config + client = self._test_client( + code_gen_service=mock_codegen, extra_config=extra_config + ) + with client.application.app_context(): + selection_dict = { + "docker_image": "sslhep/servicex_science_image_topcp:custom" + } + request = self._generate_transformation_request( + selection=json.dumps(selection_dict) ) - with client.application.app_context(): - selection_dict = { - "docker_image": "sslhep/servicex_science_image_topcp:custom" - } - request = self._generate_transformation_request( - selection=json.dumps(selection_dict) - ) - response = client.post( - "/servicex/transformation", json=request, headers=self.fake_header() - ) - assert response.status_code == 400 + response = client.post( + "/servicex/transformation", json=request, headers=self.fake_header() + ) + assert response.status_code == 400 From 1f20afe347d363010c965efa35361f433dd224b3 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Mon, 10 Nov 2025 17:32:45 -0600 Subject: [PATCH 33/54] resolve flake8 --- .../servicex_app_test/resources/transformation/test_submit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index 31addbf00..393eaaa1c 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -609,7 +609,9 @@ def test_validate_with_matching_prefix(self): def test_validate_with_multiple_prefixes(self): """Test validation with multiple allowed prefixes""" extra_config = { - "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + "TOPCP_ALLOWED_IMAGES": ( + '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' + ) } client = self._test_client(extra_config=extra_config) with client.application.app_context(): From f8b8dde7ca851f0600d0d4f0f540696063d225ea Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 11 Nov 2025 14:08:27 -0600 Subject: [PATCH 34/54] remove local dev related changes --- code_generator_TopCPToolkit/Dockerfile | 7 +++-- code_generator_TopCPToolkit/boot.sh | 10 +++++++ code_generator_TopCPToolkit/servicex/boot.sh | 22 --------------- .../templates/codegen/deployment.yaml | 28 ------------------- helm/servicex/values.yaml | 2 +- 5 files changed, 15 insertions(+), 54 deletions(-) create mode 100755 code_generator_TopCPToolkit/boot.sh delete mode 100755 code_generator_TopCPToolkit/servicex/boot.sh diff --git a/code_generator_TopCPToolkit/Dockerfile b/code_generator_TopCPToolkit/Dockerfile index 98136d379..ec8adf0ce 100644 --- a/code_generator_TopCPToolkit/Dockerfile +++ b/code_generator_TopCPToolkit/Dockerfile @@ -17,6 +17,9 @@ RUN poetry config virtualenvs.create false && \ RUN pip install gunicorn +COPY boot.sh ./ +RUN chmod 755 boot.sh + COPY transformer_capabilities.json ./ RUN chmod 644 transformer_capabilities.json @@ -26,11 +29,9 @@ RUN chmod 777 -R servicex COPY app.conf . RUN chmod 755 app.conf -RUN chmod 755 /home/servicex/servicex/boot.sh - USER servicex ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf" EXPOSE 5000 -ENTRYPOINT ["/home/servicex/servicex/boot.sh"] +ENTRYPOINT ["./boot.sh"] diff --git a/code_generator_TopCPToolkit/boot.sh b/code_generator_TopCPToolkit/boot.sh new file mode 100755 index 000000000..4616242e8 --- /dev/null +++ b/code_generator_TopCPToolkit/boot.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Running the web server? +action=${1:-web_service} +if [ "$action" = "web_service" ] ; then + mkdir instance + exec gunicorn -b :5000 --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" +else + echo "Unknown action '$action'" +fi \ No newline at end of file diff --git a/code_generator_TopCPToolkit/servicex/boot.sh b/code_generator_TopCPToolkit/servicex/boot.sh deleted file mode 100755 index 76cd19549..000000000 --- a/code_generator_TopCPToolkit/servicex/boot.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Initialize reload flag -RELOAD="" - -# Parse command line arguments -for arg in "$@" -do - if [ "$arg" = "--reload" ]; then - RELOAD="--reload" - break - fi -done - -# Running the web server? -action=${1:-web_service} -if [ "$action" = "web_service" ] ; then - mkdir instance - exec gunicorn -b :5000 $RELOAD --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" -else - echo "Unknown action '$action'" -fi diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index b6b4d97e1..97d225ae9 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -19,13 +19,6 @@ spec: containers: - name: {{ $.Release.Name }}-code-gen-{{ $codeGenName }} image: {{ .image }}:{{ .tag }} - {{- if eq $.Values.app.environment "dev" }} - {{- if eq $codeGenName "topcp" }} - {{- if $.Values.app.reload }} - command: [ "./servicex/boot.sh", "web_service", "--reload" ] - {{- end }} - {{- end }} - {{- end }} env: - name: INSTANCE_NAME value: {{ $.Release.Name }} @@ -42,26 +35,5 @@ spec: imagePullPolicy: {{ .pullPolicy }} ports: - containerPort: 5000 - {{- if eq $codeGenName "topcp" }} - {{- if eq $.Values.app.environment "dev" }} - {{- if $.Values.app.mountLocal }} - volumeMounts: - - name: host-topcp-volume - mountPath: /home/servicex - {{- end }} - {{- end }} - {{- end }} - - {{- if eq $codeGenName "topcp" }} - {{- if eq $.Values.app.environment "dev" }} - {{- if $.Values.app.mountLocal }} - volumes: - - name: host-topcp-volume - hostPath: - path: /mnt/topcp - type: DirectoryOrCreate - {{- end }} - {{- end }} - {{- end }} {{- end }} {{- end }} diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index 80d99dce7..c7d556b57 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -104,7 +104,7 @@ codeGen: topcp: enabled: true image: sslhep/servicex_code_gen_topcp - pullPolicy: IfNotPresent + pullPolicy: Always tag: develop defaultScienceContainerImage: sslhep/servicex_science_image_topcp defaultScienceContainerTag: 2.17.0-25.2.45 From d03f8ba96f7dd6712563c76a396aa06e560a2fa5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Nov 2025 20:10:40 +0000 Subject: [PATCH 35/54] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- code_generator_TopCPToolkit/boot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code_generator_TopCPToolkit/boot.sh b/code_generator_TopCPToolkit/boot.sh index 4616242e8..599ab4610 100755 --- a/code_generator_TopCPToolkit/boot.sh +++ b/code_generator_TopCPToolkit/boot.sh @@ -7,4 +7,4 @@ if [ "$action" = "web_service" ] ; then exec gunicorn -b :5000 --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" else echo "Unknown action '$action'" -fi \ No newline at end of file +fi From 109808409d6b2670d0af41b19007ac6e02628e7a Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 11 Nov 2025 14:30:18 -0600 Subject: [PATCH 36/54] remove Procfile changes from this PR --- Procfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Procfile b/Procfile index 8b0636d50..cfe64a9c2 100644 --- a/Procfile +++ b/Procfile @@ -1,5 +1,4 @@ -minikube-mount-app: minikube mount $LOCAL_DIR:/mnt/servicex -minikube-mount-topcp: minikube mount $LOCAL_DIR/code_generator_TopCPToolkit:/mnt/topcp +minikube-mount: minikube mount $LOCAL_DIR:/mnt/servicex helm-install: sleep 5; cd $CHART_DIR && helm install -f $VALUES_FILE servicex . && tail -f /dev/null port-forward-app: sleep 30 && cd $LOCAL_DIR && bash local/port-forward.sh app port-forward-minio: sleep 20 && cd $LOCAL_DIR && bash local/port-forward.sh minio From cdc29c2d71622cf1615fb92f8e787e3c3126a99c Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 11 Nov 2025 14:44:22 -0600 Subject: [PATCH 37/54] add topcp codegen values to reference.md --- docs/deployment/reference.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/deployment/reference.md b/docs/deployment/reference.md index fd4ce6669..efb2899c8 100644 --- a/docs/deployment/reference.md +++ b/docs/deployment/reference.md @@ -25,8 +25,8 @@ parameters for the [rabbitMQ](https://github.com/bitnami/charts/tree/master/bitn | `app.replicas` | Number of App pods to start. Experimental! | 1 | | `app.auth` | Enable authentication or allow unfettered access (Python boolean string) | `false` | | `app.oauthMetadataURL` | OpenID Connect provider well known configuration endpoint URL | - | -| `app.oauthClientID` | OpenID Connect application Client ID | - | -| `app.oauthClientSecret` | OpenID Connect application Client Secret | - | +| `app.oauthClientID` | OpenID Connect application Client ID | - | +| `app.oauthClientSecret` | OpenID Connect application Client Secret | - | | `app.adminEmail` | Email address for initial admin user | | | `app.tokenExpires` | Seconds until the ServiceX API tokens (JWT refresh tokens) expire | False (never) | | `app.authExpires` | Seconds until the JWT access tokens expire | 21600 (six hours) | @@ -75,6 +75,13 @@ parameters for the [rabbitMQ](https://github.com/bitnami/charts/tree/master/bitn | `codegen.atlasr21.enabled` | Deploy the ATLAS FuncADL Release 21 code generator? - also all of the code gen settings above are available | true | | `codegen.atlasr22.enabled` | Deploy the ATLAS FuncADL Release 22 code generator? - also all of the code gen settings above are available | true | | `codegen.python.enabled` | Deploy the python uproot code generator? - also all of the code gen settings, above are available | true | +| `codegen.topcp.enabled` | Deploy the TopCP code generator? | true | +| `codegen.topcp.image` | TopCP ode generator image | `sslhep/servicex_code_gen_topcp` | +| `codegen.topcp.pullPolicy` | TopCP code generator image pull policy | true | +| `codegen.topcp.tag` | TopCP code generator image tag | develop | +| `codegen.topcp.defaultScienceContainerImage` | The default image used by a TopCP transformer container | sslhep/servicex_science_image_topcp | +| `codegen.topcp.defaultScienceContainerTag` | The default tag used by a TopCP transformer container | 2.17.0-25.2.45 | +| `codegen.topcp.allowedImages` | A list of strings, of which one must be a valid prefix of a submitted custom TopCP docker image | - "sslhep/servicex_science_image_topcp:" | | `x509Secrets.image` | X509 Secret Service image name | `sslhep/x509-secrets` | | `x509Secrets.tag` | X509 Secret Service image tag | `latest` | | `x509Secrets.pullPolicy` | X509 Secret Service image pull policy | `Always` | @@ -94,9 +101,9 @@ parameters for the [rabbitMQ](https://github.com/bitnami/charts/tree/master/bitn | `minio.apiIngress.enabled` | Should minio chart deploy an ingress to the service? | false | | `minio.apiIngress.hostname` | Hostname associate with ingress controller | nil | | `transformer.cachePrefix` | Prefix string to stick in front of file paths. Useful for XCache. If `transformer.cacheVPSSite` is also set, this will be ignored | nil | -| `transformer.cacheVPSSite` | Specify a Virtual Placement Service site whose XCaches we should use. Will update automatically if list changes. If set, takes priority over `transformer.cachePrefix` | nil | -| `transformer.cacheVPSCheckInterval` | How frequency should the Virtual Placement Service be consulted for the list of XCaches (in seconds) | 1800 | -| `transformer.cacheVPSLivenessURL` | URL from which Virtual Placement Service site info can be obtained | https://vps.cern.ch/liveness | +| `transformer.cacheVPSSite` | Specify a Virtual Placement Service site whose XCaches we should use. Will update automatically if list changes. If set, takes priority over `transformer.cachePrefix` | nil | +| `transformer.cacheVPSCheckInterval` | How frequency should the Virtual Placement Service be consulted for the list of XCaches (in seconds) | 1800 | +| `transformer.cacheVPSLivenessURL` | URL from which Virtual Placement Service site info can be obtained | https://vps.cern.ch/liveness | | `transformer.autoscaler.enabled` | Enable/disable horizontal pod autoscaler for transformers | True | | `transformer.autoscaler.cpuScaleThreshold` | CPU percentage threshold for pod scaling | 30 | | `transformer.autoscaler.minReplicas` | Minimum number of transformer pods per request | 1 | From e2e076c832042b4589db04cc905504a8e471aaaa Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Tue, 11 Nov 2025 14:45:56 -0600 Subject: [PATCH 38/54] correct topcp pullpolicy --- docs/deployment/reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment/reference.md b/docs/deployment/reference.md index efb2899c8..9a4e26f3e 100644 --- a/docs/deployment/reference.md +++ b/docs/deployment/reference.md @@ -77,7 +77,7 @@ parameters for the [rabbitMQ](https://github.com/bitnami/charts/tree/master/bitn | `codegen.python.enabled` | Deploy the python uproot code generator? - also all of the code gen settings, above are available | true | | `codegen.topcp.enabled` | Deploy the TopCP code generator? | true | | `codegen.topcp.image` | TopCP ode generator image | `sslhep/servicex_code_gen_topcp` | -| `codegen.topcp.pullPolicy` | TopCP code generator image pull policy | true | +| `codegen.topcp.pullPolicy` | TopCP code generator image pull policy | Always | | `codegen.topcp.tag` | TopCP code generator image tag | develop | | `codegen.topcp.defaultScienceContainerImage` | The default image used by a TopCP transformer container | sslhep/servicex_science_image_topcp | | `codegen.topcp.defaultScienceContainerTag` | The default tag used by a TopCP transformer container | 2.17.0-25.2.45 | From 1b140d0d612002f7c4b00d82194fb7801761d6eb Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 12:14:37 -0600 Subject: [PATCH 39/54] move custom image logic to topcp codegen --- .../TopCP_code_generator/query_translate.py | 5 +-- .../request_translator.py | 35 ++++++++++++++++-- .../{ => servicex}/boot.sh | 0 .../templates/codegen/deployment.yaml | 4 +++ .../resources/transformation/submit.py | 36 +------------------ 5 files changed, 39 insertions(+), 41 deletions(-) rename code_generator_TopCPToolkit/{ => servicex}/boot.sh (100%) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py index 4923b0d6a..925c64d71 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/query_translate.py @@ -1,4 +1,3 @@ -import json import os options = { @@ -44,9 +43,7 @@ } -def generate_files_from_query(query, query_file_path): - jquery = json.loads(query) - +def generate_files_from_query(jquery: dict, query_file_path): runTopCommand = [ "runTop_el.py", "-i", diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index 3a7ef9021..46fea07fd 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -26,7 +26,9 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os +import json import shutil + from . import query_translate from servicex_codegen.code_generator import ( CodeGenerator, @@ -34,6 +36,25 @@ GenerateCodeException, ) +def validate_custom_docker_image(image_name: str) -> bool: + allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES") + print(allowed_images_json) + + if not allowed_images_json: + raise GenerateCodeException("Custom Docker images are not allowed.") + + try: + allowed_prefixes = json.loads(allowed_images_json) + if not isinstance(allowed_prefixes, list): + raise GenerateCodeException("TopCP allowed images are improperly configured.") + for prefix in allowed_prefixes: + if image_name.startswith(prefix): + return True + + raise GenerateCodeException(f"Custom Docker image '{image_name}' not allowed.") + + except json.JSONDecodeError: + raise GenerateCodeException("TopCP allowed images are improperly configured.") class TopCPTranslator(CodeGenerator): # Generate the code. Ignoring caching for now @@ -63,11 +84,21 @@ def generate_code(self, query, cache_path: str): "CAPABILITIES_PATH", "/home/servicex/transformer_capabilities.json" ) - query_translate.generate_files_from_query(query, query_file_path) + jquery = json.loads(query) + query_translate.generate_files_from_query(jquery, query_file_path) shutil.copyfile( capabilities_path, os.path.join(query_file_path, "transformer_capabilities.json"), ) - return GeneratedFileResult(_hash, query_file_path) + results = GeneratedFileResult(_hash, query_file_path) + + if "docker_image" in jquery: + docker_image = jquery["docker_image"] + + validate_custom_docker_image(docker_image) + + results.image = docker_image + + return results diff --git a/code_generator_TopCPToolkit/boot.sh b/code_generator_TopCPToolkit/servicex/boot.sh similarity index 100% rename from code_generator_TopCPToolkit/boot.sh rename to code_generator_TopCPToolkit/servicex/boot.sh diff --git a/helm/servicex/templates/codegen/deployment.yaml b/helm/servicex/templates/codegen/deployment.yaml index 97d225ae9..09a8d8228 100644 --- a/helm/servicex/templates/codegen/deployment.yaml +++ b/helm/servicex/templates/codegen/deployment.yaml @@ -22,6 +22,10 @@ spec: env: - name: INSTANCE_NAME value: {{ $.Release.Name }} + {{- if eq $codeGenName "topcp" }} + - name: TOPCP_ALLOWED_IMAGES + value: {{ $.Values.codeGen.topcp.allowedImages | toJson | quote }} + {{- end }} - name: TRANSFORMER_SCIENCE_IMAGE value: {{ .defaultScienceContainerImage }}:{{ .defaultScienceContainerTag }} {{- $compressionAlgorithm := .compressionAlgorithm | default "ZSTD" }} diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 6209d8750..a353d39b8 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -40,26 +40,6 @@ from werkzeug.exceptions import BadRequest, HTTPException -def validate_custom_docker_image(image_name: str) -> bool: - allowed_images_json = current_app.config.get("TOPCP_ALLOWED_IMAGES") - - if not allowed_images_json: - raise BadRequest("Custom Docker images are not allowed.") - - try: - allowed_prefixes = json.loads(allowed_images_json) - if not isinstance(allowed_prefixes, list): - raise BadRequest("TopCP allowed images are improperly configured.") - for prefix in allowed_prefixes: - if image_name.startswith(prefix): - return True - - raise BadRequest(f"Custom Docker image '{image_name}' not allowed.") - - except json.JSONDecodeError: - raise BadRequest("TopCP allowed images are improperly configured.") - - class SubmitTransformationRequest(ServiceXResource): @classmethod def make_api( @@ -241,21 +221,7 @@ def post(self): ) = self.code_gen_service.generate_code_for_selection( request_rec, namespace, user_codegen_name ) - - custom_docker_image = None - if user_codegen_name == "topcp": - try: - selection = json.loads(args["selection"]) - if "docker_image" in selection: - custom_docker_image = selection["docker_image"] - validate_custom_docker_image(custom_docker_image) - except json.decoder.JSONDecodeError: - raise BadRequest("Malformed JSON submitted") - - if custom_docker_image: - request_rec.image = custom_docker_image - else: - request_rec.image = codegen_transformer_image + request_rec.image = codegen_transformer_image # Check to make sure the transformer docker image actually exists (if enabled) if config["TRANSFORMER_VALIDATE_DOCKER_IMAGE"]: From eea313ba0acdff1e6caf1f7ad1ac71f273852e27 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 12:18:24 -0600 Subject: [PATCH 40/54] move boot.sh back --- code_generator_TopCPToolkit/{servicex => }/boot.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename code_generator_TopCPToolkit/{servicex => }/boot.sh (100%) diff --git a/code_generator_TopCPToolkit/servicex/boot.sh b/code_generator_TopCPToolkit/boot.sh similarity index 100% rename from code_generator_TopCPToolkit/servicex/boot.sh rename to code_generator_TopCPToolkit/boot.sh From e348931432eee38b3cc122f652c6062ec7a94e67 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 12:19:13 -0600 Subject: [PATCH 41/54] remove comment --- .../servicex/TopCP_code_generator/request_translator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index 46fea07fd..0aa11eb70 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -38,7 +38,6 @@ def validate_custom_docker_image(image_name: str) -> bool: allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES") - print(allowed_images_json) if not allowed_images_json: raise GenerateCodeException("Custom Docker images are not allowed.") From 2ba1a2a9468f6d4f9ae8a72b9c7da326c8796942 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 14:11:24 -0600 Subject: [PATCH 42/54] move docker image validation tests to codegen --- .../request_translator.py | 2 - .../test_validate_custom_docker_image.py | 65 ++++ .../resources/transformation/test_submit.py | 353 +++++++----------- 3 files changed, 199 insertions(+), 221 deletions(-) create mode 100644 code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index 0aa11eb70..ce87175bb 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -95,9 +95,7 @@ def generate_code(self, query, cache_path: str): if "docker_image" in jquery: docker_image = jquery["docker_image"] - validate_custom_docker_image(docker_image) - results.image = docker_image return results diff --git a/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py new file mode 100644 index 000000000..f420d26f9 --- /dev/null +++ b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py @@ -0,0 +1,65 @@ +import pytest +from pytest import MonkeyPatch +from servicex.TopCP_code_generator.request_translator import validate_custom_docker_image +from servicex_codegen.code_generator import GenerateCodeException + +class TestValidateCustomDockerImage: + """Tests for the validate_custom_docker_image function""" + + def test_validate_with_matching_prefix(self, monkeypatch: MonkeyPatch): + monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", '["sslhep/servicex_science_image_topcp:"]') + result = validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) + assert result is True + + def test_validate_with_multiple_prefixes(self, monkeypatch: MonkeyPatch): + """Test validation with multiple allowed prefixes""" + monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]') + assert ( + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:latest" + ) + is True + ) + assert validate_custom_docker_image("docker.io/ssl-hep/custom:v1") is True + + def test_validate_with_no_matching_prefix(self, monkeypatch: MonkeyPatch): + """Test validation fails when image doesn't match any allowed prefix""" + monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", '["sslhep/servicex_science_image_topcp:"]') + with pytest.raises(GenerateCodeException, match="not allowed"): + validate_custom_docker_image("unauthorized/image:latest") + + def test_validate_with_no_env_variable(self, monkeypatch: MonkeyPatch): + """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" + monkeypatch.delenv("TOPCP_ALLOWED_IMAGES") + with pytest.raises( + GenerateCodeException, match="Custom Docker images are not allowed" + ): + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) + + def test_validate_with_invalid_json(self, monkeypatch: MonkeyPatch): + """Test validation fails with invalid JSON in env variable""" + monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", "not-valid-json") + with pytest.raises(GenerateCodeException, match="improperly configured"): + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) + + def test_validate_with_non_list_json(self, monkeypatch: MonkeyPatch): + """Test validation fails when JSON is not a list""" + monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", '{"key": "value"}') + with pytest.raises(GenerateCodeException, match="improperly configured"): + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) + + def test_validate_with_empty_list(self, monkeypatch: MonkeyPatch): + """Test validation fails when allowed list is empty""" + monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", "[]") + with pytest.raises(GenerateCodeException, match="not allowed"): + validate_custom_docker_image( + "sslhep/servicex_science_image_topcp:2.17.0" + ) \ No newline at end of file diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index 393eaaa1c..fdc673a26 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -39,7 +39,6 @@ from servicex_app.dataset_manager import DatasetManager from servicex_app.models import Dataset from servicex_app.models import TransformRequest, DatasetStatus, TransformStatus -from servicex_app.resources.transformation.submit import validate_custom_docker_image from servicex_app.transformer_manager import TransformerManager from servicex_app_test.resource_test_base import ResourceTestBase @@ -590,221 +589,137 @@ def test_submit_transformation_with_title( assert saved_obj assert saved_obj.title == title - -class TestValidateCustomDockerImage(ResourceTestBase): - """Tests for the validate_custom_docker_image function""" - - def test_validate_with_matching_prefix(self): - """Test validation succeeds when image matches an allowed prefix""" - extra_config = { - "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]' - } - client = self._test_client(extra_config=extra_config) - with client.application.app_context(): - result = validate_custom_docker_image( - "sslhep/servicex_science_image_topcp:2.17.0" - ) - assert result is True - - def test_validate_with_multiple_prefixes(self): - """Test validation with multiple allowed prefixes""" - extra_config = { - "TOPCP_ALLOWED_IMAGES": ( - '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]' - ) - } - client = self._test_client(extra_config=extra_config) - with client.application.app_context(): - assert ( - validate_custom_docker_image( - "sslhep/servicex_science_image_topcp:latest" - ) - is True - ) - assert validate_custom_docker_image("docker.io/ssl-hep/custom:v1") is True - - def test_validate_with_no_matching_prefix(self): - """Test validation fails when image doesn't match any allowed prefix""" - extra_config = { - "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]' - } - client = self._test_client(extra_config=extra_config) - with client.application.app_context(): - with pytest.raises(BadRequest, match="not allowed"): - validate_custom_docker_image("unauthorized/image:latest") - - def test_validate_with_no_env_variable(self): - """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" - client = self._test_client() - with client.application.app_context(): - with pytest.raises( - BadRequest, match="Custom Docker images are not allowed" - ): - validate_custom_docker_image( - "sslhep/servicex_science_image_topcp:2.17.0" - ) - - def test_validate_with_invalid_json(self): - """Test validation fails with invalid JSON in env variable""" - extra_config = {"TOPCP_ALLOWED_IMAGES": "not-valid-json"} - client = self._test_client(extra_config=extra_config) - with client.application.app_context(): - with pytest.raises(BadRequest, match="improperly configured"): - validate_custom_docker_image( - "sslhep/servicex_science_image_topcp:2.17.0" - ) - - def test_validate_with_non_list_json(self): - """Test validation fails when JSON is not a list""" - extra_config = {"TOPCP_ALLOWED_IMAGES": '{"key": "value"}'} - client = self._test_client(extra_config=extra_config) - with client.application.app_context(): - with pytest.raises(BadRequest, match="improperly configured"): - validate_custom_docker_image( - "sslhep/servicex_science_image_topcp:2.17.0" - ) - - def test_validate_with_empty_list(self): - """Test validation fails when allowed list is empty""" - extra_config = {"TOPCP_ALLOWED_IMAGES": "[]"} - client = self._test_client(extra_config=extra_config) - with client.application.app_context(): - with pytest.raises(BadRequest, match="not allowed"): - validate_custom_docker_image( - "sslhep/servicex_science_image_topcp:2.17.0" - ) - - -class TestSubmitTransformationRequestCustomImage(ResourceTestBase): - """Tests for custom Docker image feature in transformation requests""" - - @staticmethod - def _generate_transformation_request(**kwargs): - request = { - "did": "123-45-678", - "selection": "test-string", - "result-destination": "object-store", - "result-format": "root-file", - "workers": 10, - "codegen": "topcp", - } - request.update(kwargs) - return request - - @fixture - def mock_dataset_manager_from_did(self, mocker): - dm = mocker.Mock() - dm.dataset = Dataset( - name="rucio://123-45-678", - did_finder="rucio", - lookup_status=DatasetStatus.looking, - last_used=datetime.now(tz=timezone.utc), - last_updated=datetime.fromtimestamp(0), - ) - dm.name = "rucio://123-45-678" - dm.id = 42 - mock_from_did = mocker.patch.object(DatasetManager, "from_did", return_value=dm) - return mock_from_did - - @fixture - def mock_codegen(self, mocker): - mock_code_gen = mocker.MagicMock(CodeGenAdapter) - mock_code_gen.generate_code_for_selection.return_value = ( - "my-cm", - "sslhep/servicex_science_image_topcp:2.17.0", - "bash", - "echo", - ) - return mock_code_gen - - def test_submit_topcp_with_custom_docker_image( - self, mock_dataset_manager_from_did, mock_codegen, mock_app_version - ): - """Test submitting a TopCP transformation with a valid custom docker image""" - extra_config = { - "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"}, - "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]', - } - client = self._test_client( - code_gen_service=mock_codegen, extra_config=extra_config - ) - with client.application.app_context(): - selection_dict = { - "docker_image": "sslhep/servicex_science_image_topcp:custom" - } - request = self._generate_transformation_request( - selection=json.dumps(selection_dict) - ) - - response = client.post( - "/servicex/transformation", json=request, headers=self.fake_header() - ) - assert response.status_code == 200 - request_id = response.json["request_id"] - - saved_obj = TransformRequest.lookup(request_id) - assert saved_obj - assert saved_obj.image == "sslhep/servicex_science_image_topcp:custom" - - def test_submit_topcp_with_invalid_custom_docker_image( - self, mock_dataset_manager_from_did, mock_codegen - ): - """Submitting a TopCP transformation with an invalid custom docker image fails""" - extra_config = { - "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"}, - "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]', - } - client = self._test_client( - code_gen_service=mock_codegen, extra_config=extra_config - ) - with client.application.app_context(): - selection_dict = {"docker_image": "unauthorized/image:latest"} - request = self._generate_transformation_request( - selection=json.dumps(selection_dict) - ) - - response = client.post( - "/servicex/transformation", json=request, headers=self.fake_header() - ) - assert response.status_code == 400 - - def test_submit_topcp_with_non_json_selection( - self, mock_dataset_manager_from_did, mock_codegen, mock_app_version - ): - """Test submitting a TopCP transformation with non-JSON selection string""" - extra_config = { - "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} - } - client = self._test_client( - code_gen_service=mock_codegen, extra_config=extra_config - ) - with client.application.app_context(): - request = self._generate_transformation_request(selection="not-json-string") - - response = client.post( - "/servicex/transformation", json=request, headers=self.fake_header() - ) - assert response.status_code == 400 - - def test_submit_custom_topcp_without_env_var( - self, mock_dataset_manager_from_did, mock_codegen - ): - """Test submitting TopCP with custom image when TOPCP_ALLOWED_IMAGES is not set""" - extra_config = { - "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} - } - client = self._test_client( - code_gen_service=mock_codegen, extra_config=extra_config - ) - with client.application.app_context(): - selection_dict = { - "docker_image": "sslhep/servicex_science_image_topcp:custom" - } - request = self._generate_transformation_request( - selection=json.dumps(selection_dict) - ) - - response = client.post( - "/servicex/transformation", json=request, headers=self.fake_header() - ) - assert response.status_code == 400 +# +# class TestSubmitTransformationRequestCustomImage(ResourceTestBase): +# """Tests for custom Docker image feature in transformation requests""" +# +# @staticmethod +# def _generate_transformation_request(**kwargs): +# request = { +# "did": "123-45-678", +# "selection": "test-string", +# "result-destination": "object-store", +# "result-format": "root-file", +# "workers": 10, +# "codegen": "topcp", +# } +# request.update(kwargs) +# return request +# +# @fixture +# def mock_dataset_manager_from_did(self, mocker): +# dm = mocker.Mock() +# dm.dataset = Dataset( +# name="rucio://123-45-678", +# did_finder="rucio", +# lookup_status=DatasetStatus.looking, +# last_used=datetime.now(tz=timezone.utc), +# last_updated=datetime.fromtimestamp(0), +# ) +# dm.name = "rucio://123-45-678" +# dm.id = 42 +# mock_from_did = mocker.patch.object(DatasetManager, "from_did", return_value=dm) +# return mock_from_did +# +# @fixture +# def mock_codegen(self, mocker): +# mock_code_gen = mocker.MagicMock(CodeGenAdapter) +# mock_code_gen.generate_code_for_selection.return_value = ( +# "my-cm", +# "sslhep/servicex_science_image_topcp:2.17.0", +# "bash", +# "echo", +# ) +# return mock_code_gen +# +# def test_submit_topcp_with_custom_docker_image( +# self, mock_dataset_manager_from_did, mock_codegen, mock_app_version +# ): +# """Test submitting a TopCP transformation with a valid custom docker image""" +# extra_config = { +# "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"}, +# "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]', +# } +# client = self._test_client( +# code_gen_service=mock_codegen, extra_config=extra_config +# ) +# with client.application.app_context(): +# selection_dict = { +# "docker_image": "sslhep/servicex_science_image_topcp:custom" +# } +# request = self._generate_transformation_request( +# selection=json.dumps(selection_dict) +# ) +# +# response = client.post( +# "/servicex/transformation", json=request, headers=self.fake_header() +# ) +# assert response.status_code == 200 +# request_id = response.json["request_id"] +# +# saved_obj = TransformRequest.lookup(request_id) +# assert saved_obj +# assert saved_obj.image == "sslhep/servicex_science_image_topcp:custom" +# +# def test_submit_topcp_with_invalid_custom_docker_image( +# self, mock_dataset_manager_from_did, mock_codegen +# ): +# """Submitting a TopCP transformation with an invalid custom docker image fails""" +# extra_config = { +# "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"}, +# "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]', +# } +# client = self._test_client( +# code_gen_service=mock_codegen, extra_config=extra_config +# ) +# with client.application.app_context(): +# selection_dict = {"docker_image": "unauthorized/image:latest"} +# request = self._generate_transformation_request( +# selection=json.dumps(selection_dict) +# ) +# +# response = client.post( +# "/servicex/transformation", json=request, headers=self.fake_header() +# ) +# assert response.status_code == 400 +# +# def test_submit_topcp_with_non_json_selection( +# self, mock_dataset_manager_from_did, mock_codegen, mock_app_version +# ): +# """Test submitting a TopCP transformation with non-JSON selection string""" +# extra_config = { +# "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} +# } +# client = self._test_client( +# code_gen_service=mock_codegen, extra_config=extra_config +# ) +# with client.application.app_context(): +# request = self._generate_transformation_request(selection="not-json-string") +# +# response = client.post( +# "/servicex/transformation", json=request, headers=self.fake_header() +# ) +# assert response.status_code == 400 +# +# def test_submit_custom_topcp_without_env_var( +# self, mock_dataset_manager_from_did, mock_codegen +# ): +# """Test submitting TopCP with custom image when TOPCP_ALLOWED_IMAGES is not set""" +# extra_config = { +# "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} +# } +# client = self._test_client( +# code_gen_service=mock_codegen, extra_config=extra_config +# ) +# with client.application.app_context(): +# selection_dict = { +# "docker_image": "sslhep/servicex_science_image_topcp:custom" +# } +# request = self._generate_transformation_request( +# selection=json.dumps(selection_dict) +# ) +# +# response = client.post( +# "/servicex/transformation", json=request, headers=self.fake_header() +# ) +# assert response.status_code == 400 From 50c542c880b9f212aa2cdba29be4d5bb84f9e6bd Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 14:16:44 -0600 Subject: [PATCH 43/54] flake8/black --- .../request_translator.py | 6 ++- .../test_validate_custom_docker_image.py | 38 +++++++++---------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index ce87175bb..2cbb46456 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -36,6 +36,7 @@ GenerateCodeException, ) + def validate_custom_docker_image(image_name: str) -> bool: allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES") @@ -45,7 +46,9 @@ def validate_custom_docker_image(image_name: str) -> bool: try: allowed_prefixes = json.loads(allowed_images_json) if not isinstance(allowed_prefixes, list): - raise GenerateCodeException("TopCP allowed images are improperly configured.") + raise GenerateCodeException( + "TopCP allowed images are improperly configured." + ) for prefix in allowed_prefixes: if image_name.startswith(prefix): return True @@ -55,6 +58,7 @@ def validate_custom_docker_image(image_name: str) -> bool: except json.JSONDecodeError: raise GenerateCodeException("TopCP allowed images are improperly configured.") + class TopCPTranslator(CodeGenerator): # Generate the code. Ignoring caching for now def generate_code(self, query, cache_path: str): diff --git a/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py index f420d26f9..872a5e879 100644 --- a/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py +++ b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py @@ -1,13 +1,18 @@ import pytest from pytest import MonkeyPatch -from servicex.TopCP_code_generator.request_translator import validate_custom_docker_image +from servicex.TopCP_code_generator.request_translator import ( + validate_custom_docker_image, +) from servicex_codegen.code_generator import GenerateCodeException + class TestValidateCustomDockerImage: """Tests for the validate_custom_docker_image function""" def test_validate_with_matching_prefix(self, monkeypatch: MonkeyPatch): - monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", '["sslhep/servicex_science_image_topcp:"]') + monkeypatch.setenv( + "TOPCP_ALLOWED_IMAGES", '["sslhep/servicex_science_image_topcp:"]' + ) result = validate_custom_docker_image( "sslhep/servicex_science_image_topcp:2.17.0" ) @@ -15,18 +20,21 @@ def test_validate_with_matching_prefix(self, monkeypatch: MonkeyPatch): def test_validate_with_multiple_prefixes(self, monkeypatch: MonkeyPatch): """Test validation with multiple allowed prefixes""" - monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]') + monkeypatch.setenv( + "TOPCP_ALLOWED_IMAGES", + '["sslhep/servicex_science_image_topcp:", "docker.io/ssl-hep/"]', + ) assert ( - validate_custom_docker_image( - "sslhep/servicex_science_image_topcp:latest" - ) + validate_custom_docker_image("sslhep/servicex_science_image_topcp:latest") is True ) assert validate_custom_docker_image("docker.io/ssl-hep/custom:v1") is True def test_validate_with_no_matching_prefix(self, monkeypatch: MonkeyPatch): """Test validation fails when image doesn't match any allowed prefix""" - monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", '["sslhep/servicex_science_image_topcp:"]') + monkeypatch.setenv( + "TOPCP_ALLOWED_IMAGES", '["sslhep/servicex_science_image_topcp:"]' + ) with pytest.raises(GenerateCodeException, match="not allowed"): validate_custom_docker_image("unauthorized/image:latest") @@ -36,30 +44,22 @@ def test_validate_with_no_env_variable(self, monkeypatch: MonkeyPatch): with pytest.raises( GenerateCodeException, match="Custom Docker images are not allowed" ): - validate_custom_docker_image( - "sslhep/servicex_science_image_topcp:2.17.0" - ) + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") def test_validate_with_invalid_json(self, monkeypatch: MonkeyPatch): """Test validation fails with invalid JSON in env variable""" monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", "not-valid-json") with pytest.raises(GenerateCodeException, match="improperly configured"): - validate_custom_docker_image( - "sslhep/servicex_science_image_topcp:2.17.0" - ) + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") def test_validate_with_non_list_json(self, monkeypatch: MonkeyPatch): """Test validation fails when JSON is not a list""" monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", '{"key": "value"}') with pytest.raises(GenerateCodeException, match="improperly configured"): - validate_custom_docker_image( - "sslhep/servicex_science_image_topcp:2.17.0" - ) + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") def test_validate_with_empty_list(self, monkeypatch: MonkeyPatch): """Test validation fails when allowed list is empty""" monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", "[]") with pytest.raises(GenerateCodeException, match="not allowed"): - validate_custom_docker_image( - "sslhep/servicex_science_image_topcp:2.17.0" - ) \ No newline at end of file + validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") From 5fc9d75cc6f9e90af69b66ac73c5e7ef86c6c10a Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 14:23:06 -0600 Subject: [PATCH 44/54] flake8 --- .../tests/test_validate_custom_docker_image.py | 3 +-- servicex_app/servicex_app/resources/transformation/submit.py | 1 - .../servicex_app_test/resources/transformation/test_submit.py | 3 --- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py index 872a5e879..7889868d3 100644 --- a/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py +++ b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py @@ -38,9 +38,8 @@ def test_validate_with_no_matching_prefix(self, monkeypatch: MonkeyPatch): with pytest.raises(GenerateCodeException, match="not allowed"): validate_custom_docker_image("unauthorized/image:latest") - def test_validate_with_no_env_variable(self, monkeypatch: MonkeyPatch): + def test_validate_with_no_env_variable(self): """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" - monkeypatch.delenv("TOPCP_ALLOWED_IMAGES") with pytest.raises( GenerateCodeException, match="Custom Docker images are not allowed" ): diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index a353d39b8..5c1f80973 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -26,7 +26,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import uuid -import json from datetime import datetime, timezone from typing import Optional, List diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index fdc673a26..7c48dd528 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -25,14 +25,11 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import json from datetime import datetime, timezone from unittest.mock import ANY -import pytest from celery import Celery from pytest import fixture -from werkzeug.exceptions import BadRequest from servicex_app import LookupResultProcessor from servicex_app.code_gen_adapter import CodeGenAdapter From c45ad9fcbf451e8a1bf6fa2eb1ac8cb84f238dbf Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 14:30:26 -0600 Subject: [PATCH 45/54] revert submit.py --- .../servicex_app/resources/transformation/submit.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/servicex_app/servicex_app/resources/transformation/submit.py b/servicex_app/servicex_app/resources/transformation/submit.py index 5c1f80973..51e57668e 100644 --- a/servicex_app/servicex_app/resources/transformation/submit.py +++ b/servicex_app/servicex_app/resources/transformation/submit.py @@ -36,7 +36,7 @@ from servicex_app.did_parser import DIDParser from servicex_app.models import TransformRequest, db, TransformStatus from servicex_app.resources.servicex_resource import ServiceXResource -from werkzeug.exceptions import BadRequest, HTTPException +from werkzeug.exceptions import BadRequest class SubmitTransformationRequest(ServiceXResource): @@ -99,6 +99,7 @@ def _initialize_dataset_manager( request_id: str, config: dict, ) -> DatasetManager: + # did xor file_list if bool(did) == bool(file_list): raise BadRequest("Must provide did or file-list but not both") @@ -150,6 +151,7 @@ def post(self): uuid.uuid4() ) # make sure we have a request id for all messages try: + try: args = self.parser.parse_args() except BadRequest as bad_request: @@ -220,6 +222,7 @@ def post(self): ) = self.code_gen_service.generate_code_for_selection( request_rec, namespace, user_codegen_name ) + request_rec.image = codegen_transformer_image # Check to make sure the transformer docker image actually exists (if enabled) @@ -267,8 +270,6 @@ def post(self): "Transformation request submitted!", extra={"requestId": request_id} ) return {"request_id": str(request_id)} - except HTTPException: - raise except Exception as eek: current_app.logger.exception( "Got exception while submitting transformation request", From 714163756338fbc1b2319b79ee6a76ec8de2c167 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 14:36:16 -0600 Subject: [PATCH 46/54] remove TOPCP_CUSTOM_IMAGES from app template --- helm/servicex/templates/app/configmap.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index 59add3396..b328e69bf 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -183,11 +183,6 @@ data: CODE_GEN_IMAGES = { {{ join "," $code_gen_images }} } {{- end }} - {{- if .Values.codeGen.topcp.enabled }} - # TopCP custom image configuration - TOPCP_ALLOWED_IMAGES = {{ .Values.codeGen.topcp.allowedImages | toJson | quote }} - {{- end }} - {{- $didFinders := list }} {{- if .Values.didFinder.CERNOpenData.enabled }} {{- $didFinders = append $didFinders "cernopendata" }} From 1d9fab9ddd181360da1c36bf66b81ff6b54108a6 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 14:39:17 -0600 Subject: [PATCH 47/54] remove custom_image from generate_code_for_selection --- servicex_app/servicex_app/code_gen_adapter.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/servicex_app/servicex_app/code_gen_adapter.py b/servicex_app/servicex_app/code_gen_adapter.py index c088f3c06..62752174d 100644 --- a/servicex_app/servicex_app/code_gen_adapter.py +++ b/servicex_app/servicex_app/code_gen_adapter.py @@ -45,11 +45,7 @@ def post_request(self, post_url, post_obj): return result def generate_code_for_selection( - self, - request_record: TransformRequest, - namespace: str, - user_codegen_name: str, - custom_image: Optional[str] = None, + self, request_record: TransformRequest, namespace: str, user_codegen_name: str ) -> tuple[str, str, str, str]: """ Generates the C++ code for a request's selection string. From 4cca0aade9c30803b29cd82628af2c8ff575cb9f Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 14:41:32 -0600 Subject: [PATCH 48/54] flake8 --- servicex_app/servicex_app/code_gen_adapter.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/servicex_app/servicex_app/code_gen_adapter.py b/servicex_app/servicex_app/code_gen_adapter.py index 62752174d..ea88516c5 100644 --- a/servicex_app/servicex_app/code_gen_adapter.py +++ b/servicex_app/servicex_app/code_gen_adapter.py @@ -25,8 +25,6 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from typing import Optional - import requests from requests_toolbelt.multipart import decoder From c688ece855648258c1ce66efc928419a460e8b32 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 16:09:41 -0600 Subject: [PATCH 49/54] add new codegen topcp custom docker image test --- code_generator_TopCPToolkit/tests/test_src.py | 104 ++++++++++++++ .../resources/transformation/test_submit.py | 135 ------------------ 2 files changed, 104 insertions(+), 135 deletions(-) diff --git a/code_generator_TopCPToolkit/tests/test_src.py b/code_generator_TopCPToolkit/tests/test_src.py index edc0bcd94..ae157b6bb 100755 --- a/code_generator_TopCPToolkit/tests/test_src.py +++ b/code_generator_TopCPToolkit/tests/test_src.py @@ -88,6 +88,110 @@ def test_generate_code(): translator.generate_code(query, tmpdirname) +def test_generate_code_with_custom_docker_image(): + os.environ["TEMPLATE_PATH"] = "servicex/templates/transform_single_file.py" + os.environ["CAPABILITIES_PATH"] = "transformer_capabilities.json" + os.environ["TOPCP_ALLOWED_IMAGES"] = '["sslhep/custom_image:"]' + + with tempfile.TemporaryDirectory() as tmpdirname: + translator = TopCPTranslator() + query = ( + '{"reco": "CommonServices:\\n systematicsHistogram: \'listOfSystematics\'\\n\\n' + "PileupReweighting: {}\\n\\nEventCleaning:\\n runEventCleaning: False\\n" + " runGRL: False\\n\\nElectrons:\\n - containerName: 'AnaElectrons'\\n" + " crackVeto: True\\n IFFClassification: {}\\n WorkingPoint:\\n" + " - selectionName: 'loose'\\n identificationWP: 'TightLH'\\n" + " isolationWP: 'NonIso'\\n noEffSF: True\\n" + " - selectionName: 'tight'\\n identificationWP: 'TightLH'\\n" + " isolationWP: 'Tight_VarRad'\\n noEffSF: True\\n" + " PtEtaSelection:\\n minPt: 25000.0\\n maxEta: 2.47\\n" + " useClusterEta: True\\n\\n" + "# After configuring each container, many variables will be saved automatically.\\n" + "Output:\\n treeName: 'reco'\\n vars: []\\n metVars: []\\n containers:\\n" + " # Format should follow: ':'\\n" + " el_: 'AnaElectrons'\\n '': 'EventInfo'\\n commands:\\n" + " # Turn output branches on and off with 'enable' and 'disable'\\n\\n" + 'AddConfigBlocks: []\\n", "parton": null, "particle": null, "max_events": 100, ' + '"no_systematics": true, "no_filter": false, ' + '"docker_image": "sslhep/custom_image:test"}' + ) + + expected_hash = "f30db9cc91520d3fc08cffd95b072634" + result = translator.generate_code(query, tmpdirname) + + # is the generated code at least syntactically valid Python? + try: + exec( + open(os.path.join(result.output_dir, "generated_transformer.py")).read() + ) + except SyntaxError: + pytest.fail("Generated Python is not valid code") + + assert result.hash == expected_hash + assert result.image == "sslhep/custom_image:test" + assert result.output_dir == os.path.join(tmpdirname, expected_hash) + + +def test_generate_code_fails_with_unknown_selection_key(): + os.environ["TEMPLATE_PATH"] = "servicex/templates/transform_single_file.py" + os.environ["CAPABILITIES_PATH"] = "transformer_capabilities.json" + + with tempfile.TemporaryDirectory() as tmpdirname: + translator = TopCPTranslator() + query = ( + '{"reco": "CommonServices:\\n systematicsHistogram: \'listOfSystematics\'\\n\\n' + "PileupReweighting: {}\\n\\nEventCleaning:\\n runEventCleaning: False\\n" + " runGRL: False\\n\\nElectrons:\\n - containerName: 'AnaElectrons'\\n" + " crackVeto: True\\n IFFClassification: {}\\n WorkingPoint:\\n" + " - selectionName: 'loose'\\n identificationWP: 'TightLH'\\n" + " isolationWP: 'NonIso'\\n noEffSF: True\\n" + " - selectionName: 'tight'\\n identificationWP: 'TightLH'\\n" + " isolationWP: 'Tight_VarRad'\\n noEffSF: True\\n" + " PtEtaSelection:\\n minPt: 25000.0\\n maxEta: 2.47\\n" + " useClusterEta: True\\n\\n" + "# After configuring each container, many variables will be saved automatically.\\n" + "Output:\\n treeName: 'reco'\\n vars: []\\n metVars: []\\n containers:\\n" + " # Format should follow: ':'\\n" + " el_: 'AnaElectrons'\\n '': 'EventInfo'\\n commands:\\n" + " # Turn output branches on and off with 'enable' and 'disable'\\n\\n" + 'AddConfigBlocks: []\\n", "parton": null, "particle": null, "max_events": 100, ' + '"no_systematics": true, "no_filter": false, "unknown_key": "unknown_value"}' + ) + + with pytest.raises(KeyError): + translator.generate_code(query, tmpdirname) + + +def test_generate_code_fails_with_missing_required_selection_key(): + os.environ["TEMPLATE_PATH"] = "servicex/templates/transform_single_file.py" + os.environ["CAPABILITIES_PATH"] = "transformer_capabilities.json" + + with tempfile.TemporaryDirectory() as tmpdirname: + translator = TopCPTranslator() + query = ( + '{"reco": "CommonServices:\\n systematicsHistogram: \'listOfSystematics\'\\n\\n' + "PileupReweighting: {}\\n\\nEventCleaning:\\n runEventCleaning: False\\n" + " runGRL: False\\n\\nElectrons:\\n - containerName: 'AnaElectrons'\\n" + " crackVeto: True\\n IFFClassification: {}\\n WorkingPoint:\\n" + " - selectionName: 'loose'\\n identificationWP: 'TightLH'\\n" + " isolationWP: 'NonIso'\\n noEffSF: True\\n" + " - selectionName: 'tight'\\n identificationWP: 'TightLH'\\n" + " isolationWP: 'Tight_VarRad'\\n noEffSF: True\\n" + " PtEtaSelection:\\n minPt: 25000.0\\n maxEta: 2.47\\n" + " useClusterEta: True\\n\\n" + "# After configuring each container, many variables will be saved automatically.\\n" + "Output:\\n treeName: 'reco'\\n vars: []\\n metVars: []\\n containers:\\n" + " # Format should follow: ':'\\n" + " el_: 'AnaElectrons'\\n '': 'EventInfo'\\n commands:\\n" + " # Turn output branches on and off with 'enable' and 'disable'\\n\\n" + 'AddConfigBlocks: []\\n", "parton": null, "particle": null, "max_events": 100, ' + '"no_systematics": true}' + ) + + with pytest.raises(ValueError): + translator.generate_code(query, tmpdirname) + + def test_app(): import servicex.TopCP_code_generator diff --git a/servicex_app/servicex_app_test/resources/transformation/test_submit.py b/servicex_app/servicex_app_test/resources/transformation/test_submit.py index 7c48dd528..79662b398 100644 --- a/servicex_app/servicex_app_test/resources/transformation/test_submit.py +++ b/servicex_app/servicex_app_test/resources/transformation/test_submit.py @@ -585,138 +585,3 @@ def test_submit_transformation_with_title( saved_obj = TransformRequest.lookup(request_id) assert saved_obj assert saved_obj.title == title - -# -# class TestSubmitTransformationRequestCustomImage(ResourceTestBase): -# """Tests for custom Docker image feature in transformation requests""" -# -# @staticmethod -# def _generate_transformation_request(**kwargs): -# request = { -# "did": "123-45-678", -# "selection": "test-string", -# "result-destination": "object-store", -# "result-format": "root-file", -# "workers": 10, -# "codegen": "topcp", -# } -# request.update(kwargs) -# return request -# -# @fixture -# def mock_dataset_manager_from_did(self, mocker): -# dm = mocker.Mock() -# dm.dataset = Dataset( -# name="rucio://123-45-678", -# did_finder="rucio", -# lookup_status=DatasetStatus.looking, -# last_used=datetime.now(tz=timezone.utc), -# last_updated=datetime.fromtimestamp(0), -# ) -# dm.name = "rucio://123-45-678" -# dm.id = 42 -# mock_from_did = mocker.patch.object(DatasetManager, "from_did", return_value=dm) -# return mock_from_did -# -# @fixture -# def mock_codegen(self, mocker): -# mock_code_gen = mocker.MagicMock(CodeGenAdapter) -# mock_code_gen.generate_code_for_selection.return_value = ( -# "my-cm", -# "sslhep/servicex_science_image_topcp:2.17.0", -# "bash", -# "echo", -# ) -# return mock_code_gen -# -# def test_submit_topcp_with_custom_docker_image( -# self, mock_dataset_manager_from_did, mock_codegen, mock_app_version -# ): -# """Test submitting a TopCP transformation with a valid custom docker image""" -# extra_config = { -# "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"}, -# "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]', -# } -# client = self._test_client( -# code_gen_service=mock_codegen, extra_config=extra_config -# ) -# with client.application.app_context(): -# selection_dict = { -# "docker_image": "sslhep/servicex_science_image_topcp:custom" -# } -# request = self._generate_transformation_request( -# selection=json.dumps(selection_dict) -# ) -# -# response = client.post( -# "/servicex/transformation", json=request, headers=self.fake_header() -# ) -# assert response.status_code == 200 -# request_id = response.json["request_id"] -# -# saved_obj = TransformRequest.lookup(request_id) -# assert saved_obj -# assert saved_obj.image == "sslhep/servicex_science_image_topcp:custom" -# -# def test_submit_topcp_with_invalid_custom_docker_image( -# self, mock_dataset_manager_from_did, mock_codegen -# ): -# """Submitting a TopCP transformation with an invalid custom docker image fails""" -# extra_config = { -# "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"}, -# "TOPCP_ALLOWED_IMAGES": '["sslhep/servicex_science_image_topcp:"]', -# } -# client = self._test_client( -# code_gen_service=mock_codegen, extra_config=extra_config -# ) -# with client.application.app_context(): -# selection_dict = {"docker_image": "unauthorized/image:latest"} -# request = self._generate_transformation_request( -# selection=json.dumps(selection_dict) -# ) -# -# response = client.post( -# "/servicex/transformation", json=request, headers=self.fake_header() -# ) -# assert response.status_code == 400 -# -# def test_submit_topcp_with_non_json_selection( -# self, mock_dataset_manager_from_did, mock_codegen, mock_app_version -# ): -# """Test submitting a TopCP transformation with non-JSON selection string""" -# extra_config = { -# "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} -# } -# client = self._test_client( -# code_gen_service=mock_codegen, extra_config=extra_config -# ) -# with client.application.app_context(): -# request = self._generate_transformation_request(selection="not-json-string") -# -# response = client.post( -# "/servicex/transformation", json=request, headers=self.fake_header() -# ) -# assert response.status_code == 400 -# -# def test_submit_custom_topcp_without_env_var( -# self, mock_dataset_manager_from_did, mock_codegen -# ): -# """Test submitting TopCP with custom image when TOPCP_ALLOWED_IMAGES is not set""" -# extra_config = { -# "CODE_GEN_IMAGES": {"topcp": "sslhep/servicex_code_gen_topcp:develop"} -# } -# client = self._test_client( -# code_gen_service=mock_codegen, extra_config=extra_config -# ) -# with client.application.app_context(): -# selection_dict = { -# "docker_image": "sslhep/servicex_science_image_topcp:custom" -# } -# request = self._generate_transformation_request( -# selection=json.dumps(selection_dict) -# ) -# -# response = client.post( -# "/servicex/transformation", json=request, headers=self.fake_header() -# ) -# assert response.status_code == 400 From c956c90a132f799eaa8494bac39639664ad18458 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 16:23:23 -0600 Subject: [PATCH 50/54] update test --- .../tests/test_validate_custom_docker_image.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py index 7889868d3..581532eb2 100644 --- a/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py +++ b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py @@ -38,11 +38,10 @@ def test_validate_with_no_matching_prefix(self, monkeypatch: MonkeyPatch): with pytest.raises(GenerateCodeException, match="not allowed"): validate_custom_docker_image("unauthorized/image:latest") - def test_validate_with_no_env_variable(self): + def test_validate_with_no_env_variable(self, monkeypatch: MonkeyPatch): """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" - with pytest.raises( - GenerateCodeException, match="Custom Docker images are not allowed" - ): + monkeypatch.delenv("TOPCP_ALLOWED_IMAGES") + with pytest.raises(GenerateCodeException): validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") def test_validate_with_invalid_json(self, monkeypatch: MonkeyPatch): From 8a1d8a53816528c5c0a289fc3034db26f2fa9507 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 16:42:26 -0600 Subject: [PATCH 51/54] add json parsing check to topcp codegen startup --- .../servicex/TopCP_code_generator/__init__.py | 8 ++++++++ .../servicex/TopCP_code_generator/request_translator.py | 4 ---- .../tests/test_validate_custom_docker_image.py | 8 +------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/__init__.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/__init__.py index 90ddf5c40..a865e6daf 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/__init__.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/__init__.py @@ -27,11 +27,19 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import os +import json import servicex_codegen from servicex.TopCP_code_generator.request_translator import TopCPTranslator def create_app(test_config=None, provided_translator=None): + allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES") + allowed_images = json.loads(allowed_images_json) + assert isinstance(allowed_images, list) + for item in allowed_images: + assert isinstance(item, str) + return servicex_codegen.create_app( test_config, provided_translator=( diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py index 2cbb46456..46348a893 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/request_translator.py @@ -45,10 +45,6 @@ def validate_custom_docker_image(image_name: str) -> bool: try: allowed_prefixes = json.loads(allowed_images_json) - if not isinstance(allowed_prefixes, list): - raise GenerateCodeException( - "TopCP allowed images are improperly configured." - ) for prefix in allowed_prefixes: if image_name.startswith(prefix): return True diff --git a/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py index 581532eb2..e1748f6ab 100644 --- a/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py +++ b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py @@ -41,7 +41,7 @@ def test_validate_with_no_matching_prefix(self, monkeypatch: MonkeyPatch): def test_validate_with_no_env_variable(self, monkeypatch: MonkeyPatch): """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" monkeypatch.delenv("TOPCP_ALLOWED_IMAGES") - with pytest.raises(GenerateCodeException): + with pytest.raises(GenerateCodeException, match="Custom Docker images are not allowed."): validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") def test_validate_with_invalid_json(self, monkeypatch: MonkeyPatch): @@ -50,12 +50,6 @@ def test_validate_with_invalid_json(self, monkeypatch: MonkeyPatch): with pytest.raises(GenerateCodeException, match="improperly configured"): validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") - def test_validate_with_non_list_json(self, monkeypatch: MonkeyPatch): - """Test validation fails when JSON is not a list""" - monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", '{"key": "value"}') - with pytest.raises(GenerateCodeException, match="improperly configured"): - validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") - def test_validate_with_empty_list(self, monkeypatch: MonkeyPatch): """Test validation fails when allowed list is empty""" monkeypatch.setenv("TOPCP_ALLOWED_IMAGES", "[]") From 2a5f73051bb2700cf989c12e37f181c9a9398904 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 16:44:47 -0600 Subject: [PATCH 52/54] check to make sure allowed_images_json is truthy --- .../servicex/TopCP_code_generator/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/__init__.py b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/__init__.py index a865e6daf..bdbeb82c2 100644 --- a/code_generator_TopCPToolkit/servicex/TopCP_code_generator/__init__.py +++ b/code_generator_TopCPToolkit/servicex/TopCP_code_generator/__init__.py @@ -35,10 +35,12 @@ def create_app(test_config=None, provided_translator=None): allowed_images_json = os.environ.get("TOPCP_ALLOWED_IMAGES") - allowed_images = json.loads(allowed_images_json) - assert isinstance(allowed_images, list) - for item in allowed_images: - assert isinstance(item, str) + + if allowed_images_json: + allowed_images = json.loads(allowed_images_json) + assert isinstance(allowed_images, list) + for item in allowed_images: + assert isinstance(item, str) return servicex_codegen.create_app( test_config, From 32293b52fc6b010e39681b1413cbc176df21bcdb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Nov 2025 00:27:18 +0000 Subject: [PATCH 53/54] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tests/test_validate_custom_docker_image.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py index e1748f6ab..4320e5f2c 100644 --- a/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py +++ b/code_generator_TopCPToolkit/tests/test_validate_custom_docker_image.py @@ -41,7 +41,9 @@ def test_validate_with_no_matching_prefix(self, monkeypatch: MonkeyPatch): def test_validate_with_no_env_variable(self, monkeypatch: MonkeyPatch): """Test validation fails when TOPCP_ALLOWED_IMAGES is not set""" monkeypatch.delenv("TOPCP_ALLOWED_IMAGES") - with pytest.raises(GenerateCodeException, match="Custom Docker images are not allowed."): + with pytest.raises( + GenerateCodeException, match="Custom Docker images are not allowed." + ): validate_custom_docker_image("sslhep/servicex_science_image_topcp:2.17.0") def test_validate_with_invalid_json(self, monkeypatch: MonkeyPatch): From 19601a7efa7aaf4bbe3782c94e0700743e02ae61 Mon Sep 17 00:00:00 2001 From: Matt Shirley Date: Thu, 13 Nov 2025 18:28:26 -0600 Subject: [PATCH 54/54] cleanup merge conflict --- docs/deployment/reference.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/deployment/reference.md b/docs/deployment/reference.md index 77e62c00c..cee82e634 100644 --- a/docs/deployment/reference.md +++ b/docs/deployment/reference.md @@ -74,6 +74,13 @@ parameters for the [rabbitMQ](https://github.com/bitnami/charts/tree/master/bitn | `codegen.cmssw-5-3-32.enabled` | Deploy the CMS AOD code generator? - also all of the code gen settings above are available | true | | `codegen.atlasr21.enabled` | Deploy the ATLAS FuncADL Release 21 code generator? - also all of the code gen settings above are available | true | | `codegen.atlasr22.enabled` | Deploy the ATLAS FuncADL Release 22 code generator? - also all of the code gen settings above are available | true | +| `codegen.topcp.enabled` | Deploy the TopCP code generator? | true | +| `codegen.topcp.image` | TopCP ode generator image | `sslhep/servicex_code_gen_topcp` | +| `codegen.topcp.pullPolicy` | TopCP code generator image pull policy | Always | +| `codegen.topcp.tag` | TopCP code generator image tag | develop | +| `codegen.topcp.defaultScienceContainerImage` | The default image used by a TopCP transformer container | sslhep/servicex_science_image_topcp | +| `codegen.topcp.defaultScienceContainerTag` | The default tag used by a TopCP transformer container | 2.17.0-25.2.45 | +| `codegen.topcp.allowedImages` | A list of strings, of which one must be a valid prefix of a submitted custom TopCP docker image | - "sslhep/servicex_science_image_topcp:" | | `codegen.python.enabled` | Deploy the python uproot code generator? - also all of the code gen settings, above are available | true | | `x509Secrets.image` | X509 Secret Service image name | `sslhep/x509-secrets` | | `x509Secrets.tag` | X509 Secret Service image tag | `latest` |