diff --git a/AIVision/genai.py b/AIVision/genai.py index e8aeecf..ac012cd 100644 --- a/AIVision/genai.py +++ b/AIVision/genai.py @@ -24,15 +24,19 @@ from openai import OpenAI import os import base64 +from io import BytesIO +from PIL import Image from .attachments import AttachmentProcessor class AIPlatform: """Configuration class for AI platform settings.""" DEFAULT_IMG_DETAIL = "high" + DEFAULT_IMAGE_DPI = 72 def __init__(self, platform: Platforms = None, base_url: str = None, - api_key: str = None, model: str = None, image_detail: str = DEFAULT_IMG_DETAIL): + api_key: str = None, model: str = None, image_detail: str = DEFAULT_IMG_DETAIL, + image_dpi: int = DEFAULT_IMAGE_DPI): """ Initialize AI platform configuration. @@ -42,11 +46,13 @@ def __init__(self, platform: Platforms = None, base_url: str = None, api_key: API key for authentication model: Model name (overrides platform default) image_detail: Image detail level for vision models + image_dpi: DPI used when encoding images for AI model attachments """ self.platform = platform self.base_url = base_url or (platform.value["default_base_url"] if platform else None) self.model = model or (platform.value["default_model"] if platform else None) self.detail = image_detail + self.image_dpi = image_dpi self.api_key = api_key self.supports_vision = platform.value.get("supports_vision", False) if platform else False @@ -110,7 +116,8 @@ class GenAI: def __init__(self, platform: Platforms = Platforms.Ollama, base_url: str = None, api_key: str = None, model: str = None, image_detail: str = None, - simple_response: bool = True, initialize: bool = True, + image_dpi: int = AIPlatform.DEFAULT_IMAGE_DPI, simple_response: bool = True, + initialize: bool = True, system_prompt: str = AUTOMATOR_INSTRUCTION): """ Initialize GenAI instance. @@ -121,6 +128,7 @@ def __init__(self, platform: Platforms = Platforms.Ollama, base_url: str = None, api_key: API key for authentication model: Model name to use image_detail: Detail level for image processing + image_dpi: DPI used when encoding images for AI model attachments simple_response: Return simplified responses initialize: Initialize client immediately system_prompt: Main AI System prompt specifying Gen AI behavior @@ -138,7 +146,8 @@ def __init__(self, platform: Platforms = Platforms.Ollama, base_url: str = None, base_url=base_url, api_key=api_key, model=model, - image_detail=image_detail + image_detail=image_detail, + image_dpi=image_dpi ) self.attachments = AttachmentProcessor(supports_vision=self.ai_platform.supports_vision) @@ -277,7 +286,10 @@ def _format_messages_for_openai(self, messages): # Convert image file to base64 data URI image_path = item.get("image_path") if image_path and self.ai_platform.supports_vision: - image_data = self._encode_image_to_base64(image_path) + image_data = self._encode_image_to_base64( + image_path, + dpi=self.ai_platform.image_dpi, + ) formatted_content.append({ "type": "image_url", "image_url": { @@ -299,29 +311,57 @@ def _format_messages_for_openai(self, messages): return formatted_messages @staticmethod - def _encode_image_to_base64(image_path: str) -> str: + def _encode_image_to_base64(image_path: str, dpi: int = AIPlatform.DEFAULT_IMAGE_DPI) -> str: """ Encode image file to base64 data URI. Args: image_path: Path to image file + dpi: DPI value stored in the encoded image metadata Returns: Base64-encoded data URI string """ - with open(image_path, "rb") as image_file: - image_data = base64.b64encode(image_file.read()).decode('utf-8') - - # Detect image format from file extension ext = os.path.splitext(image_path)[1].lower() mime_types = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.gif': 'image/gif', - '.webp': 'image/webp' + '.webp': 'image/webp', + '.bmp': 'image/bmp', + '.tiff': 'image/tiff', + '.tif': 'image/tiff', + } + pil_formats = { + '.png': 'PNG', + '.jpg': 'JPEG', + '.jpeg': 'JPEG', + '.gif': 'GIF', + '.webp': 'WEBP', + '.bmp': 'BMP', + '.tiff': 'TIFF', + '.tif': 'TIFF', } mime_type = mime_types.get(ext, 'image/png') + pil_format = pil_formats.get(ext) + + if pil_format: + try: + with Image.open(image_path) as image_file: + image_to_save = image_file + if pil_format == "JPEG" and image_file.mode not in ("L", "RGB"): + image_to_save = image_file.convert("RGB") + + buffer = BytesIO() + image_to_save.save(buffer, format=pil_format, dpi=(dpi, dpi)) + image_data = base64.b64encode(buffer.getvalue()).decode("utf-8") + return f"data:{mime_type};base64,{image_data}" + except Exception: + pass + + with open(image_path, "rb") as image_file: + image_data = base64.b64encode(image_file.read()).decode('utf-8') return f"data:{mime_type};base64,{image_data}" diff --git a/AIVision/library.py b/AIVision/library.py index f70ba6e..bf77317 100644 --- a/AIVision/library.py +++ b/AIVision/library.py @@ -58,11 +58,11 @@ class AIVision: OUTPUT_DIR = _get_rf_output_dir() def __init__(self, base_url: str = None, api_key: str = None, platform: Platforms = Platforms.Ollama, - model: str = None, image_detail: str = None, simple_response: bool = True, + model: str = None, image_detail: str = None, image_dpi: int = 72, simple_response: bool = True, initialize: bool = True, system_prompt: str = None): self.genai = GenAI(base_url=base_url, api_key=api_key, platform=platform, - model=model, image_detail=image_detail, + model=model, image_detail=image_detail, image_dpi=image_dpi, simple_response=simple_response, initialize=initialize, system_prompt=system_prompt) self.OUTPUT_DIR = _get_rf_output_dir() @@ -132,6 +132,7 @@ def _is_image_path(path): @keyword def verify_screenshot_matches_look_and_feel_template(self, screenshot_path, template_path, override_instructions: str = None, + additional_instructions: str = None, create_combined_image: bool = True): """Verifies that the screenshot matches the look and feel template @@ -143,6 +144,9 @@ def verify_screenshot_matches_look_and_feel_template(self, screenshot_path, temp ``override_instructions``: (optional) If specified, it will override the built-in assertion instructions + ``additional_instructions``: (optional) If specified, additional user specified instructions will be added + to the system instructions + ``create_combined_image``: (optional) default is _True_. If _True_, combined image will be created and saved _Return Value_ is the path of the saved image @@ -167,6 +171,10 @@ def verify_screenshot_matches_look_and_feel_template(self, screenshot_path, temp """ if override_instructions: instructions = override_instructions + + if additional_instructions: + instructions = f"{instructions} {additional_instructions}" + response = self.genai.generate_ai_response( instructions=instructions, image_paths=[screenshot_path, template_path]) @@ -320,7 +328,7 @@ def generate_image_name(prefix="Snap", extension="png"): @keyword def combine_images_on_paths_side_by_side(self, image_path1, image_path2, watermark1=None, watermark2=None, - mode="RGB", save=True): + mode="RGB", save=True, align=True): """Combines two images specified by file path to one big image side-by-side Input parameters: @@ -335,6 +343,9 @@ def combine_images_on_paths_side_by_side(self, image_path1, image_path2, waterma ``mode``: (optional) default is _RGB_. + ``align``: (optional) default is _True_. + If _True_, the wider image is resized to match the narrower image width. + _Return Value_ is combined image as PIL Image format *Examples*: @@ -346,7 +357,7 @@ def combine_images_on_paths_side_by_side(self, image_path1, image_path2, waterma img2 = self.open_image(image_path2, mode=mode) combined_img = self.combine_images_side_by_side(img1, img2, watermark1=watermark1, watermark2=watermark2, - mode=mode) + mode=mode, align=align) if save: self.save_image(combined_img) @@ -356,7 +367,7 @@ def combine_images_on_paths_side_by_side(self, image_path1, image_path2, waterma # pylint: disable=too-many-arguments,too-many-positional-arguments @keyword def combine_images_side_by_side( - self, image1, image2, watermark1=None, watermark2=None, mode="RGB" + self, image1, image2, watermark1=None, watermark2=None, mode="RGB", align=True ): """Combines two images to one big image side-by-side @@ -376,6 +387,9 @@ def combine_images_side_by_side( Supported modes can be seen [https://pillow.readthedocs.io/en/3.0.x/handbook/concepts.html#modes|here]. + ``align``: (optional) default is _True_. + If _True_, the wider image is resized to match the narrower image width. + _Return Value_ is combined image as PIL Image format *Examples*: @@ -383,6 +397,21 @@ def combine_images_side_by_side( | ${image} = | Combine Images Side By Side | ${image1} | ${image2} |RGBA | """ try: + if align and image1.size[0] != image2.size[0]: + target_width = min(image1.size[0], image2.size[0]) + resample = Image.Resampling.LANCZOS if hasattr(Image, "Resampling") else Image.LANCZOS + + if image1.size[0] > target_width: + image1 = image1.resize( + (target_width, max(1, int(image1.size[1] * (target_width / image1.size[0])))), + resample=resample, + ) + else: + image2 = image2.resize( + (target_width, max(1, int(image2.size[1] * (target_width / image2.size[0])))), + resample=resample, + ) + # Create empty image for both images to fit combined_image = Image.new( mode, diff --git a/CHANGES.txt b/CHANGES.txt index 31158bd..8bd5535 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,4 @@ +1.2.0, 2026-02-24 -- Image size optimizations, additional_instructions can now be passed to look and feel check keyword 1.1.2, 2026-02-09 -- Improved debug logging 1.1.1, 2026-02-09 -- Pass/Fail AI response optimized 1.1.0, 2026-02-08 -- Added support for (text) files attachments diff --git a/README.md b/README.md index 2677640..c3367ed 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Runtime dependencies include Robot Framework, Pillow, and the `openai` Python cl ## Configuration -Import the library in Robot Framework and choose a provider using `platform` plus optional overrides (`base_url`, `api_key`, `model`, `image_detail`). +Import the library in Robot Framework and choose a provider using `platform` plus optional overrides (`base_url`, `api_key`, `model`, `image_detail`, `image_dpi`). ### Robot Framework import examples @@ -100,7 +100,7 @@ All keywords below are implemented in `AIVision` and are available after importi | Keyword | Purpose | |---|---| | `Verify That` | Send one or more screenshots and/or file attachments with instructions to the model, parse the `RESULT` and raise `AssertionError` on failure. | -| `Verify Screenshot Matches Look And Feel Template` | Compare a screenshot against a reference template with a built-in instruction set; optional combined image creation. | +| `Verify Screenshot Matches Look And Feel Template` | Compare a screenshot against a reference template with built-in instructions, optional `override_instructions`, optional `additional_instructions`, and optional combined image creation. | | `Open Image` | Open an image (and optionally convert mode, default `RGB`). | | `Save Image` | Save a PIL image to a path (defaults to RF output directory) with optional watermark. | | `Generate Image Name` | Create a unique timestamp-based filename with prefix/extension. | @@ -161,3 +161,19 @@ Home page matches template - custom rules ... ${CURDIR}/templates/home_expected.png ... override_instructions=Verify layout, spacing, typography, and brand colors match the template exactly. ``` + +### Add additional template instructions + +```robotframework +*** Settings *** +Library AIVision + +*** Test Cases *** +Home page matches template - ignore dynamic parts + Verify Screenshot Matches Look And Feel Template + ... ${CURDIR}/screens/home_actual.png + ... ${CURDIR}/templates/home_expected.png + ... additional_instructions=Ignore the live clock and rotating promotional banner. +``` + +`additional_instructions` is appended to the default template checks (or to `override_instructions` if you provide both). diff --git a/docs/index.html b/docs/index.html index 99a91fd..8e54ba5 100644 --- a/docs/index.html +++ b/docs/index.html @@ -8,7 +8,7 @@ diff --git a/unittests/test_genai_attachments.py b/unittests/test_genai_attachments.py index 1f40eae..e73f80e 100644 --- a/unittests/test_genai_attachments.py +++ b/unittests/test_genai_attachments.py @@ -22,6 +22,7 @@ from AIVision.genai import GenAI from AIVision.attachments import AttachmentProcessor +from unittest.mock import patch def test_prepare_prompt_includes_text_attachment(tmp_path): @@ -97,3 +98,23 @@ def test_pdf_fallback_renders_images_when_no_text(tmp_path, monkeypatch): assert any("format: pdf images" in item["text"] for item in text_items) assert [item["image_path"] for item in image_items] == ["page1.png", "page2.png"] + + +def test_format_messages_uses_default_image_dpi(): + genai = GenAI(initialize=False) + messages = [{"role": "user", "content": [{"type": "image", "image_path": "/tmp/image.png"}]}] + + with patch.object(GenAI, "_encode_image_to_base64", return_value="data:image/png;base64,abc") as mock_encode: + genai._format_messages_for_openai(messages) + + mock_encode.assert_called_once_with("/tmp/image.png", dpi=72) + + +def test_format_messages_uses_custom_image_dpi(): + genai = GenAI(initialize=False, image_dpi=144) + messages = [{"role": "user", "content": [{"type": "image", "image_path": "/tmp/image.png"}]}] + + with patch.object(GenAI, "_encode_image_to_base64", return_value="data:image/png;base64,abc") as mock_encode: + genai._format_messages_for_openai(messages) + + mock_encode.assert_called_once_with("/tmp/image.png", dpi=144) diff --git a/unittests/test_lib.py b/unittests/test_lib.py index b2818d6..d527894 100644 --- a/unittests/test_lib.py +++ b/unittests/test_lib.py @@ -52,6 +52,21 @@ def test_verify_screenshot_matches_look_and_feel_template(aivison_library): aivison_library._assert_result.assert_called_once_with("response") +def test_verify_screenshot_matches_look_and_feel_template_with_additional_instructions(aivison_library): + aivison_library.genai.generate_ai_response = MagicMock(return_value="response") + aivison_library._assert_result = MagicMock() + with patch.object(aivison_library, 'combine_images_on_paths_side_by_side', return_value=None): + aivison_library.verify_screenshot_matches_look_and_feel_template( + "path/to/screenshot.png", + "path/to/template.png", + additional_instructions="Ignore status badge." + ) + instructions = aivison_library.genai.generate_ai_response.call_args[1]["instructions"] + assert "First image is showing actual application view" in instructions + assert instructions.endswith("Ignore status badge.") + aivison_library._assert_result.assert_called_once_with("response") + + def test_open_image(aivison_library): with patch("PIL.Image.open", return_value=Image.new("RGB", (100, 100))): image = aivison_library.open_image("path/to/image.png") diff --git a/unittests/test_library.py b/unittests/test_library.py index f18bd91..ebf3af4 100644 --- a/unittests/test_library.py +++ b/unittests/test_library.py @@ -88,11 +88,20 @@ def test_init(self, mock_genai): platform=Platforms.Ollama, model="test_model", image_detail="high", + image_dpi=72, simple_response=True, initialize=True, system_prompt=None ) + def test_init_with_custom_image_dpi(self, mock_genai): + """Test initialization with custom image DPI""" + with patch('AIVision.library._get_rf_output_dir', return_value='/mock/output/dir'): + AIVision(api_key="test_key", image_dpi=144) + + from AIVision.library import GenAI + assert GenAI.call_args[1]["image_dpi"] == 144 + def test_verify_that_single_path(self, library, mock_genai, mock_logger): """Test verify_that method with a single screenshot path""" library.verify_that("/path/to/image.png", "Contains green logo in top right corner") @@ -193,6 +202,40 @@ def test_verify_screenshot_matches_look_and_feel_template_with_override(self, li image_paths=["/path/to/screenshot.png", "/path/to/template.png"] ) + def test_verify_screenshot_matches_look_and_feel_template_with_additional_instructions(self, library, mock_genai): + """Test verify_screenshot_matches_look_and_feel_template method with additional instructions""" + with patch.object(library, 'combine_images_on_paths_side_by_side'): + library.verify_screenshot_matches_look_and_feel_template( + "/path/to/screenshot.png", + "/path/to/template.png", + additional_instructions="Ignore clock in header." + ) + + mock_genai.generate_ai_response.assert_called_once() + instructions = mock_genai.generate_ai_response.call_args[1]["instructions"] + assert "First image is showing actual application view" in instructions + assert instructions.endswith("Ignore clock in header.") + assert mock_genai.generate_ai_response.call_args[1]["image_paths"] == [ + "/path/to/screenshot.png", + "/path/to/template.png", + ] + + def test_verify_screenshot_matches_look_and_feel_template_with_override_and_additional_instructions(self, library, + mock_genai): + """Test verify_screenshot_matches_look_and_feel_template method with override and additional instructions""" + with patch.object(library, 'combine_images_on_paths_side_by_side'): + library.verify_screenshot_matches_look_and_feel_template( + "/path/to/screenshot.png", + "/path/to/template.png", + override_instructions="Custom instructions", + additional_instructions="Focus on button colors." + ) + + mock_genai.generate_ai_response.assert_called_once_with( + instructions="Custom instructions Focus on button colors.", + image_paths=["/path/to/screenshot.png", "/path/to/template.png"] + ) + def test_verify_screenshot_matches_look_and_feel_template_no_combine(self, library, mock_genai): """Test verify_screenshot_matches_look_and_feel_template method without combining images""" library.verify_screenshot_matches_look_and_feel_template( @@ -352,7 +395,8 @@ def test_combine_images_on_paths_side_by_side(self, library, mock_image, mock_lo mock_image, watermark1="Watermark1", watermark2="Watermark2", - mode="RGB" + mode="RGB", + align=True ) mock_save.assert_called_once_with(mock_image) @@ -409,6 +453,22 @@ def test_combine_images_side_by_side_failure(self, library, mock_image): assert "Could not create combined image" in str(exc.value) assert "Combine error" in str(exc.value) + def test_combine_images_side_by_side_align_resizes_wider_image(self, library): + image1 = Image.new("RGB", (200, 100)) + image2 = Image.new("RGB", (100, 100)) + + result = library.combine_images_side_by_side(image1, image2) + + assert result.size == (201, 100) + + def test_combine_images_side_by_side_align_disabled_keeps_original_widths(self, library): + image1 = Image.new("RGB", (200, 100)) + image2 = Image.new("RGB", (100, 100)) + + result = library.combine_images_side_by_side(image1, image2, align=False) + + assert result.size == (301, 100) + def test_add_watermark_to_image(self, library, mock_image, mock_logger): """Test add_watermark_to_image method""" mock_font = MagicMock()