diff --git a/.github/workflows/data-dictionary-tests.yml b/.github/workflows/data-dictionary-tests.yml
index 71d393d06..6ceca5654 100644
--- a/.github/workflows/data-dictionary-tests.yml
+++ b/.github/workflows/data-dictionary-tests.yml
@@ -83,7 +83,7 @@ jobs:
# Extract variables from the api_test_counts.txt file
while IFS= read -r line; do
echo "::set-output name=${line%=*}::${line#*=}"
- done < data_dictionary_test_counts_${{ matrix.environment }}.txt
+ done < data_dictionary_tests_counts_${{ matrix.environment }}.txt
- name: Archive test results
id: artifact-upload-step
diff --git a/.github/workflows/power-bi-tests.yml b/.github/workflows/power-bi-tests.yml
index 3482c1b85..795cde5b6 100644
--- a/.github/workflows/power-bi-tests.yml
+++ b/.github/workflows/power-bi-tests.yml
@@ -73,7 +73,7 @@ jobs:
echo "matrix environment: ${{ matrix.environment }}"
echo "NOW=$(date +'%m-%d %H:%M')" >> $GITHUB_ENV
echo ${{env.NOW}}
- pytest -v --rootdir= Features/Powerbi_integration_exports/testCases -n 0 --dist=loadfile --reruns 1 --html=power_bi_reports_${{ matrix.environment }}.html
+ pytest -v --rootdir= Features/Powerbi_integration_exports/testCases -n 1 --dist=loadfile --reruns 1 --html=power_bi_reports_${{ matrix.environment }}.html
- name: Parse test counts
diff --git a/.gitignore b/.gitignore
index f47195c5a..d3b95614a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,6 +33,8 @@ MANIFEST
*.manifest
*.spec
+*/slack_charts/*
+
#Ignoring compliled files of RequestAPI
.idea/
pytest_html_report.html
@@ -238,6 +240,17 @@ Features/Lookuptable/settings.cfg
Features/Lookuptable/report.html
Features/Lookuptable/testCases/report.html
+#Ignoring compliled files of DataDictionary
+Features/DataDictionary/testCases/report.html
+Features/DataDictionary/testCases/slack_charts
+Features/DataDictionary/test_cases/report.html
+
+#Ignoring compliled files of Powerbi_integration_exports
+Features/Powerbi_integration_exports/settings.cfg
+Features/Powerbi_integration_exports/report.html
+Features/Powerbi_integration_exports/test_cases/report.html
+Features/Powerbi_integration_exports/testCases/slack_charts
+
#Ignoring compliled files of QA_Requests
QA_Requests/BHAStressTest/settings.cfg
QA_Requests/BHAStressTest/user_inputs/*.csv
@@ -277,6 +290,8 @@ LocustScripts/update-scripts/project-config/co-carecoordination-perf/mobile_work
/POCs/PercyWebApps/settings.cfg
/POCs/VisualComparison/settings.cfg
+
+
USH_Apps/WeeklyBHACleanup/testCases/report.html
USH_Apps/WeeklyBHACleanup/report.html
USH_Apps/WeeklyBHACleanup/report_*
diff --git a/Features/DataDictionary/README.md b/Features/DataDictionary/README.md
new file mode 100644
index 000000000..88116b467
--- /dev/null
+++ b/Features/DataDictionary/README.md
@@ -0,0 +1,57 @@
+## Commcare Data Dictionary Test Script
+
+These tests ensure that the Data Dictionary (https://dimagi.atlassian.net/wiki/spaces/commcarepublic/pages/2143944977/Data+Dictionary)features work as expected and that there are no regressions
+The automated tests comprises of the Data dictionary functionality.](https://docs.google.com/spreadsheets/d/1Ixw1PC9PVpqYOlP7aq9a86pC0wZaJVgGNVL_4h9KLbM/edit#gid=195915568)
+## Executing Scripts
+
+### On Local Machine
+
+#### Setting up test environment
+
+```sh
+
+# create and activate a virtualenv using your preferred method. Example:
+python -m venv venv
+source venv/bin/activate
+
+
+# install requirements
+pip install -r requires.txt
+
+```
+
+[More on setting up virtual environments](https://confluence.dimagi.com/display/GTD/QA+and+Python+Virtual+Environments)
+
+
+#### Running Tests
+
+
+ - Copy `settings-sample.cfg` to `settings.cfg` and populate `settings.cfg` for
+the environment you want to test.
+- Run tests using pytest command like:
+
+```sh
+
+# To execute all the test cases
+pytest -v --rootdir= Features/DataDictionary/testCases
+
+```
+- You could also pass the following arguments
+ - ` -n 3 --dist=loadfile` - This will run the tests parallelly in 3 instances. The number of reruns is configurable.
+ - ` --reruns 1` - This will re-run the tests once in case of failures.The number of reruns is configurable too.
+
+### Trigger Manually on Gitaction
+
+
+
+To manually trigger the script,
+ - Go to [DataDictionary](https://github.com/dimagi/dimagi-qa/actions/lookuptable.yml)
+ - Run workflow
+ - Use workflow from ```master```
+ - Run!
+
+If you are a part of the QA team, you'll receive emails for the result of the run after it's complete.
+
+
+
+Besides, you should be able to find the zipped results in the **Artifacts** section, of the corresponding run (after it's complete).
diff --git a/Features/DataDictionary/__init__.py b/Features/DataDictionary/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Features/DataDictionary/requires.txt b/Features/DataDictionary/requires.txt
new file mode 100644
index 000000000..0fc230cfa
--- /dev/null
+++ b/Features/DataDictionary/requires.txt
@@ -0,0 +1,22 @@
+## Stores information about all the libraries, modules, and packages that are used in this project.
+
+
+flake8>=3.8.4
+pandas>=1.2.2
+pytest
+py>=1.10.0
+pytest-html>=3.1.1
+pytest-json-report
+selenium == 4.11.0
+openpyxl
+matplotlib >= 3.3.4
+pytest-rerunfailures
+pytest-xdist
+pytest-xdist[psutil]
+pytest-order
+requests
+imap-tools
+beautifulsoup4
+html5lib
+pytest-metadata
+pyotp >=2.6.0
\ No newline at end of file
diff --git a/Features/DataDictionary/settings-sample.cfg b/Features/DataDictionary/settings-sample.cfg
new file mode 100644
index 000000000..b6bb402bb
--- /dev/null
+++ b/Features/DataDictionary/settings-sample.cfg
@@ -0,0 +1,12 @@
+[default]
+# This is the environment url of commcare
+url = https://www.commcarehq.org/
+# Login username of the webuser
+login_username =
+# Login password of the webuser
+login_password =
+# This is a preconfigured authentication key used for 2FA tests on staging - If 2FA enabled on staging.
+staging_auth_key =
+# This is a preconfigured authentication key used for 2FA tests on prod
+prod_auth_key =
+
diff --git a/Features/DataDictionary/testCases/__init__.py b/Features/DataDictionary/testCases/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/Features/DataDictionary/testCases/conftest.py b/Features/DataDictionary/testCases/conftest.py
new file mode 100644
index 000000000..01615e904
--- /dev/null
+++ b/Features/DataDictionary/testCases/conftest.py
@@ -0,0 +1,295 @@
+import os
+
+from configparser import ConfigParser
+from pathlib import Path
+from common_utilities.fixtures import *
+
+""""This file provides fixture functions for driver initialization"""
+
+global driver
+
+
+@pytest.fixture(scope="session")
+def environment_settings_lookup():
+ """Load settings from os.environ
+
+ Names of environment variables:
+ DIMAGIQA_URL
+ DIMAGIQA_LOGIN_USERNAME
+ DIMAGIQA_LOGIN_PASSWORD
+ DIMAGIQA_MAIL_USERNAME
+ DIMAGIQA_MAIL_PASSWORD
+
+ See https://docs.github.com/en/actions/reference/encrypted-secrets
+ for instructions on how to set them.
+ """
+ settings = {}
+ for name in ["url", "login_username", "login_password", "staging_auth_key", "prod_auth_key"]:
+
+ var = f"DIMAGIQA_{name.upper()}"
+ if var in os.environ:
+ settings[name] = os.environ[var]
+ if "url" not in settings:
+ env = os.environ.get("DIMAGIQA_ENV") or "staging"
+ subdomain = "www" if env == "production" else env
+ # updates the url with the project domain while testing in CI
+ project = "a/qa-automation-prod" if env == "production" else "a/qa-automation"
+ settings["url"] = f"https://{subdomain}.commcarehq.org/{project}"
+ return settings
+
+
+@pytest.fixture(scope="session", autouse=True)
+def settings(environment_settings_lookup):
+ if os.environ.get("CI") == "true":
+ settings = environment_settings_lookup
+ settings["CI"] = "true"
+ if any(x not in settings for x in ["url", "login_username", "login_password",
+ "staging_auth_key", "prod_auth_key"]):
+ lines = environment_settings_lookup.__doc__.splitlines()
+ vars_ = "\n ".join(line.strip() for line in lines if "DIMAGIQA_" in line)
+ raise RuntimeError(
+ f"Environment variables not set:\n {vars_}\n\n"
+ "See https://docs.github.com/en/actions/reference/encrypted-secrets "
+ "for instructions on how to set them."
+ )
+ return settings
+ path = Path(__file__).parent.parent / "settings.cfg"
+ if not path.exists():
+ raise RuntimeError(
+ f"Not found: {path}\n\n"
+ "Copy settings-sample.cfg to settings.cfg and populate "
+ "it with values for the environment you want to test."
+ )
+ settings = ConfigParser()
+ settings.read(path)
+ # updates the url with the project domain while testing in local
+ if settings["default"]["url"] == "https://www.commcarehq.org/":
+ settings["default"]["url"] = f"{settings['default']['url']}a/qa-automation-prod"
+ else:
+ settings["default"]["url"] = f"{settings['default']['url']}a/qa-automation"
+ return settings["default"]
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config):
+ # Collect test counts
+ passed = terminalreporter.stats.get('passed', [])
+ failed = terminalreporter.stats.get('failed', [])
+ error = terminalreporter.stats.get('error', [])
+ skipped = terminalreporter.stats.get('skipped', [])
+ xfail = terminalreporter.stats.get('xfail', [])
+
+ env = os.environ.get("DIMAGIQA_ENV", "default_env")
+
+ # Define the filename based on the environment
+ filename = f'data_dictionary_tests_counts_{env}.txt'
+
+ # Write the counts to a file
+ with open(filename, 'w') as f:
+ f.write(f'PASSED={len(passed)}\n')
+ f.write(f'FAILED={len(failed)}\n')
+ f.write(f'ERROR={len(error)}\n')
+ f.write(f'SKIPPED={len(skipped)}\n')
+ f.write(f'XFAIL={len(xfail)}\n')
+
+# conftest.py
+import pytest
+import matplotlib.pyplot as plt
+import base64
+from io import BytesIO
+
+_test_stats = {}
+
+def pytest_sessionfinish(session, exitstatus):
+ """Collect stats at the end of the test session."""
+ tr = session.config.pluginmanager.get_plugin("terminalreporter")
+ global _test_stats
+ _test_stats = {
+ "passed": len(tr.stats.get("passed", [])),
+ "failed": len(tr.stats.get("failed", [])),
+ "skipped": len(tr.stats.get("skipped", [])),
+ "error": len(tr.stats.get("error", [])),
+ "xfail": len(tr.stats.get("xfail", [])),
+ "reruns": len(tr.stats.get("rerun", [])),
+ }
+ save_summary_charts(_test_stats)
+
+import base64
+
+def save_summary_charts(stats):
+ from pathlib import Path
+ out_dir = Path("slack_charts")
+ out_dir.mkdir(exist_ok=True)
+
+ passed = stats.get("passed", 0)
+ failed = stats.get("failed", 0)
+ skipped = stats.get("skipped", 0)
+ reruns = stats.get("reruns", 0)
+
+ # --- Pie chart with legend ---
+ pie_labels = ["Passed", "Failed", "Skipped"]
+ pie_sizes = [passed, failed, skipped]
+ pie_colors = ["#66bb6a", "#ef5350", "#fad000"]
+
+ fig, ax = plt.subplots()
+ wedges, texts = ax.pie(
+ pie_sizes,
+ labels=None,
+ colors=pie_colors,
+ startangle=90,
+ wedgeprops=dict(width=0.4)
+ )
+ ax.axis("equal")
+ ax.set_title("Test Summary")
+
+ # Add legend with counts
+ ax.legend(
+ [f"Passed: {passed}", f"Failed: {failed}", f"Skipped: {skipped}"],
+ loc="lower center",
+ ncol=3,
+ bbox_to_anchor=(0.5, -0.15)
+ )
+
+ fig.savefig(out_dir / "summary_pie.png", bbox_inches="tight")
+ plt.close(fig)
+
+ # --- Bar chart with labels + legend ---
+ bar_path = None
+ if failed > 0 or reruns > 0: # ✅ only generate if needed
+ fig, ax = plt.subplots()
+ bars = ax.bar(
+ ["Failed", "Reruns"],
+ [failed, reruns],
+ color=["#ef5350", "#ffa726"]
+ )
+ ax.set_ylabel("Number of Tests")
+ ax.set_title("Failures and Reruns")
+
+ # Add counts above bars
+ for bar in bars:
+ height = bar.get_height()
+ ax.text(
+ bar.get_x() + bar.get_width() / 2,
+ height + 0.05,
+ str(int(height)),
+ ha="center",
+ va="bottom",
+ fontsize=10,
+ fontweight="bold"
+ )
+
+ # Legend with counts
+ ax.legend(
+ [f"Failed: {failed}", f"Reruns: {reruns}"],
+ loc="lower center",
+ ncol=2,
+ bbox_to_anchor=(0.5, -0.15)
+ )
+
+ bar_path = out_dir / "summary_bar.png"
+ fig.savefig(bar_path, bbox_inches="tight")
+ plt.close(fig)
+
+ # --- Combine ---
+ combine_charts(
+ pie_path=out_dir / "summary_pie.png",
+ bar_path=bar_path,
+ combined_path=out_dir / "summary_combined.png"
+ )
+
+
+import matplotlib.pyplot as plt
+from PIL import Image
+
+def combine_charts(pie_path="slack_charts/summary_pie.png",
+ bar_path=None,
+ combined_path="slack_charts/summary_combined.png"):
+ """Combine pie and bar charts side by side if bar exists, else only pie."""
+ from PIL import Image
+
+ pie = Image.open(pie_path)
+
+ if bar_path and Path(bar_path).exists():
+ bar = Image.open(bar_path)
+ bar = bar.resize((bar.width * pie.height // bar.height, pie.height))
+ combined = Image.new("RGB", (pie.width + bar.width, pie.height), (255, 255, 255))
+ combined.paste(pie, (0, 0))
+ combined.paste(bar, (pie.width, 0))
+ else:
+ # Only pie chart
+ combined = pie.copy()
+
+ combined.save(combined_path)
+ print(f"✅ Combined chart saved to {combined_path}")
+
+
+
+def _matplotlib_img(fig) -> str:
+ """Convert a matplotlib figure to base64 string."""
+ buf = BytesIO()
+ plt.tight_layout()
+ fig.savefig(buf, format="png")
+ plt.close(fig)
+ buf.seek(0)
+ return base64.b64encode(buf.read()).decode("utf-8")
+
+def pytest_html_results_summary(prefix, summary, postfix, session):
+ """Inject donut pie + bar chart with reruns support (parallel-safe)."""
+ tr = session.config.pluginmanager.get_plugin("terminalreporter")
+ stats = tr.stats if tr and hasattr(tr, "stats") else {}
+
+ passed = len(stats.get("passed", []))
+ failed = len(stats.get("failed", []))
+ skipped = len(stats.get("skipped", []))
+
+ # Reruns are recorded separately by pytest-rerunfailures
+ reruns = len(stats.get("rerun", []))
+
+ # --- Donut Pie Chart (Passed, Failed, Skipped) ---
+ pie_labels = ["Passed", "Failed", "Skipped"]
+ pie_sizes = [passed, failed, skipped]
+ pie_colors = ["#66bb6a", "#ef5350", "#fad000"]
+
+ fig, ax = plt.subplots()
+ wedges, texts = ax.pie(
+ pie_sizes,
+ labels=None,
+ colors=pie_colors,
+ startangle=90,
+ wedgeprops=dict(width=0.4)
+ )
+ ax.axis("equal")
+
+ # Legend below the donut
+ plt.legend(
+ wedges,
+ [f"{l}: {v}" for l, v in zip(pie_labels, pie_sizes)],
+ title="Results",
+ loc="upper center",
+ bbox_to_anchor=(0.5, -0.08),
+ ncol=len(pie_labels)
+ )
+ pie_img = _matplotlib_img(fig)
+
+ # --- Bar Chart (Failures + Reruns) ---
+ bar_img = None
+ if failed > 0 or reruns > 0:
+ fig, ax = plt.subplots()
+ bars = ax.bar(["Failed", "Reruns"], [failed, reruns], color=["#ef5350", "#ff9933"])
+ ax.set_title("Failures and Reruns")
+ ax.set_ylabel("Number of Tests")
+ plt.legend(
+ bars,
+ [f"Failed: {failed}", f"Reruns: {reruns}"],
+ loc="upper center",
+ bbox_to_anchor=(0.5, -0.12),
+ ncol=2
+ )
+ bar_img = _matplotlib_img(fig)
+
+ # --- Embed in HTML report ---
+ html = ""
+ html += f"
Test Summary
"
+ if bar_img:
+ html += f"
Failures and Reruns
"
+ html += "
"
+ html += f"
Test Summary
"
+ if bar_img:
+ html += f"
Failures and Reruns
"
+ html += "