Merge pull request #142 from nold-ai/dev #182
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json | |
| name: publish-modules | |
| on: | |
| push: | |
| branches: [dev, main] | |
| workflow_dispatch: | |
| inputs: | |
| bundles: | |
| description: "Comma-separated bundle names (for example: specfact-backlog,specfact-project). Empty = auto-detect." | |
| required: false | |
| default: "" | |
| dry_run: | |
| description: "Prepare registry changes but do not push commit" | |
| required: false | |
| default: false | |
| type: boolean | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| concurrency: | |
| group: publish-modules-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| publish: | |
| if: github.actor != 'github-actions[bot]' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install publish dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install pyyaml packaging | |
| - name: Resolve publish bundle set | |
| id: bundles | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| DISPATCH_BUNDLES: ${{ github.event.inputs.bundles }} | |
| BEFORE_SHA: ${{ github.event.before }} | |
| AFTER_SHA: ${{ github.sha }} | |
| TARGET_REGISTRY_BASE_REF: ${{ github.event.repository.default_branch }} | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import os | |
| import subprocess | |
| from pathlib import Path | |
| import yaml | |
| from publish_bundle_selection import determine_registry_baseline_ref, resolve_publish_bundles | |
| def normalize_bundle(name: str) -> str: | |
| cleaned = name.strip() | |
| if not cleaned: | |
| return "" | |
| if not cleaned.startswith("specfact-"): | |
| cleaned = f"specfact-{cleaned}" | |
| return cleaned | |
| def load_registry_versions(payload: dict) -> dict[str, str]: | |
| modules = payload.get("modules") | |
| if not isinstance(modules, list): | |
| return {} | |
| result: dict[str, str] = {} | |
| for entry in modules: | |
| if not isinstance(entry, dict): | |
| continue | |
| module_id = str(entry.get("id") or "").strip() | |
| latest_version = str(entry.get("latest_version") or "").strip() | |
| if not module_id or not latest_version: | |
| continue | |
| bundle_name = module_id.split("/", 1)[-1] | |
| if not bundle_name.startswith("specfact-"): | |
| continue | |
| result[bundle_name] = latest_version | |
| return result | |
| def load_manifest_versions(repo_root: Path) -> dict[str, str]: | |
| result: dict[str, str] = {} | |
| for manifest_path in sorted((repo_root / "packages").glob("*/module-package.yaml")): | |
| manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) | |
| if not isinstance(manifest, dict): | |
| continue | |
| bundle_name = manifest_path.parent.name | |
| version = str(manifest.get("version") or "").strip() | |
| if bundle_name.startswith("specfact-") and version: | |
| result[bundle_name] = version | |
| return result | |
| def load_baseline_registry(repo_root: Path, base_ref: str) -> dict: | |
| current_branch = os.environ.get("GITHUB_REF_NAME", "").strip() | |
| normalized = determine_registry_baseline_ref( | |
| current_branch=current_branch, | |
| default_branch=(base_ref or "main"), | |
| ) | |
| if normalized == current_branch: | |
| return json.loads((repo_root / "registry" / "index.json").read_text(encoding="utf-8")) | |
| subprocess.run(["git", "fetch", "origin", normalized, "--depth=1"], check=True) | |
| raw = subprocess.check_output( | |
| ["git", "show", f"origin/{normalized}:registry/index.json"], | |
| text=True, | |
| ) | |
| return json.loads(raw) | |
| event_name = os.environ.get("EVENT_NAME", "") | |
| dispatch_bundles = os.environ.get("DISPATCH_BUNDLES", "").strip() | |
| repo_root = Path(".").resolve() | |
| bundles: list[str] = [] | |
| changed_paths: list[str] = [] | |
| if event_name == "workflow_dispatch" and dispatch_bundles: | |
| bundles = [normalize_bundle(part) for part in dispatch_bundles.split(",")] | |
| bundles = sorted({bundle for bundle in bundles if bundle}) | |
| else: | |
| before_sha = os.environ.get("BEFORE_SHA", "").strip() | |
| after_sha = os.environ.get("AFTER_SHA", "").strip() or "HEAD" | |
| if not before_sha or before_sha == "0000000000000000000000000000000000000000": | |
| before_sha = "HEAD~1" | |
| try: | |
| changed_paths = subprocess.check_output( | |
| ["git", "diff", "--name-only", before_sha, after_sha, "--", "packages"], | |
| text=True, | |
| ).splitlines() | |
| except subprocess.CalledProcessError: | |
| changed_paths = [] | |
| manifest_versions = load_manifest_versions(repo_root) | |
| baseline_registry = load_baseline_registry(repo_root, os.environ.get("TARGET_REGISTRY_BASE_REF", "main")) | |
| registry_versions = load_registry_versions(baseline_registry) | |
| selection = resolve_publish_bundles( | |
| changed_paths=changed_paths, | |
| manifest_versions=manifest_versions, | |
| registry_versions=registry_versions, | |
| ) | |
| bundles = sorted(selection) | |
| reason_map = {bundle: sorted(reasons) for bundle, reasons in selection.items()} | |
| print(f"Resolved bundle reasons: {reason_map}") | |
| if event_name == "workflow_dispatch" and dispatch_bundles: | |
| reason_map = {bundle: ["manual-dispatch"] for bundle in bundles} | |
| out_path = Path(os.environ["GITHUB_OUTPUT"]) | |
| out_path.write_text( | |
| f"bundles_json={json.dumps(bundles)}\ncount={len(bundles)}\nbundle_reasons_json={json.dumps(reason_map)}\n", | |
| encoding="utf-8", | |
| ) | |
| print(f"Resolved bundles: {bundles}") | |
| PY | |
| - name: Publish changed bundles into registry | |
| if: steps.bundles.outputs.count != '0' | |
| env: | |
| BUNDLES_JSON: ${{ steps.bundles.outputs.bundles_json }} | |
| BUNDLE_REASONS_JSON: ${{ steps.bundles.outputs.bundle_reasons_json }} | |
| TARGET_REGISTRY_BASE_REF: ${{ github.event.repository.default_branch }} | |
| run: | | |
| python - <<'PY' | |
| import hashlib | |
| import json | |
| import subprocess | |
| import tarfile | |
| from pathlib import Path | |
| import yaml | |
| from publish_bundle_selection import determine_registry_baseline_ref | |
| repo_root = Path(".").resolve() | |
| import os | |
| bundles = json.loads(os.environ["BUNDLES_JSON"]) | |
| bundle_reasons = json.loads(os.environ.get("BUNDLE_REASONS_JSON", "{}")) | |
| registry_index_path = repo_root / "registry" / "index.json" | |
| registry_modules_dir = repo_root / "registry" / "modules" | |
| registry_signatures_dir = repo_root / "registry" / "signatures" | |
| registry_modules_dir.mkdir(parents=True, exist_ok=True) | |
| registry_signatures_dir.mkdir(parents=True, exist_ok=True) | |
| registry = json.loads(registry_index_path.read_text(encoding="utf-8")) | |
| modules = registry.get("modules") | |
| if not isinstance(modules, list): | |
| raise ValueError("registry/index.json must contain a 'modules' list") | |
| ignored_dir_names = {".git", "tests", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", "logs"} | |
| ignored_suffixes = {".pyc", ".pyo"} | |
| skipped_bundles: list[str] = [] | |
| current_branch = os.environ.get("GITHUB_REF_NAME", "").strip() | |
| baseline_ref = determine_registry_baseline_ref( | |
| current_branch=current_branch, | |
| default_branch=(os.environ.get("TARGET_REGISTRY_BASE_REF") or "main"), | |
| ) | |
| baseline_index_path = repo_root / ".publish-registry-baseline.json" | |
| if baseline_ref == current_branch: | |
| baseline_index_path.write_text(registry_index_path.read_text(encoding="utf-8"), encoding="utf-8") | |
| else: | |
| subprocess.run(["git", "fetch", "origin", baseline_ref, "--depth=1"], check=True) | |
| raw_baseline = subprocess.check_output( | |
| ["git", "show", f"origin/{baseline_ref}:registry/index.json"], | |
| text=True, | |
| ) | |
| baseline_index_path.write_text(raw_baseline, encoding="utf-8") | |
| for bundle in bundles: | |
| reasons = bundle_reasons.get(bundle, []) | |
| print(f"Processing {bundle} because of: {', '.join(reasons) if reasons else 'unspecified'}") | |
| # Keep existing monotonic version guard logic. | |
| publish_check = subprocess.run( | |
| [ | |
| "python", | |
| "scripts/publish-module.py", | |
| "--bundle", | |
| bundle, | |
| "--repo-root", | |
| ".", | |
| "--registry-index-path", | |
| str(baseline_index_path), | |
| ], | |
| check=False, | |
| capture_output=True, | |
| text=True, | |
| ) | |
| combined_output = "\n".join( | |
| part.strip() for part in (publish_check.stdout, publish_check.stderr) if part and part.strip() | |
| ) | |
| if publish_check.returncode != 0: | |
| if "Bundle version must be greater than registry latest_version" in combined_output: | |
| print(f"Skipping {bundle}: registry already at manifest version.") | |
| if combined_output: | |
| print(combined_output) | |
| skipped_bundles.append(bundle) | |
| continue | |
| if combined_output: | |
| print(combined_output) | |
| raise subprocess.CalledProcessError( | |
| publish_check.returncode, | |
| publish_check.args, | |
| output=publish_check.stdout, | |
| stderr=publish_check.stderr, | |
| ) | |
| bundle_dir = repo_root / "packages" / bundle | |
| manifest_path = bundle_dir / "module-package.yaml" | |
| if not manifest_path.exists(): | |
| raise FileNotFoundError(f"Missing manifest: {manifest_path}") | |
| manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) | |
| if not isinstance(manifest, dict): | |
| raise ValueError(f"Invalid manifest content: {manifest_path}") | |
| module_id = str(manifest.get("name") or f"nold-ai/{bundle}") | |
| version = str(manifest.get("version") or "").strip() | |
| if not version: | |
| raise ValueError(f"Manifest missing version: {manifest_path}") | |
| artifact_name = f"{bundle}-{version}.tar.gz" | |
| artifact_path = registry_modules_dir / artifact_name | |
| signature_path = registry_signatures_dir / f"{bundle}-{version}.tar.sig" | |
| with tarfile.open(artifact_path, mode="w:gz") as tar: | |
| for path in sorted(bundle_dir.rglob("*")): | |
| if not path.is_file(): | |
| continue | |
| rel = path.relative_to(bundle_dir) | |
| if any(part in ignored_dir_names for part in rel.parts): | |
| continue | |
| if path.suffix.lower() in ignored_suffixes: | |
| continue | |
| tar.add(path, arcname=f"{bundle}/{rel.as_posix()}") | |
| digest = hashlib.sha256(artifact_path.read_bytes()).hexdigest() | |
| (artifact_path.with_suffix(artifact_path.suffix + ".sha256")).write_text(f"{digest}\n", encoding="utf-8") | |
| integrity = manifest.get("integrity") | |
| if isinstance(integrity, dict): | |
| signature_text = str(integrity.get("signature") or "").strip() | |
| if signature_text: | |
| signature_path.write_text(signature_text + "\n", encoding="utf-8") | |
| entry = next( | |
| ( | |
| item | |
| for item in modules | |
| if isinstance(item, dict) and str(item.get("id") or "").strip() == module_id | |
| ), | |
| None, | |
| ) | |
| if entry is None: | |
| entry = {"id": module_id} | |
| modules.append(entry) | |
| entry["latest_version"] = version | |
| entry["download_url"] = f"modules/{artifact_name}" | |
| entry["checksum_sha256"] = digest | |
| if "tier" in manifest: | |
| entry["tier"] = manifest["tier"] | |
| if "publisher" in manifest: | |
| entry["publisher"] = manifest["publisher"] | |
| if "bundle_dependencies" in manifest: | |
| entry["bundle_dependencies"] = manifest["bundle_dependencies"] | |
| if "description" in manifest: | |
| entry["description"] = manifest["description"] | |
| print(f"Published registry artifact for {module_id} v{version}") | |
| if skipped_bundles: | |
| print(f"Skipped already-published bundles: {skipped_bundles}") | |
| registry_index_path.write_text(json.dumps(registry, indent=2) + "\n", encoding="utf-8") | |
| PY | |
| - name: Commit registry updates via PR branch | |
| if: steps.bundles.outputs.count != '0' | |
| env: | |
| DRY_RUN: ${{ github.event.inputs.dry_run }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BUNDLE_REASONS_JSON: ${{ steps.bundles.outputs.bundle_reasons_json }} | |
| run: | | |
| if git diff --quiet -- registry/index.json registry/modules registry/signatures; then | |
| echo "No registry changes to commit." | |
| exit 0 | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| TARGET_BRANCH="${GITHUB_REF_NAME}" | |
| PUBLISH_BRANCH="auto/publish-${TARGET_BRANCH}-${GITHUB_RUN_ID}" | |
| git checkout -b "${PUBLISH_BRANCH}" | |
| git add registry/index.json registry/modules registry/signatures | |
| git commit -m "chore(registry): publish changed modules [skip ci]" | |
| if [ "${DRY_RUN:-false}" = "true" ]; then | |
| echo "Dry run enabled; skipping push." | |
| exit 0 | |
| fi | |
| git push origin "${PUBLISH_BRANCH}" | |
| REASON_LINES=$(python - <<'PY' | |
| import json | |
| import os | |
| reason_map = json.loads(os.environ.get("BUNDLE_REASONS_JSON", "{}")) | |
| lines = [] | |
| for bundle in sorted(reason_map): | |
| reasons = ", ".join(reason_map[bundle]) | |
| lines.append(f"- `{bundle}`: {reasons}") | |
| print("\n".join(lines)) | |
| PY | |
| ) | |
| gh pr create \ | |
| --base "${TARGET_BRANCH}" \ | |
| --head "${PUBLISH_BRANCH}" \ | |
| --title "chore(registry): publish changed modules" \ | |
| --body "Automated registry publish update from workflow run ${GITHUB_RUN_ID}. | |
| Bundle selection reasons: | |
| ${REASON_LINES}" |