feat: Add GitHub CI/CD versioning support with env profile

Add native support for NiFi's GitHub Flow Registry Client to enable
CI/CD workflows for versioned NiFi flows.

New Git-specific versioning helpers:
- list_git_registry_buckets, get_git_registry_bucket
- list_git_registry_flows, get_git_registry_flow
- list_git_registry_flow_versions, deploy_git_registry_flow
- ensure_registry_client, update_registry_client

Profile system enhancements:
- Add 'env' profile for pure environment variable configuration
- Ideal for GitHub Actions, containers, and CI/CD pipelines
- No profiles file required when using nipyapi.profiles.switch('env')

Controller service management:
- Add schedule_all_controllers for bulk enable/disable operations
- Uses NiFi's native bulk activation API with descendant support

Bug fixes:
- Fix test_create_controller leaving orphaned controller services

Documentation:
- Add 1.1.0 release notes to history
- Add env profile usage guide to profiles documentation

Related: nipyapi-actions and nipyapi-workflow companion repositories

Author: Chaffelson

.gitignore

@@ -190,3 +190,4 @@ htmlcov/
 
 # setuptools-scm generated version file
 nipyapi/_version.py
+.cursor/

Makefile

@@ -4,6 +4,10 @@
 # Default NiFi/Registry version for docker compose profiles
 NIFI_VERSION ?= 2.6.0
 
+# Load .env file if it exists (for secrets like GITHUB_REGISTRY_TOKEN)
+-include .env
+export
+
 # Python command for cross-platform compatibility
 # Defaults to 'python' for conda/venv users, override with PYTHON=python3 for system installs
 PYTHON ?= python
@@ -227,17 +231,17 @@ extract-jks: ## extract PEM certificates from JKS: make extract-jks JKS_FILE=tru
 	fi
 	@echo "✅ Certificates extracted to resources/certs/extracted/"
 
-up: ensure-certs # bring up docker profile: make up NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc (uses NIFI_VERSION=$(NIFI_VERSION))
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi
+up: ensure-certs # bring up docker profile: make up NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd (uses NIFI_VERSION=$(NIFI_VERSION))
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi
 	$(DC) --profile $(NIPYAPI_PROFILE) up -d
 
 down: ## bring down all docker services
 	@echo "Bringing down Docker services (NIFI_VERSION=$(NIFI_VERSION))"
-	@$(DC) --profile single-user --profile secure-ldap --profile secure-mtls --profile secure-oidc down -v --remove-orphans || true
+	@$(DC) --profile single-user --profile secure-ldap --profile secure-mtls --profile secure-oidc --profile github-cicd down -v --remove-orphans || true
 	@echo "Verifying expected containers are stopped/removed:"
 	@COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME) NIFI_VERSION=$(NIFI_VERSION) docker compose -f $(COMPOSE_FILE) ps --format "table {{.Name}}\t{{.State}}" | tail -n +2 | awk '{print " - " $$1 ": " $$2}' || true
 
-wait-ready: ## wait for readiness using profile configuration; requires NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc
+wait-ready: ## wait for readiness using profile configuration; requires NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd
 	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "ERROR: NIPYAPI_PROFILE is required"; exit 1; fi
 	@echo "Waiting for $(NIPYAPI_PROFILE) infrastructure to be ready..."
 	@NIPYAPI_PROFILE=$(NIPYAPI_PROFILE) $(PYTHON) resources/scripts/wait_ready.py
@@ -269,7 +273,7 @@ gen-clients: ## generate NiFi and Registry clients from specs (use wv_spec_varia
 # Individual testing
 
 test: ## run pytest with provided NIPYAPI_PROFILE; config resolved by tests/conftest.py
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi; \
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi; \
 	NIPYAPI_PROFILE=$(NIPYAPI_PROFILE) PYTHONPATH=$(PWD):$$PYTHONPATH pytest -q
 
 test-su: ## shortcut: NIPYAPI_PROFILE=single-user pytest
@@ -285,7 +289,7 @@ test-oidc: check-certs ## shortcut: NIPYAPI_PROFILE=secure-oidc pytest (requires
 	NIPYAPI_PROFILE=secure-oidc $(MAKE) test
 
 test-specific: ## run specific pytest with provided NIPYAPI_PROFILE and TEST_ARGS
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi; \
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi; \
 	if [ -z "$(TEST_ARGS)" ]; then echo "TEST_ARGS is required (e.g., tests/test_utils.py::test_dump -v)"; exit 1; fi; \
 	NIPYAPI_PROFILE=$(NIPYAPI_PROFILE) PYTHONPATH=$(PWD):$$PYTHONPATH pytest -q $(TEST_ARGS)
 
@@ -333,8 +337,8 @@ test-all: ensure-certs ## run full e2e tests across automated profiles (requires
 	done
 	@echo "All profiles tested successfully"
 
-sandbox: ensure-certs ## create isolated environment with sample objects: make sandbox NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc
-	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "ERROR: NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc)"; exit 1; fi
+sandbox: ensure-certs ## create isolated environment with sample objects: make sandbox NIPYAPI_PROFILE=single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd
+	@if [ -z "$(NIPYAPI_PROFILE)" ]; then echo "ERROR: NIPYAPI_PROFILE is required (single-user|secure-ldap|secure-mtls|secure-oidc|github-cicd)"; exit 1; fi
 	@echo "🏗️ Setting up NiPyAPI sandbox with profile: $(NIPYAPI_PROFILE)"
 	@echo "=== 1/4: Starting infrastructure ==="
 	$(MAKE) up NIPYAPI_PROFILE=$(NIPYAPI_PROFILE)

docs/history.rst

@@ -2,6 +2,47 @@
 History
 =======
 
+1.1.0 (2025-12-XX)
+-------------------
+
+| GitHub CI/CD Integration - Native support for NiFi's GitHub Flow Registry Client
+
+**GitHub Flow Registry Support**
+
+- **Git-specific versioning helpers**: New functions to work with NiFi's native GitHub Flow Registry Client
+    - ``list_git_registry_buckets``: List buckets (folders) in a Git-backed registry
+    - ``get_git_registry_bucket``: Get a specific bucket by name
+    - ``list_git_registry_flows``: List flows in a bucket
+    - ``get_git_registry_flow``: Get a specific flow by name
+    - ``list_git_registry_flow_versions``: List all versions (commits) of a flow
+    - ``deploy_git_registry_flow``: Deploy a versioned flow from GitHub to the NiFi canvas
+- **Registry client management**: ``ensure_registry_client`` and ``update_registry_client`` for idempotent registry configuration
+
+**Profile System Enhancements**
+
+- **"env" profile for CI/CD**: New special profile name that configures nipyapi entirely from environment variables
+    - No profiles file required - ideal for GitHub Actions, containers, and CI/CD pipelines
+    - Uses ``nipyapi.profiles.switch('env')`` to activate
+    - All standard environment variable mappings (``NIFI_API_ENDPOINT``, ``NIFI_USERNAME``, etc.) apply
+- **Documentation**: Added "env" profile usage guide to profiles documentation
+
+**Controller Service Management**
+
+- **Bulk controller service operations**: ``schedule_all_controllers(pg_id, scheduled)`` to enable/disable all controller services in a process group
+    - Uses NiFi's native bulk activation API
+    - Handles all descendant controller services automatically
+    - Simplifies flow start/stop operations in CI/CD workflows
+
+**Bug Fixes**
+
+- Fixed ``test_create_controller`` leaving orphaned ADLS controller services after test runs
+- Improved test cleanup fixtures for better isolation
+
+**Related Projects**
+
+- **nipyapi-actions**: Companion GitHub Action for NiFi CI/CD workflows (separate repository)
+- **nipyapi-workflow**: Demo repository showing PR-based flow testing patterns
+
 1.0.1 (2025-11-10)
 -------------------
 
@@ -195,41 +236,6 @@ History
 * If Client is not instantiated, optimistically instantiate for version checking
 * add socks proxy support
 
-0.16.3 (2021-10-11)
--------------------
-
-| Removed force reset of configuration.password and configuration.username to empty string. This was not increasing security, and was causing unexpected errors for users connecting to multiple services in a single script.
-| Add greedy control to versioning.get_registry_bucket and versioning.get_flow_in_bucket to avoid undesirable partial string match.
-
-* Update readme to reflect switch from 'master' branch naming to 'main'.
-* Update tox to pin testing to Python 3.8, as Python 3.9 is producing unexpected and unrelated SSL failures
-* Minor lint formatting improvements
-
-0.16.2 (2021-02-10)
--------------------
-
-| NOTE: If you are using secured Registry, this release will enforce access controls for the swagger interface which is used to determine which version of Registry is connected in order to correctly provide features - you may have to update your authorizations
-
-* Update requirements.txt to unpin future and lxml
-* Update lxml to 4.6.2 or newer to resolve vulnerability
-* Pin watchdog to <1.0.0 per their docs to maintain Python2.7 compatibility
-* Revert 0.14.3 changes to Authentication handling which introduced basicAuth support but resulted in some NiFi connections appearing incorrectly as Anonymous
-* Added simpler basicAuth control to force it via a config switch without changing tokenAuth and other Authorization header behavior during normal usage
-* nipyapi.config.global_force_basic_auth is now available for use for this purpose
-* Secured Registry users will now require the authorization policy to retrieve the swagger so we may use it to validate which version of
-* Registry is in use for feature enablement
-* Moved all Security controls in config.py to a common area at the foot of the file
-* Removed auth_type from security.service_login as it is now redundant
-* Added controls to handle certificate checking behavior which has become more strict in recently versions of Python3, ssl_verify and check_hostname are now handled
-* security.set_service_auth_token now has an explicit flag for ssl host checking as well
-* Fix oversight where improved model serialisation logic was not correctly applied to Registry
-* Removed unusused parameter refresh from parameters.update_parameter_context
-* Reduced unecessary complexity in utils.dump with no change in functionality
-* Updated client gen mustache templates to reflect refactored security and api client code
-* Minor linting and docstring and codestyle improvements
-* Set pyUp to ignore Watchdog as it must stay between versions to statisfy py2 and py3 compatibility
-* If Client is not instantiated, optimistically instantiate for version checking
-* add socks proxy support
 
 0.15.0 (2020-11-06)
 -------------------

docs/profiles.rst

@@ -37,6 +37,7 @@ Quick Start
 - ``secure-ldap`` - LDAP authentication over TLS
 - ``secure-mtls`` - Mutual TLS certificate authentication
 - ``secure-oidc`` - OpenID Connect (OAuth2) authentication
+- ``env`` - Pure environment variable configuration (no profiles file required)
 
 Why Use Profiles?
 =================
@@ -342,6 +343,34 @@ NiFi CLI properties file integration with OIDC Client Credentials flow:
 
 **Use case**: Integration with existing NiFi CLI configurations, client credentials OIDC
 
+env (Environment-Only Configuration)
+------------------------------------
+
+Pure environment variable configuration without requiring a profiles file:
+
+.. code-block:: python
+
+    nipyapi.profiles.switch('env')
+
+**Authentication method**: Auto-detected from environment variables
+
+**Required environment variables** (minimum for NiFi connection):
+  - ``NIFI_API_ENDPOINT`` - NiFi API URL
+  - ``NIFI_USERNAME`` - NiFi username (for basic auth)
+  - ``NIFI_PASSWORD`` - NiFi password (for basic auth)
+
+**Optional environment variables**:
+  - ``REGISTRY_API_ENDPOINT`` - Registry API URL
+  - ``REGISTRY_USERNAME`` / ``REGISTRY_PASSWORD`` - Registry credentials
+  - ``TLS_CA_CERT_PATH`` - CA certificate path
+  - ``MTLS_CLIENT_CERT`` / ``MTLS_CLIENT_KEY`` - mTLS certificates
+  - ``OIDC_TOKEN_ENDPOINT`` / ``OIDC_CLIENT_ID`` / ``OIDC_CLIENT_SECRET`` - OIDC configuration
+  - All other environment variables listed in the Environment Variable Overrides section
+
+**Use case**: CI/CD pipelines, containerized deployments, GitHub Actions, any environment where configuration is injected via environment variables rather than files.
+
+**Key benefit**: No profiles file needed. The ``env`` profile starts with all-null defaults and populates values entirely from environment variables using the standard ``ENV_VAR_MAPPINGS``. This keeps secrets out of files and allows dynamic configuration in deployment environments.
+
 Environment Variable Overrides
 ===============================
 
@@ -504,6 +533,12 @@ You don't have to specify values that are otherwise null unless required for tha
 
 You don't have to use profiles. You can also directly configure the NiPyAPI client using ``nipyapi.config`` and ``nipyapi.utils.set_endpoint()`` as shown in earlier examples.
 
+GitHub Flow Registry Client
+===========================
+
+NiFi 2.x supports versioning flows directly to GitHub repositories using the GitHub Flow Registry Client.
+For CI/CD workflows with GitHub Actions, see the `nipyapi-actions <https://github.com/Chaffelson/nipyapi-actions>`_ repository which provides reusable actions for deploying NiFi flows from Git repositories.
+
 Integration with Examples
 =========================
 

examples/profiles.yml

@@ -85,6 +85,18 @@ secure-oidc:
   oidc_client_id: "nipyapi-client"
   oidc_client_secret: "nipyapi-secret"
 
+# GitHub CI/CD profile - NiFi only, for testing Git-based registry workflows
+# Uses GitHub Registry Client instead of NiFi Registry
+# GitHub registry configuration is handled by nipyapi-actions, not nipyapi profiles
+github-cicd:
+  nifi_url: https://localhost:9447/nifi-api
+  nifi_user: einstein
+  nifi_pass: password1234
+  nifi_verify_ssl: false
+  suppress_ssl_warnings: true
+  # No NiFi Registry - using GitHub as flow registry
+  registry_url: null
+
 # Example profile using NiFi CLI properties file integration
 cli-properties:
   # Load configuration from existing NiFi CLI properties file

nipyapi/canvas.py

@@ -46,6 +46,7 @@
     "delete_controller",
     "update_controller",
     "schedule_controller",
+    "schedule_all_controllers",
     "get_controller",
     "list_all_controller_types",
     "list_all_by_kind",
@@ -1216,6 +1217,33 @@ def _schedule_controller_state(cont_id, tgt_state):
     raise ValueError("Scheduling request timed out")
 
 
+def schedule_all_controllers(pg_id, scheduled):
+    """
+    Enable or Disable all Controller Services in a Process Group.
+
+    Uses NiFi's native bulk controller service activation API which handles
+    all descendant controller services automatically.
+
+    Args:
+        pg_id (str): The UUID of the Process Group
+        scheduled (bool): True to enable, False to disable
+
+    Returns:
+        ActivateControllerServicesEntity: The result of the operation
+
+    """
+    assert isinstance(pg_id, str)
+    assert isinstance(scheduled, bool)
+
+    target_state = "ENABLED" if scheduled else "DISABLED"
+
+    with nipyapi.utils.rest_exceptions():
+        return nipyapi.nifi.FlowApi().activate_controller_services(
+            id=pg_id,
+            body=nipyapi.nifi.ActivateControllerServicesEntity(id=pg_id, state=target_state),
+        )
+
+
 def get_controller(
     identifier, identifier_type="name", bool_response=False, include_reporting_tasks=True
 ):

nipyapi/config.py

@@ -69,6 +69,13 @@
     "ControllerServiceEntity": {"id": ["id"], "name": ["component", "name"]},
     "ParameterContextEntity": {"id": ["id"], "name": ["component", "name"]},
     "ReportingTaskEntity": {"id": ["id"], "name": ["component", "name"]},
+    # Git-based Flow Registry types (GitHub, GitLab, Bitbucket, Azure DevOps)
+    "FlowRegistryBucketEntity": {"id": ["id"], "name": ["bucket", "name"]},
+    "VersionedFlowEntity": {
+        "flow_id": ["versioned_flow", "flow_id"],
+        "flow_name": ["versioned_flow", "flow_name"],
+    },
+    "VersionedFlowDTO": {"flow_id": ["flow_id"], "flow_name": ["flow_name"]},
 }