CI: add gpMgmt Behave test

.github/workflows/build-cloudberry.yml

@@ -223,6 +223,7 @@ jobs:
           DEFAULT_ENABLE_CGROUPS=false
           DEFAULT_ENABLE_CORE_CHECK=true
           DEFAULT_PG_SETTINGS_OPTIMIZER=""
+          DEFAULT_TEST_KIND="installcheck"
 
           # Define base test configurations
           ALL_TESTS='{
@@ -338,6 +339,69 @@ jobs:
               },
               {"test":"ic-cbdb-parallel",
                "make_configs":["src/test/regress:installcheck-cbdb-parallel"]
+              },
+              {"test":"ic-behave-state",
+               "make_configs":["gpMgmt/:behave"],
+               "test_kind":"behave",
+               "behave_features":[
+                 "test/behave/mgmt_utils/analyzedb.feature",
+                 "test/behave/mgmt_utils/gp_bash_functions.feature",
+                 "test/behave/mgmt_utils/gpcheckcat.feature",
+                 "test/behave/mgmt_utils/gpcheckperf.feature",
+                 "test/behave/mgmt_utils/gpconfig.feature",
+                 "test/behave/mgmt_utils/minirepro.feature",
+                 "test/behave/mgmt_utils/gpstart.feature",
+                 "test/behave/mgmt_utils/gpstate.feature",
+                 "test/behave/mgmt_utils/gpstop.feature"
+               ],
+               "enable_core_check":false
+              },
+              {"test":"ic-behave-init",
+               "make_configs":["gpMgmt/:behave"],
+               "test_kind":"behave",
+               "behave_features":[
+                 "test/behave/mgmt_utils/gpactivatestandby.feature",
+                 "test/behave/mgmt_utils/gpinitsystem.feature",
+                 "test/behave/mgmt_utils/gpinitstandby.feature"
+               ],
+               "enable_core_check":false
+              },
+              {"test":"ic-behave-mirror",
+               "make_configs":["gpMgmt/:behave"],
+               "test_kind":"behave",
+               "behave_features":[
+                 "test/behave/mgmt_utils/gpaddmirrors.feature",
+                 "test/behave/mgmt_utils/gpmovemirrors.feature",
+                 "test/behave/mgmt_utils/gprecoverseg.feature",
+                 "test/behave/mgmt_utils/gpreload.feature",
+                 "test/behave/mgmt_utils/replication_slots.feature"
+               ],
+               "enable_core_check":false
+              },
+              {"test":"ic-behave-expand",
+               "make_configs":["gpMgmt/:behave"],
+               "test_kind":"behave",
+               "behave_features":[
+                 "test/behave/mgmt_utils/gpexpand.feature"
+               ],
+               "enable_core_check":false
+              },
+              {"test":"ic-behave-package",
+               "make_configs":["gpMgmt/:behave"],
+               "test_kind":"behave",
+               "behave_features":[
+                 "test/behave/mgmt_utils/gppkg.feature"
+               ],
+               "enable_core_check":false
+              },
+              {"test":"ic-behave-ssh",
+               "make_configs":["gpMgmt/:behave"],
+               "test_kind":"behave",
+               "behave_features":[
+                 "test/behave/mgmt_utils/gpssh.feature",
+                 "test/behave/mgmt_utils/gpssh_exkeys.feature"
+               ],
+               "enable_core_check":false
               }
             ]
           }'
@@ -348,11 +412,13 @@ jobs:
                            --argjson ec  "$DEFAULT_ENABLE_CGROUPS" \
                            --argjson ecc "$DEFAULT_ENABLE_CORE_CHECK" \
                            --arg     opt "$DEFAULT_PG_SETTINGS_OPTIMIZER" \
+                           --arg     tk  "$DEFAULT_TEST_KIND" \
               'def get_defaults:
                 {
                   num_primary_mirror_pairs: ($npm|tonumber),
                   enable_cgroups: $ec,
                   enable_core_check: $ecc,
+                  test_kind: $tk,
                   pg_settings: {
                     optimizer: $opt
                   }
@@ -1448,6 +1514,31 @@ jobs:
             config="${configs[$i]}"
             IFS=':' read -r dir target <<< "$config"
 
+            if [ "${{ matrix.test_kind }}" == "behave" ]; then
+              echo "=== Executing Behave test ==="
+              config_log="build-logs/details/make-${{ matrix.test }}-config$i.log"
+              behave_targets="${{ join(matrix.behave_features, ' ') }}"
+
+              echo "Installing OS dependencies for Behave..."
+              dnf install -y libffi-devel || echo "Warning: failed to install libffi-devel"
+
+              echo "Installing Python dependencies for Behave..."
+              su - gpadmin -c "pip3 install --user -r ${SRC_DIR}/gpMgmt/requirements-dev.txt || pip install --user -r ${SRC_DIR}/gpMgmt/requirements-dev.txt"
+
+              echo "Behave feature group:"
+              for feature in $behave_targets; do
+                echo "- $feature"
+              done
+
+              if ! time su - gpadmin -c "cd ${SRC_DIR}/gpMgmt && source /usr/local/cloudberry-db/cloudberry-env.sh && source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && PYTHONPATH=${SRC_DIR}/gpMgmt:\$PYTHONPATH behave $behave_targets" \
+                   2>&1 | tee -a "$config_log"; then
+                echo "::warning::Test execution failed for Behave"
+                overall_status=1
+              fi
+              echo "=== End configuration $((i+1)) execution ==="
+              continue
+            fi
+
             echo "=== Executing configuration $((i+1))/${#configs[@]} ==="
             echo "Make command: make -C $dir $target"
             echo "Environment:"
@@ -1574,16 +1665,53 @@ jobs:
             fi
 
             # Parse this configuration's results
+            if [ "${{ matrix.test_kind }}" == "behave" ]; then
+              scenarios_line=$(grep -E '^[0-9]+ scenarios passed, [0-9]+ failed, [0-9]+ skipped, [0-9]+ untested$' "$config_log" | tail -n 1)
+              features_line=$(grep -E '^[0-9]+ features passed, [0-9]+ failed, [0-9]+ skipped$' "$config_log" | tail -n 1)
+              steps_line=$(grep -E '^[0-9]+ steps passed, [0-9]+ failed, [0-9]+ skipped, [0-9]+ undefined, [0-9]+ untested$' "$config_log" | tail -n 1)
+
+              if [[ -z "$scenarios_line" ]]; then
+                status_code=2
+              else
+                read -r scenarios_passed scenarios_failed scenarios_skipped scenarios_untested <<< "$(echo "$scenarios_line" | sed -E 's/^([0-9]+) scenarios passed, ([0-9]+) failed, ([0-9]+) skipped, ([0-9]+) untested$/\1 \2 \3 \4/')"
+                total_scenarios=$((scenarios_passed + scenarios_failed + scenarios_skipped))
 
-            MAKE_NAME="${{ matrix.test }}-config$i" \
-            "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/parse-test-results.sh "$config_log"
-            status_code=$?
+                {
+                  if [[ "$scenarios_failed" -eq 0 ]]; then
+                    echo "STATUS=passed"
+                  else
+                    echo "STATUS=failed"
+                  fi
+                  echo "TOTAL_TESTS=${total_scenarios}"
+                  echo "FAILED_TESTS=${scenarios_failed}"
+                  echo "PASSED_TESTS=${scenarios_passed}"
+                  echo "IGNORED_TESTS=${scenarios_skipped}"
+                  echo "BEHAVE_UNTESTED_SCENARIOS=${scenarios_untested}"
+                  echo "BEHAVE_FEATURES_SUMMARY=\"${features_line:-unavailable}\""
+                  echo "BEHAVE_SCENARIOS_SUMMARY=\"${scenarios_line}\""
+                  echo "BEHAVE_STEPS_SUMMARY=\"${steps_line:-unavailable}\""
+                  echo "SUITE_NAME=${{ matrix.test }}"
+                  echo "DIR=${dir}"
+                  echo "TARGET=${target}"
+                } > test_results.txt
+
+                if [[ "$scenarios_failed" -eq 0 ]]; then
+                  status_code=0
+                else
+                  status_code=1
+                fi
+              fi
+            else
+              MAKE_NAME="${{ matrix.test }}-config$i" \
+              "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/parse-test-results.sh "$config_log"
+              status_code=$?
 
-            {
-                echo "SUITE_NAME=${{ matrix.test }}"
-                echo "DIR=${dir}"
-                echo "TARGET=${target}"
-            } >> test_results.txt
+              {
+                  echo "SUITE_NAME=${{ matrix.test }}"
+                  echo "DIR=${dir}"
+                  echo "TARGET=${target}"
+              } >> test_results.txt
+            fi
 
             # Process return code
             case $status_code in
@@ -1787,6 +1915,17 @@ jobs:
               echo "| Failed Tests | ${FAILED_TESTS:-0} |"
               echo "| Ignored Tests | ${IGNORED_TESTS:-0} |"
 
+              if [[ -n "${BEHAVE_SCENARIOS_SUMMARY:-}" ]]; then
+                echo ""
+                echo "#### Behave Summary"
+                echo "| Metric | Summary |"
+                echo "|--------|---------|"
+                echo "| Features | ${BEHAVE_FEATURES_SUMMARY:-unavailable} |"
+                echo "| Scenarios | ${BEHAVE_SCENARIOS_SUMMARY} |"
+                echo "| Steps | ${BEHAVE_STEPS_SUMMARY:-unavailable} |"
+                echo "| Untested Scenarios | ${BEHAVE_UNTESTED_SCENARIOS:-0} |"
+              fi
+
               # Add failed tests if any
               if [[ -n "${FAILED_TEST_NAMES:-}" && "${FAILED_TESTS:-0}" != "0" ]]; then
                 echo ""

gpMgmt/test/behave/mgmt_utils/gprecoverseg.feature

@@ -1,7 +1,7 @@
 @gprecoverseg
 Feature: gprecoverseg tests
 
-    Scenario: incremental recovery works with tablespaces
+    Scenario Outline: incremental recovery works with tablespaces
         Given the database is running
           And a tablespace is created with data
           And user stops all primary processes

gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py

@@ -489,14 +489,6 @@ def impl(context):
     else:
         return
 
-@then( 'verify if the gprecoverseg.lock directory is present in coordinator_data_directory')
-def impl(context):
-    gprecoverseg_lock_file = "%s/gprecoverseg.lock" % gp.get_coordinatordatadir()
-    if not os.path.exists(gprecoverseg_lock_file):
-        raise Exception('gprecoverseg.lock directory does not exist')
-    else:
-        return
-
 
 @then('verify that lines from recovery_progress.file are present in segment progress files in {logdir}')
 def impl(context, logdir):
@@ -671,11 +663,6 @@ def impl(context, process_name, signal_name):
     command = "ps ux | grep bin/{0} | awk '{{print $2}}' | xargs kill -{1}".format(process_name, sig.value)
     run_async_command(context, command)
 
-@when('the user asynchronously sets up to end {process_name} process with SIGHUP')
-def impl(context, process_name):
-    command = "ps ux | grep bin/%s | awk '{print $2}' | xargs kill -9" % (process_name)
-    run_async_command(context, command)
-
 @when('the user asynchronously sets up to end gpcreateseg process when it starts')
 def impl(context):
     # We keep trying to find the gpcreateseg process using ps,grep

gpMgmt/test/behave/mgmt_utils/steps/recoverseg_mgmt_utils.py

@@ -4,15 +4,17 @@
 from time import sleep
 
 from contextlib import closing
+from gppylib import gplog
 from gppylib.commands.base import Command, ExecutionError, REMOTE, WorkerPool
+from gppylib.commands.gp import RECOVERY_REWIND_APPNAME
 from gppylib.db import dbconn
 from gppylib.gparray import GpArray, ROLE_PRIMARY, ROLE_MIRROR
-from gppylib.programs.clsRecoverSegment_triples import get_segments_with_running_basebackup, is_pg_rewind_running
-from gppylib.operations.get_segments_in_recovery import is_seg_in_backup_mode
 from test.behave_utils.utils import *
 import platform, shutil
 from behave import given, when, then
 
+logger = gplog.get_default_logger()
+
 #TODO remove duplication of these functions
 def _get_gpAdminLogs_directory():
     return "%s/gpAdminLogs" % os.path.expanduser("~")
@@ -23,6 +25,76 @@ def lines_matching_both(in_str, str_1, str_2):
     return [line for line in lines if line.count(str_1) and line.count(str_2)]
 
 
+def get_segments_with_running_basebackup():
+    """
+    Returns a set of content ids whose source segments currently have
+    a running pg_basebackup.
+    """
+    sql = "select gp_segment_id from gp_stat_replication where application_name = 'pg_basebackup'"
+
+    try:
+        with closing(dbconn.connect(dbconn.DbURL())) as conn:
+            rows = dbconn.query(conn, sql).fetchall()
+    except Exception as e:
+        raise Exception("Failed to query gp_stat_replication: %s" % str(e))
+
+    segments_with_running_basebackup = {row[0] for row in rows}
+
+    if len(segments_with_running_basebackup) == 0:
+        logger.debug("No basebackup running")
+
+    return segments_with_running_basebackup
+
+
+def is_pg_rewind_running(hostname, port):
+    """
+    Returns true if a pg_rewind process is running for the given segment.
+    """
+    sql = "SELECT count(*) FROM pg_stat_activity WHERE application_name = '{}'".format(
+        RECOVERY_REWIND_APPNAME
+    )
+
+    try:
+        url = dbconn.DbURL(hostname=hostname, port=port, dbname='template1')
+        with closing(dbconn.connect(url, utility=True)) as conn:
+            return dbconn.querySingleton(conn, sql) > 0
+    except Exception as e:
+        raise Exception(
+            "Failed to query pg_stat_activity for segment hostname: {}, port: {}, error: {}".format(
+                hostname, str(port), str(e)
+            )
+        )
+
+
+def is_seg_in_backup_mode(hostname, port):
+    """
+    Returns true if the source segment is already in backup mode.
+
+    Differential recovery uses pg_start_backup() on the source segment, so
+    a source that is already in backup mode indicates differential recovery
+    may already be in progress.
+    """
+    logger.debug(
+        "Checking if backup is already in progress for the source server with host {} and port {}".format(
+            hostname, port
+        )
+    )
+
+    sql = "SELECT pg_is_in_backup()"
+    try:
+        url = dbconn.DbURL(hostname=hostname, port=port, dbname='template1')
+        with closing(dbconn.connect(url, utility=True)) as conn:
+            res = dbconn.querySingleton(conn, sql)
+    except Exception as e:
+        raise Exception(
+            "Failed to query pg_is_in_backup() for segment with hostname {}, port {}, error: {}".format(
+                hostname, str(port), str(e)
+            )
+        )
+
+    return res
+
+
 @given('the information of contents {contents} is saved')
 @when('the information of contents {contents} is saved')
 @then('the information of contents {contents} is saved')