From a92fb9130b7dfebbe2d3a05834d04411f1bfc36d Mon Sep 17 00:00:00 2001 From: Martin Kolman Date: Mon, 25 Jan 2021 20:37:06 +0100 Subject: [PATCH] Add additional repository support Make it possible to add additional repo for a kickstart test run. At the moment we assume the repository is a regular RPM repository, as for example created by running createrepo_c on a directory containing a bunch of RPMs. The main aim of this functionality is to make it possible to add additional packages to the RPM transaction Anaconda runs during a regular RPM based installation. Also by adding packages with a higher version number than those in the normal package repositories it is possible to override them with the package from the additional repo. This can be useful to test Anaconda and/or Initial Setup scratchbuilds. To use this feature use the -a or --additional-repo flag to the appropriate runner script and pass either a local filesystem path to a RPM repository or to a remote RPM repository. If a local path is detected a localhost web server on the runner will be used to serve the repo folder to the VM running the test. If a remote path to a RPM repository is passed, it will be passed directly to the VM. In both cases the additional repositories are passed via the inst.addrepo boot option as "inst.addrepo=KSTEST_ADDITIONAL_REPOSITORY,". --- containers/runner/launch | 16 +++++++++-- containers/runner/run-kstest | 7 ++++- scripts/launcher/lib/conf/configuration.py | 9 ++++++ scripts/launcher/lib/conf/runner_parser.py | 6 ++++ scripts/launcher/run_one_test.py | 3 ++ scripts/run_kickstart_tests.sh | 33 +++++++++++++++++++--- 6 files changed, 67 insertions(+), 7 deletions(-) diff --git a/containers/runner/launch b/containers/runner/launch index 53835b462..e154f6a98 100755 --- a/containers/runner/launch +++ b/containers/runner/launch @@ -34,6 +34,7 @@ Options: -t, --testtype TYPE Only run TYPE tests -s, --skip-testtypes TYPE[,TYPE..] Don't run tests with given types -u, --updates PATH|URL Set updates.img path or URL + -a, --additional-repo PATH|URL Set additional repo path or URL -r, --retry Retry failed tests once, to guard against random infrastructure failures --daily-iso TOKEN_FILE Download and use daily boot.iso instead of rawhide's @@ -46,7 +47,7 @@ EOF } # parse options -eval set -- "$(getopt -o j:p:t:s:u:rh --long jobs:,platform:,testtype:,skip-testtypes:,updates:,retry,daily-iso:,defaults:,run-args:,help -- "$@")" +eval set -- "$(getopt -o j:p:t:s:u:a:rh --long jobs:,platform:,testtype:,skip-testtypes:,updates:additional-repo:,retry,daily-iso:,defaults:,run-args:,help -- "$@")" while true; do case "${1:-}" in @@ -55,6 +56,7 @@ while true; do -t|--testtype) shift; TESTTYPE="$1" ;; -s|--skip-testtypes) shift; SKIP_TESTTYPES="$1" ;; -u|--updates) shift; UPDATES_IMAGE="$1" ;; + -a|--additional-repo) shift; ADDITIONAL_REPO="$1" ;; -r|--retry) TEST_RETRY=1 ;; --daily-iso) shift; DAILY_ISO_TOKEN="$1" ;; --defaults) shift; DEFAULTS_SH="$1" ;; @@ -103,6 +105,15 @@ elif [ -n "${UPDATES_IMAGE:-}" ]; then UPDATES_IMG_ARGS="--env UPDATES_IMAGE=$UPDATES_IMAGE" fi +# support both path and URL for additional repo +if [ -e "${ADDITIONAL_REPO:-}" ]; then + # local folder; bind mount into container + ADDITIONAL_REPO_ARGS="-v $ADDITIONAL_REPO:/kstest_additional_repo:ro,Z --env ADDITIONAL_REPO=/kstest_additional_repo" +elif [ -n "${ADDITIONAL_REPO:-}" ]; then + # URL, pass through + ADDITIONAL_REPO_ARGS="--env ADDITIONAL_REPO=$ADDITIONAL_REPO" +fi + if [ -n "${DEFAULTS_SH:-}" ]; then DEFAULTS_SH_ARGS="-v $DEFAULTS_SH:/home/kstest/.kstests.defaults.sh:ro,z" fi @@ -126,6 +137,7 @@ fi set -x $CRUN run -it --rm --device=/dev/kvm --publish 127.0.0.1::16509 $PODMAN_SELINUX_FIX \ --env KSTESTS_TEST="$KSTESTS_TEST" --env TESTTYPE="${TESTTYPE:-}" --env SKIP_TESTTYPES="${SKIP_TESTTYPES:-}" \ - --env TEST_JOBS="$TEST_JOBS" --env PLATFORM="${PLATFORM:-}" --env TEST_RETRY="${TEST_RETRY:-}" ${UPDATES_IMG_ARGS:-} ${CONTAINER_RUN_ARGS:-} \ + --env TEST_JOBS="$TEST_JOBS" --env PLATFORM="${PLATFORM:-}" --env TEST_RETRY="${TEST_RETRY:-}" \ + ${UPDATES_IMG_ARGS:-} ${ADDITIONAL_REPO_ARGS:-} ${CONTAINER_RUN_ARGS:-} \ ${VAR_TMP:-} -v "$PWD/data:/opt/kstest/data:z" -v "$BASEDIR:/kickstart-tests:ro,z" ${DEFAULTS_SH_ARGS:-} \ $CONTAINER /kickstart-tests/containers/runner/run-kstest diff --git a/containers/runner/run-kstest b/containers/runner/run-kstest index c2d7cfff5..0fa3eb47e 100755 --- a/containers/runner/run-kstest +++ b/containers/runner/run-kstest @@ -30,6 +30,11 @@ if [ -n "${UPDATES_IMAGE}" ]; then UPDATES_IMAGE_ARG="-u ${UPDATES_IMAGE}" fi +ADDITIONAL_REPO_ARG="" +if [ -n "${ADDITIONAL_REPO}" ]; then + ADDITIONAL_REPO_ARG="-a ${ADDITIONAL_REPO}" +fi + PLATFORM_ARG="" if [ -n "${PLATFORM}" ]; then PLATFORM_ARG="-p ${PLATFORM}" @@ -74,7 +79,7 @@ fi TEST_LOG=/var/tmp/kstest.log pushd ${KSTESTS_DIR} set +e -scripts/run_kickstart_tests.sh -k ${KSTESTS_KEEP} -i ${ISO_DIR}/${BOOT_ISO} ${UPDATES_IMAGE_ARG} ${PLATFORM_ARG} ${TESTTYPE_ARG} ${SKIP_TESTTYPES_ARG} ${RETRY_ARG} ${KSTESTS_TEST} 2>&1 | tee $TEST_LOG +scripts/run_kickstart_tests.sh -k ${KSTESTS_KEEP} -i ${ISO_DIR}/${BOOT_ISO} ${UPDATES_IMAGE_ARG} ${ADDITIONAL_REPO_ARG} ${PLATFORM_ARG} ${TESTTYPE_ARG} ${SKIP_TESTTYPES_ARG} ${RETRY_ARG} ${KSTESTS_TEST} 2>&1 | tee $TEST_LOG RC=$? set -e popd diff --git a/scripts/launcher/lib/conf/configuration.py b/scripts/launcher/lib/conf/configuration.py index 83de530a1..d7c19338a 100644 --- a/scripts/launcher/lib/conf/configuration.py +++ b/scripts/launcher/lib/conf/configuration.py @@ -83,6 +83,7 @@ def __init__(self): self._image_path = "" self._keep_option = KeepLevel.NOTHING self._updates_img_path = "" + self._additional_repo_path = "" self._append_host_id = False self._hung_task_timeout_secs = 1200 @@ -138,6 +139,14 @@ def updates_img_path(self): def updates_img_path(self, val): self._updates_img_path = val + @property + def additional_repo_path(self): + return self._additional_repo_path + + @additional_repo_path.setter + def additional_repo_path(self, val): + self._additional_repo_path = val + @property def script_path(self): path = os.path.join(__file__, "..") diff --git a/scripts/launcher/lib/conf/runner_parser.py b/scripts/launcher/lib/conf/runner_parser.py index 949a225ee..ecf37553b 100644 --- a/scripts/launcher/lib/conf/runner_parser.py +++ b/scripts/launcher/lib/conf/runner_parser.py @@ -85,6 +85,9 @@ def _configure_parser(self): self._parser.add_argument("--updates", '-u', metavar="Path", type=str, dest="updates_path", help="Updates image path used in the test") + self._parser.add_argument("--additional_repo", '-a', metavar="Path", + type=str, dest="additional_repo_path", + help="Additionl repo path used in the test") self._parser.add_argument("--append-host-id", default=False, action="store_true", dest="append_host_id", help="append an id of the host running the test to the result") @@ -115,6 +118,9 @@ def get_configuration(self): if ns.updates_path: conf.updates_img_path = ns.updates_path + if ns.additional_repo_path: + conf.additional_repo_path = ns.additional_repo_path + if ns.append_host_id: conf.append_host_id = ns.append_host_id diff --git a/scripts/launcher/run_one_test.py b/scripts/launcher/run_one_test.py index a8837f1d7..8c04b81ca 100755 --- a/scripts/launcher/run_one_test.py +++ b/scripts/launcher/run_one_test.py @@ -135,6 +135,9 @@ def _create_virtual_conf(self, log_path) -> VirtualConfiguration: if self._conf.updates_img_path: kernel_args += " inst.updates={}".format(self._conf.updates_img_path) + if self._conf.additional_repo_path: + kernel_args += " inst.addrepo=KSTEST_ADDITIONAL_REPO,{}".format(self._conf.additional_repo_path) + if self._conf.hung_task_timeout_secs: kernel_args += " inst.kernel.hung_task_timeout_secs={}".format( self._conf.hung_task_timeout_secs) diff --git a/scripts/run_kickstart_tests.sh b/scripts/run_kickstart_tests.sh index f2098938f..ee9773963 100755 --- a/scripts/run_kickstart_tests.sh +++ b/scripts/run_kickstart_tests.sh @@ -61,10 +61,13 @@ KEEPIT=${KEEPIT:-0} # responsibility, this can break tests. UPDATES_IMG="" +# Local or remote path to an additional repo that will be added for the installation transaction. +ADDITIONAL_REPO="" + TESTTYPE="" SKIP_TESTTYPES="" -while getopts ":i:k:t:s:u:b:p:o:r" opt; do +while getopts ":i:k:t:s:u:a:b:p:o:r" opt; do case $opt in i) # If this wasn't set from the environment, set it from the command line @@ -97,6 +100,12 @@ while getopts ":i:k:t:s:u:b:p:o:r" opt; do # This may not be compatible with all the tests. UPDATES_IMG=$OPTARG ;; + a) + # Path to a RPM repo on a server or in local directory. This will be added as + # a kernel parameter inst.addrepo=KSTEST_ADDITIONAL_REPO, to the VM boot options. + # This may not be compatible with all the tests. + ADDITIONAL_REPO=$OPTARG + ;; b) # Use additional boot options. Will be added to kernel_args from .sh file. BOOT_ARGS=$OPTARG @@ -118,7 +127,7 @@ while getopts ":i:k:t:s:u:b:p:o:r" opt; do RETRY=--retry ;; *) - echo "Usage: run_kickstart_tests.sh [-i boot.iso] [-k 0|1|2] [-t test_type_to_run] [-s test_types_to_ignore] [-u link_to_updates.img] [-b additional_boot_options] [-p platform_name] [-o ksappend_overrides] [tests]" + echo "Usage: run_kickstart_tests.sh [-i boot.iso] [-k 0|1|2] [-t test_type_to_run] [-s test_types_to_ignore] [-u link_to_updates.img] [-a local_or_remote_repo_path] [-b additional_boot_options] [-p platform_name] [-o ksappend_overrides] [tests]" exit 1 ;; esac @@ -351,6 +360,22 @@ if [[ -n "$UPDATES_IMG" ]]; then UPDATES_ARG="-u ${UPDATES_IMG}" fi +# set up additional package repo +ADDITIONAL_REPO_ARG="" +if [[ -n "$ADDITIONAL_REPO" ]]; then + if [ -e "$ADDITIONAL_REPO" ]; then + # set up a local web server for the local runner local + # folder containing the repo + python3 -m http.server --directory ${ADDITIONAL_REPO} 9999 & + # stop it when this script exits + trap "kill $!" EXIT INT QUIT PIPE + # SLIRP networking address as seem from QEMU guests + ADDITIONAL_REPO="http://10.0.2.2:9999/${ADDITIONAL_REPO}" + fi + + ADDITIONAL_REPO_ARG="-a ${ADDITIONAL_REPO}" +fi + BOOT_ARG="" if [[ -n "$BOOT_ARGS" ]]; then BOOT_ARG="-b \"${BOOT_ARGS}\"" @@ -416,7 +441,7 @@ if [[ "$TEST_REMOTES" != "" ]]; then -i ../install_images/${_IMAGE} \ -k ${KEEPIT} \ --append-host-id \ - ${RETRY} ${UPDATES_ARG} ${BOOT_ARG} {} ::: ${tests} + ${RETRY} ${UPDATES_ARG} ${ADDITIONAL_REPO_ARG} ${BOOT_ARG} {} ::: ${tests} rc=$? cd - @@ -448,7 +473,7 @@ else -i ${IMAGE} \ -k ${KEEPIT} \ --append-host-id \ - ${RETRY} ${UPDATES_ARG} ${BOOT_ARG} {} ::: ${tests} + ${RETRY} ${UPDATES_ARG} ${ADDITIONAL_REPO_ARG} ${BOOT_ARG} {} ::: ${tests} rc=$? fi