diff --git a/build.sh b/build.sh index 6bd8dd43..264578e2 100755 --- a/build.sh +++ b/build.sh @@ -11,11 +11,14 @@ function setGlobals() { SKIP_TOOLKIT="${SKIP_TOOLKIT:-false}" RUN_REMOTE_EXAMPLES="${RUN_REMOTE_EXAMPLES:-false}" PYTHON_VERSION="3.8" + SK_PANDAS_VERSION="sklearn-pandas==2.2.0" + XGBOOST_VERSION="xgboost==1.7.2" SCRIPT_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P )" ARTIFACTS_DIR="${SCRIPT_PATH}/artifacts" TOOLKIT_PATH="${ARTIFACTS_DIR}/certifai_toolkit.zip" TOOLKIT_WORK_DIR="${ARTIFACTS_DIR}/toolkit" PACKAGES_DIR="${TOOLKIT_WORK_DIR}/packages" + CONTAINERIZED_EXAMPLES_DIR="${SCRIPT_PATH}/models/containerized_model/examples" TEMPLATES_DIR="${SCRIPT_PATH}/models/containerized_model" BASE_IMAGES_DIR="${SCRIPT_PATH}/models/containerized_model/base_images" NOTEBOOK_DIR="${SCRIPT_PATH}/notebooks" @@ -157,6 +160,10 @@ function _installModelRequirements() { pip install -r "${TEMPLATES_DIR}/requirements.txt" } +function _installLocalModelRequirements() { + pip install "$SK_PANDAS_VERSION" "$XGBOOST_VERSION" +} + function buildModelDeploymentImages() { # Builds Docker images for the example Containerized Model Types (Scikit, H2O, Proxy, R). These are images are used @@ -292,11 +299,13 @@ function buildPredictionServiceBaseImages() { echo "{\"python38\": \"${py38_image}\", \"python39\": \"${py39_image}\"}" > "${BASE_IMAGE_BUILD_REPORT_JSON}" } -function test() { +function testAll() { testMarkdownLinks testModels testNotebooks testTutorials + # Requires Docker/Minikube - so skipped in pipeline + #testContainerizedModels } function testMarkdownLinks() { @@ -328,12 +337,44 @@ function testMarkdownLinks() { } function testModels() { - echo "TODO: automate subset of model examples - " - # for each - # - train the models - # - start the app in one process, - # - run the test in another process - # - assert both processes exit successfully + MODELS_DIR="${SCRIPT_PATH}/models" + # run tests for each individual example + cd "$MODELS_DIR"/german_credit/ + python -m unittest -v test.py + + cd "$MODELS_DIR"/german_credit_pandas + python -m unittest -v test.py + + cd "$MODELS_DIR"/income_prediction + python -m unittest -v test.py + + cd "$MODELS_DIR"/iris + python -m unittest -v test.py + + cd "$MODELS_DIR"/patient_readmission + python -m unittest -v test.py + + # Go back to root directory + cd "$SCRIPT_PATH" + + # TODO: Run other examples (see https://github.com/CognitiveScale/certifai/issues/4870) + # - h2o_dai_german_credit + # - h2o_dai_regression_auto_insurance + # - r-models +} + +function testContainerizedModels() { + # run base of set of containerized model examples locally (with docker) + cd "$CONTAINERIZED_EXAMPLES_DIR" + TOOLKIT_PATH="$TOOLKIT_WORK_DIR" ./run_test.sh "local" + + # TODO: Add 'RUN_H2O=true' to test other examples (see https://github.com/CognitiveScale/certifai/issues/4870) + # - h2o_dai_german_credit + # - h2o_dai_regression_auto_insurance + # - r-models + + # Go back to root directory + cd "$SCRIPT_PATH" } function testTutorials() { @@ -512,7 +553,7 @@ function _sagemakerNotebook() { function _xgboostModel() { # xgboost-model cd "${NOTEBOOK_DIR}" - pip install xgboost + pip install "$XGBOOST_VERSION" _runNotebookInPlace "${NOTEBOOK_DIR}/xgboost-model/xgboostDmatrixExample.ipynb" } @@ -522,7 +563,9 @@ function main() { setGlobals activateConda installToolkit - test + _installModelRequirements + _installLocalModelRequirements + testAll rm -rf "${TOOLKIT_WORK_DIR}" ;; docker) @@ -558,6 +601,14 @@ function main() { activateConda testMarkdownLinks ;; + models) + setGlobals + activateConda + installToolkit + _installModelRequirements + _installLocalModelRequirements + testModels + ;; notebook) setGlobals activateConda diff --git a/models/README.md b/models/README.md index 6bf68a3f..80de67f3 100644 --- a/models/README.md +++ b/models/README.md @@ -19,5 +19,7 @@ for detailed information about Cortex Certifai. | [german_credit_pandas](./german_credit_pandas) | Illustrates using the Certifai Model SDK to run a models in a service, where the models expect as input a [pandas DataFrame](https://pandas.pydata.org/) instead of a [numpy array](https://numpy.org/). | Binary classification | python | sklearn | | [h2o_dai_auto_insurance](./h2o_dai_regression_auto_insurance) | Illustrates using the Certifai Model SDK to create a gunicorn prediction service from a regression H2O MOJO model, and scan it for trust scores. | Regression | python | H2O MOJO | | [h2o_dai_german_credit](./h2o_dai_german_credit) | Illustrates using the Certifai Model SDK to create a development or gunicorn prediction service from a binary classification H2O MOJO, and scan it for trust scores or for explanations. | Binary classification | python | H2O MOJO | +| [income_prediction](./income_prediction) | Illustrates using the Certifai Model SDK to run a single binary-classification XGBoost model in a service, using a customized model wrapper. | Binary classification | python | sklearn
xgboost | | [iris](./iris) | Illustrates using the Certifai Model SDK to run a single multi-class model in a service, using a customized model wrapper. | Multi-class classification | python | sklearn
xgboost | +| [patient_readmission](./patient_readmission) | Illustrates using the Certifai Model SDK to run a single binary-classification model in a service, using a customized model wrapper, for creation of fast (bulk) explanations. | Binary classification | python | sklearn
xgboost | | [r-models](https://github.com/CognitiveScale/cortex-certifai-examples/tree/master/models/r-models) | Illustrates running a R model in a service using plumber. | Binary classification | R | randomForest | diff --git a/models/base_test.py b/models/base_test.py new file mode 100644 index 00000000..8afcfa1f --- /dev/null +++ b/models/base_test.py @@ -0,0 +1,140 @@ +import time +import contextlib +import subprocess +import tempfile +import unittest +from typing import Optional, Sequence + + +def capture_err_and_out(stderr, stdout): + if stderr is not None: + print("\n---------------------- (main) stderr: ----------------------") + print(stderr, end="") + print("\n------------------------------------------------------------\n") + if stdout is not None: + print("\n---------------------- (main) stdout: ----------------------") + print(stdout, end="") + print("\n------------------------------------------------------------") + + +def capture_output(stdout, stderr, limit=100): + count = 0 + print("\n---------------------- (service) stdout: ----------------------\n") + with open(stdout, 'r+') as f: + for line in f: + if count > limit: + break + print(line, end="") + limit += 1 + print("\n------------------------------------------------------------\n") + print() + + count = 0 + print("\n---------------------- (service) stderr: ----------------------\n") + with open(stderr, 'r+') as f: + for line in f: + if count > limit: + break + print(line, end="") + limit += 1 + print("\n------------------------------------------------------------\n") + + +class ModelTest(unittest.TestCase): + """Base class for testing Certifai Prediction Service Examples. Each example will typically include multiple + scenarios where: + + 1) a flask server is launched as a background process (via the Certifai Model SDK) + 2) a Certifai Scan is launched (or just a plain Python script) is launched in the foreground that calls (1) + + Each process that is launched in the foreground, (2), is expected to complete with a 0 exit code. Each process + launched in the background (1) are expected to be run until explicitly killed. + + The following functions should cover scenarios that run plain Python Scripts:: + + run_standalone_python_script(python_script) + run_python_app_test(model_app, python_script) + + The following functions should cover scenarios that involve running a Certifai Scan:: + + run_model_and_definition_test('app_dtree.py', 'my-definition.yaml') + run_model_and_scan('app_dtree.py', 'my-definition.yaml') + run_model_and_explain('app_dtree.py', 'my-definition.yaml', fast=True) + """ + SLEEP_TIME = 5 # 5 seconds + TERMINATION_TIME = 5 # 5 seconds + DEFAULT_TEST_TIMEOUT = 2 * 60 # 2 minutes + DEFAULT_SCAN_TIMEOUT = 60 * 60 * 1 # 1 hour + PRECALCULATE_TIMEOUT = DEFAULT_SCAN_TIMEOUT * 3 # 3 hours + bg = None + + def _run_in_foreground(self, command: Sequence[str], timeout: Optional[int] = None): + try: + # Run process and wait until it completes + process = subprocess.run(command, shell=False, capture_output=True, timeout=timeout, text=True) + process.check_returncode() + except subprocess.TimeoutExpired as te: + error = f"\nProcess did not finish within expected time (command={te.cmd}, timeout={te.timeout} seconds). Error: {str(te)}" + capture_err_and_out(te.stderr, te.stdout) + self.fail(error) + except subprocess.CalledProcessError as ce: + error = f"\nProcess finished with non-zero exit code (command={ce.cmd}, code={ce.returncode}). Error: {str(ce)}" + capture_err_and_out(ce.stderr, ce.stdout) + self.fail(error) + + @contextlib.contextmanager + def _run_in_background(self, command: Sequence[str]): + with tempfile.NamedTemporaryFile(mode='w+') as stdout, tempfile.NamedTemporaryFile(mode='w+') as stderr: + try: + p = subprocess.Popen(command, shell=False, stdout=stdout, stderr=stderr, stdin=subprocess.DEVNULL, + close_fds=True, text=True) + yield + except Exception: + # WARNING: Killing the subprocess may not kill any workers spawned by the process (e.g. gunicorn!) + p.kill() + p.wait() + capture_output(stdout.name, stderr.name) + raise + finally: + # WARNING: Killing the subprocess may not kill any workers spawned by the process (e.g. gunicorn!) + p.kill() + p.wait() + + # Outward facing API + + def run_python_app_test(self, model_app: str, test_script: str): + # Run a Python Model (flask app) in the background, give it a couple seconds to start up, before running test + with self._run_in_background(["python", model_app]): + time.sleep(self.SLEEP_TIME) + self._run_in_foreground(["python", test_script], timeout=self.DEFAULT_TEST_TIMEOUT) + + def run_standalone_python_script(self, script: str): + # Run the standalone test script + self._run_in_foreground(["python", script], timeout=self.DEFAULT_SCAN_TIMEOUT) + + def run_model_and_definition_test(self, model_app: str, definition: str): + # Run a Python Model (flask app) in the background, give it a couple seconds to start up, before running test + with self._run_in_background(["python", model_app]): + time.sleep(self.SLEEP_TIME) + self._run_in_foreground(["certifai", "definition-test", "-f", definition], timeout=self.DEFAULT_SCAN_TIMEOUT) + + def run_model_and_scan(self, model_app: str, definition: str): + # Run a Python Model (flask app) in the background, give it a couple seconds to start up, before running test + with self._run_in_background(f"python {model_app}".split()): + time.sleep(self.SLEEP_TIME) + self._run_in_foreground(["certifai", "scan", "-f", definition], timeout=self.DEFAULT_SCAN_TIMEOUT) + + def run_model_and_explain(self, model_app: str, definition: str, fast: bool = False): + # Run a Python Model (flask app) in the background, give it a couple seconds to start up. + with self._run_in_background(f"python {model_app}".split()): + time.sleep(self.SLEEP_TIME) + if fast: + # Run the precalculate step prior to the fast explain + pre_calc_command = ["certifai", "explain", "-f", definition, "--precalculate"] + self._run_in_foreground(pre_calc_command, timeout=self.PRECALCULATE_TIMEOUT) + command = ["certifai", "explain", "-f", definition, "--fast"] + else: + command = ["certifai", "explain", "-f", definition] + # Run the explanation scan + self._run_in_foreground(command, timeout=self.DEFAULT_SCAN_TIMEOUT) + diff --git a/models/containerized_model/README.md b/models/containerized_model/README.md index 0d11219f..510857ea 100644 --- a/models/containerized_model/README.md +++ b/models/containerized_model/README.md @@ -242,7 +242,7 @@ Add respective cloud storage credentials and `MODEL_PATH` to `generated-containe ### Step 5 - Add extra-dependencies (optional) The dependencies work out of the box with a standard scikit-learn model, -providing the model was trained with version 0.23.2 of scikit-learn. If +providing the model was trained with a version `1.0.2` of scikit-learn. If you are using a different version, you should update `generated-container-model/requirements.txt`. diff --git a/models/containerized_model/examples/README.md b/models/containerized_model/examples/README.md index 4173c05f..86eee591 100644 --- a/models/containerized_model/examples/README.md +++ b/models/containerized_model/examples/README.md @@ -30,12 +30,12 @@ The following files must exist in this folder: * certifai_toolkit/ - certifai toolkit v1.3.6 or above The current conda environment has been setup with: -* python 3.6 (if 3.7 or 3.8, update PYTHON_VERSION in `run_test.sh`) +* python 3.8 (otherwise, update PYTHON_VERSION in `run_test.sh`) * pip install -U Jinja2 To train and test the models: * Certifai toolkit installed in the current conda environment -* conda install -c conda-forge xgboost==1.2.0 +* conda install -c conda-forge xgboost==1.7.2 To build/run the prediction services: Docker @@ -70,3 +70,5 @@ storage by changing the environment.yml. The tests exit on any error, printing out the prediction service log and deleting the running prediction service container. + +For more options, run: `sh run_test.sh -h` diff --git a/models/containerized_model/examples/run_test.sh b/models/containerized_model/examples/run_test.sh old mode 100644 new mode 100755 index 9e7be220..1caa89a3 --- a/models/containerized_model/examples/run_test.sh +++ b/models/containerized_model/examples/run_test.sh @@ -1,51 +1,68 @@ # -# Copyright (c) 2020. Cognitive Scale Inc. All rights reserved. +# Copyright (c) 2023. Cognitive Scale Inc. All rights reserved. # Licensed under CognitiveScale Example Code License https://github.com/CognitiveScale/cortex-certifai-examples/blob/master/LICENSE.md # #!/usr/bin/env bash set -e -target=${1:-local} +# - ask in Trusted AI chat if we still have an h2o license? -THIS_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P )" -GEN_DIR="${THIS_DIR}/generated-container-model" -PYTHON_VERSION=${PYTHON_VERSION:-3.8} # or 3.7 -NAMESPACE=certifai-models +function set_globals() { + THIS_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P )" + GEN_DIR="${THIS_DIR}/generated-container-model" + PYTHON_VERSION=${PYTHON_VERSION:-3.8} # or 3.7 + XGBOOST_VERSION="xgboost==1.7.2" + NAMESPACE=certifai-models + MODEL_DIR=${THIS_DIR}/../.. + RUN_H2O=${RUN_H2O-"false"} + MINIO="mc" + + # Default toolkit to './certifai_toolkit' + TOOLKIT_PATH="${TOOLKIT_PATH:-$THIS_DIR/certifai_toolkit}" + PACKAGES_DIR="${TOOLKIT_PATH}/packages" +} + +function check_minio_installed() { + if ! command -v $MINIO &> /dev/null + then + echo "'$MINIO' CLI could not be found! Install Minio Client: https://min.io/docs/minio/linux/reference/minio-mc.html" + exit 1 + fi +} function base_setup() { model_type=$1 image_name=$2 model_file=$3 echo "***Generating ${model_type}***" - rm -rf ${GEN_DIR} - sh ${THIS_DIR}/../generate.sh -i ${image_name}:latest -m ${model_type} -d ${GEN_DIR} + rm -rf "${GEN_DIR}" + "${THIS_DIR}/../generate.sh" -i "${image_name}":latest -m "${model_type}" -d "${GEN_DIR}" -t "$TOOLKIT_PATH" all_dir=${GEN_DIR}/packages/all - mkdir -p ${all_dir} - cp ${THIS_DIR}/certifai_toolkit/packages/all/cortex-certifai-common*.zip ${all_dir} - cp ${THIS_DIR}/certifai_toolkit/packages/all/cortex-certifai-model*.zip ${all_dir} + mkdir -p "${all_dir}" + cp "${PACKAGES_DIR}"/all/cortex-certifai-common*.zip "${all_dir}" + cp "${PACKAGES_DIR}"/all/cortex-certifai-model*.zip "${all_dir}" } function h2o_setup() { - base_setup $1 $2 pipeline.mojo - cp ${THIS_DIR}/license.txt ${GEN_DIR}/license/license.txt - cp ${THIS_DIR}/daimojo*linux_x86_64.whl ${GEN_DIR}/ext_packages/ + base_setup "$1" "$2" pipeline.mojo + cp "${THIS_DIR}"/license.txt "${GEN_DIR}"/license/license.txt + cp "${THIS_DIR}"/daimojo*linux_x86_64.whl "${GEN_DIR}"/ext_packages/ } -MODEL_DIR=${THIS_DIR}/../.. function train_models() { - (cd ${MODEL_DIR}/german_credit && python train.py) - cp ${MODEL_DIR}/german_credit/german_credit_dtree.pkl ${THIS_DIR}/sklearn_german_credit/model/model.pkl - (cd ${MODEL_DIR}/income_prediction && python train.py) - cp ${MODEL_DIR}/income_prediction/adult_income_xgb.pkl ${THIS_DIR}/xgboost_dmatrix_income/model/model.pkl - (cd ${MODEL_DIR}/iris && python train.py) - cp ${MODEL_DIR}/iris/iris_xgb.pkl ${THIS_DIR}/xgboost_iris/model/model.pkl + (cd "${MODEL_DIR}"/german_credit && python train.py) + cp "${MODEL_DIR}"/german_credit/models/german_credit_dtree.pkl "${THIS_DIR}"/sklearn_german_credit/model/model.pkl + (cd "${MODEL_DIR}"/income_prediction && python train.py) + cp "${MODEL_DIR}"/income_prediction/adult_income_xgb.pkl "${THIS_DIR}"/xgboost_dmatrix_income/model/model.pkl + (cd "${MODEL_DIR}"/iris && python train.py) + cp "${MODEL_DIR}"/iris/iris_xgb.pkl "${THIS_DIR}"/xgboost_iris/model/model.pkl } function python_setup() { - base_setup $1 $2 model.pkl + base_setup "$1" "$2" model.pkl } function wait_for() { @@ -54,19 +71,19 @@ function wait_for() { until [ $next_wait_time -eq 30 ] || $(command); do sleep $(( next_wait_time=next_wait_time+5 )) done - [ $next_wait_time -lt 30 ] + [ "$next_wait_time" -lt 30 ] } function end_prediction_service() { result=${1:-failed} - if [ ${result} = 'failed' ] + if [ "${result}" = 'failed' ] then echo "!!!TEST FAILED for ${name}!!!" fi echo "Removing running prediction service, if any" - if [ $target == "local" ]; then + if [ "$target" == "local" ]; then end_prediction_service_local - elif [ $target == "minikube" ]; then + elif [ "$target" == "minikube" ]; then end_prediction_service_minikube fi result=failed # set for next test, until it explicitly succeeds @@ -76,51 +93,56 @@ function end_prediction_service() { function end_prediction_service_local() { if [ -n "$container_id" ] then - if [ ${result} = 'failed' ] + if [ "${result}" = 'failed' ] then - docker logs ${container_id} + docker logs "${container_id}" fi - docker stop ${container_id} - docker rm -f ${container_id} + docker stop "${container_id}" + docker rm -f "${container_id}" fi unset container_id } function end_prediction_service_minikube() { - if [ ${result} = 'failed' ] + if [ "${result}" = 'failed' ] then - kubectl logs -l app=${resource_name} --namespace $NAMESPACE + kubectl logs -l app="${resource_name}" --namespace $NAMESPACE fi - kubectl delete service ${resource_name} --ignore-not-found --namespace $NAMESPACE - kubectl delete deployment ${resource_name} --ignore-not-found --namespace $NAMESPACE + kubectl delete service "${resource_name}" --ignore-not-found --namespace $NAMESPACE + kubectl delete deployment "${resource_name}" --ignore-not-found --namespace $NAMESPACE } function build() { image_name=$1 echo "***Building ${image_name}***" - sh ${GEN_DIR}/container_util.sh build + sh "${GEN_DIR}"/container_util.sh build } function minikube_setup() { eval $(minikube docker-env) # build images in shared registry # setup minio server on local port 9000 set +e - kubectl create namespace $NAMESPACE - kubectl apply -f ${THIS_DIR}/minikube/test-minio.yml - kubectl get svc test-minio 2>&1 > /dev/null - if [ "$?" -ne 0 ]; then + kubectl create namespace "$NAMESPACE" + kubectl apply -f "${THIS_DIR}"/minikube/test-minio.yml + sleep 5 # wait for service to come up + #kubectl get svc test-minio 2>&1 > /dev/null + #if [ "$?" -ne 0 ]; then + if kubectl get svc test-minio >> /dev/null; then kubectl expose deployment test-minio --type=LoadBalancer --port 9000 --target-port 9000 fi - echo "***Setting up minio command line (mc) and certifai bucket***" - mc config host list minikube 2>&1 > /dev/null + echo "***Setting up minio command line ($MINIO) and certifai bucket***" + $MINIO config host list minikube 2>&1 > /dev/null if [ "$?" -ne 0 ]; then - mc config host add minikube http://127.0.0.1:9000 minio minio123 + $MINIO config host add minikube http://127.0.0.1:9000 minio minio123 fi - mc ls minikube/certifai 2>&1 > /dev/null + $MINIO ls minikube/certifai 2>&1 > /dev/null if [ "$?" -ne 0 ]; then - mc mb minikube/certifai + $MINIO mb minikube/certifai + fi + + if [[ "$RUN_H2O" == "true" ]]; then + $MINIO cp "${THIS_DIR}"/license.txt minikube/certifai/files/license.txt fi - mc cp ${THIS_DIR}/license.txt minikube/certifai/files/license.txt set -e } @@ -128,15 +150,15 @@ function data_setup() { local_name=$1 model_file=$2 model_use_case_id=$3 - if [ $target == "local" ]; then + if [ "$target" == "local" ]; then return 0 fi echo "***Setting up prediction service data in minio for ${local_name}***" - mc config host add minikube http://127.0.0.1:9000 minio minio123 + $MINIO config host add minikube http://127.0.0.1:9000 minio minio123 model_data_path="minikube/certifai/${model_use_case_id}/models" local_path="${THIS_DIR}/${local_name}/model" - mc cp ${local_path}/${model_file} ${model_data_path}/${model_file} - mc cp ${local_path}/metadata.yml ${model_data_path}/metadata.yml + $MINIO cp "${local_path}"/"${model_file}" "${model_data_path}"/"${model_file}" + $MINIO cp "${local_path}"/metadata.yml "${model_data_path}"/metadata.yml } function run_and_test() { @@ -144,89 +166,150 @@ function run_and_test() { model_file=$2 image_name=$3 resource_name=$4 - if [ $target == "local" ]; then + if [ "$target" == "local" ]; then echo "***Running ${local_name}***" - container_id=$(docker run -d -p 8551:8551 -v ${THIS_DIR}/${local_name}/model:/tmp/model \ - -e MODEL_PATH=/tmp/model/${model_file} \ - -e METADATA_PATH=/tmp/model/metadata.yml -t ${image_name}) - elif [ $target == "minikube" ]; then + container_id=$(docker run -d -p 8551:8551 -v "${THIS_DIR}"/"${local_name}"/model:/tmp/model \ + -e MODEL_PATH=/tmp/model/"${model_file}" \ + -e METADATA_PATH=/tmp/model/metadata.yml -t "${image_name}") + elif [ "$target" == "minikube" ]; then echo "***Generating deployment definition for ${resource_name}***" - sh ${GEN_DIR}/config_deploy.sh -c ${THIS_DIR}/${local_name}/model/deployment_config.yml - kubectl apply -f ${GEN_DIR}/deployment.yml - kubectl wait --for=condition=ready --timeout=300s pod -l app=${resource_name} -n ${NAMESPACE} + sh "${GEN_DIR}"/config_deploy.sh -c "${THIS_DIR}"/"${local_name}"/model/deployment_config.yml + kubectl apply -f "${GEN_DIR}"/deployment.yml + kubectl wait --for=condition=ready --timeout=300s pod -l app="${resource_name}" -n ${NAMESPACE} # we need to delete the created service so expose can create it - kubectl delete svc ${resource_name} --ignore-not-found --namespace certifai-models - kubectl expose deployment ${resource_name} --type=LoadBalancer \ + kubectl delete svc "${resource_name}" --ignore-not-found --namespace certifai-models + kubectl expose deployment "${resource_name}" --type=LoadBalancer \ --port 8551 --target-port 8551 --namespace certifai-models fi # Wait until health endpoint is available wait_for 'curl -X GET http://127.0.0.1:8551/health' echo "***Testing ${local_name}***" - certifai scan -f ${THIS_DIR}/${local_name}/explain_def.yml + certifai scan -f "${THIS_DIR}"/"${local_name}"/explain_def.yml echo "***Successfully tested ${local_name}***" end_prediction_service succeeded } +function install_toolkit() { + # Install the toolkit, if its not already installed + if ! command -v certifai &> /dev/null + then + echo "Installing Certifai Toolkit!" + pip install "${THIS_DIR}"/certifai_toolkit/packages/all/* + pip install "${THIS_DIR}"/certifai_toolkit/packages/python"${PYTHON_VERSION}"/* + fi +} + + +function run_sklearn_model() { + # Predict service for Sklearn + python_setup python sklearn_predict + build sklearn_predict + data_setup sklearn_german_credit model.pkl test_german_credit + run_and_test sklearn_german_credit model.pkl sklearn_predict test-german-credit-dtree +} + +function run_xgboost_iris_model() { + # Predict service for XGBClassifier or XGBRegressor + python_setup python xgboost_predict + # Add xgboost to requirements + echo "\n$XGBOOST_VERSION\n" >> "${GEN_DIR}"/requirements.txt + build xgboost_predict + data_setup xgboost_iris model.pkl test_iris + run_and_test xgboost_iris model.pkl xgboost_predict test-iris-xgb-iris +} + +function run_xgboost_income_prediction() { + # Predict service for xgboost using DMatrix + python_setup python_xgboost_dmatrix xgboost_dmatrix_predict + build xgboost_dmatrix_predict + data_setup xgboost_dmatrix_income model.pkl test_income + run_and_test xgboost_dmatrix_income model.pkl xgboost_dmatrix_predict test-income-xgboost +} + +function run_h2o_models() { + # Predict service for H2O MOJO + h2o_setup h2o_mojo h2o_mojo_predict + build h2o_mojo_predict + for name in auto_insurance german_credit iris + do + local_name="h2o_${name}" + muc_id="test_${name}" + model_id="dai-mojo" + data_setup ${local_name} pipeline.mojo ${muc_id} + name_dashed=$(echo ${muc_id}-${model_id} | tr '_' '-') + run_and_test "${local_name}" pipeline.mojo h2o_mojo_predict "${name_dashed}" + done +} -# Install the toolkit, if its not already installed -if ! command -v certifai &> /dev/null -then - pip install ${THIS_DIR}/certifai_toolkit/packages/all/* - pip install ${THIS_DIR}/certifai_toolkit/packages/python${PYTHON_VERSION}/* -fi + +function printUsage() { + local prog_name + prog_name="$(basename "$0")" + + local usage + usage="$prog_name [-h|--help] [local | minikube] + +Test Containerized Model examples using the Certifai Model SDK. + +Options: + + local - Run local prediction service tests + + minikube - Run minikube prediction service tests. Requires Minikube to already be running. + +Environment Variables: + + RUN_H2O - if 'true', then h2o examples will be tested and the 'license.txt' must contain a valid h2o license. + Defaults to 'false'. + + TOOLKIT_PATH - The path to the (unzipped) Certifai Toolkit, defaults to: ./certifai_toolkit + +Examples: + + TOOLKIT_PATH=./toolkit ./run_test.sh local + + TOOLKIT_PATH=./toolkit ./run_test.sh minikube +" + echo "$usage" +} # # MAIN EXECUTION STARTS HERE # -if [ $target == "local" ]; then - echo "Running local prediction service tests" -elif [ $target == "minikube" ]; then - echo "Running minikube prediction service tests" - minikube_setup -else - echo "Invalid target environment" - exit 1 -fi - -set -exv -trap end_prediction_service EXIT - -# Predict service for H2O MOJO -h2o_setup h2o_mojo h2o_mojo_predict -build h2o_mojo_predict -for name in auto_insurance german_credit iris -do - local_name="h2o_${name}" - muc_id="test_${name}" - model_id="dai-mojo" - data_setup ${local_name} pipeline.mojo ${muc_id} - name_dashed=$(echo ${muc_id}-${model_id} | tr '_' '-') - run_and_test ${local_name} pipeline.mojo h2o_mojo_predict "${name_dashed}" -done - -train_models # For the non-H2O models -# -# Predict service for Sklearn -python_setup python sklearn_predict -build sklearn_predict -data_setup sklearn_german_credit model.pkl test_german_credit -run_and_test sklearn_german_credit model.pkl sklearn_predict test-german-credit-dtree - -# Predict service for XGBClassifier or XGBRegressor -python_setup python xgboost_predict -# Add xgboost to requirements -echo "\nxgboost==1.2.0\n" >> ${GEN_DIR}/requirements.txt -build xgboost_predict -data_setup xgboost_iris model.pkl test_iris -run_and_test xgboost_iris model.pkl xgboost_predict test-iris-xgb-iris -# -# -# Predict service for xgboost using DMatrix -python_setup python_xgboost_dmatrix xgboost_dmatrix_predict -build xgboost_dmatrix_predict -data_setup xgboost_dmatrix_income model.pkl test_income -run_and_test xgboost_dmatrix_income model.pkl xgboost_dmatrix_predict test-income-xgboost - -echo "***All tests completed successfully***" -trap - EXIT +function main() { + target=${1:-local} + set -exv + set_globals + install_toolkit + if [ "$target" == "local" ]; then + echo "Running local prediction service tests" + elif [ "$target" == "minikube" ]; then + echo "Running minikube prediction service tests" + check_minio_installed + minikube_setup + elif [ "$target" == "-h" ] || [ "$target" == "--help" ]; then + printUsage + exit 0 + else + echo "Invalid target environment" + printUsage + exit 1 + fi + + # catch any EXIT signals & clean prediction services + trap end_prediction_service EXIT + if [[ "$RUN_H2O" == "true" ]]; then + run_h2o_models + fi + # Train the non-H2O models + train_models + # Run the non-H2O models + run_sklearn_model + run_xgboost_iris_model + run_xgboost_income_prediction + echo "***All tests completed successfully***" + trap - EXIT +} + +main "$@" diff --git a/models/containerized_model/templates/python/requirements.txt b/models/containerized_model/templates/python/requirements.txt index e6153084..a66d672b 100644 --- a/models/containerized_model/templates/python/requirements.txt +++ b/models/containerized_model/templates/python/requirements.txt @@ -1,4 +1,4 @@ # add python pip install dependencies below pyyaml # required by prediction service - do not remove scikit-learn==1.0.2 -#xgboost==1.2.0 # uncomment if using xgboost and pin to same version as model +#xgboost==1.7.2 # uncomment if using xgboost and pin to same version as model diff --git a/models/containerized_model/templates/python_xgboost_dmatrix/requirements.txt b/models/containerized_model/templates/python_xgboost_dmatrix/requirements.txt index 1899193f..3a1724a7 100644 --- a/models/containerized_model/templates/python_xgboost_dmatrix/requirements.txt +++ b/models/containerized_model/templates/python_xgboost_dmatrix/requirements.txt @@ -1,3 +1,3 @@ # add python pip install dependencies below pyyaml # required by prediction service - do not remove -xgboost==1.2.0 # pin to match the environment in which the model was pickled +xgboost==1.7.2 # pin to match the environment in which the model was pickled diff --git a/models/german_credit/app_mlp_soft_scoring.py b/models/german_credit/app_mlp_soft_scoring.py index 29395a72..e2444cdc 100644 --- a/models/german_credit/app_mlp_soft_scoring.py +++ b/models/german_credit/app_mlp_soft_scoring.py @@ -24,4 +24,5 @@ endpoint_url='/german_credit_mlp/predict') # to start production ready gunicorn server use `production=True` -app.run(production=True) +#app.run(production=True) +app.run() diff --git a/models/german_credit/composed_app.py b/models/german_credit/composed_app.py index b9eb15c3..febced39 100644 --- a/models/german_credit/composed_app.py +++ b/models/german_credit/composed_app.py @@ -40,4 +40,5 @@ composed_app.add_wrapped_model('/german_credit_logit', logit_app) composed_app.add_wrapped_model('/german_credit_svm', svm_app) composed_app.add_wrapped_model('/german_credit_mlp', mlp_app) +#composed_app.run(production=True) composed_app.run() diff --git a/models/german_credit/test.py b/models/german_credit/test.py new file mode 100644 index 00000000..fb2d1c77 --- /dev/null +++ b/models/german_credit/test.py @@ -0,0 +1,35 @@ +import sys +import os +import unittest + +# Add the file to classpath for relative import +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +from base_test import ModelTest + + +class GermanCreditTest(ModelTest): + + def setUp(self): + self.run_standalone_python_script("train.py") + + def test_single_app(self): + self.run_python_app_test("app_dtree.py", "app_test.py") + + def test_composed_app(self): + self.run_python_app_test("composed_app.py", "composed_app_test.py") + + def test_trust_scan(self): + self.run_model_and_scan("composed_app.py", "german_credit_scanner_definition.yaml") + + def test_explain_scan(self): + self.run_standalone_python_script("explain.py") + + def test_soft_scoring_app(self): + self.run_python_app_test("app_mlp_soft_scoring.py", "app_mlp_soft_scoring_test.py") + + def test_soft_scoring_scan(self): + self.run_model_and_scan("app_mlp_soft_scoring.py", "german_credit_shap_explanation_scanner_definition.yaml") + + +if __name__ == '__main__': + unittest.main() diff --git a/models/german_credit_pandas/app_mlp_soft_scoring.py b/models/german_credit_pandas/app_mlp_soft_scoring.py index 3138b759..7219df0b 100644 --- a/models/german_credit_pandas/app_mlp_soft_scoring.py +++ b/models/german_credit_pandas/app_mlp_soft_scoring.py @@ -25,4 +25,5 @@ endpoint_url='/german_credit_mlp/predict', pandas_kwargs={'columns': columns}) # to start production ready gunicorn server use `production=True` -app.run(production=True) +#app.run(production=True) +app.run() diff --git a/models/german_credit_pandas/test.py b/models/german_credit_pandas/test.py new file mode 100644 index 00000000..e06fd4f0 --- /dev/null +++ b/models/german_credit_pandas/test.py @@ -0,0 +1,32 @@ +import sys +import os +import unittest + +# Add the file to classpath for relative import +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +from base_test import ModelTest + + +class GermanCreditPandasTest(ModelTest): + + def setUp(self): + self.run_standalone_python_script("train_pandas.py") + + def test_single_app(self): + self.run_python_app_test("app_dtree.py", "app_test.py") + + def test_composed_app(self): + self.run_python_app_test("composed_app.py", "composed_app_test.py") + + def test_trust_scan(self): + self.run_model_and_scan("composed_app.py", "german_credit_scanner_definition.yaml") + + def test_soft_scoring_app(self): + self.run_python_app_test("app_mlp_soft_scoring.py", "app_mlp_soft_scoring_test.py") + + def test_soft_scoring_scan(self): + self.run_model_and_scan("app_mlp_soft_scoring.py", "german_credit_shap_explanation_scanner_definition.yaml") + + +if __name__ == '__main__': + unittest.main() diff --git a/models/income_prediction/test.py b/models/income_prediction/test.py new file mode 100644 index 00000000..70a6411c --- /dev/null +++ b/models/income_prediction/test.py @@ -0,0 +1,23 @@ +import sys +import os +import unittest + +# Add the file to classpath for relative import +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +from base_test import ModelTest + + +class IncomePredictionTest(ModelTest): + + def setUp(self): + self.run_standalone_python_script("train.py") + + def test_single_app(self): + self.run_python_app_test("app_xgb.py", "app_test.py") + + def test_explain_scan(self): + self.run_model_and_scan("app_xgb.py", "income_explain_definition.yaml") + + +if __name__ == '__main__': + unittest.main() diff --git a/models/iris/test.py b/models/iris/test.py new file mode 100644 index 00000000..c0096c0b --- /dev/null +++ b/models/iris/test.py @@ -0,0 +1,28 @@ +import sys +import os +import unittest + +# Add the file to classpath for relative import +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +from base_test import ModelTest + + +class IrisTest(ModelTest): + + def setUp(self): + # Train all models at the start of each test - possibly excessive, but gives full flexibility to run + # a single test in isolation + self.run_standalone_python_script("train.py") + + def test_single_app(self): + self.run_python_app_test("app_svm.py", "app_test.py") + + def test_single_app_xgboost(self): + self.run_python_app_test("app_xgb.py", "app_test.py") + + def test_scan(self): + self.run_model_and_scan("app_svm.py", "iris_scanner_definition.yaml") + + +if __name__ == '__main__': + unittest.main() diff --git a/models/patient_readmission/README.md b/models/patient_readmission/README.md index f10a0cee..07d80bf7 100644 --- a/models/patient_readmission/README.md +++ b/models/patient_readmission/README.md @@ -11,12 +11,6 @@ Specifically, it shows how to use the Certifai toolkit to: * generate explanations for the model's predictions using the CLI * set up fast-explanation, for generating large numbers of explanations at scale -*Note*: Fast explanation is currently a beta feature. To enable it you will need to -edit your `~/.certifai/certifai_conf.ini` file to add: -``` -[scanner] -support_fast_explanations_beta = True -``` ## Wrap a single model as a service 1. Make sure you have activated your Certifai toolkit environment: diff --git a/models/patient_readmission/app.py b/models/patient_readmission/app.py index 2fc314de..29b667d8 100644 --- a/models/patient_readmission/app.py +++ b/models/patient_readmission/app.py @@ -14,4 +14,5 @@ encoder = saved.get('encoder', None) app = SimpleModelWrapper(model=model, encoder=encoder) -app.run(production=True) +#app.run(production=True) +app.run() diff --git a/models/patient_readmission/test.py b/models/patient_readmission/test.py new file mode 100644 index 00000000..970650d4 --- /dev/null +++ b/models/patient_readmission/test.py @@ -0,0 +1,29 @@ +import sys +import os +import unittest + +# Add the file to classpath for relative import +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +from base_test import ModelTest + + +class PatientReadmissionTest(ModelTest): + + def setUp(self): + self.run_standalone_python_script("train.py") + + def test_single_app(self): + self.run_python_app_test("app.py", "app_test.py") + + def test_definition_test(self): + self.run_model_and_definition_test("app.py", "explain_def.yml") + + def test_fast_explain(self): + self.run_model_and_explain("app.py", "explain_def.yml", fast=True) + + def test_traditional_explain(self): + self.run_model_and_explain("app.py", "explain_def.yml", fast=False) + + +if __name__ == '__main__': + unittest.main() diff --git a/notebooks/azureml_model_headers_demo/german_credit_azure_ml_demo.ipynb b/notebooks/azureml_model_headers_demo/german_credit_azure_ml_demo.ipynb index b9d21a21..dbb88867 100644 --- a/notebooks/azureml_model_headers_demo/german_credit_azure_ml_demo.ipynb +++ b/notebooks/azureml_model_headers_demo/german_credit_azure_ml_demo.ipynb @@ -21,7 +21,7 @@ "source": [ "## CONTENTS\n", "\n", - "1. In this tutorial we will create sklearn models to classify [german credit loan risk](https://archive.ics.uci.edu/ml/datasets/Statlog+%28German+Credit+Data%29) (predict whether loan will be granted or not)\n", + "1. In this tutorial we will create sklearn models to classify [german credit loan risk](https://archive.ics.uci.edu/dataset/144/statlog+german+credit+data) (predict whether loan will be granted or not)\n", "\n", "2. Register model and deploy as webservice in ACI (AZURE CONTAINER INSTANCE) with authentication\n", "\n", @@ -45,7 +45,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Training Scikit-learn models on [UCI German credit data](https://archive.ics.uci.edu/ml/datasets/Statlog+%28German+Credit+Data%29)" + "## Training Scikit-learn models on [UCI German credit data](https://archive.ics.uci.edu/dataset/144/statlog+german+credit+data)" ] }, { diff --git a/notebooks/target_encoded/dataset_generation/README.md b/notebooks/target_encoded/dataset_generation/README.md index 6301ae1c..520abfa6 100644 --- a/notebooks/target_encoded/dataset_generation/README.md +++ b/notebooks/target_encoded/dataset_generation/README.md @@ -3,7 +3,7 @@ ## Notebooks -- `german_credit_multiclass_dataset_generation`: Creates a neural network model with soft-scoring to transform binary outcome-labels (loan granted/loan denied (1/2) ) [German-Credit-Dataset](https://archive.ics.uci.edu/ml/datasets/Statlog+%28German+Credit+Data%29) into multiclass problem with three outcome-labels (loan granted/loan denied/further inspection (1/2/3) ) and saves the dataset as csv +- `german_credit_multiclass_dataset_generation`: Creates a neural network model with soft-scoring to transform binary outcome-labels (loan granted/loan denied (1/2) ) [German-Credit-Dataset](https://archive.ics.uci.edu/dataset/144/statlog+german+credit+data) into multiclass problem with three outcome-labels (loan granted/loan denied/further inspection (1/2/3) ) and saves the dataset as csv - `german_credit_multiclass_dataset_encoding`: Encodes the above generated dataset into target encoded and one-hot encoded feature columns and dumps the generated mappings to disk as json diff --git a/scan-manager/docs/README.md b/scan-manager/docs/README.md index 7a98eeba..8aa6ebe0 100644 --- a/scan-manager/docs/README.md +++ b/scan-manager/docs/README.md @@ -49,7 +49,7 @@ artifacts include deployment templates and .yaml config file(s); and are provide The `deployment` folder contains: - Deployment templates: `.yaml` templates that provide the configuration templates for deploying each of the specified model types on Kubernetes: - - `scikit_0.23`: Uses a `python3.8` base image with `scikit-learn v0.23` pre-installed. + - `python`: Uses a `python3.8` base image with `scikit-learn` pre-installed. - `h2o_mojo`: Uses a `python3.8` base image with `daimojo-2.4.8-cp36` whl pre-installed. - `r_model`: Uses `rocker/r-ver:latest` base image with `r-cran-randomforest` pre-installed. - `hosted_model`: Uses a `python3.8` base image for wrapping an already hosted model service. diff --git a/scan-manager/docs/setup_artifacts/deployment/config.yml b/scan-manager/docs/setup_artifacts/deployment/config.yml index f4ed1aa3..a1eeb0a5 100644 --- a/scan-manager/docs/setup_artifacts/deployment/config.yml +++ b/scan-manager/docs/setup_artifacts/deployment/config.yml @@ -1,11 +1,13 @@ -scikit_0.23: - deployment: scikit_0.23_deployment.yml +python: + deployment: python_deployment.yml default_base_image: name: python38_scikit - value: c12e/cortex-certifai-model-scikit:v4-1.3.16-25-g527dfe93 + value: c12e/cortex-certifai-model-scikit:base-py38-1.3.17-63-g2ecf19f0-d357535 available_base_images: - name: python38_scikit - value: c12e/cortex-certifai-model-scikit:v4-1.3.16-25-g527dfe93 + value: c12e/cortex-certifai-model-scikit:base-py38-1.3.17-63-g2ecf19f0-d357535 + - name: python39_scikit + value: c12e/cortex-certifai-model-scikit:base-py39-1.3.17-63-g2ecf19f0-d357535 h2o_mojo: deployment: h2o_mojo_deployment.yml default_base_image: diff --git a/scan-manager/docs/setup_artifacts/deployment/scikit_0.23_deployment.yml b/scan-manager/docs/setup_artifacts/deployment/python_deployment.yml similarity index 100% rename from scan-manager/docs/setup_artifacts/deployment/scikit_0.23_deployment.yml rename to scan-manager/docs/setup_artifacts/deployment/python_deployment.yml