diff --git a/build.sh b/build.sh
index 6bd8dd43..264578e2 100755
--- a/build.sh
+++ b/build.sh
@@ -11,11 +11,14 @@ function setGlobals() {
SKIP_TOOLKIT="${SKIP_TOOLKIT:-false}"
RUN_REMOTE_EXAMPLES="${RUN_REMOTE_EXAMPLES:-false}"
PYTHON_VERSION="3.8"
+ SK_PANDAS_VERSION="sklearn-pandas==2.2.0"
+ XGBOOST_VERSION="xgboost==1.7.2"
SCRIPT_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P )"
ARTIFACTS_DIR="${SCRIPT_PATH}/artifacts"
TOOLKIT_PATH="${ARTIFACTS_DIR}/certifai_toolkit.zip"
TOOLKIT_WORK_DIR="${ARTIFACTS_DIR}/toolkit"
PACKAGES_DIR="${TOOLKIT_WORK_DIR}/packages"
+ CONTAINERIZED_EXAMPLES_DIR="${SCRIPT_PATH}/models/containerized_model/examples"
TEMPLATES_DIR="${SCRIPT_PATH}/models/containerized_model"
BASE_IMAGES_DIR="${SCRIPT_PATH}/models/containerized_model/base_images"
NOTEBOOK_DIR="${SCRIPT_PATH}/notebooks"
@@ -157,6 +160,10 @@ function _installModelRequirements() {
pip install -r "${TEMPLATES_DIR}/requirements.txt"
}
+function _installLocalModelRequirements() {
+ pip install "$SK_PANDAS_VERSION" "$XGBOOST_VERSION"
+}
+
function buildModelDeploymentImages() {
# Builds Docker images for the example Containerized Model Types (Scikit, H2O, Proxy, R). These are images are used
@@ -292,11 +299,13 @@ function buildPredictionServiceBaseImages() {
echo "{\"python38\": \"${py38_image}\", \"python39\": \"${py39_image}\"}" > "${BASE_IMAGE_BUILD_REPORT_JSON}"
}
-function test() {
+function testAll() {
testMarkdownLinks
testModels
testNotebooks
testTutorials
+ # Requires Docker/Minikube - so skipped in pipeline
+ #testContainerizedModels
}
function testMarkdownLinks() {
@@ -328,12 +337,44 @@ function testMarkdownLinks() {
}
function testModels() {
- echo "TODO: automate subset of model examples - "
- # for each
- # - train the models
- # - start the app in one process,
- # - run the test in another process
- # - assert both processes exit successfully
+ MODELS_DIR="${SCRIPT_PATH}/models"
+ # run tests for each individual example
+ cd "$MODELS_DIR"/german_credit/
+ python -m unittest -v test.py
+
+ cd "$MODELS_DIR"/german_credit_pandas
+ python -m unittest -v test.py
+
+ cd "$MODELS_DIR"/income_prediction
+ python -m unittest -v test.py
+
+ cd "$MODELS_DIR"/iris
+ python -m unittest -v test.py
+
+ cd "$MODELS_DIR"/patient_readmission
+ python -m unittest -v test.py
+
+ # Go back to root directory
+ cd "$SCRIPT_PATH"
+
+ # TODO: Run other examples (see https://github.com/CognitiveScale/certifai/issues/4870)
+ # - h2o_dai_german_credit
+ # - h2o_dai_regression_auto_insurance
+ # - r-models
+}
+
+function testContainerizedModels() {
+ # run base of set of containerized model examples locally (with docker)
+ cd "$CONTAINERIZED_EXAMPLES_DIR"
+ TOOLKIT_PATH="$TOOLKIT_WORK_DIR" ./run_test.sh "local"
+
+ # TODO: Add 'RUN_H2O=true' to test other examples (see https://github.com/CognitiveScale/certifai/issues/4870)
+ # - h2o_dai_german_credit
+ # - h2o_dai_regression_auto_insurance
+ # - r-models
+
+ # Go back to root directory
+ cd "$SCRIPT_PATH"
}
function testTutorials() {
@@ -512,7 +553,7 @@ function _sagemakerNotebook() {
function _xgboostModel() {
# xgboost-model
cd "${NOTEBOOK_DIR}"
- pip install xgboost
+ pip install "$XGBOOST_VERSION"
_runNotebookInPlace "${NOTEBOOK_DIR}/xgboost-model/xgboostDmatrixExample.ipynb"
}
@@ -522,7 +563,9 @@ function main() {
setGlobals
activateConda
installToolkit
- test
+ _installModelRequirements
+ _installLocalModelRequirements
+ testAll
rm -rf "${TOOLKIT_WORK_DIR}"
;;
docker)
@@ -558,6 +601,14 @@ function main() {
activateConda
testMarkdownLinks
;;
+ models)
+ setGlobals
+ activateConda
+ installToolkit
+ _installModelRequirements
+ _installLocalModelRequirements
+ testModels
+ ;;
notebook)
setGlobals
activateConda
diff --git a/models/README.md b/models/README.md
index 6bf68a3f..80de67f3 100644
--- a/models/README.md
+++ b/models/README.md
@@ -19,5 +19,7 @@ for detailed information about Cortex Certifai.
| [german_credit_pandas](./german_credit_pandas) | Illustrates using the Certifai Model SDK to run a models in a service, where the models expect as input a [pandas DataFrame](https://pandas.pydata.org/) instead of a [numpy array](https://numpy.org/). | Binary classification | python | sklearn |
| [h2o_dai_auto_insurance](./h2o_dai_regression_auto_insurance) | Illustrates using the Certifai Model SDK to create a gunicorn prediction service from a regression H2O MOJO model, and scan it for trust scores. | Regression | python | H2O MOJO |
| [h2o_dai_german_credit](./h2o_dai_german_credit) | Illustrates using the Certifai Model SDK to create a development or gunicorn prediction service from a binary classification H2O MOJO, and scan it for trust scores or for explanations. | Binary classification | python | H2O MOJO |
+| [income_prediction](./income_prediction) | Illustrates using the Certifai Model SDK to run a single binary-classification XGBoost model in a service, using a customized model wrapper. | Binary classification | python | sklearn
xgboost |
| [iris](./iris) | Illustrates using the Certifai Model SDK to run a single multi-class model in a service, using a customized model wrapper. | Multi-class classification | python | sklearn
xgboost |
+| [patient_readmission](./patient_readmission) | Illustrates using the Certifai Model SDK to run a single binary-classification model in a service, using a customized model wrapper, for creation of fast (bulk) explanations. | Binary classification | python | sklearn
xgboost |
| [r-models](https://github.com/CognitiveScale/cortex-certifai-examples/tree/master/models/r-models) | Illustrates running a R model in a service using plumber. | Binary classification | R | randomForest |
diff --git a/models/base_test.py b/models/base_test.py
new file mode 100644
index 00000000..8afcfa1f
--- /dev/null
+++ b/models/base_test.py
@@ -0,0 +1,140 @@
+import time
+import contextlib
+import subprocess
+import tempfile
+import unittest
+from typing import Optional, Sequence
+
+
+def capture_err_and_out(stderr, stdout):
+ if stderr is not None:
+ print("\n---------------------- (main) stderr: ----------------------")
+ print(stderr, end="")
+ print("\n------------------------------------------------------------\n")
+ if stdout is not None:
+ print("\n---------------------- (main) stdout: ----------------------")
+ print(stdout, end="")
+ print("\n------------------------------------------------------------")
+
+
+def capture_output(stdout, stderr, limit=100):
+ count = 0
+ print("\n---------------------- (service) stdout: ----------------------\n")
+ with open(stdout, 'r+') as f:
+ for line in f:
+ if count > limit:
+ break
+ print(line, end="")
+ limit += 1
+ print("\n------------------------------------------------------------\n")
+ print()
+
+ count = 0
+ print("\n---------------------- (service) stderr: ----------------------\n")
+ with open(stderr, 'r+') as f:
+ for line in f:
+ if count > limit:
+ break
+ print(line, end="")
+ limit += 1
+ print("\n------------------------------------------------------------\n")
+
+
+class ModelTest(unittest.TestCase):
+ """Base class for testing Certifai Prediction Service Examples. Each example will typically include multiple
+ scenarios where:
+
+ 1) a flask server is launched as a background process (via the Certifai Model SDK)
+ 2) a Certifai Scan is launched (or just a plain Python script) is launched in the foreground that calls (1)
+
+ Each process that is launched in the foreground, (2), is expected to complete with a 0 exit code. Each process
+ launched in the background (1) are expected to be run until explicitly killed.
+
+ The following functions should cover scenarios that run plain Python Scripts::
+
+ run_standalone_python_script(python_script)
+ run_python_app_test(model_app, python_script)
+
+ The following functions should cover scenarios that involve running a Certifai Scan::
+
+ run_model_and_definition_test('app_dtree.py', 'my-definition.yaml')
+ run_model_and_scan('app_dtree.py', 'my-definition.yaml')
+ run_model_and_explain('app_dtree.py', 'my-definition.yaml', fast=True)
+ """
+ SLEEP_TIME = 5 # 5 seconds
+ TERMINATION_TIME = 5 # 5 seconds
+ DEFAULT_TEST_TIMEOUT = 2 * 60 # 2 minutes
+ DEFAULT_SCAN_TIMEOUT = 60 * 60 * 1 # 1 hour
+ PRECALCULATE_TIMEOUT = DEFAULT_SCAN_TIMEOUT * 3 # 3 hours
+ bg = None
+
+ def _run_in_foreground(self, command: Sequence[str], timeout: Optional[int] = None):
+ try:
+ # Run process and wait until it completes
+ process = subprocess.run(command, shell=False, capture_output=True, timeout=timeout, text=True)
+ process.check_returncode()
+ except subprocess.TimeoutExpired as te:
+ error = f"\nProcess did not finish within expected time (command={te.cmd}, timeout={te.timeout} seconds). Error: {str(te)}"
+ capture_err_and_out(te.stderr, te.stdout)
+ self.fail(error)
+ except subprocess.CalledProcessError as ce:
+ error = f"\nProcess finished with non-zero exit code (command={ce.cmd}, code={ce.returncode}). Error: {str(ce)}"
+ capture_err_and_out(ce.stderr, ce.stdout)
+ self.fail(error)
+
+ @contextlib.contextmanager
+ def _run_in_background(self, command: Sequence[str]):
+ with tempfile.NamedTemporaryFile(mode='w+') as stdout, tempfile.NamedTemporaryFile(mode='w+') as stderr:
+ try:
+ p = subprocess.Popen(command, shell=False, stdout=stdout, stderr=stderr, stdin=subprocess.DEVNULL,
+ close_fds=True, text=True)
+ yield
+ except Exception:
+ # WARNING: Killing the subprocess may not kill any workers spawned by the process (e.g. gunicorn!)
+ p.kill()
+ p.wait()
+ capture_output(stdout.name, stderr.name)
+ raise
+ finally:
+ # WARNING: Killing the subprocess may not kill any workers spawned by the process (e.g. gunicorn!)
+ p.kill()
+ p.wait()
+
+ # Outward facing API
+
+ def run_python_app_test(self, model_app: str, test_script: str):
+ # Run a Python Model (flask app) in the background, give it a couple seconds to start up, before running test
+ with self._run_in_background(["python", model_app]):
+ time.sleep(self.SLEEP_TIME)
+ self._run_in_foreground(["python", test_script], timeout=self.DEFAULT_TEST_TIMEOUT)
+
+ def run_standalone_python_script(self, script: str):
+ # Run the standalone test script
+ self._run_in_foreground(["python", script], timeout=self.DEFAULT_SCAN_TIMEOUT)
+
+ def run_model_and_definition_test(self, model_app: str, definition: str):
+ # Run a Python Model (flask app) in the background, give it a couple seconds to start up, before running test
+ with self._run_in_background(["python", model_app]):
+ time.sleep(self.SLEEP_TIME)
+ self._run_in_foreground(["certifai", "definition-test", "-f", definition], timeout=self.DEFAULT_SCAN_TIMEOUT)
+
+ def run_model_and_scan(self, model_app: str, definition: str):
+ # Run a Python Model (flask app) in the background, give it a couple seconds to start up, before running test
+ with self._run_in_background(f"python {model_app}".split()):
+ time.sleep(self.SLEEP_TIME)
+ self._run_in_foreground(["certifai", "scan", "-f", definition], timeout=self.DEFAULT_SCAN_TIMEOUT)
+
+ def run_model_and_explain(self, model_app: str, definition: str, fast: bool = False):
+ # Run a Python Model (flask app) in the background, give it a couple seconds to start up.
+ with self._run_in_background(f"python {model_app}".split()):
+ time.sleep(self.SLEEP_TIME)
+ if fast:
+ # Run the precalculate step prior to the fast explain
+ pre_calc_command = ["certifai", "explain", "-f", definition, "--precalculate"]
+ self._run_in_foreground(pre_calc_command, timeout=self.PRECALCULATE_TIMEOUT)
+ command = ["certifai", "explain", "-f", definition, "--fast"]
+ else:
+ command = ["certifai", "explain", "-f", definition]
+ # Run the explanation scan
+ self._run_in_foreground(command, timeout=self.DEFAULT_SCAN_TIMEOUT)
+
diff --git a/models/containerized_model/README.md b/models/containerized_model/README.md
index 0d11219f..510857ea 100644
--- a/models/containerized_model/README.md
+++ b/models/containerized_model/README.md
@@ -242,7 +242,7 @@ Add respective cloud storage credentials and `MODEL_PATH` to `generated-containe
### Step 5 - Add extra-dependencies (optional)
The dependencies work out of the box with a standard scikit-learn model,
-providing the model was trained with version 0.23.2 of scikit-learn. If
+providing the model was trained with a version `1.0.2` of scikit-learn. If
you are using a different version, you should update
`generated-container-model/requirements.txt`.
diff --git a/models/containerized_model/examples/README.md b/models/containerized_model/examples/README.md
index 4173c05f..86eee591 100644
--- a/models/containerized_model/examples/README.md
+++ b/models/containerized_model/examples/README.md
@@ -30,12 +30,12 @@ The following files must exist in this folder:
* certifai_toolkit/ - certifai toolkit v1.3.6 or above
The current conda environment has been setup with:
-* python 3.6 (if 3.7 or 3.8, update PYTHON_VERSION in `run_test.sh`)
+* python 3.8 (otherwise, update PYTHON_VERSION in `run_test.sh`)
* pip install -U Jinja2
To train and test the models:
* Certifai toolkit installed in the current conda environment
-* conda install -c conda-forge xgboost==1.2.0
+* conda install -c conda-forge xgboost==1.7.2
To build/run the prediction services: Docker
@@ -70,3 +70,5 @@ storage by changing the environment.yml.
The tests exit on any error, printing out the prediction service log
and deleting the running prediction service container.
+
+For more options, run: `sh run_test.sh -h`
diff --git a/models/containerized_model/examples/run_test.sh b/models/containerized_model/examples/run_test.sh
old mode 100644
new mode 100755
index 9e7be220..1caa89a3
--- a/models/containerized_model/examples/run_test.sh
+++ b/models/containerized_model/examples/run_test.sh
@@ -1,51 +1,68 @@
#
-# Copyright (c) 2020. Cognitive Scale Inc. All rights reserved.
+# Copyright (c) 2023. Cognitive Scale Inc. All rights reserved.
# Licensed under CognitiveScale Example Code License https://github.com/CognitiveScale/cortex-certifai-examples/blob/master/LICENSE.md
#
#!/usr/bin/env bash
set -e
-target=${1:-local}
+# - ask in Trusted AI chat if we still have an h2o license?
-THIS_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P )"
-GEN_DIR="${THIS_DIR}/generated-container-model"
-PYTHON_VERSION=${PYTHON_VERSION:-3.8} # or 3.7
-NAMESPACE=certifai-models
+function set_globals() {
+ THIS_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P )"
+ GEN_DIR="${THIS_DIR}/generated-container-model"
+ PYTHON_VERSION=${PYTHON_VERSION:-3.8} # or 3.7
+ XGBOOST_VERSION="xgboost==1.7.2"
+ NAMESPACE=certifai-models
+ MODEL_DIR=${THIS_DIR}/../..
+ RUN_H2O=${RUN_H2O-"false"}
+ MINIO="mc"
+
+ # Default toolkit to './certifai_toolkit'
+ TOOLKIT_PATH="${TOOLKIT_PATH:-$THIS_DIR/certifai_toolkit}"
+ PACKAGES_DIR="${TOOLKIT_PATH}/packages"
+}
+
+function check_minio_installed() {
+ if ! command -v $MINIO &> /dev/null
+ then
+ echo "'$MINIO' CLI could not be found! Install Minio Client: https://min.io/docs/minio/linux/reference/minio-mc.html"
+ exit 1
+ fi
+}
function base_setup() {
model_type=$1
image_name=$2
model_file=$3
echo "***Generating ${model_type}***"
- rm -rf ${GEN_DIR}
- sh ${THIS_DIR}/../generate.sh -i ${image_name}:latest -m ${model_type} -d ${GEN_DIR}
+ rm -rf "${GEN_DIR}"
+ "${THIS_DIR}/../generate.sh" -i "${image_name}":latest -m "${model_type}" -d "${GEN_DIR}" -t "$TOOLKIT_PATH"
all_dir=${GEN_DIR}/packages/all
- mkdir -p ${all_dir}
- cp ${THIS_DIR}/certifai_toolkit/packages/all/cortex-certifai-common*.zip ${all_dir}
- cp ${THIS_DIR}/certifai_toolkit/packages/all/cortex-certifai-model*.zip ${all_dir}
+ mkdir -p "${all_dir}"
+ cp "${PACKAGES_DIR}"/all/cortex-certifai-common*.zip "${all_dir}"
+ cp "${PACKAGES_DIR}"/all/cortex-certifai-model*.zip "${all_dir}"
}
function h2o_setup() {
- base_setup $1 $2 pipeline.mojo
- cp ${THIS_DIR}/license.txt ${GEN_DIR}/license/license.txt
- cp ${THIS_DIR}/daimojo*linux_x86_64.whl ${GEN_DIR}/ext_packages/
+ base_setup "$1" "$2" pipeline.mojo
+ cp "${THIS_DIR}"/license.txt "${GEN_DIR}"/license/license.txt
+ cp "${THIS_DIR}"/daimojo*linux_x86_64.whl "${GEN_DIR}"/ext_packages/
}
-MODEL_DIR=${THIS_DIR}/../..
function train_models() {
- (cd ${MODEL_DIR}/german_credit && python train.py)
- cp ${MODEL_DIR}/german_credit/german_credit_dtree.pkl ${THIS_DIR}/sklearn_german_credit/model/model.pkl
- (cd ${MODEL_DIR}/income_prediction && python train.py)
- cp ${MODEL_DIR}/income_prediction/adult_income_xgb.pkl ${THIS_DIR}/xgboost_dmatrix_income/model/model.pkl
- (cd ${MODEL_DIR}/iris && python train.py)
- cp ${MODEL_DIR}/iris/iris_xgb.pkl ${THIS_DIR}/xgboost_iris/model/model.pkl
+ (cd "${MODEL_DIR}"/german_credit && python train.py)
+ cp "${MODEL_DIR}"/german_credit/models/german_credit_dtree.pkl "${THIS_DIR}"/sklearn_german_credit/model/model.pkl
+ (cd "${MODEL_DIR}"/income_prediction && python train.py)
+ cp "${MODEL_DIR}"/income_prediction/adult_income_xgb.pkl "${THIS_DIR}"/xgboost_dmatrix_income/model/model.pkl
+ (cd "${MODEL_DIR}"/iris && python train.py)
+ cp "${MODEL_DIR}"/iris/iris_xgb.pkl "${THIS_DIR}"/xgboost_iris/model/model.pkl
}
function python_setup() {
- base_setup $1 $2 model.pkl
+ base_setup "$1" "$2" model.pkl
}
function wait_for() {
@@ -54,19 +71,19 @@ function wait_for() {
until [ $next_wait_time -eq 30 ] || $(command); do
sleep $(( next_wait_time=next_wait_time+5 ))
done
- [ $next_wait_time -lt 30 ]
+ [ "$next_wait_time" -lt 30 ]
}
function end_prediction_service() {
result=${1:-failed}
- if [ ${result} = 'failed' ]
+ if [ "${result}" = 'failed' ]
then
echo "!!!TEST FAILED for ${name}!!!"
fi
echo "Removing running prediction service, if any"
- if [ $target == "local" ]; then
+ if [ "$target" == "local" ]; then
end_prediction_service_local
- elif [ $target == "minikube" ]; then
+ elif [ "$target" == "minikube" ]; then
end_prediction_service_minikube
fi
result=failed # set for next test, until it explicitly succeeds
@@ -76,51 +93,56 @@ function end_prediction_service() {
function end_prediction_service_local() {
if [ -n "$container_id" ]
then
- if [ ${result} = 'failed' ]
+ if [ "${result}" = 'failed' ]
then
- docker logs ${container_id}
+ docker logs "${container_id}"
fi
- docker stop ${container_id}
- docker rm -f ${container_id}
+ docker stop "${container_id}"
+ docker rm -f "${container_id}"
fi
unset container_id
}
function end_prediction_service_minikube() {
- if [ ${result} = 'failed' ]
+ if [ "${result}" = 'failed' ]
then
- kubectl logs -l app=${resource_name} --namespace $NAMESPACE
+ kubectl logs -l app="${resource_name}" --namespace $NAMESPACE
fi
- kubectl delete service ${resource_name} --ignore-not-found --namespace $NAMESPACE
- kubectl delete deployment ${resource_name} --ignore-not-found --namespace $NAMESPACE
+ kubectl delete service "${resource_name}" --ignore-not-found --namespace $NAMESPACE
+ kubectl delete deployment "${resource_name}" --ignore-not-found --namespace $NAMESPACE
}
function build() {
image_name=$1
echo "***Building ${image_name}***"
- sh ${GEN_DIR}/container_util.sh build
+ sh "${GEN_DIR}"/container_util.sh build
}
function minikube_setup() {
eval $(minikube docker-env) # build images in shared registry
# setup minio server on local port 9000
set +e
- kubectl create namespace $NAMESPACE
- kubectl apply -f ${THIS_DIR}/minikube/test-minio.yml
- kubectl get svc test-minio 2>&1 > /dev/null
- if [ "$?" -ne 0 ]; then
+ kubectl create namespace "$NAMESPACE"
+ kubectl apply -f "${THIS_DIR}"/minikube/test-minio.yml
+ sleep 5 # wait for service to come up
+ #kubectl get svc test-minio 2>&1 > /dev/null
+ #if [ "$?" -ne 0 ]; then
+ if kubectl get svc test-minio >> /dev/null; then
kubectl expose deployment test-minio --type=LoadBalancer --port 9000 --target-port 9000
fi
- echo "***Setting up minio command line (mc) and certifai bucket***"
- mc config host list minikube 2>&1 > /dev/null
+ echo "***Setting up minio command line ($MINIO) and certifai bucket***"
+ $MINIO config host list minikube 2>&1 > /dev/null
if [ "$?" -ne 0 ]; then
- mc config host add minikube http://127.0.0.1:9000 minio minio123
+ $MINIO config host add minikube http://127.0.0.1:9000 minio minio123
fi
- mc ls minikube/certifai 2>&1 > /dev/null
+ $MINIO ls minikube/certifai 2>&1 > /dev/null
if [ "$?" -ne 0 ]; then
- mc mb minikube/certifai
+ $MINIO mb minikube/certifai
+ fi
+
+ if [[ "$RUN_H2O" == "true" ]]; then
+ $MINIO cp "${THIS_DIR}"/license.txt minikube/certifai/files/license.txt
fi
- mc cp ${THIS_DIR}/license.txt minikube/certifai/files/license.txt
set -e
}
@@ -128,15 +150,15 @@ function data_setup() {
local_name=$1
model_file=$2
model_use_case_id=$3
- if [ $target == "local" ]; then
+ if [ "$target" == "local" ]; then
return 0
fi
echo "***Setting up prediction service data in minio for ${local_name}***"
- mc config host add minikube http://127.0.0.1:9000 minio minio123
+ $MINIO config host add minikube http://127.0.0.1:9000 minio minio123
model_data_path="minikube/certifai/${model_use_case_id}/models"
local_path="${THIS_DIR}/${local_name}/model"
- mc cp ${local_path}/${model_file} ${model_data_path}/${model_file}
- mc cp ${local_path}/metadata.yml ${model_data_path}/metadata.yml
+ $MINIO cp "${local_path}"/"${model_file}" "${model_data_path}"/"${model_file}"
+ $MINIO cp "${local_path}"/metadata.yml "${model_data_path}"/metadata.yml
}
function run_and_test() {
@@ -144,89 +166,150 @@ function run_and_test() {
model_file=$2
image_name=$3
resource_name=$4
- if [ $target == "local" ]; then
+ if [ "$target" == "local" ]; then
echo "***Running ${local_name}***"
- container_id=$(docker run -d -p 8551:8551 -v ${THIS_DIR}/${local_name}/model:/tmp/model \
- -e MODEL_PATH=/tmp/model/${model_file} \
- -e METADATA_PATH=/tmp/model/metadata.yml -t ${image_name})
- elif [ $target == "minikube" ]; then
+ container_id=$(docker run -d -p 8551:8551 -v "${THIS_DIR}"/"${local_name}"/model:/tmp/model \
+ -e MODEL_PATH=/tmp/model/"${model_file}" \
+ -e METADATA_PATH=/tmp/model/metadata.yml -t "${image_name}")
+ elif [ "$target" == "minikube" ]; then
echo "***Generating deployment definition for ${resource_name}***"
- sh ${GEN_DIR}/config_deploy.sh -c ${THIS_DIR}/${local_name}/model/deployment_config.yml
- kubectl apply -f ${GEN_DIR}/deployment.yml
- kubectl wait --for=condition=ready --timeout=300s pod -l app=${resource_name} -n ${NAMESPACE}
+ sh "${GEN_DIR}"/config_deploy.sh -c "${THIS_DIR}"/"${local_name}"/model/deployment_config.yml
+ kubectl apply -f "${GEN_DIR}"/deployment.yml
+ kubectl wait --for=condition=ready --timeout=300s pod -l app="${resource_name}" -n ${NAMESPACE}
# we need to delete the created service so expose can create it
- kubectl delete svc ${resource_name} --ignore-not-found --namespace certifai-models
- kubectl expose deployment ${resource_name} --type=LoadBalancer \
+ kubectl delete svc "${resource_name}" --ignore-not-found --namespace certifai-models
+ kubectl expose deployment "${resource_name}" --type=LoadBalancer \
--port 8551 --target-port 8551 --namespace certifai-models
fi
# Wait until health endpoint is available
wait_for 'curl -X GET http://127.0.0.1:8551/health'
echo "***Testing ${local_name}***"
- certifai scan -f ${THIS_DIR}/${local_name}/explain_def.yml
+ certifai scan -f "${THIS_DIR}"/"${local_name}"/explain_def.yml
echo "***Successfully tested ${local_name}***"
end_prediction_service succeeded
}
+function install_toolkit() {
+ # Install the toolkit, if its not already installed
+ if ! command -v certifai &> /dev/null
+ then
+ echo "Installing Certifai Toolkit!"
+ pip install "${THIS_DIR}"/certifai_toolkit/packages/all/*
+ pip install "${THIS_DIR}"/certifai_toolkit/packages/python"${PYTHON_VERSION}"/*
+ fi
+}
+
+
+function run_sklearn_model() {
+ # Predict service for Sklearn
+ python_setup python sklearn_predict
+ build sklearn_predict
+ data_setup sklearn_german_credit model.pkl test_german_credit
+ run_and_test sklearn_german_credit model.pkl sklearn_predict test-german-credit-dtree
+}
+
+function run_xgboost_iris_model() {
+ # Predict service for XGBClassifier or XGBRegressor
+ python_setup python xgboost_predict
+ # Add xgboost to requirements
+ echo "\n$XGBOOST_VERSION\n" >> "${GEN_DIR}"/requirements.txt
+ build xgboost_predict
+ data_setup xgboost_iris model.pkl test_iris
+ run_and_test xgboost_iris model.pkl xgboost_predict test-iris-xgb-iris
+}
+
+function run_xgboost_income_prediction() {
+ # Predict service for xgboost using DMatrix
+ python_setup python_xgboost_dmatrix xgboost_dmatrix_predict
+ build xgboost_dmatrix_predict
+ data_setup xgboost_dmatrix_income model.pkl test_income
+ run_and_test xgboost_dmatrix_income model.pkl xgboost_dmatrix_predict test-income-xgboost
+}
+
+function run_h2o_models() {
+ # Predict service for H2O MOJO
+ h2o_setup h2o_mojo h2o_mojo_predict
+ build h2o_mojo_predict
+ for name in auto_insurance german_credit iris
+ do
+ local_name="h2o_${name}"
+ muc_id="test_${name}"
+ model_id="dai-mojo"
+ data_setup ${local_name} pipeline.mojo ${muc_id}
+ name_dashed=$(echo ${muc_id}-${model_id} | tr '_' '-')
+ run_and_test "${local_name}" pipeline.mojo h2o_mojo_predict "${name_dashed}"
+ done
+}
-# Install the toolkit, if its not already installed
-if ! command -v certifai &> /dev/null
-then
- pip install ${THIS_DIR}/certifai_toolkit/packages/all/*
- pip install ${THIS_DIR}/certifai_toolkit/packages/python${PYTHON_VERSION}/*
-fi
+
+function printUsage() {
+ local prog_name
+ prog_name="$(basename "$0")"
+
+ local usage
+ usage="$prog_name [-h|--help] [local | minikube]
+
+Test Containerized Model examples using the Certifai Model SDK.
+
+Options:
+
+ local - Run local prediction service tests
+
+ minikube - Run minikube prediction service tests. Requires Minikube to already be running.
+
+Environment Variables:
+
+ RUN_H2O - if 'true', then h2o examples will be tested and the 'license.txt' must contain a valid h2o license.
+ Defaults to 'false'.
+
+ TOOLKIT_PATH - The path to the (unzipped) Certifai Toolkit, defaults to: ./certifai_toolkit
+
+Examples:
+
+ TOOLKIT_PATH=./toolkit ./run_test.sh local
+
+ TOOLKIT_PATH=./toolkit ./run_test.sh minikube
+"
+ echo "$usage"
+}
#
# MAIN EXECUTION STARTS HERE
#
-if [ $target == "local" ]; then
- echo "Running local prediction service tests"
-elif [ $target == "minikube" ]; then
- echo "Running minikube prediction service tests"
- minikube_setup
-else
- echo "Invalid target environment"
- exit 1
-fi
-
-set -exv
-trap end_prediction_service EXIT
-
-# Predict service for H2O MOJO
-h2o_setup h2o_mojo h2o_mojo_predict
-build h2o_mojo_predict
-for name in auto_insurance german_credit iris
-do
- local_name="h2o_${name}"
- muc_id="test_${name}"
- model_id="dai-mojo"
- data_setup ${local_name} pipeline.mojo ${muc_id}
- name_dashed=$(echo ${muc_id}-${model_id} | tr '_' '-')
- run_and_test ${local_name} pipeline.mojo h2o_mojo_predict "${name_dashed}"
-done
-
-train_models # For the non-H2O models
-#
-# Predict service for Sklearn
-python_setup python sklearn_predict
-build sklearn_predict
-data_setup sklearn_german_credit model.pkl test_german_credit
-run_and_test sklearn_german_credit model.pkl sklearn_predict test-german-credit-dtree
-
-# Predict service for XGBClassifier or XGBRegressor
-python_setup python xgboost_predict
-# Add xgboost to requirements
-echo "\nxgboost==1.2.0\n" >> ${GEN_DIR}/requirements.txt
-build xgboost_predict
-data_setup xgboost_iris model.pkl test_iris
-run_and_test xgboost_iris model.pkl xgboost_predict test-iris-xgb-iris
-#
-#
-# Predict service for xgboost using DMatrix
-python_setup python_xgboost_dmatrix xgboost_dmatrix_predict
-build xgboost_dmatrix_predict
-data_setup xgboost_dmatrix_income model.pkl test_income
-run_and_test xgboost_dmatrix_income model.pkl xgboost_dmatrix_predict test-income-xgboost
-
-echo "***All tests completed successfully***"
-trap - EXIT
+function main() {
+ target=${1:-local}
+ set -exv
+ set_globals
+ install_toolkit
+ if [ "$target" == "local" ]; then
+ echo "Running local prediction service tests"
+ elif [ "$target" == "minikube" ]; then
+ echo "Running minikube prediction service tests"
+ check_minio_installed
+ minikube_setup
+ elif [ "$target" == "-h" ] || [ "$target" == "--help" ]; then
+ printUsage
+ exit 0
+ else
+ echo "Invalid target environment"
+ printUsage
+ exit 1
+ fi
+
+ # catch any EXIT signals & clean prediction services
+ trap end_prediction_service EXIT
+ if [[ "$RUN_H2O" == "true" ]]; then
+ run_h2o_models
+ fi
+ # Train the non-H2O models
+ train_models
+ # Run the non-H2O models
+ run_sklearn_model
+ run_xgboost_iris_model
+ run_xgboost_income_prediction
+ echo "***All tests completed successfully***"
+ trap - EXIT
+}
+
+main "$@"
diff --git a/models/containerized_model/templates/python/requirements.txt b/models/containerized_model/templates/python/requirements.txt
index e6153084..a66d672b 100644
--- a/models/containerized_model/templates/python/requirements.txt
+++ b/models/containerized_model/templates/python/requirements.txt
@@ -1,4 +1,4 @@
# add python pip install dependencies below
pyyaml # required by prediction service - do not remove
scikit-learn==1.0.2
-#xgboost==1.2.0 # uncomment if using xgboost and pin to same version as model
+#xgboost==1.7.2 # uncomment if using xgboost and pin to same version as model
diff --git a/models/containerized_model/templates/python_xgboost_dmatrix/requirements.txt b/models/containerized_model/templates/python_xgboost_dmatrix/requirements.txt
index 1899193f..3a1724a7 100644
--- a/models/containerized_model/templates/python_xgboost_dmatrix/requirements.txt
+++ b/models/containerized_model/templates/python_xgboost_dmatrix/requirements.txt
@@ -1,3 +1,3 @@
# add python pip install dependencies below
pyyaml # required by prediction service - do not remove
-xgboost==1.2.0 # pin to match the environment in which the model was pickled
+xgboost==1.7.2 # pin to match the environment in which the model was pickled
diff --git a/models/german_credit/app_mlp_soft_scoring.py b/models/german_credit/app_mlp_soft_scoring.py
index 29395a72..e2444cdc 100644
--- a/models/german_credit/app_mlp_soft_scoring.py
+++ b/models/german_credit/app_mlp_soft_scoring.py
@@ -24,4 +24,5 @@
endpoint_url='/german_credit_mlp/predict')
# to start production ready gunicorn server use `production=True`
-app.run(production=True)
+#app.run(production=True)
+app.run()
diff --git a/models/german_credit/composed_app.py b/models/german_credit/composed_app.py
index b9eb15c3..febced39 100644
--- a/models/german_credit/composed_app.py
+++ b/models/german_credit/composed_app.py
@@ -40,4 +40,5 @@
composed_app.add_wrapped_model('/german_credit_logit', logit_app)
composed_app.add_wrapped_model('/german_credit_svm', svm_app)
composed_app.add_wrapped_model('/german_credit_mlp', mlp_app)
+#composed_app.run(production=True)
composed_app.run()
diff --git a/models/german_credit/test.py b/models/german_credit/test.py
new file mode 100644
index 00000000..fb2d1c77
--- /dev/null
+++ b/models/german_credit/test.py
@@ -0,0 +1,35 @@
+import sys
+import os
+import unittest
+
+# Add the file to classpath for relative import
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+from base_test import ModelTest
+
+
+class GermanCreditTest(ModelTest):
+
+ def setUp(self):
+ self.run_standalone_python_script("train.py")
+
+ def test_single_app(self):
+ self.run_python_app_test("app_dtree.py", "app_test.py")
+
+ def test_composed_app(self):
+ self.run_python_app_test("composed_app.py", "composed_app_test.py")
+
+ def test_trust_scan(self):
+ self.run_model_and_scan("composed_app.py", "german_credit_scanner_definition.yaml")
+
+ def test_explain_scan(self):
+ self.run_standalone_python_script("explain.py")
+
+ def test_soft_scoring_app(self):
+ self.run_python_app_test("app_mlp_soft_scoring.py", "app_mlp_soft_scoring_test.py")
+
+ def test_soft_scoring_scan(self):
+ self.run_model_and_scan("app_mlp_soft_scoring.py", "german_credit_shap_explanation_scanner_definition.yaml")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/models/german_credit_pandas/app_mlp_soft_scoring.py b/models/german_credit_pandas/app_mlp_soft_scoring.py
index 3138b759..7219df0b 100644
--- a/models/german_credit_pandas/app_mlp_soft_scoring.py
+++ b/models/german_credit_pandas/app_mlp_soft_scoring.py
@@ -25,4 +25,5 @@
endpoint_url='/german_credit_mlp/predict', pandas_kwargs={'columns': columns})
# to start production ready gunicorn server use `production=True`
-app.run(production=True)
+#app.run(production=True)
+app.run()
diff --git a/models/german_credit_pandas/test.py b/models/german_credit_pandas/test.py
new file mode 100644
index 00000000..e06fd4f0
--- /dev/null
+++ b/models/german_credit_pandas/test.py
@@ -0,0 +1,32 @@
+import sys
+import os
+import unittest
+
+# Add the file to classpath for relative import
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+from base_test import ModelTest
+
+
+class GermanCreditPandasTest(ModelTest):
+
+ def setUp(self):
+ self.run_standalone_python_script("train_pandas.py")
+
+ def test_single_app(self):
+ self.run_python_app_test("app_dtree.py", "app_test.py")
+
+ def test_composed_app(self):
+ self.run_python_app_test("composed_app.py", "composed_app_test.py")
+
+ def test_trust_scan(self):
+ self.run_model_and_scan("composed_app.py", "german_credit_scanner_definition.yaml")
+
+ def test_soft_scoring_app(self):
+ self.run_python_app_test("app_mlp_soft_scoring.py", "app_mlp_soft_scoring_test.py")
+
+ def test_soft_scoring_scan(self):
+ self.run_model_and_scan("app_mlp_soft_scoring.py", "german_credit_shap_explanation_scanner_definition.yaml")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/models/income_prediction/test.py b/models/income_prediction/test.py
new file mode 100644
index 00000000..70a6411c
--- /dev/null
+++ b/models/income_prediction/test.py
@@ -0,0 +1,23 @@
+import sys
+import os
+import unittest
+
+# Add the file to classpath for relative import
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+from base_test import ModelTest
+
+
+class IncomePredictionTest(ModelTest):
+
+ def setUp(self):
+ self.run_standalone_python_script("train.py")
+
+ def test_single_app(self):
+ self.run_python_app_test("app_xgb.py", "app_test.py")
+
+ def test_explain_scan(self):
+ self.run_model_and_scan("app_xgb.py", "income_explain_definition.yaml")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/models/iris/test.py b/models/iris/test.py
new file mode 100644
index 00000000..c0096c0b
--- /dev/null
+++ b/models/iris/test.py
@@ -0,0 +1,28 @@
+import sys
+import os
+import unittest
+
+# Add the file to classpath for relative import
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+from base_test import ModelTest
+
+
+class IrisTest(ModelTest):
+
+ def setUp(self):
+ # Train all models at the start of each test - possibly excessive, but gives full flexibility to run
+ # a single test in isolation
+ self.run_standalone_python_script("train.py")
+
+ def test_single_app(self):
+ self.run_python_app_test("app_svm.py", "app_test.py")
+
+ def test_single_app_xgboost(self):
+ self.run_python_app_test("app_xgb.py", "app_test.py")
+
+ def test_scan(self):
+ self.run_model_and_scan("app_svm.py", "iris_scanner_definition.yaml")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/models/patient_readmission/README.md b/models/patient_readmission/README.md
index f10a0cee..07d80bf7 100644
--- a/models/patient_readmission/README.md
+++ b/models/patient_readmission/README.md
@@ -11,12 +11,6 @@ Specifically, it shows how to use the Certifai toolkit to:
* generate explanations for the model's predictions using the CLI
* set up fast-explanation, for generating large numbers of explanations at scale
-*Note*: Fast explanation is currently a beta feature. To enable it you will need to
-edit your `~/.certifai/certifai_conf.ini` file to add:
-```
-[scanner]
-support_fast_explanations_beta = True
-```
## Wrap a single model as a service
1. Make sure you have activated your Certifai toolkit environment:
diff --git a/models/patient_readmission/app.py b/models/patient_readmission/app.py
index 2fc314de..29b667d8 100644
--- a/models/patient_readmission/app.py
+++ b/models/patient_readmission/app.py
@@ -14,4 +14,5 @@
encoder = saved.get('encoder', None)
app = SimpleModelWrapper(model=model, encoder=encoder)
-app.run(production=True)
+#app.run(production=True)
+app.run()
diff --git a/models/patient_readmission/test.py b/models/patient_readmission/test.py
new file mode 100644
index 00000000..970650d4
--- /dev/null
+++ b/models/patient_readmission/test.py
@@ -0,0 +1,29 @@
+import sys
+import os
+import unittest
+
+# Add the file to classpath for relative import
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+from base_test import ModelTest
+
+
+class PatientReadmissionTest(ModelTest):
+
+ def setUp(self):
+ self.run_standalone_python_script("train.py")
+
+ def test_single_app(self):
+ self.run_python_app_test("app.py", "app_test.py")
+
+ def test_definition_test(self):
+ self.run_model_and_definition_test("app.py", "explain_def.yml")
+
+ def test_fast_explain(self):
+ self.run_model_and_explain("app.py", "explain_def.yml", fast=True)
+
+ def test_traditional_explain(self):
+ self.run_model_and_explain("app.py", "explain_def.yml", fast=False)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/notebooks/azureml_model_headers_demo/german_credit_azure_ml_demo.ipynb b/notebooks/azureml_model_headers_demo/german_credit_azure_ml_demo.ipynb
index b9d21a21..dbb88867 100644
--- a/notebooks/azureml_model_headers_demo/german_credit_azure_ml_demo.ipynb
+++ b/notebooks/azureml_model_headers_demo/german_credit_azure_ml_demo.ipynb
@@ -21,7 +21,7 @@
"source": [
"## CONTENTS\n",
"\n",
- "1. In this tutorial we will create sklearn models to classify [german credit loan risk](https://archive.ics.uci.edu/ml/datasets/Statlog+%28German+Credit+Data%29) (predict whether loan will be granted or not)\n",
+ "1. In this tutorial we will create sklearn models to classify [german credit loan risk](https://archive.ics.uci.edu/dataset/144/statlog+german+credit+data) (predict whether loan will be granted or not)\n",
"\n",
"2. Register model and deploy as webservice in ACI (AZURE CONTAINER INSTANCE) with authentication\n",
"\n",
@@ -45,7 +45,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "## Training Scikit-learn models on [UCI German credit data](https://archive.ics.uci.edu/ml/datasets/Statlog+%28German+Credit+Data%29)"
+ "## Training Scikit-learn models on [UCI German credit data](https://archive.ics.uci.edu/dataset/144/statlog+german+credit+data)"
]
},
{
diff --git a/notebooks/target_encoded/dataset_generation/README.md b/notebooks/target_encoded/dataset_generation/README.md
index 6301ae1c..520abfa6 100644
--- a/notebooks/target_encoded/dataset_generation/README.md
+++ b/notebooks/target_encoded/dataset_generation/README.md
@@ -3,7 +3,7 @@
## Notebooks
-- `german_credit_multiclass_dataset_generation`: Creates a neural network model with soft-scoring to transform binary outcome-labels (loan granted/loan denied (1/2) ) [German-Credit-Dataset](https://archive.ics.uci.edu/ml/datasets/Statlog+%28German+Credit+Data%29) into multiclass problem with three outcome-labels (loan granted/loan denied/further inspection (1/2/3) ) and saves the dataset as csv
+- `german_credit_multiclass_dataset_generation`: Creates a neural network model with soft-scoring to transform binary outcome-labels (loan granted/loan denied (1/2) ) [German-Credit-Dataset](https://archive.ics.uci.edu/dataset/144/statlog+german+credit+data) into multiclass problem with three outcome-labels (loan granted/loan denied/further inspection (1/2/3) ) and saves the dataset as csv
- `german_credit_multiclass_dataset_encoding`: Encodes the above generated dataset into target encoded and one-hot encoded feature columns and dumps the generated mappings to disk as json
diff --git a/scan-manager/docs/README.md b/scan-manager/docs/README.md
index 7a98eeba..8aa6ebe0 100644
--- a/scan-manager/docs/README.md
+++ b/scan-manager/docs/README.md
@@ -49,7 +49,7 @@ artifacts include deployment templates and .yaml config file(s); and are provide
The `deployment` folder contains:
- Deployment templates: `.yaml` templates that provide the configuration templates for deploying each of the specified model types on Kubernetes:
- - `scikit_0.23`: Uses a `python3.8` base image with `scikit-learn v0.23` pre-installed.
+ - `python`: Uses a `python3.8` base image with `scikit-learn` pre-installed.
- `h2o_mojo`: Uses a `python3.8` base image with `daimojo-2.4.8-cp36` whl pre-installed.
- `r_model`: Uses `rocker/r-ver:latest` base image with `r-cran-randomforest` pre-installed.
- `hosted_model`: Uses a `python3.8` base image for wrapping an already hosted model service.
diff --git a/scan-manager/docs/setup_artifacts/deployment/config.yml b/scan-manager/docs/setup_artifacts/deployment/config.yml
index f4ed1aa3..a1eeb0a5 100644
--- a/scan-manager/docs/setup_artifacts/deployment/config.yml
+++ b/scan-manager/docs/setup_artifacts/deployment/config.yml
@@ -1,11 +1,13 @@
-scikit_0.23:
- deployment: scikit_0.23_deployment.yml
+python:
+ deployment: python_deployment.yml
default_base_image:
name: python38_scikit
- value: c12e/cortex-certifai-model-scikit:v4-1.3.16-25-g527dfe93
+ value: c12e/cortex-certifai-model-scikit:base-py38-1.3.17-63-g2ecf19f0-d357535
available_base_images:
- name: python38_scikit
- value: c12e/cortex-certifai-model-scikit:v4-1.3.16-25-g527dfe93
+ value: c12e/cortex-certifai-model-scikit:base-py38-1.3.17-63-g2ecf19f0-d357535
+ - name: python39_scikit
+ value: c12e/cortex-certifai-model-scikit:base-py39-1.3.17-63-g2ecf19f0-d357535
h2o_mojo:
deployment: h2o_mojo_deployment.yml
default_base_image:
diff --git a/scan-manager/docs/setup_artifacts/deployment/scikit_0.23_deployment.yml b/scan-manager/docs/setup_artifacts/deployment/python_deployment.yml
similarity index 100%
rename from scan-manager/docs/setup_artifacts/deployment/scikit_0.23_deployment.yml
rename to scan-manager/docs/setup_artifacts/deployment/python_deployment.yml