diff --git a/.github/workflows/custom-script-ci.yml b/.github/workflows/custom-script-ci.yml index 0c63542a..3122c286 100644 --- a/.github/workflows/custom-script-ci.yml +++ b/.github/workflows/custom-script-ci.yml @@ -5,9 +5,9 @@ on: branches: - v2-preview paths: - - 00_Setup_AML_Workspace.ipynb - - 01_Data_Preparation.ipynb - - Custom_Script/** + - notebooks/00_Setup_AML_Workspace.ipynb + - notebooks/01_Data_Preparation.ipynb + - notebooks/Custom_Script/** env: SUBSCRIPTION_ID: ${{ secrets.SUBSCRIPTION_ID }} RESOURCE_GROUP: ${{ secrets.RESOURCE_GROUP }} @@ -34,13 +34,13 @@ jobs: architecture: 'x64' - name: Install dependencies run: | - python -m pip install -r Custom_Script/requirements.txt + python -m pip install -r notebooks/Custom_Script/requirements.txt - name: Convert notebooks to python run: | - jupyter nbconvert --to script 00_Setup_AML_Workspace.ipynb - jupyter nbconvert --to script 01_Data_Preparation.ipynb - jupyter nbconvert --to script Custom_Script/02_CustomScript_Training_Pipeline.ipynb - jupyter nbconvert --to script Custom_Script/03_CustomScript_Forecasting_Pipeline.ipynb + jupyter nbconvert --to script notebooks/00_Setup_AML_Workspace.ipynb + jupyter nbconvert --to script notebooks/01_Data_Preparation.ipynb + jupyter nbconvert --to script notebooks/Custom_Script/02_CustomScript_Training_Pipeline.ipynb + jupyter nbconvert --to script notebooks/Custom_Script/03_CustomScript_Forecasting_Pipeline.ipynb - name: Login via Az module uses: azure/login@v1.1 with: @@ -54,6 +54,7 @@ jobs: echo "************************************" echo "Using workspace: $Env:WORKSPACE_NAME" echo "************************************" + cd notebooks python 00_Setup_AML_Workspace.py $resource = Get-AzResource -Name $Env:WORKSPACE_NAME -ResourceGroupName $Env:RESOURCE_GROUP $expirationTime = (Get-Date).AddHours(24).ToUniversalTime() @@ -61,12 +62,13 @@ jobs: azPSVersion: 'latest' - name: Data preparation run: | + cd notebooks ipython 01_Data_Preparation.py - name: Run training run: | - cd Custom_Script - python 02_CustomScript_Training_Pipeline.py + cd notebooks/Custom_Script + ipython 02_CustomScript_Training_Pipeline.py - name: Run prediction run: | - cd Custom_Script - python 03_CustomScript_Forecasting_Pipeline.py + cd notebooks/Custom_Script + ipython 03_CustomScript_Forecasting_Pipeline.py diff --git a/images/01_userfilesupdate.PNG b/.images/01_userfilesupdate.PNG similarity index 100% rename from images/01_userfilesupdate.PNG rename to .images/01_userfilesupdate.PNG diff --git a/.images/Flow_map.png b/.images/Flow_map.png new file mode 100644 index 00000000..2b22eada Binary files /dev/null and b/.images/Flow_map.png differ diff --git a/images/ai show.gif b/.images/ai show.gif similarity index 100% rename from images/ai show.gif rename to .images/ai show.gif diff --git a/images/computes_view.png b/.images/computes_view.png similarity index 100% rename from images/computes_view.png rename to .images/computes_view.png diff --git a/images/create_notebook_vm.png b/.images/create_notebook_vm.png similarity index 100% rename from images/create_notebook_vm.png rename to .images/create_notebook_vm.png diff --git a/.images/mlops_pipeline1.png b/.images/mlops_pipeline1.png new file mode 100644 index 00000000..40ebd907 Binary files /dev/null and b/.images/mlops_pipeline1.png differ diff --git a/.images/mlops_pipeline2.png b/.images/mlops_pipeline2.png new file mode 100644 index 00000000..d7592002 Binary files /dev/null and b/.images/mlops_pipeline2.png differ diff --git a/.images/mlops_pipeline3.png b/.images/mlops_pipeline3.png new file mode 100644 index 00000000..808e3b11 Binary files /dev/null and b/.images/mlops_pipeline3.png differ diff --git a/.images/mlops_pipeline4.png b/.images/mlops_pipeline4.png new file mode 100644 index 00000000..fa4a2f19 Binary files /dev/null and b/.images/mlops_pipeline4.png differ diff --git a/.images/mlops_pipeline5.png b/.images/mlops_pipeline5.png new file mode 100644 index 00000000..2167c0bc Binary files /dev/null and b/.images/mlops_pipeline5.png differ diff --git a/.images/mlops_pipeline_1_setup.png b/.images/mlops_pipeline_1_setup.png new file mode 100644 index 00000000..8fb311c5 Binary files /dev/null and b/.images/mlops_pipeline_1_setup.png differ diff --git a/.images/mlops_pipeline_2_modeling.png b/.images/mlops_pipeline_2_modeling.png new file mode 100644 index 00000000..e721157d Binary files /dev/null and b/.images/mlops_pipeline_2_modeling.png differ diff --git a/.images/mlops_pipeline_3_batchforecasting.png b/.images/mlops_pipeline_3_batchforecasting.png new file mode 100644 index 00000000..c95648bb Binary files /dev/null and b/.images/mlops_pipeline_3_batchforecasting.png differ diff --git a/images/mmsa-overview.png b/.images/mmsa-overview.png similarity index 100% rename from images/mmsa-overview.png rename to .images/mmsa-overview.png diff --git a/images/mmsa.png b/.images/mmsa.png similarity index 100% rename from images/mmsa.png rename to .images/mmsa.png diff --git a/images/terminal.png b/.images/terminal.png similarity index 100% rename from images/terminal.png rename to .images/terminal.png diff --git a/Automated_ML/mlops-pipelines/1-setup/deploy-infra/deploy-infra.template.yml b/Automated_ML/mlops-pipelines/1-setup/deploy-infra/deploy-infra.template.yml deleted file mode 100644 index aff8c842..00000000 --- a/Automated_ML/mlops-pipelines/1-setup/deploy-infra/deploy-infra.template.yml +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Azure Pipeline Template for ML Workspace Resources Deployment - -parameters: -- name: environment - type: string -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: resourcesLocation - type: string -- name: storageAccountName - type: string -- name: keyVaultName - type: string -- name: appInsightsName - type: string -- name: containerRegistryName - type: string -- name: amlWorkspaceName - type: string - - -jobs: - -- job: iac_build - displayName: 'IaC Build' - steps: - - task: CopyFiles@2 - displayName: 'Copy ARM templates' - inputs: - sourceFolder: 'Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates' - targetFolder: '$(Build.ArtifactStagingDirectory)' - - publish: '$(Build.ArtifactStagingDirectory)' - artifact: infratemplates - -- deployment: iac_deployment - displayName: 'IaC Deployment' - environment: ${{parameters.environment}} - strategy: - runOnce: - deploy: - steps: - - download: current - artifact: infratemplates - - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy Storage Account' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Pipeline.Workspace)/infratemplates/storage.template.json' - csmParametersFile: '$(Pipeline.Workspace)/infratemplates/storage.parameters.json' - overrideParameters: '-name ${{parameters.storageAccountName}} -location ${{parameters.resourcesLocation}}' - - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy Key Vault' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Pipeline.Workspace)/infratemplates/keyvault.template.json' - csmParametersFile: '$(Pipeline.Workspace)/infratemplates/keyvault.parameters.json' - overrideParameters: '-name ${{parameters.keyVaultName}} -location ${{parameters.resourcesLocation}}' - - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy Application Insights' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Pipeline.Workspace)/infratemplates/appinsights.template.json' - overrideParameters: '-name ${{parameters.appInsightsName}} -location ${{parameters.resourcesLocation}}' - - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy Container Registry' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Pipeline.Workspace)/infratemplates/containerregistry.template.json' - overrideParameters: '-name ${{parameters.containerRegistryName}} -location ${{parameters.resourcesLocation}}' - - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy AML Workspace' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Pipeline.Workspace)/infratemplates/mlworkspace.template.json' - overrideParameters: '-workspaceName ${{parameters.amlWorkspaceName}} -keyVaultName ${{parameters.keyVaultName}} -appInsightsName ${{parameters.appInsightsName}} -containerRegistryName ${{parameters.containerRegistryName}} -storageAccountName ${{parameters.storageAccountName}}' diff --git a/Automated_ML/mlops-pipelines/1-setup/environment-setup/environment-setup.template.yml b/Automated_ML/mlops-pipelines/1-setup/environment-setup/environment-setup.template.yml deleted file mode 100644 index fea18dea..00000000 --- a/Automated_ML/mlops-pipelines/1-setup/environment-setup/environment-setup.template.yml +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Azure Pipeline Template for AutoML Workspace Setup - -parameters: -- name: sdkVersion - type: string -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: resourcesLocation - type: string -- name: amlWorkspaceName - type: string -- name: amlComputeName - type: string -- name: aksName - type: string -- name: aksResourceGroup - type: string -- name: amlAksName - type: string -- name: amlTrainDatasetName - type: string -- name: amlTestDatasetName - type: string -- name: maxFiles - type: string -- name: containerName - type: string -- name: accountName - type: string -- name: accountKey - type: string -- name: trainDataPathPrefix - type: string -- name: testDataPathPrefix - type: string - - -jobs: - -- job: aml_compute - displayName: 'Deploy AML Compute' - steps: - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy AML Compute' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Build.SourcesDirectory)/Automated_ML/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.template.json' - csmParametersFile: '$(Build.SourcesDirectory)/Automated_ML/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.parameters.json' - overrideParameters: '-workspaceName ${{parameters.amlWorkspaceName}} -clusterName ${{parameters.amlComputeName}}' - - -- job: aml_aks - displayName: 'Attach AKS Target to AML' - steps: - - task: AzureCLI@1 - displayName: 'Attach AKS Target to AML' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - workspace_params="--workspace-name ${{parameters.amlWorkspaceName}} --resource-group ${{parameters.resourceGroup}}" - # Install ML extension - az extension add -n azure-cli-ml - # Check if AKS target is already attached and attach if not - az ml computetarget show --name ${{parameters.amlAksName}} $workspace_params - if [ $? -eq 1 ]; then - # Get AKS resource id - aks_id=$(az resource list -n ${{parameters.aksName}} -g ${{parameters.aksResourceGroup}} --query "[0].id" -o tsv) - # Attach AKS - az ml computetarget attach aks --name ${{parameters.amlAksName}} --compute-resource-id $aks_id $workspace_params - fi - - -- job: sample_files - displayName: 'Sample Files Setup' - steps: - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - - task: AzureCLI@1 - displayName: 'Download Sample Files and Upload to AML datastore' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - # Install dependencies - python -m pip install --upgrade pip && python -m pip install azureml-sdk==${{parameters.sdkVersion}} - # Download sample files and upload to blob store - python Automated_ML/mlops-pipelines/scripts/download_data.py \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} \ - --container-name ${{parameters.containerName}} \ - --account-name ${{parameters.accountName}} \ - --account-key ${{parameters.accountKey}} \ - --train-data-path-prefix ${{parameters.trainDataPathPrefix}} \ - --test-data-path-prefix ${{parameters.testDataPathPrefix}} \ - --train-target-path ${{parameters.trainDataPathPrefix}} \ - --test-target-path ${{parameters.testDataPathPrefix}} \ - --subscription-id $(az account show --query id -o tsv) - register_dataset_script=Automated_ML/mlops-pipelines/scripts/register_or_update_dataset.py - # Register dataset - python $register_dataset_script --path ${{parameters.trainDataPathPrefix}} \ - --name ${{parameters.amlTrainDatasetName}} \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} - python $register_dataset_script --path ${{parameters.testDataPathPrefix}} \ - --name ${{parameters.amlTestDatasetName}} \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} - name: download_files - failOnStderr: true diff --git a/Automated_ML/mlops-pipelines/1-setup/setup-pipeline.yml b/Automated_ML/mlops-pipelines/1-setup/setup-pipeline.yml deleted file mode 100644 index ab6920d3..00000000 --- a/Automated_ML/mlops-pipelines/1-setup/setup-pipeline.yml +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pipeline that configures the environment for this solution accelerator - -# trigger: -# branches: -# include: -# - v2-preview -# paths: -# include: -# - Automated_ML/mlops-pipelines/1-setup/* -# - Automated_ML/mlops-pipelines/scripts/download_data.py -# - Automated_ML/mlops-pipelines/scripts/register_dataset.py - -variables: -- template: ../many-models-variables.yml -- group: manymodels-vg - -pool: - vmImage: ubuntu-latest - -stages: -- stage: deploy_infra - displayName: 'Deploy Infra' - jobs: - - template: deploy-infra/deploy-infra.template.yml - parameters: - environment: '$(ENVIRONMENT)' - serviceConnection: '$(SERVICECONNECTION_GROUP)' - resourceGroup: '$(RESOURCE_GROUP)' - resourcesLocation: '$(LOCATION)' - storageAccountName: '$(STORAGEACCOUNT_NAME)' - keyVaultName: '$(KEYVAULT_NAME)' - appInsightsName: '$(APPINSIGHTS_NAME)' - containerRegistryName: '$(CONTAINERREGISTRY_NAME)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - -- stage: environment_setup - displayName: 'Environment Setup' - jobs: - - template: environment-setup/environment-setup.template.yml - parameters: - sdkVersion: '$(SDK_VERSION)' - serviceConnection: '$(SERVICECONNECTION_GROUP)' - resourceGroup: '$(RESOURCE_GROUP)' - resourcesLocation: '$(LOCATION)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - amlComputeName: '$(AML_COMPUTE_NAME)' - aksName: '$(AKS_NAME)' - aksResourceGroup: '$(AKS_RESOURCE_GROUP)' - amlAksName: '$(AML_AKS_NAME)' - amlTrainDatasetName: '$(AML_TRAIN_DATASET_NAME)' - amlTestDatasetName: '$(AML_TEST_DATASET_NAME)' - maxFiles: '$(DATASET_MAXFILES)' - containerName: '$(CONTAINER_NAME)' - accountName: '$(ACCOUNT_NAME)' - accountKey: '$(ACCOUNT_KEY)' - trainDataPathPrefix: '$(TRAIN_DATA_PATH_PREFIX)' - testDataPathPrefix: '$(TEST_DATA_PATH_PREFIX)' \ No newline at end of file diff --git a/Automated_ML/mlops-pipelines/2-training-code-build/training-code-build-pipeline.yml b/Automated_ML/mlops-pipelines/2-training-code-build/training-code-build-pipeline.yml deleted file mode 100644 index a770c6d2..00000000 --- a/Automated_ML/mlops-pipelines/2-training-code-build/training-code-build-pipeline.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pipeline that publishes de AML Pipeline that will train the models - -# trigger: -# branches: -# include: -# - v2-preview -# paths: -# include: -# - Automated_ML/mlops-pipelines/2-training-code-build/* -# - Automated_ML/mlops-pipelines/scripts/create_training_pipeline.py -# - Automated_ML/02b_Train_AutoML/scripts/* -# - Automated_ML/common/scripts/* - -variables: -- template: ../many-models-variables.yml -- group: manymodels-vg - -pool: - vmImage: ubuntu-latest - -stages: -- stage: build_training - displayName: 'Publish Training AML Pipeline' - jobs: - - job: build_training - displayName: 'Publish Training AML Pipeline' - steps: - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - - task: AzureCLI@1 - displayName: 'Publish Training AML Pipeline' - inputs: - azureSubscription: $(SERVICECONNECTION_WORKSPACE) - scriptLocation: inlineScript - inlineScript: | - # Install dependencies - python -m pip install --upgrade pip && python -m pip install azureml-sdk[automl]==$SDK_VERSION - # Create/update training pipeline - trainingbuild_script=Automated_ML/mlops-pipelines/scripts/create_training_pipeline.py - python $trainingbuild_script --pipeline-name $(AML_TRAINING_PIPELINE_NAME) \ - --version $(Build.BuildId) \ - --dataset $(AML_TRAIN_DATASET_NAME) \ - --compute $(AML_COMPUTE_NAME) \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group $(RESOURCE_GROUP) \ - --workspace-name $(AMLWORKSPACE_NAME) - diff --git a/Automated_ML/mlops-pipelines/3-modeling/deploy-models/deploy-models.template.yml b/Automated_ML/mlops-pipelines/3-modeling/deploy-models/deploy-models.template.yml deleted file mode 100644 index fd02cfcc..00000000 --- a/Automated_ML/mlops-pipelines/3-modeling/deploy-models/deploy-models.template.yml +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Azure Pipeline Template for deploying models to ACI - -parameters: -- name: sdkVersion - type: string -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: amlWorkspaceName - type: string -- name: deploymentType - type: string -- name: amlAksName - type: string -- name: sortingTags - type: string -- name: splittingTags - type: string -- name: webservicePrefix - type: string -- name: containerSize - type: string -- name: routingModelName - type: string -- name: routingModelTagName - type: string -- name: routingModelTagValue - type: string -- name: routingServiceName - type: string - - -jobs: - -- job: deploy_models - displayName: 'Deploy Models' - timeoutInMinutes: 0 - steps: - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - - task: AzureCLI@1 - displayName: 'Deploy Models in Groups' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - # Install dependencies - python -m pip install --upgrade pip && python -m pip install joblib pyyaml azureml-sdk[automl]==${{parameters.sdkVersion}} - # Create/update training pipeline - deployment_config_path="Automated_ML/mlops-pipelines/3-modeling/deploy-models/" - deployment_config_file="$deployment_config_path/forecasting-deployment-config-${{parameters.deploymentType}}.yml" - deploy_script="Automated_ML/mlops-pipelines/scripts/deploy_or_update_models.py" - python $deploy_script --splitting-tags '${{parameters.splittingTags}}' \ - --sorting-tags '${{parameters.sortingTags}}' \ - --routing-model-name '${{parameters.routingModelName}}' \ - --output 'models_deployed.pkl' \ - --deploy-config-file $deployment_config_file \ - --aks-target '${{parameters.amlAksName}}' \ - --service-prefix '${{parameters.webservicePrefix}}' \ - --container-size '${{parameters.containerSize}}' \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} - - - publish: 'models_deployed.pkl' - artifact: modelsdeployed-${{parameters.deploymentType}} - displayName: 'Publish Models Deployed Artifact' - - -- job: routing_model - displayName: 'Register Routing Model' - dependsOn: deploy_models - steps: - - - download: current - artifact: modelsdeployed-${{parameters.deploymentType}} - - - task: AzureCLI@1 - displayName: 'Register Routing Model' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - # Install ML extension - az extension add -n azure-cli-ml - # Register model - az ml model register --name ${{parameters.routingModelName}} \ - --model-path '$(Pipeline.Workspace)/modelsdeployed-${{parameters.deploymentType}}/models_deployed.pkl' \ - --tag '${{parameters.routingModelTagName}}=${{parameters.routingModelTagValue}}' \ - --output-metadata-file routing_model.json \ - --workspace-name ${{parameters.amlWorkspaceName}} \ - --resource-group ${{parameters.resourceGroup}} - - - publish: 'routing_model.json' - artifact: routingmodel-${{parameters.deploymentType}} - displayName: 'Publish Routing Model Artifact' - - -- job: routing_webservice - displayName: 'Deploy Routing Webservice' - dependsOn: routing_model - steps: - - - download: current - artifact: routingmodel-${{parameters.deploymentType}} - - - task: AzureCLI@1 - displayName: 'Deploy Routing Webservice' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - # Install ML extension - az extension add -n azure-cli-ml - # Deployment config - deployment_config_path="Automated_ML/mlops-pipelines/3-modeling/deploy-models/" - if [ "${{parameters.deploymentType}}" == "aci" ]; then - deployment_params="--deploy-config-file $deployment_config_path/routing-deployment-config-aci.yml" - elif [ "${{parameters.deploymentType}}" == "aks" ]; then - deployment_params="--compute-target ${{parameters.amlAksName}} --deploy-config-file $deployment_config_path/routing-deployment-config-aks.yml" - fi - # Deploy model - az ml model deploy $deployment_params \ - --name ${{parameters.routingServiceName}} \ - --model-metadata-file '$(Pipeline.Workspace)/routingmodel-${{parameters.deploymentType}}/routing_model.json' \ - --runtime python \ - --source-directory Automated_ML/03b_Forecasting_Pipeline/scripts \ - --entry-script routing_webservice.py \ - --conda-file routing_webservice.conda.yml \ - --overwrite \ - --workspace-name ${{parameters.amlWorkspaceName}} \ - --resource-group ${{parameters.resourceGroup}} diff --git a/Automated_ML/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aci.yml b/Automated_ML/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aci.yml deleted file mode 100644 index ef897d9d..00000000 --- a/Automated_ML/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aci.yml +++ /dev/null @@ -1,4 +0,0 @@ -computeType: ACI -containerResourceRequirements: - cpu_cores: 1 - memory_gb: 1 diff --git a/Automated_ML/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aks.yml b/Automated_ML/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aks.yml deleted file mode 100644 index 3306414d..00000000 --- a/Automated_ML/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aks.yml +++ /dev/null @@ -1,4 +0,0 @@ -computeType: AKS -containerResourceRequirements: - cpu_cores: 1 - memory_gb: 1 diff --git a/Automated_ML/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aci.yml b/Automated_ML/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aci.yml deleted file mode 100644 index 274607ca..00000000 --- a/Automated_ML/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aci.yml +++ /dev/null @@ -1,4 +0,0 @@ -computeType: ACI -containerResourceRequirements: - cpu: 0.1 - memoryInGB: 0.5 diff --git a/Automated_ML/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aks.yml b/Automated_ML/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aks.yml deleted file mode 100644 index bd532dfa..00000000 --- a/Automated_ML/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aks.yml +++ /dev/null @@ -1,4 +0,0 @@ -computeType: AKS -containerResourceRequirements: - cpu: 0.1 - memoryInGB: 0.5 diff --git a/Automated_ML/mlops-pipelines/3-modeling/modeling-pipeline.yml b/Automated_ML/mlops-pipelines/3-modeling/modeling-pipeline.yml deleted file mode 100644 index 907e0e9a..00000000 --- a/Automated_ML/mlops-pipelines/3-modeling/modeling-pipeline.yml +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pipeline that runs AML Pipeline for model training - -# trigger: -# branches: -# include: -# - v2-preview -# paths: -# include: -# - Automated_ML/mlops-pipelines/3-modeling/* -# - Automated_ML/mlops-pipelines/scripts/deploy_or_update_models.py -# - Automated_ML/03b_Forecasting_Pipeline/scripts/forecast_webservice.py -# - Automated_ML/03b_Forecasting_Pipeline/scripts/routing_webservice.py - - -# schedules: -# - cron: "0 0 * * 1" -# displayName: 'Weekly training at midnight on Mondays' -# branches: -# include: -# - v2-preview -# always: true - -variables: -- template: ../many-models-variables.yml -- group: manymodels-vg - -pool: - vmImage: ubuntu-latest - -stages: - -- stage: update_data - displayName: 'Update Data for Training/Retraining' - jobs: - - template: update-data/update-data.template.yml - parameters: - sdkVersion: '$(SDK_VERSION)' - serviceConnection: '$(SERVICECONNECTION_WORKSPACE)' - resourceGroup: '$(RESOURCE_GROUP)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - amlTrainDatasetName: '$(AML_TRAIN_DATASET_NAME)' - amlTestDatasetName: '$(AML_TEST_DATASET_NAME)' - containerName: '$(CONTAINER_NAME)' - accountName: '$(ACCOUNT_NAME)' - accountKey: '$(ACCOUNT_KEY)' - trainDataPathPrefix: '$(TRAIN_DATA_PATH_PREFIX)' - testDataPathPrefix: '$(TEST_DATA_PATH_PREFIX)' - -- stage: run_training - displayName: 'Run Model Training' - dependsOn: update_data - jobs: - - template: run-training/run-training.template.yml - parameters: - serviceConnection: '$(SERVICECONNECTION_WORKSPACE)' - resourceGroup: '$(RESOURCE_GROUP)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - trainingPipelineName: '$(AML_TRAINING_PIPELINE_NAME)' - -- stage: deploy_models_aci - displayName: 'Deploy Models to ACI' - dependsOn: run_training - condition: eq(variables['DEPLOY_ACI'], 'true') - jobs: - - template: deploy-models/deploy-models.template.yml - parameters: - sdkVersion: '$(SDK_VERSION)' - serviceConnection: '$(SERVICECONNECTION_WORKSPACE)' - resourceGroup: '$(RESOURCE_GROUP)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - deploymentType: aci - amlAksName: '' - splittingTags: '$(AML_MODEL_SPLITTING_TAGS)' - sortingTags: '$(AML_MODEL_SORTING_TAGS)' - webservicePrefix: test-'$(AML_WEBSERVICE_PREFIX)' - containerSize: '$(AML_MAX_CONTAINER_SIZE)' - routingModelName: test-'$(AML_ROUTING_MODEL_NAME)' - routingModelTagName: '$(AML_ROUTING_MODEL_TAG_NAME)' - routingModelTagValue: '$(AML_ROUTING_MODEL_TAG_VALUE)' - routingServiceName: test-'$(AML_ROUTING_WEBSERVICE)' - -- stage: deploy_models_aks - displayName: 'Deploy Models to AKS' - dependsOn: run_training - condition: and(eq(variables['DEPLOY_AKS'], 'true'), variables['AML_AKS_NAME']) - jobs: - - template: deploy-models/deploy-models.template.yml - parameters: - sdkVersion: '$(SDK_VERSION)' - serviceConnection: '$(SERVICECONNECTION_WORKSPACE)' - resourceGroup: '$(RESOURCE_GROUP)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - deploymentType: aks - amlAksName: '$(AML_AKS_NAME)' - splittingTags: '$(AML_MODEL_SPLITTING_TAGS)' - sortingTags: '$(AML_MODEL_SORTING_TAGS)' - webservicePrefix: '$(AML_WEBSERVICE_PREFIX)' - containerSize: '$(AML_MAX_CONTAINER_SIZE)' - routingModelName: '$(AML_ROUTING_MODEL_NAME)' - routingModelTagName: '$(AML_ROUTING_MODEL_TAG_NAME)' - routingModelTagValue: '$(AML_ROUTING_MODEL_TAG_VALUE)' - routingServiceName: '$(AML_ROUTING_WEBSERVICE)' diff --git a/Automated_ML/mlops-pipelines/3-modeling/run-training/run-training.template.yml b/Automated_ML/mlops-pipelines/3-modeling/run-training/run-training.template.yml deleted file mode 100644 index 409a61ba..00000000 --- a/Automated_ML/mlops-pipelines/3-modeling/run-training/run-training.template.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Azure Pipeline Template for running the training pipeline - -parameters: -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: amlWorkspaceName - type: string -- name: trainingPipelineName - type: string - - -jobs: - -- job: pipeline_id - displayName: 'Get Training Pipeline ID' - steps: - - - task: AzureCLI@1 - name: get_pipeline_id - displayName: 'Get Training Pipeline ID' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - # Install ML extension - az extension add -n azure-cli-ml - # Get training pipeline ID - workspace_params="--workspace-name ${{parameters.amlWorkspaceName}} --resource-group ${{parameters.resourceGroup}}" - pipeline_id=$(az ml pipeline list $workspace_params --query "[?Name == '${{parameters.trainingPipelineName}}'].Id" -o tsv) - echo "##vso[task.setvariable variable=pipeline_id;isOutput=true;]$pipeline_id" - -- job: run_training - displayName: 'Run Training' - pool: server - timeoutInMinutes: 0 - dependsOn: pipeline_id - variables: - pipeline_id: $[ dependencies.pipeline_id.outputs['get_pipeline_id.pipeline_id'] ] - steps: - - - task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0 - displayName: 'Invoke AML Training Pipeline' - inputs: - azureSubscription: '${{parameters.serviceConnection}}' - PipelineId: '$(PIPELINE_ID)' - ExperimentName: '${{parameters.trainingPipelineName}}' - - diff --git a/Automated_ML/mlops-pipelines/3-modeling/update-data/update-data.template.yml b/Automated_ML/mlops-pipelines/3-modeling/update-data/update-data.template.yml deleted file mode 100644 index 5eab630c..00000000 --- a/Automated_ML/mlops-pipelines/3-modeling/update-data/update-data.template.yml +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Azure Pipeline Template for updating training dataset -# The dataset in this example is updated for illustration purposes only, the data downloaded is the same - - -parameters: -- name: sdkVersion - type: string -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: amlWorkspaceName - type: string -- name: amlTrainDatasetName - type: string -- name: amlTestDatasetName - type: string -- name: containerName - type: string -- name: accountName - type: string -- name: accountKey - type: string -- name: trainDataPathPrefix - type: string -- name: testDataPathPrefix - type: string - - - -jobs: - -- job: download_new_data - displayName: 'Download New Sample Files' - steps: - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - - task: AzureCLI@1 - displayName: 'Download Sample Files and Upload to AML datastore' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - # Install dependencies - python -m pip install --upgrade pip && python -m pip install azureml-sdk==${{parameters.sdkVersion}} - python Automated_ML/mlops-pipelines/scripts/download_data.py \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} \ - --container-name ${{parameters.containerName}} \ - --account-name ${{parameters.accountName}} \ - --account-key ${{parameters.accountKey}} \ - --train-data-path-prefix ${{parameters.trainDataPathPrefix}} \ - --test-data-path-prefix ${{parameters.testDataPathPrefix}} \ - --train-target-path ${{parameters.trainDataPathPrefix}} \ - --test-target-path ${{parameters.testDataPathPrefix}} \ - --subscription-id $(az account show --query id -o tsv) - register_dataset_script=Automated_ML/mlops-pipelines/scripts/register_or_update_dataset.py - # Register dataset - python $register_dataset_script --path ${{parameters.trainDataPathPrefix}} \ - --name ${{parameters.amlTrainDatasetName}} \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} - python $register_dataset_script --path ${{parameters.testDataPathPrefix}} \ - --name ${{parameters.amlTestDatasetName}} \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} - name: download_files \ No newline at end of file diff --git a/Automated_ML/mlops-pipelines/4-batch-forecasting-code-build/batch-forecasting-code-build-pipeline.yml b/Automated_ML/mlops-pipelines/4-batch-forecasting-code-build/batch-forecasting-code-build-pipeline.yml deleted file mode 100644 index ac35ef17..00000000 --- a/Automated_ML/mlops-pipelines/4-batch-forecasting-code-build/batch-forecasting-code-build-pipeline.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pipeline that publishes de AML Pipeline that will generate the forecast - -# trigger: -# branches: -# include: -# - v2-preview -# paths: -# include: -# - Automated_ML/mlops-pipelines/4-batch-forecasting-code-build/* -# - Automated_ML/mlops-pipelines/scripts/create_forecasting_pipeline.py -# - Automated_ML/03b_Forecasting_Pipeline/scripts/forecast.py -# - Automated_ML/03b_Forecasting_Pipeline/scripts/helper.py - -variables: -- template: ../many-models-variables.yml -- group: manymodels-vg - -pool: - vmImage: ubuntu-latest - -stages: -- stage: build_forecasting - displayName: 'Publish Forecasting AML Pipeline' - jobs: - - job: build_forecasting - displayName: 'Publish Forecasting AML Pipeline' - steps: - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - - task: AzureCLI@1 - displayName: 'Publish Forecasting AML Pipeline' - inputs: - azureSubscription: $(SERVICECONNECTION_WORKSPACE) - scriptLocation: inlineScript - inlineScript: | - # Install dependencies - python -m pip install --upgrade pip && python -m pip install azureml-sdk[automl]==$SDK_VERSION - # Create/update forecasting pipeline - forecastingbuild_script=Automated_ML/mlops-pipelines/scripts/create_forecasting_pipeline.py - python $forecastingbuild_script --pipeline-name $(AML_FORECASTING_PIPELINE_NAME) \ - --version $(Build.BuildId) \ - --dataset $(AML_TEST_DATASET_NAME) \ - --compute $(AML_COMPUTE_NAME) \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group $(RESOURCE_GROUP) \ - --workspace-name $(AMLWORKSPACE_NAME) - diff --git a/Automated_ML/mlops-pipelines/5-batch-forecasting/batch-forecasting-pipeline.yml b/Automated_ML/mlops-pipelines/5-batch-forecasting/batch-forecasting-pipeline.yml deleted file mode 100644 index f5ea22cd..00000000 --- a/Automated_ML/mlops-pipelines/5-batch-forecasting/batch-forecasting-pipeline.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pipeline that runs AML Pipeline for model training - -# trigger: -# branches: -# include: -# - v2-preview -# paths: -# include: -# - Automated_ML/mlops-pipelines/5-batch-forecasting/* - -# schedules: -# - cron: "30 7 * * *" -# displayName: 'Daily forecasting at 07:30 AM' -# branches: -# include: -# - v2-preview -# always: true - -variables: -- template: ../many-models-variables.yml -- group: manymodels-vg - -pool: - vmImage: ubuntu-latest - -stages: - -# There is no data to update in this example, but in the general case -# the registered dataset should be updated before forecasting. -# - stage: update_data -# displayName: 'Update Data for Forecasting' -# jobs: -# - template: update-data/update-data.template.yml - -- stage: run_forecasting - displayName: 'Run Model Forecasting' - jobs: - - template: run-forecasting/run-forecasting.template.yml - parameters: - serviceConnection: '$(SERVICECONNECTION_WORKSPACE)' - resourceGroup: '$(RESOURCE_GROUP)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - forecastingPipelineName: '$(AML_FORECASTING_PIPELINE_NAME)' diff --git a/Automated_ML/mlops-pipelines/5-batch-forecasting/run-forecasting/run-forecasting.template.yml b/Automated_ML/mlops-pipelines/5-batch-forecasting/run-forecasting/run-forecasting.template.yml deleted file mode 100644 index c4207893..00000000 --- a/Automated_ML/mlops-pipelines/5-batch-forecasting/run-forecasting/run-forecasting.template.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Azure Pipeline Template for running the forecasting pipeline - -parameters: -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: amlWorkspaceName - type: string -- name: forecastingPipelineName - type: string - - -jobs: - -- job: pipeline_id - displayName: 'Get Forecasting Pipeline ID' - steps: - - - task: AzureCLI@1 - name: get_pipeline_id - displayName: 'Get Forecasting Pipeline ID' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - # Install ML extension - az extension add -n azure-cli-ml - # Get forecasting pipeline ID - workspace_params="--workspace-name ${{parameters.amlWorkspaceName}} --resource-group ${{parameters.resourceGroup}}" - pipeline_id=$(az ml pipeline list $workspace_params --query "[?Name == '${{parameters.forecastingPipelineName}}'].Id" -o tsv) - echo "##vso[task.setvariable variable=pipeline_id;isOutput=true;]$pipeline_id" - -- job: run_forecasting - displayName: 'Run Forecasting' - pool: server - timeoutInMinutes: 0 - dependsOn: pipeline_id - variables: - pipeline_id: $[ dependencies.pipeline_id.outputs['get_pipeline_id.pipeline_id'] ] - steps: - - - task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0 - displayName: 'Invoke AML Forecasting Pipeline' - inputs: - azureSubscription: '${{parameters.serviceConnection}}' - PipelineId: '$(PIPELINE_ID)' - ExperimentName: '${{parameters.forecastingPipelineName}}' diff --git a/Automated_ML/mlops-pipelines/5-batch-forecasting/update-data/update-data.template.yml b/Automated_ML/mlops-pipelines/5-batch-forecasting/update-data/update-data.template.yml deleted file mode 100644 index 621d7130..00000000 --- a/Automated_ML/mlops-pipelines/5-batch-forecasting/update-data/update-data.template.yml +++ /dev/null @@ -1 +0,0 @@ -# Azure Pipeline Template for updating forecasting dataset diff --git a/Automated_ML/mlops-pipelines/many-models-variables.yml b/Automated_ML/mlops-pipelines/many-models-variables.yml deleted file mode 100644 index 28180673..00000000 --- a/Automated_ML/mlops-pipelines/many-models-variables.yml +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Definition of common runtime environment variables - -variables: - - # General - - name: ENVIRONMENT - value: DEV - - name: SDK_VERSION - value: 1.8.0 - - # Infra - - name: AMLWORKSPACE_NAME - value: $(NAMESPACE)aml - - name: STORAGEACCOUNT_NAME - value: $(NAMESPACE)sa - - name: KEYVAULT_NAME - value: $(NAMESPACE)kv - - name: APPINSIGHTS_NAME - value: $(NAMESPACE)ai - - name: CONTAINERREGISTRY_NAME - value: $(NAMESPACE)cr - - # Azure ML workspace - - name: AML_COMPUTE_NAME - value: cpu-compute - - - # Training and Inferencing data - - name: AML_TRAIN_DATASET_NAME - value: oj_sales_data - - name: AML_TEST_DATASET_NAME - value: oj_inference_data - - name: CONTAINER_NAME - value: automl-sample-notebook-data - - name: ACCOUNT_NAME - value: automlsamplenotebookdata - - name: ACCOUNT_KEY - value: None - - name: TRAIN_DATA_PATH_PREFIX - value: oj_data_small - - name: TEST_DATA_PATH_PREFIX - value: oj_inference_small - - # Training - - name: AML_TRAINING_PIPELINE_NAME - value: automl-training-pipeline - - # Real-time inferencing - - name: AML_AKS_NAME - value: manymodels-aks - - name: AML_MODEL_SPLITTING_TAGS - value: Store - - name: AML_MODEL_SORTING_TAGS - value: Store - - name: AML_WEBSERVICE_PREFIX - value: manymodels- - - name: AML_MAX_CONTAINER_SIZE - value: 50 - - name: AML_ROUTING_MODEL_NAME - value: deployed_models_info - - name: AML_ROUTING_MODEL_TAG_NAME - value: ModelType - - name: AML_ROUTING_MODEL_TAG_VALUE - value: _meta_ - - name: AML_ROUTING_WEBSERVICE - value: routing-manymodels - - # Batch forecasting - - name: AML_FORECASTING_PIPELINE_NAME - value: automl-forecasting-pipeline diff --git a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/appinsights.template.json b/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/appinsights.template.json deleted file mode 100644 index 2a3995fa..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/appinsights.template.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "http://schema.management.azure.com/schemas/2014-04-01-preview/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "type": "string" - }, - "location": { - "type": "string" - } - }, - "resources": [ - { - "type": "Microsoft.Insights/components", - "apiVersion": "2015-05-01", - "name": "[parameters('name')]", - "location": "[parameters('location')]", - "kind": "web", - "properties": { - "Application_Type": "web" - } - } - ] -} \ No newline at end of file diff --git a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/containerregistry.template.json b/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/containerregistry.template.json deleted file mode 100644 index 459f3040..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/containerregistry.template.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "type": "string" - }, - "location": { - "type": "string" - } - }, - "variables": {}, - "resources": [ - { - "type": "Microsoft.ContainerRegistry/registries", - "apiVersion": "2017-10-01", - "name": "[parameters('name')]", - "location": "[parameters('location')]", - "sku": { - "name": "Basic", - "tier": "Basic" - }, - "properties": { - "adminUserEnabled": true - }, - "scale": null, - "tags": {} - } - ] -} diff --git a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/keyvault.parameters.json b/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/keyvault.parameters.json deleted file mode 100644 index e816ccef..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/keyvault.parameters.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "createMode": { - "value": "default" - } - } -} \ No newline at end of file diff --git a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/keyvault.template.json b/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/keyvault.template.json deleted file mode 100644 index 9d8d643e..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/keyvault.template.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "type": "string" - }, - "location": { - "type": "string" - }, - "createMode": { - "type": "string" - } - }, - "variables": { - }, - "resources": [ - { - "type": "Microsoft.KeyVault/vaults", - "apiVersion": "2018-02-14", - "name": "[parameters('name')]", - "location": "[parameters('location')]", - "properties": { - "sku": { - "family": "A", - "name": "standard" - }, - "tenantId": "[subscription().tenantId]", - "createMode": "[parameters('createMode')]", - "enabledForTemplateDeployment": true, - "accessPolicies": [] - }, - "scale": null, - "tags": {} - } - ] -} \ No newline at end of file diff --git a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/mlworkspace.template.json b/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/mlworkspace.template.json deleted file mode 100644 index be400388..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/mlworkspace.template.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "workspaceName": { - "type": "string" - }, - "keyVaultName": { - "type": "string" - }, - "appInsightsName": { - "type": "string" - }, - "containerRegistryName": { - "type": "string" - }, - "storageAccountName": { - "type": "string" - } - }, - "resources": [ - { - "type": "Microsoft.MachineLearningServices/workspaces", - "apiVersion": "2018-11-19", - "name": "[parameters('workspaceName')]", - "location": "[resourceGroup().location]", - "identity": { - "type": "systemAssigned" - }, - "properties": { - "keyVault": "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]", - "applicationInsights": "[resourceId('Microsoft.Insights/components', parameters('appInsightsName'))]", - "containerRegistry": "[resourceId('Microsoft.ContainerRegistry/registries', parameters('containerRegistryName'))]", - "storageAccount": "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]" - } - } - ] -} \ No newline at end of file diff --git a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/storage.parameters.json b/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/storage.parameters.json deleted file mode 100644 index fb6c5c9a..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/storage.parameters.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "accountType": { - "value": "Standard_RAGRS" - }, - "kind": { - "value": "StorageV2" - }, - "accessTier": { - "value": "Hot" - } - } -} \ No newline at end of file diff --git a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/storage.template.json b/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/storage.template.json deleted file mode 100644 index ff111579..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates/storage.template.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "type": "string" - }, - "location": { - "type": "string" - }, - "accountType": { - "type": "string" - }, - "kind": { - "type": "string" - }, - "accessTier": { - "type": "string" - } - }, - "variables": {}, - "resources": [ - { - "type": "Microsoft.Storage/storageAccounts", - "apiVersion": "2018-07-01", - "name": "[parameters('name')]", - "location": "[parameters('location')]", - "sku": { - "name": "[parameters('accountType')]" - }, - "kind": "[parameters('kind')]", - "properties": { - "accessTier": "[parameters('accessTier')]", - "supportsHttpsTrafficOnly": true - } - } - ] -} \ No newline at end of file diff --git a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/deploy-infra.template.yml b/Custom_Script/mlops-pipelines/1-setup/deploy-infra/deploy-infra.template.yml deleted file mode 100644 index b93c9a37..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/deploy-infra/deploy-infra.template.yml +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Azure Pipeline Template for ML Workspace Resources Deployment - -parameters: -- name: environment - type: string -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: resourcesLocation - type: string -- name: storageAccountName - type: string -- name: keyVaultName - type: string -- name: appInsightsName - type: string -- name: containerRegistryName - type: string -- name: amlWorkspaceName - type: string - - -jobs: - -- job: iac_build - displayName: 'IaC Build' - steps: - - task: CopyFiles@2 - displayName: 'Copy ARM templates' - inputs: - sourceFolder: 'Custom_Script/mlops-pipelines/1-setup/deploy-infra/arm-templates' - targetFolder: '$(Build.ArtifactStagingDirectory)' - - publish: '$(Build.ArtifactStagingDirectory)' - artifact: infratemplates - -- deployment: iac_deployment - displayName: 'IaC Deployment' - environment: ${{parameters.environment}} - strategy: - runOnce: - deploy: - steps: - - download: current - artifact: infratemplates - - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy Storage Account' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Pipeline.Workspace)/infratemplates/storage.template.json' - csmParametersFile: '$(Pipeline.Workspace)/infratemplates/storage.parameters.json' - overrideParameters: '-name ${{parameters.storageAccountName}} -location ${{parameters.resourcesLocation}}' - - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy Key Vault' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Pipeline.Workspace)/infratemplates/keyvault.template.json' - csmParametersFile: '$(Pipeline.Workspace)/infratemplates/keyvault.parameters.json' - overrideParameters: '-name ${{parameters.keyVaultName}} -location ${{parameters.resourcesLocation}}' - - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy Application Insights' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Pipeline.Workspace)/infratemplates/appinsights.template.json' - overrideParameters: '-name ${{parameters.appInsightsName}} -location ${{parameters.resourcesLocation}}' - - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy Container Registry' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Pipeline.Workspace)/infratemplates/containerregistry.template.json' - overrideParameters: '-name ${{parameters.containerRegistryName}} -location ${{parameters.resourcesLocation}}' - - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy AML Workspace' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Pipeline.Workspace)/infratemplates/mlworkspace.template.json' - overrideParameters: '-workspaceName ${{parameters.amlWorkspaceName}} -keyVaultName ${{parameters.keyVaultName}} -appInsightsName ${{parameters.appInsightsName}} -containerRegistryName ${{parameters.containerRegistryName}} -storageAccountName ${{parameters.storageAccountName}}' diff --git a/Custom_Script/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.parameters.json b/Custom_Script/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.parameters.json deleted file mode 100644 index 02ab0df0..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.parameters.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "vmSize": { - "value": "STANDARD_D13_V2" - }, - "minNodeCount": { - "value": 0 - }, - "maxNodeCount": { - "value": 5 - }, - "scaleDownTime": { - "value": "PT15M" - } - } -} \ No newline at end of file diff --git a/Custom_Script/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.template.json b/Custom_Script/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.template.json deleted file mode 100644 index def9ef09..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.template.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "workspaceName": { - "type": "string" - }, - "clusterName": { - "type": "string" - }, - "vmSize": { - "type": "string" - }, - "minNodeCount": { - "type": "int" - }, - "maxNodeCount": { - "type": "int" - }, - "scaleDownTime": { - "type": "string" - } - }, - "variables": {}, - "resources": [ - { - "type": "Microsoft.MachineLearningServices/workspaces/computes", - "apiVersion": "2018-11-19", - "name": "[concat(parameters('workspaceName'), '/', parameters('clusterName'))]", - "location" : "[resourceGroup().location]", - "properties": { - "computeType": "AmlCompute", - "computeLocation" : "[resourceGroup().location]", - "properties": { - "scaleSettings": { - "minNodeCount" : "[parameters('minNodeCount')]", - "maxNodeCount" : "[parameters('maxNodeCount')]", - "nodeIdleTimeBeforeScaleDown": "[parameters('scaleDownTime')]" - }, - "vmPriority": "Dedicated", - "vmSize" : "[parameters('vmSize')]" - } - } - } - ] -} \ No newline at end of file diff --git a/Custom_Script/mlops-pipelines/1-setup/environment-setup/environment-setup.template.yml b/Custom_Script/mlops-pipelines/1-setup/environment-setup/environment-setup.template.yml deleted file mode 100644 index 63036de1..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/environment-setup/environment-setup.template.yml +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Azure Pipeline Template for ML Workspace Setup - -parameters: -- name: sdkVersion - type: string -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: resourcesLocation - type: string -- name: amlWorkspaceName - type: string -- name: amlComputeName - type: string -- name: aksName - type: string -- name: aksResourceGroup - type: string -- name: amlAksName - type: string -- name: amlTrainDatasetName - type: string -- name: amlInferenceDatasetName - type: string -- name: maxFiles - type: string - - -jobs: - -- job: aml_compute - displayName: 'Deploy AML Compute' - steps: - - task: AzureResourceGroupDeployment@2 - displayName: 'Deploy AML Compute' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - resourceGroupName: ${{parameters.resourceGroup}} - location: ${{parameters.resourcesLocation}} - csmFile: '$(Build.SourcesDirectory)/Custom_Script/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.template.json' - csmParametersFile: '$(Build.SourcesDirectory)/Custom_Script/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.parameters.json' - overrideParameters: '-workspaceName ${{parameters.amlWorkspaceName}} -clusterName ${{parameters.amlComputeName}}' - - -- job: aml_aks - displayName: 'Attach AKS Target to AML' - steps: - - task: AzureCLI@1 - displayName: 'Attach AKS Target to AML' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - workspace_params="--workspace-name ${{parameters.amlWorkspaceName}} --resource-group ${{parameters.resourceGroup}}" - # Install ML extension - az extension add -n azure-cli-ml - # Check if AKS target is already attached and attach if not - az ml computetarget show --name ${{parameters.amlAksName}} $workspace_params - if [ $? -eq 1 ]; then - # Get AKS resource id - aks_id=$(az resource list -n ${{parameters.aksName}} -g ${{parameters.aksResourceGroup}} --query "[0].id" -o tsv) - # Attach AKS - az ml computetarget attach aks --name ${{parameters.amlAksName}} --compute-resource-id $aks_id $workspace_params - fi - - -- job: sample_files - displayName: 'Sample Files Setup' - steps: - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - - bash: | - # Install dependencies - dependencies="azureml-opendatasets==${{parameters.sdkVersion}} pandas" - python -m pip install --upgrade pip && python -m pip install $dependencies --use-feature=2020-resolver - # Download sample files - datapath_train="sample-data-train" - datapath_inference="sample-data-inference" - data_script="Custom_Script/mlops-pipelines/scripts/download_data.py" - python $data_script --maxfiles ${{parameters.maxFiles}} --train-path $datapath_train --inference-path $datapath_inference - echo "##vso[task.setvariable variable=datapath_train;isOutput=true;]$datapath_train" - echo "##vso[task.setvariable variable=datapath_inference;isOutput=true;]$datapath_inference" - name: download_files - displayName: 'Download Sample Files' - failOnStderr: true - - - task: AzureCLI@1 - displayName: 'Upload files to AML datastore' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - workspace_params="--workspace-name ${{parameters.amlWorkspaceName}} --resource-group ${{parameters.resourceGroup}}" - # Install ML extension - az extension add -n azure-cli-ml - # Get default datastore - datastore=$(az ml datastore show-default $workspace_params --query name -o tsv) - # Upload train files - az ml datastore upload --name $datastore \ - --src-path $(download_files.datapath_train) \ - --target-path $(download_files.datapath_train) \ - --overwrite true \ - $workspace_params - # Upload inference files - az ml datastore upload --name $datastore \ - --src-path $(download_files.datapath_inference) \ - --target-path $(download_files.datapath_inference) \ - --overwrite true \ - $workspace_params - - -- job: register_dataset - displayName: 'Register Dataset' - dependsOn: sample_files - variables: - datapath_train: $[ dependencies.sample_files.outputs['download_files.datapath_train'] ] - datapath_inference: $[ dependencies.sample_files.outputs['download_files.datapath_inference'] ] - steps: - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - - task: AzureCLI@1 - displayName: 'Register dataset' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - # Install dependencies - python -m pip install --upgrade pip && python -m pip install azureml-sdk==${{parameters.sdkVersion}} - # Register train dataset - register_dataset_script=Custom_Script/mlops-pipelines/scripts/register_or_update_dataset.py - python $register_dataset_script --path $DATAPATH_TRAIN \ - --name ${{parameters.amlTrainDatasetName}} \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} - # Register inference dataset - python $register_dataset_script --path $DATAPATH_INFERENCE \ - --name ${{parameters.amlInferenceDatasetName}} \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} diff --git a/Custom_Script/mlops-pipelines/1-setup/setup-pipeline.yml b/Custom_Script/mlops-pipelines/1-setup/setup-pipeline.yml deleted file mode 100644 index 0feba7b5..00000000 --- a/Custom_Script/mlops-pipelines/1-setup/setup-pipeline.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pipeline that configures the environment for this solution accelerator - -# trigger: -# branches: -# include: -# - v2-preview -# paths: -# include: -# - Custom_Script/mlops-pipelines/1-setup/* -# - Custom_Script/mlops-pipelines/scripts/download_data.py -# - Custom_Script/mlops-pipelines/scripts/register_dataset.py - -variables: -- template: ../many-models-variables.yml -- group: manymodels-vg - -pool: - vmImage: ubuntu-latest - -stages: -- stage: deploy_infra - displayName: 'Deploy Infra' - jobs: - - template: deploy-infra/deploy-infra.template.yml - parameters: - environment: '$(ENVIRONMENT)' - serviceConnection: '$(SERVICECONNECTION_GROUP)' - resourceGroup: '$(RESOURCE_GROUP)' - resourcesLocation: '$(LOCATION)' - storageAccountName: '$(STORAGEACCOUNT_NAME)' - keyVaultName: '$(KEYVAULT_NAME)' - appInsightsName: '$(APPINSIGHTS_NAME)' - containerRegistryName: '$(CONTAINERREGISTRY_NAME)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - -- stage: environment_setup - displayName: 'Environment Setup' - jobs: - - template: environment-setup/environment-setup.template.yml - parameters: - sdkVersion: '$(SDK_VERSION)' - serviceConnection: '$(SERVICECONNECTION_GROUP)' - resourceGroup: '$(RESOURCE_GROUP)' - resourcesLocation: '$(LOCATION)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - amlComputeName: '$(AML_COMPUTE_NAME)' - aksName: '$(AKS_NAME)' - aksResourceGroup: '$(AKS_RESOURCE_GROUP)' - amlAksName: '$(AML_AKS_NAME)' - amlTrainDatasetName: '$(AML_DATASET_NAME)_train' - amlInferenceDatasetName: '$(AML_DATASET_NAME)_inference' - maxFiles: '$(DATASET_MAXFILES)' diff --git a/Custom_Script/mlops-pipelines/2-training-code-build/training-code-build-pipeline.yml b/Custom_Script/mlops-pipelines/2-training-code-build/training-code-build-pipeline.yml deleted file mode 100644 index 84e780fc..00000000 --- a/Custom_Script/mlops-pipelines/2-training-code-build/training-code-build-pipeline.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pipeline that publishes de AML Pipeline that will train the models - -# trigger: -# branches: -# include: -# - v2-preview -# paths: -# include: -# - Custom_Script/mlops-pipelines/2-training-code-build/* -# - Custom_Script/mlops-pipelines/scripts/create_training_pipeline.py -# - Custom_Script/scripts/train.py -# - Custom_Script/scripts/train.conda.yml - -variables: -- template: ../many-models-variables.yml -- group: manymodels-vg - -pool: - vmImage: ubuntu-latest - -stages: -- stage: build_training - displayName: 'Publish Training AML Pipeline' - jobs: - - job: build_training - displayName: 'Publish Training AML Pipeline' - steps: - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - - task: AzureCLI@1 - displayName: 'Publish Training AML Pipeline' - inputs: - azureSubscription: $(SERVICECONNECTION_WORKSPACE) - scriptLocation: inlineScript - inlineScript: | - # Install dependencies - python -m pip install --upgrade pip && python -m pip install azureml-sdk[contrib]==$SDK_VERSION - # Create/update training pipeline - trainingbuild_script=Custom_Script/mlops-pipelines/scripts/create_training_pipeline.py - python $trainingbuild_script --name $(AML_TRAINING_PIPELINE_NAME) \ - --version $(Build.BuildId) \ - --dataset $(AML_DATASET_NAME)_train \ - --compute $(AML_COMPUTE_NAME) \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group $(RESOURCE_GROUP) \ - --workspace-name $(AMLWORKSPACE_NAME) - diff --git a/Custom_Script/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aks.yml b/Custom_Script/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aks.yml deleted file mode 100644 index 28afccd8..00000000 --- a/Custom_Script/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aks.yml +++ /dev/null @@ -1,9 +0,0 @@ -computeType: AKS -containerResourceRequirements: - cpu_cores: 0.1 - memory_gb: 0.5 -environmentVariables: - WORKER_COUNT: 8 - MKL_NUM_THREADS: 1 - OMP_NUM_THREADS: 1 - WORKER_PRELOAD: False diff --git a/Custom_Script/mlops-pipelines/3-modeling/modeling-pipeline.yml b/Custom_Script/mlops-pipelines/3-modeling/modeling-pipeline.yml deleted file mode 100644 index 6b37f61d..00000000 --- a/Custom_Script/mlops-pipelines/3-modeling/modeling-pipeline.yml +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pipeline that runs AML Pipeline for model training - -# trigger: -# branches: -# include: -# - v2-preview -# paths: -# include: -# - Custom_Script/mlops-pipelines/3-modeling/* -# - Custom_Script/mlops-pipelines/scripts/deploy_grouped_models.py -# - Custom_Script/scripts/forecast_webservice.py -# - Custom_Script/scripts/routing_webservice.py -# - Custom_Script/scripts/routing_webservice.conda.yml - -# schedules: -# - cron: "0 0 * * 1" -# displayName: 'Weekly training at midnight on Mondays' -# branches: -# include: -# - v2-preview -# always: true - -variables: -- template: ../many-models-variables.yml -- group: manymodels-vg - -pool: - vmImage: ubuntu-latest - -stages: - -- stage: update_data - displayName: 'Update Data for Training/Retraining' - jobs: - - template: update-data/update-data.template.yml - parameters: - sdkVersion: '$(SDK_VERSION)' - serviceConnection: '$(SERVICECONNECTION_WORKSPACE)' - resourceGroup: '$(RESOURCE_GROUP)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - amlTrainDatasetName: '$(AML_DATASET_NAME)_train' - amlInferenceDatasetName: '$(AML_DATASET_NAME)_inference' - maxFiles: '$(DATASET_MAXFILES)' - -- stage: run_training - displayName: 'Run Model Training' - dependsOn: update_data - jobs: - - template: run-training/run-training.template.yml - parameters: - serviceConnection: '$(SERVICECONNECTION_WORKSPACE)' - resourceGroup: '$(RESOURCE_GROUP)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - trainingPipelineName: '$(AML_TRAINING_PIPELINE_NAME)' - -- stage: deploy_models_aci - displayName: 'Deploy Models to ACI' - dependsOn: run_training - condition: and(succeeded(), eq(variables['DEPLOY_ACI'], 'true')) - jobs: - - template: deploy-models/deploy-models.template.yml - parameters: - sdkVersion: '$(SDK_VERSION)' - serviceConnection: '$(SERVICECONNECTION_WORKSPACE)' - resourceGroup: '$(RESOURCE_GROUP)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - deploymentType: aci - amlAksName: '' - splittingTags: '$(AML_MODEL_SPLITTING_TAGS)' - sortingTags: '$(AML_MODEL_SORTING_TAGS)' - webservicePrefix: test-'$(AML_WEBSERVICE_PREFIX)' - containerSize: '$(MAX_CONTAINER_SIZE)' - resetContainers: '$(RESET_DEPLOYMENT)' - routingModelName: test-'$(AML_ROUTING_MODEL_NAME)' - routingModelTagName: '$(AML_ROUTING_MODEL_TAG_NAME)' - routingModelTagValue: '$(AML_ROUTING_MODEL_TAG_VALUE)' - routingServiceName: test-'$(AML_ROUTING_WEBSERVICE)' - -- stage: deploy_models_aks - displayName: 'Deploy Models to AKS' - dependsOn: run_training - condition: and(succeeded(), eq(variables['DEPLOY_AKS'], 'true'), variables['AML_AKS_NAME']) - jobs: - - template: deploy-models/deploy-models.template.yml - parameters: - sdkVersion: '$(SDK_VERSION)' - serviceConnection: '$(SERVICECONNECTION_WORKSPACE)' - resourceGroup: '$(RESOURCE_GROUP)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - deploymentType: aks - amlAksName: '$(AML_AKS_NAME)' - splittingTags: '$(AML_MODEL_SPLITTING_TAGS)' - sortingTags: '$(AML_MODEL_SORTING_TAGS)' - webservicePrefix: '$(AML_WEBSERVICE_PREFIX)' - containerSize: '$(MAX_CONTAINER_SIZE)' - resetContainers: '$(RESET_DEPLOYMENT)' - routingModelName: '$(AML_ROUTING_MODEL_NAME)' - routingModelTagName: '$(AML_ROUTING_MODEL_TAG_NAME)' - routingModelTagValue: '$(AML_ROUTING_MODEL_TAG_VALUE)' - routingServiceName: '$(AML_ROUTING_WEBSERVICE)' diff --git a/Custom_Script/mlops-pipelines/3-modeling/run-training/run-training.template.yml b/Custom_Script/mlops-pipelines/3-modeling/run-training/run-training.template.yml deleted file mode 100644 index 409a61ba..00000000 --- a/Custom_Script/mlops-pipelines/3-modeling/run-training/run-training.template.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Azure Pipeline Template for running the training pipeline - -parameters: -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: amlWorkspaceName - type: string -- name: trainingPipelineName - type: string - - -jobs: - -- job: pipeline_id - displayName: 'Get Training Pipeline ID' - steps: - - - task: AzureCLI@1 - name: get_pipeline_id - displayName: 'Get Training Pipeline ID' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - # Install ML extension - az extension add -n azure-cli-ml - # Get training pipeline ID - workspace_params="--workspace-name ${{parameters.amlWorkspaceName}} --resource-group ${{parameters.resourceGroup}}" - pipeline_id=$(az ml pipeline list $workspace_params --query "[?Name == '${{parameters.trainingPipelineName}}'].Id" -o tsv) - echo "##vso[task.setvariable variable=pipeline_id;isOutput=true;]$pipeline_id" - -- job: run_training - displayName: 'Run Training' - pool: server - timeoutInMinutes: 0 - dependsOn: pipeline_id - variables: - pipeline_id: $[ dependencies.pipeline_id.outputs['get_pipeline_id.pipeline_id'] ] - steps: - - - task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0 - displayName: 'Invoke AML Training Pipeline' - inputs: - azureSubscription: '${{parameters.serviceConnection}}' - PipelineId: '$(PIPELINE_ID)' - ExperimentName: '${{parameters.trainingPipelineName}}' - - diff --git a/Custom_Script/mlops-pipelines/3-modeling/update-data/update-data.template.yml b/Custom_Script/mlops-pipelines/3-modeling/update-data/update-data.template.yml deleted file mode 100644 index 72f0fdb3..00000000 --- a/Custom_Script/mlops-pipelines/3-modeling/update-data/update-data.template.yml +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Azure Pipeline Template for updating training dataset -# The dataset in this example is updated for illustration purposes only, the data downloaded is the same - - -parameters: -- name: sdkVersion - type: string -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: amlWorkspaceName - type: string -- name: amlTrainDatasetName - type: string -- name: amlInferenceDatasetName - type: string -- name: maxFiles - type: string - - -jobs: - -- job: download_new_data - displayName: 'Download New Sample Files' - steps: - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - - bash: | - # Install dependencies - python -m pip install --upgrade pip && python -m pip install azureml-opendatasets==${{parameters.sdkVersion}} - # Download sample files - data_script="Custom_Script/mlops-pipelines/scripts/download_data.py" - python $data_script --maxfiles ${{parameters.maxFiles}} --train-path "new-sample-data-train" --inference-path "new-sample-data-inference" - - name: download_files - displayName: 'Download Sample Files' - failOnStderr: true - - - task: AzureCLI@1 - name: upload_files - displayName: 'Upload files to AML datastore' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - workspace_params="--workspace-name ${{parameters.amlWorkspaceName}} --resource-group ${{parameters.resourceGroup}}" - # Install ML extension - az extension add -n azure-cli-ml - # Get default datastore - datastore=$(az ml datastore show-default $workspace_params --query name -o tsv) - # Upload train files - datapath_train=new-sample-data-$(date +"%Y%m%d-%H%M%S")-train - az ml datastore upload --name $datastore \ - --src-path "new-sample-data-train" \ - --target-path $datapath_train \ - --overwrite true \ - $workspace_params - # Upload inference files - datapath_inference=new-sample-data-$(date +"%Y%m%d-%H%M%S")-inference - az ml datastore upload --name $datastore \ - --src-path "new-sample-data-inference" \ - --target-path $datapath_inference \ - --overwrite true \ - $workspace_params - echo "##vso[task.setvariable variable=datapath_train;isOutput=true;]$datapath_train" - echo "##vso[task.setvariable variable=datapath_inference;isOutput=true;]$datapath_inference" - - -- job: update_dataset - displayName: 'Update Registered Dataset' - dependsOn: download_new_data - variables: - datapath_train: $[ dependencies.download_new_data.outputs['upload_files.datapath_train'] ] - datapath_inference: $[ dependencies.download_new_data.outputs['upload_files.datapath_inference'] ] - steps: - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - - task: AzureCLI@1 - displayName: 'Update dataset' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - # Install dependencies - python -m pip install --upgrade pip && python -m pip install azureml-sdk==${{parameters.sdkVersion}} - # Register train dataset - update_dataset_script=Custom_Script/mlops-pipelines/scripts/register_or_update_dataset.py - python $update_dataset_script --path $DATAPATH_TRAIN \ - --name ${{parameters.amlTrainDatasetName}} \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} - # Register inference dataset - update_dataset_script=Custom_Script/mlops-pipelines/scripts/register_or_update_dataset.py - python $update_dataset_script --path $DATAPATH_INFERENCE \ - --name ${{parameters.amlInferenceDatasetName}} \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} diff --git a/Custom_Script/mlops-pipelines/4-batch-forecasting-code-build/batch-forecasting-code-build-pipeline.yml b/Custom_Script/mlops-pipelines/4-batch-forecasting-code-build/batch-forecasting-code-build-pipeline.yml deleted file mode 100644 index 60f29fa1..00000000 --- a/Custom_Script/mlops-pipelines/4-batch-forecasting-code-build/batch-forecasting-code-build-pipeline.yml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pipeline that publishes de AML Pipeline that will generate the forecast - -# trigger: -# branches: -# include: -# - v2-preview -# paths: -# include: -# - Custom_Script/mlops-pipelines/4-batch-forecasting-code-build/* -# - Custom_Script/mlops-pipelines/scripts/create_forecasting_pipeline.py -# - Custom_Script/scripts/forecast.py -# - Custom_Script/scripts/forecast.conda.yml -# - Custom_Script/scripts/utils/forecasting.py - -variables: -- template: ../many-models-variables.yml -- group: manymodels-vg - -pool: - vmImage: ubuntu-latest - -stages: -- stage: build_forecasting - displayName: 'Publish Forecasting AML Pipeline' - jobs: - - job: build_forecasting - displayName: 'Publish Forecasting AML Pipeline' - steps: - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - inputs: - versionSpec: 3.7 - - - task: AzureCLI@1 - displayName: 'Publish Forecasting AML Pipeline' - inputs: - azureSubscription: $(SERVICECONNECTION_WORKSPACE) - scriptLocation: inlineScript - inlineScript: | - # Install dependencies - python -m pip install --upgrade pip && python -m pip install azureml-sdk[contrib]==$SDK_VERSION - # Create/update forecasting pipeline - forecastingbuild_script=Custom_Script/mlops-pipelines/scripts/create_forecasting_pipeline.py - python $forecastingbuild_script --name $(AML_FORECASTING_PIPELINE_NAME) \ - --version $(Build.BuildId) \ - --dataset $(AML_DATASET_NAME)_inference \ - --output $(PREDICTIONS_CONTAINER_NAME) \ - --compute $(AML_COMPUTE_NAME) \ - --subscription-id $(az account show --query id -o tsv) \ - --resource-group $(RESOURCE_GROUP) \ - --workspace-name $(AMLWORKSPACE_NAME) - diff --git a/Custom_Script/mlops-pipelines/5-batch-forecasting/batch-forecasting-pipeline.yml b/Custom_Script/mlops-pipelines/5-batch-forecasting/batch-forecasting-pipeline.yml deleted file mode 100644 index 17635279..00000000 --- a/Custom_Script/mlops-pipelines/5-batch-forecasting/batch-forecasting-pipeline.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Pipeline that runs AML Pipeline for model training - -# trigger: -# branches: -# include: -# - v2-preview -# paths: -# include: -# - Custom_Script/mlops-pipelines/5-batch-forecasting/* - -# schedules: -# - cron: "30 7 * * *" -# displayName: 'Daily forecasting at 07:30 AM' -# branches: -# include: -# - v2-preview -# always: true - -variables: -- template: ../many-models-variables.yml -- group: manymodels-vg - -pool: - vmImage: ubuntu-latest - -stages: - -# There is no data to update in this example, but in the general case -# the registered dataset should be updated before forecasting. -# - stage: update_data -# displayName: 'Update Data for Forecasting' -# jobs: -# - template: update-data/update-data.template.yml - -- stage: run_forecasting - displayName: 'Run Model Forecasting' - jobs: - - template: run-forecasting/run-forecasting.template.yml - parameters: - serviceConnection: '$(SERVICECONNECTION_WORKSPACE)' - resourceGroup: '$(RESOURCE_GROUP)' - amlWorkspaceName: '$(AMLWORKSPACE_NAME)' - forecastingPipelineName: '$(AML_FORECASTING_PIPELINE_NAME)' diff --git a/Custom_Script/mlops-pipelines/5-batch-forecasting/run-forecasting/run-forecasting.template.yml b/Custom_Script/mlops-pipelines/5-batch-forecasting/run-forecasting/run-forecasting.template.yml deleted file mode 100644 index c4207893..00000000 --- a/Custom_Script/mlops-pipelines/5-batch-forecasting/run-forecasting/run-forecasting.template.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# Azure Pipeline Template for running the forecasting pipeline - -parameters: -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: amlWorkspaceName - type: string -- name: forecastingPipelineName - type: string - - -jobs: - -- job: pipeline_id - displayName: 'Get Forecasting Pipeline ID' - steps: - - - task: AzureCLI@1 - name: get_pipeline_id - displayName: 'Get Forecasting Pipeline ID' - inputs: - azureSubscription: ${{parameters.serviceConnection}} - scriptLocation: inlineScript - inlineScript: | - # Install ML extension - az extension add -n azure-cli-ml - # Get forecasting pipeline ID - workspace_params="--workspace-name ${{parameters.amlWorkspaceName}} --resource-group ${{parameters.resourceGroup}}" - pipeline_id=$(az ml pipeline list $workspace_params --query "[?Name == '${{parameters.forecastingPipelineName}}'].Id" -o tsv) - echo "##vso[task.setvariable variable=pipeline_id;isOutput=true;]$pipeline_id" - -- job: run_forecasting - displayName: 'Run Forecasting' - pool: server - timeoutInMinutes: 0 - dependsOn: pipeline_id - variables: - pipeline_id: $[ dependencies.pipeline_id.outputs['get_pipeline_id.pipeline_id'] ] - steps: - - - task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0 - displayName: 'Invoke AML Forecasting Pipeline' - inputs: - azureSubscription: '${{parameters.serviceConnection}}' - PipelineId: '$(PIPELINE_ID)' - ExperimentName: '${{parameters.forecastingPipelineName}}' diff --git a/Custom_Script/mlops-pipelines/5-batch-forecasting/update-data/update-data.template.yml b/Custom_Script/mlops-pipelines/5-batch-forecasting/update-data/update-data.template.yml deleted file mode 100644 index 621d7130..00000000 --- a/Custom_Script/mlops-pipelines/5-batch-forecasting/update-data/update-data.template.yml +++ /dev/null @@ -1 +0,0 @@ -# Azure Pipeline Template for updating forecasting dataset diff --git a/Custom_Script/mlops-pipelines/README.md b/Custom_Script/mlops-pipelines/README.md deleted file mode 100644 index 8450cfcb..00000000 --- a/Custom_Script/mlops-pipelines/README.md +++ /dev/null @@ -1,131 +0,0 @@ -# Instructions - -You'll use Azure DevOps for running the MLOps pipelines. Create an [organization](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/create-organization?view=azure-devops#create-an-organization) an a [project](https://docs.microsoft.com/en-us/azure/devops/organizations/projects/create-project?view=azure-devops&tabs=preview-page#create-a-project) for the Many Models solution. - -## 0. Before creating the pipelines - -- Create an **Azure Resource Manager** [service connection](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection) to access the subscription and resource group where you plan to deploy the Many Models solution. The resource group should already be created. Mark the option to grant access permission to all pipelines. Choose any name you want, and copy it as you'll need it in the next step. - -- Create a [variable group](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=classic#create-a-variable-group) called **``manymodels-vg``**, with the following variables: - -| Variable Name | Short description | -| --------------------------- | ----------------- | -| DATASET_MAXFILES | Number of sample files to use (1 file = 1 model) | -| NAMESPACE | Unique naming prefix for created resources | -| LOCATION | [Azure location](https://azure.microsoft.com/en-us/global-infrastructure/locations/), no spaces | -| RESOURCE_GROUP | Name of the Azure Resource Group that you'll be using (should be already created) | -| SERVICECONNECTION_GROUP | Name of the connection you created in the last step | -| DEPLOY_ACI | Whether to deploy in ACI (true/false, default false) | -| DEPLOY_AKS | Whether to deploy in AKS (true/false, default false) | -| AKS_NAME | [Optional] Name of the AKS resource you'll use for deploying the models | -| AKS_RESOURCE_GROUP | [Optional] Name of the resource group where the AKS resource is located | - -## 1. Setup Pipeline - -The setup pipeline will: - -- Deploy Azure Machine Learning and the other necessary resources into the resource group you specified. - -- Set up the Azure Machine Learning worskpace, creating a compute target and attaching the AKS cluster. - -- Download as many files as you specified in the `DATASET_MAXFILES` variable and register them as a dataset in AML. - -Create the pipeline as in [here](https://github.com/microsoft/MLOpsPython/blob/master/docs/getting_started.md#create-the-iac-pipeline), selecting branch **``v2-preview``** and setting the path to [/Custom_Script/mlops-pipelines/1-setup/setup-pipeline.yml](1-setup/setup-pipeline.yml). - -The pipeline run should look like this: - -Stage Deploy Infra with two jobs: IaC Build, IaC Deployment. Followed by stage Environment Setup with four jobs: Deploy AML Compute, Attach AKS cluster to AML, Sample Files Setup, Register Dataset - -## 2. Training Code Build Pipeline - -The training code build pipeline will: - -- Create an Azure Machine Learning Pipeline that will train many models in parallel using the [train script](../scripts/train.py). - -- Publish the AML Pipeline into the AML workspace so it's ready to use whenever we want to retrain. - -Before creating the Azure DevOps pipeline: - -- Make sure the [AML extension](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.vss-services-azureml) is installed in the Azure DevOps organization. - -- Create an **Azure Resource Manager** [service connection](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection) to access the Machine Learning Workspace you created in the setup pipeline before. As you did before, mark the option to grant access permission to all pipelines, and copy the name as you'll need it in the next step. - -- Modify the **``manymodels-vg``** variable group you created before, and add this new variable: - -| Variable Name | Short description | -| --------------------------- | ----------------- | -| SERVICECONNECTION_WORKSPACE | Name of the connection to the AML Workspace you have just created | - -Then, create the pipeline as you did before, selecting branch **``v2-preview``** and setting the path to [/Custom_Script/mlops-pipelines/2-training-code-build/training-code-build-pipeline.yml](2-training-code-build/training-code-build-pipeline.yml). - -The pipeline run should look like this: - -Single job: Publish Training AML Pipeline - -## 3. Modeling Pipeline - -The modeling pipeline will: - -- Trigger the many models training by invoking the training AML Pipeline previously published. - -- Group the registered models according to specified tags and maximum container size (500 by default). - -- Deploy each group into a different webservice hosted in ACI and/or AKS. These webservices will all use the same [forecast script](../scripts/forecast_webservice.py). - -- Deploy the entry point that will route the requests to the corresponding model webservice. - -Create the pipeline as you did before, selecting branch **``v2-preview``** and setting the path to [/Custom_Script/mlops-pipelines/3-modeling/modeling-pipeline.yml](3-modeling/modeling-pipeline.yml). - -The pipeline run should look like this: - -Stage Run Model Training with two jobs: Get Training Pipeline ID, Run Training. After that two parallel stages are triggered with three identical jobs: Deploy Models to ACI and Deploy Models to AKS, with jobs Deploy Models, Register Routing Model, Deploy Routing Webservice - -The deployment stages will be triggered if you have the corresponding variables `DEPLOY_ACI`/`DEPLOY_AKS` in the variable group set to `true`. - -There are two variables you can add to the **``manymodels-vg``** variable group to customize deployments: - -| Variable Name | Short description | -| --------------------------- | ----------------- | -| MAX_CONTAINER_SIZE | Maximum number of models to fit into one webservice container | -| RESET_DEPLOYMENT | Set to `true` to reset existing containers | - -## 4. [Optional] Batch Forecasting Code Build Pipeline - -The batch forecasting code build pipeline will: - -- Create an Azure Machine Learning Pipeline that will generate batch forecasts for all the models in parallel using the [forecast script](../scripts/forecast.py). - -- Publish the AML Pipeline into the AML workspace so it's ready to use whenever we want to do batch forecasting. - -You only need to create this Azure DevOps pipeline if you want to do batch forecasting. Do it as you did before, selecting branch **``v2-preview``** and setting the path to [/Custom_Script/mlops-pipelines/4-batch-forecasting-code-build/batch-forecasting-code-build-pipeline.yml](4-batch-forecasting-code-build/batch-forecasting-code-build-pipeline.yml). - -The pipeline run should look like this: - -Single job: Publish Forecasting AML Pipeline - -## 5. [Optional] Batch Forecasting Pipeline - -The batch forecasting pipeline will: - -- Trigger the many models batch forecasting by invoking the batch forecasting AML Pipeline published in step 4. - -Create the pipeline as you did before, selecting branch **``v2-preview``** and setting the path to [/Custom_Script/mlops-pipelines/5-batch-forecasting/batch-forecasting-pipeline.yml](5-batch-forecasting/batch-forecasting-pipeline.yml). - -The pipeline run should look like this: - -Two jobs: Get Forecasting Pipeline ID, Run Forecasting diff --git a/Custom_Script/mlops-pipelines/scripts/create_forecasting_pipeline.py b/Custom_Script/mlops-pipelines/scripts/create_forecasting_pipeline.py deleted file mode 100644 index f32fa37c..00000000 --- a/Custom_Script/mlops-pipelines/scripts/create_forecasting_pipeline.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import argparse -from azureml.core import Workspace, Datastore, Dataset, Environment -from azureml.data.data_reference import DataReference -from azureml.core.compute import AmlCompute -from azureml.pipeline.core import Pipeline, PipelineData, PublishedPipeline -from azureml.pipeline.steps import PythonScriptStep -from azureml.contrib.pipeline.steps import ParallelRunConfig, ParallelRunStep - - -def main(ws, pipeline_name, pipeline_version, dataset_name, output_name, compute_name): - - # Get forecasting dataset - dataset = Dataset.get_by_name(ws, name=dataset_name) - dataset_input = dataset.as_named_input(dataset_name) - - # Set outputs - datastore = ws.get_default_datastore() - output_dir = PipelineData(name='forecasting_output', datastore=datastore) - predictions_datastore = Datastore.register_azure_blob_container( - workspace=ws, - datastore_name=output_name, - container_name=output_name, - account_name=datastore.account_name, - account_key=datastore.account_key, - create_if_not_exists=True - ) - predictions_dref = DataReference(predictions_datastore) - - # Get the compute target - compute = AmlCompute(ws, compute_name) - - # Set up ParallelRunStep - parallel_run_config = get_parallel_run_config(ws, dataset_name, compute) - parallel_run_step = ParallelRunStep( - name='many-models-parallel-forecasting', - parallel_run_config=parallel_run_config, - inputs=[dataset_input], - output=output_dir, - allow_reuse=False, - arguments=[ - '--id_columns', 'Store', 'Brand', - '--timestamp_column', 'WeekStarting', - '--model_type', 'lr' - ] - ) - - # Create step to copy predictions - upload_predictions_step = PythonScriptStep( - name='many-models-copy-predictions', - source_directory='Custom_Script/scripts/', - script_name='copy_predictions.py', - compute_target=compute, - inputs=[predictions_dref, output_dir], - allow_reuse=False, - arguments=[ - '--parallel_run_step_output', output_dir, - '--output_dir', predictions_dref, - '--id_columns', 'Store', 'Brand', - '--target_column', 'Quantity', - '--timestamp_column', 'WeekStarting' - ] - ) - - # Create the pipeline - train_pipeline = Pipeline(workspace=ws, steps=[parallel_run_step, upload_predictions_step]) - train_pipeline.validate() - - # Publish it and replace old pipeline - disable_old_pipelines(ws, pipeline_name) - published_pipeline = train_pipeline.publish( - name=pipeline_name, - description="Many Models forecasting pipeline", - version=pipeline_version, - continue_on_step_failure=False - ) - - return published_pipeline.id - - -def get_parallel_run_config(ws, dataset_name, compute, processes_per_node=8, node_count=3, timeout=180): - - # Configure environment for ParallelRunStep - forecast_env = Environment.from_conda_specification( - name='many_models_environment', - file_path='Custom_Script/scripts/forecast.conda.yml' - ) - - # Set up ParallelRunStep configuration - parallel_run_config = ParallelRunConfig( - source_directory='Custom_Script/scripts/', - entry_script='forecast.py', - mini_batch_size='1', - run_invocation_timeout=timeout, - error_threshold=25, - output_action='append_row', - environment=forecast_env, - process_count_per_node=processes_per_node, - compute_target=compute, - node_count=node_count - ) - - return parallel_run_config - - -def disable_old_pipelines(ws, pipeline_name): - for pipeline in PublishedPipeline.list(ws): - if pipeline.name == pipeline_name: - pipeline.disable() - - -def parse_args(args=None): - parser = argparse.ArgumentParser() - parser.add_argument('--subscription-id', required=True, type=str) - parser.add_argument('--resource-group', required=True, type=str) - parser.add_argument('--workspace-name', required=True, type=str) - parser.add_argument('--name', required=True, type=str) - parser.add_argument('--version', required=True, type=str) - parser.add_argument('--dataset', type=str, default='oj_sales_data') - parser.add_argument('--output', type=str, default='predictions') - parser.add_argument('--compute', type=str, default='cpu-compute') - args_parsed = parser.parse_args(args) - return args_parsed - - -if __name__ == "__main__": - args = parse_args() - - # Connect to workspace - ws = Workspace.get( - name=args.workspace_name, - subscription_id=args.subscription_id, - resource_group=args.resource_group - ) - - pipeline_id = main( - ws, - pipeline_name=args.name, - pipeline_version=args.version, - dataset_name=args.dataset, - output_name=args.output, - compute_name=args.compute - ) - - print('Forecasting pipeline {} version {} published with ID {}'.format(args.name, args.version, pipeline_id)) diff --git a/Custom_Script/mlops-pipelines/scripts/create_training_pipeline.py b/Custom_Script/mlops-pipelines/scripts/create_training_pipeline.py deleted file mode 100644 index 47d9d9a6..00000000 --- a/Custom_Script/mlops-pipelines/scripts/create_training_pipeline.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import argparse -from azureml.core import Workspace, Dataset, Environment -from azureml.core.compute import AmlCompute -from azureml.pipeline.core import Pipeline, PipelineData, PublishedPipeline -from azureml.contrib.pipeline.steps import ParallelRunConfig, ParallelRunStep - - -def main(ws, pipeline_name, pipeline_version, dataset_name, compute_name): - - # Get input dataset - dataset = Dataset.get_by_name(ws, name=dataset_name) - dataset_input = dataset.as_named_input(dataset_name) - - # Set output - datastore = ws.get_default_datastore() - output_dir = PipelineData(name='training_output', datastore=datastore) - - # Set up ParallelRunStep - parallel_run_config = get_parallel_run_config(ws, dataset_name, compute_name) - parallel_run_step = ParallelRunStep( - name='many-models-parallel-training', - parallel_run_config=parallel_run_config, - inputs=[dataset_input], - output=output_dir, - allow_reuse=False, - arguments=[ - '--id_columns', 'Store', 'Brand', - '--target_column', 'Quantity', - '--timestamp_column', 'WeekStarting', - '--tag_columns', 'StoreGroup10', 'StoreGroup100', 'StoreGroup1000', - '--drop_id', - '--drop_tags', - '--drop_columns', 'Revenue', - '--model_type', 'lr', - '--test_size', 20 - ] - ) - - # Create the pipeline - train_pipeline = Pipeline(workspace=ws, steps=[parallel_run_step]) - train_pipeline.validate() - - # Publish it and replace old pipeline - disable_old_pipelines(ws, pipeline_name) - published_pipeline = train_pipeline.publish( - name=pipeline_name, - description="Many Models training/retraining pipeline", - version=pipeline_version, - continue_on_step_failure=False - ) - - return published_pipeline.id - - -def get_parallel_run_config(ws, dataset_name, compute_name, processes_per_node=8, node_count=3, timeout=300): - - # Configure environment for ParallelRunStep - train_env = Environment.from_conda_specification( - name='many_models_environment', - file_path='Custom_Script/scripts/train.conda.yml' - ) - - # Get the compute target - compute = AmlCompute(ws, compute_name) - - # Set up ParallelRunStep configuration - parallel_run_config = ParallelRunConfig( - source_directory='Custom_Script/scripts/', - entry_script='train.py', - mini_batch_size='1', - run_invocation_timeout=timeout, - error_threshold=-1, - output_action='append_row', - environment=train_env, - process_count_per_node=processes_per_node, - compute_target=compute, - node_count=node_count - ) - - return parallel_run_config - - -def disable_old_pipelines(ws, pipeline_name): - for pipeline in PublishedPipeline.list(ws): - if pipeline.name == pipeline_name: - pipeline.disable() - - -def parse_args(args=None): - parser = argparse.ArgumentParser() - parser.add_argument('--subscription-id', required=True, type=str) - parser.add_argument('--resource-group', required=True, type=str) - parser.add_argument('--workspace-name', required=True, type=str) - parser.add_argument('--name', required=True, type=str) - parser.add_argument('--version', required=True, type=str) - parser.add_argument('--dataset', type=str, default='oj_sales_data') - parser.add_argument('--compute', type=str, default='cpu-compute') - args_parsed = parser.parse_args(args) - return args_parsed - - -if __name__ == "__main__": - args = parse_args() - - # Connect to workspace - ws = Workspace.get( - name=args.workspace_name, - subscription_id=args.subscription_id, - resource_group=args.resource_group - ) - - pipeline_id = main( - ws, - pipeline_name=args.name, - pipeline_version=args.version, - dataset_name=args.dataset, - compute_name=args.compute - ) - - print('Training pipeline {} version {} published with ID {}'.format(args.name, args.version, pipeline_id)) diff --git a/README.md b/README.md index 8f114868..98dc4b6c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![Many Models Solution Accelerator Banner](images/mmsa.png) +![Many Models Solution Accelerator Banner](.images/mmsa.png) # Many Models Solution Accelerator + ![Automl notebooks](https://github.com/microsoft/solution-accelerator-many-models/workflows/Automl%20notebooks/badge.svg) ![Custom script notebooks](https://github.com/microsoft/solution-accelerator-many-models/workflows/Custom%20script%20notebooks/badge.svg) @@ -39,73 +40,34 @@ While it's not required, a basic understanding of Azure Machine Learning will be 2. [Azure Machine Learning Tutorials](https://docs.microsoft.com/azure/machine-learning/tutorial-1st-experiment-sdk-setup) 3. [Azure Machine Learning Sample Notebooks on Github](https://github.com/Azure/MachineLearningNotebooks) -## Getting started - -### 1. Deploy Resources - -Start by deploying the resources to Azure. The button below will deploy Azure Machine Learning and its related resources: - - - - - -### 2. Configure Development Environment - -Next you'll need to configure your [development environment](https://docs.microsoft.com/azure/machine-learning/how-to-configure-environment) for Azure Machine Learning. We recommend using a [Notebook VM](https://docs.microsoft.com/azure/machine-learning/how-to-configure-environment#compute-instance) as it's the fastest way to get up and running. Follow the steps in [EnvironmentSetup.md](./EnvironmentSetup.md) to create a Notebook VM and clone the repo onto it. - -### 3. Run Notebooks - -Once your development environment is set up, run through the Jupyter Notebooks sequentially following the steps outlined. By the end, you'll know how to train, score, and make predictions using the many models pattern on Azure Machine Learning. - -There are two ways to train many models: - -1. Using a custom training script -2. Using Automated ML - -However, the steps needed to set the workspace up and prepare the datasets are the same no matter which option you choose. - -![Sequence of Notebooks](./images/mmsa-overview.png) - - ## Contents -In this repo, you'll train and score a forecasting model for each orange juice brand and for each store at a (simulated) grocery chain. By the end, you'll have forecasted sales by using up to 11,973 models to predict sales for the next few weeks. - -The data used in this sample is simulated based on the [Dominick's Orange Juice Dataset](http://www.cs.unitn.it/~taufer/QMMA/L10-OJ-Data.html#(1)), sales data from a Chicago area grocery store. - - - -The functionality is broken into the notebooks folders designed to be run sequentially. +In this repo, you'll train a forecasting model for each orange juice brand and for each store at a (simulated) grocery chain. The data used in this sample is simulated based on the [Dominick's Orange Juice Dataset](http://www.cs.unitn.it/~taufer/QMMA/L10-OJ-Data.html#(1)), sales data from a Chicago area grocery store. -### Before training the models +For training you can either use a custom script or Automated ML. Choose the custom script option if you want to write your own code for data preparation, training and performance reporting (we provide a sample custom script for demonstration purposes). On the other hand, use AutoML if you prefer these aspects to be handled internally by Azure Machine Learning. -| Notebook | Description | -|----------------|--------------------------------------------| -| `00_Setup_AML_Workspace.ipynb` | Creates and configures the AML Workspace, including deploying a compute cluster for training. | -| `01_Data_Preparation.ipynb` | Prepares the datasets that will be used during training and forecasting. | +When training completes, you'll have up to 11,973 different models registered in the AML workspace and ready to be used for forecasting. You can then choose to do batch forecasting with all of them to predict sales for the next few weeks, or deploy the models into webservices to be able to issue real-time forecasts upon request. Or you can also choose to do both if you need to. -### Using a custom training script to train the models: + -The following notebooks are located under the [`Custom_Script/`](Custom_Script/) folder. +## How to run this solution accelerator -| Notebook | Description | -|----------------|--------------------------------------------| -| `02_CustomScript_Training_Pipeline.ipynb` | Creates a pipeline to train a model for each store and orange juice brand in the dataset using a custom script. | -| `03_CustomScript_Forecasting_Pipeline.ipynb` | Creates a pipeline to forecast future orange juice sales using the models trained in the previous step.| +There are two options for running this solution accelerator: -### Using Automated ML to train the models: +- Using step-by-step notebooks +- Using automated MLOps pipelines -The following notebooks are located under the [`Automated_ML/`](Automated_ML/) folder. +For production environments, MLOps pipelines in Azure DevOps should be the preferred option. +However, we recommend running the notebooks first as they explain all the details of the solution accelerator +and will allow you to understand how everything works. +For more information about the notebooks please go to the [`notebooks`](notebooks/) folder. -| Notebook | Description | -|----------------|--------------------------------------------| -| `02_AutoML_Training_Pipeline.ipynb` | Creates a pipeline to train a model for each store and orange juice brand in the dataset using Automated ML. | -| `03_AutoML_Forecasting_Pipeline.ipynb` | Creates a pipeline to forecast future orange juice sales using the models trained in the previous step. | +For more information about the Mlops pipelines please go to the [`mlops-pipelines`](mlops-pipelines/) folder. -## How-to-videos +## How-to videos -Watch these how-to-videos for a step by step walk-through of the many model solution accelerator to learn how to setup your models using both the custom training script and Automated ML. +Watch these how-to videos for a step by step walk-through of the many model solution accelerator to learn how to setup your models using both the custom training script and Automated ML. ### Custom Script diff --git a/Troubleshooting.md b/Troubleshooting.md index 6c972ee2..f26cce67 100644 --- a/Troubleshooting.md +++ b/Troubleshooting.md @@ -8,7 +8,7 @@ When running the pipelines, the quickest way to see if the pipeline is running s While there's a lot of valuable information in logs, there's a couple of key files to look at first. -- ```70_driver_log.txt``` contains information from the controller that launches parallel run step code. This file will include any print statements that you put into ```train.py``` or ```forecast.py```. +- ```70_driver_log.txt``` contains information from the controller that launches parallel run step code. This file will include any print statements that you put into ```train.py``` or ```batch_forecasting.py```. - ```~/logs/sys/error//Process-*.txt``` is the quickest ways to see errors in your pipeline. If the ```error``` folder doesn't exist, you likely haven't hit errors in your scripts yet. If it does, there's likely a problem. diff --git a/images/Flow_map.png b/images/Flow_map.png deleted file mode 100644 index e895d0bc..00000000 Binary files a/images/Flow_map.png and /dev/null differ diff --git a/images/mlops_pipeline1.png b/images/mlops_pipeline1.png deleted file mode 100644 index 7213dda8..00000000 Binary files a/images/mlops_pipeline1.png and /dev/null differ diff --git a/images/mlops_pipeline2.png b/images/mlops_pipeline2.png deleted file mode 100644 index ea9d1113..00000000 Binary files a/images/mlops_pipeline2.png and /dev/null differ diff --git a/images/mlops_pipeline3.png b/images/mlops_pipeline3.png deleted file mode 100644 index 9226e719..00000000 Binary files a/images/mlops_pipeline3.png and /dev/null differ diff --git a/images/mlops_pipeline4.png b/images/mlops_pipeline4.png deleted file mode 100644 index 2a5fa48d..00000000 Binary files a/images/mlops_pipeline4.png and /dev/null differ diff --git a/images/mlops_pipeline5.png b/images/mlops_pipeline5.png deleted file mode 100644 index a6efa317..00000000 Binary files a/images/mlops_pipeline5.png and /dev/null differ diff --git a/mlops-pipelines/1-setup/Details.md b/mlops-pipelines/1-setup/Details.md new file mode 100644 index 00000000..a03a8670 --- /dev/null +++ b/mlops-pipelines/1-setup/Details.md @@ -0,0 +1,18 @@ +# 1 - Setup Pipeline - Details + +## Deploy Infrastructure + +The first stage of the pipeline will deploy into Azure all the necessary infrastructure to run the Many Models Solution Accelerator. + +If you want to customize the Azure resources deployed, you can modify the ARM templates and parameters under [`arm-templates/`](arm-templates). + +## Environment Setup + +During the environment setup: + +- A compute target will be created to run training and forecasting. +- An existing AKS cluster will be attached for model deployment, if defined. This will only happen if the variables related to deploying in AKS are set (see [the section about deployment](../2-modeling/Details.md#deployment-optional) in the modeling pipeline for details). + +## Data Preparation + +The pipeline will download as many files as you specified in the `DATASET_MAXFILES` variable in the [variable group](../#3-create-variable-group), split them, and register them as train and inference datasets in AML. diff --git a/mlops-pipelines/1-setup/README.md b/mlops-pipelines/1-setup/README.md new file mode 100644 index 00000000..72640a5b --- /dev/null +++ b/mlops-pipelines/1-setup/README.md @@ -0,0 +1,38 @@ +# 1 - Setup Pipeline + +The setup pipeline will: + +- Deploy Azure Machine Learning and the other necessary resources into the resource group you specified. +- Set up the Azure Machine Learning worskpace. +- Download as many files as you specified in the `DATASET_MAXFILES` variable in the [variable group](../#3-create-variable-group), and register them as a dataset in AML. + +## Instructions + +Create the pipeline as explained in [here](https://github.com/microsoft/MLOpsPython/blob/master/docs/getting_started.md#create-the-iac-pipeline), selecting branch **``v2-preview``** and setting the path to [`/mlops-pipelines/1-setup/pipeline-setup.yml`](pipeline-setup.yml). + +Then, run the pipeline and wait for it to finish. + +## Result + +The pipeline run should look like this: + +Stages and jobs as described below + +Containing the following stages and jobs: + +- Deploy Infrastructure + - IaC Build + - IaC Deployment +- Environment Setup + - Deploy AML Compute + - Attach AKS cluster to AML +- Data Preparation + - Sample Files Setup + - Register Dataset + +## Details + +If you want to learn about the details and ways to customize the setup pipeline please read the [details page](Details.md). diff --git a/Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/appinsights.template.json b/mlops-pipelines/1-setup/arm-templates/appinsights.template.json similarity index 100% rename from Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/appinsights.template.json rename to mlops-pipelines/1-setup/arm-templates/appinsights.template.json diff --git a/Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/containerregistry.template.json b/mlops-pipelines/1-setup/arm-templates/containerregistry.template.json similarity index 100% rename from Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/containerregistry.template.json rename to mlops-pipelines/1-setup/arm-templates/containerregistry.template.json diff --git a/Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/keyvault.parameters.json b/mlops-pipelines/1-setup/arm-templates/keyvault.parameters.json similarity index 100% rename from Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/keyvault.parameters.json rename to mlops-pipelines/1-setup/arm-templates/keyvault.parameters.json diff --git a/Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/keyvault.template.json b/mlops-pipelines/1-setup/arm-templates/keyvault.template.json similarity index 100% rename from Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/keyvault.template.json rename to mlops-pipelines/1-setup/arm-templates/keyvault.template.json diff --git a/Automated_ML/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.parameters.json b/mlops-pipelines/1-setup/arm-templates/mlcompute.parameters.json similarity index 100% rename from Automated_ML/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.parameters.json rename to mlops-pipelines/1-setup/arm-templates/mlcompute.parameters.json diff --git a/Automated_ML/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.template.json b/mlops-pipelines/1-setup/arm-templates/mlcompute.template.json similarity index 100% rename from Automated_ML/mlops-pipelines/1-setup/environment-setup/arm-templates/mlcompute.template.json rename to mlops-pipelines/1-setup/arm-templates/mlcompute.template.json diff --git a/Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/mlworkspace.template.json b/mlops-pipelines/1-setup/arm-templates/mlworkspace.template.json similarity index 90% rename from Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/mlworkspace.template.json rename to mlops-pipelines/1-setup/arm-templates/mlworkspace.template.json index be400388..e001280d 100644 --- a/Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/mlworkspace.template.json +++ b/mlops-pipelines/1-setup/arm-templates/mlworkspace.template.json @@ -25,13 +25,14 @@ "name": "[parameters('workspaceName')]", "location": "[resourceGroup().location]", "identity": { - "type": "systemAssigned" + "type": "SystemAssigned" }, "properties": { "keyVault": "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]", "applicationInsights": "[resourceId('Microsoft.Insights/components', parameters('appInsightsName'))]", "containerRegistry": "[resourceId('Microsoft.ContainerRegistry/registries', parameters('containerRegistryName'))]", - "storageAccount": "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]" + "storageAccount": "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]", + "sku": "enterprise" } } ] diff --git a/Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/storage.parameters.json b/mlops-pipelines/1-setup/arm-templates/storage.parameters.json similarity index 100% rename from Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/storage.parameters.json rename to mlops-pipelines/1-setup/arm-templates/storage.parameters.json diff --git a/Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/storage.template.json b/mlops-pipelines/1-setup/arm-templates/storage.template.json similarity index 100% rename from Automated_ML/mlops-pipelines/1-setup/deploy-infra/arm-templates/storage.template.json rename to mlops-pipelines/1-setup/arm-templates/storage.template.json diff --git a/mlops-pipelines/1-setup/data-prep.template.yml b/mlops-pipelines/1-setup/data-prep.template.yml new file mode 100644 index 00000000..3a7c91fa --- /dev/null +++ b/mlops-pipelines/1-setup/data-prep.template.yml @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Azure Pipeline Template for ML Workspace Setup + +jobs: + +- job: sample_files + displayName: 'Sample Files Setup' + steps: + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: 3.7 + + - bash: | + # Install dependencies + dependencies="azureml-opendatasets==$(SDK_VERSION) pandas" + python -m pip install --upgrade pip && python -m pip install $dependencies + # Download sample files + datapath_train="sample-data-train" + datapath_inference="sample-data-inference" + data_script="mlops-pipelines/scripts/download_data.py" + python $data_script --maxfiles $(DATASET_MAXFILES) \ + --train-path $datapath_train \ + --inference-path $datapath_inference + echo "##vso[task.setvariable variable=datapath_train;isOutput=true;]$datapath_train" + echo "##vso[task.setvariable variable=datapath_inference;isOutput=true;]$datapath_inference" + name: download_files + displayName: 'Download Sample Files' + failOnStderr: true + + - task: AzureCLI@1 + displayName: 'Upload files to AML datastore' + inputs: + azureSubscription: $(SERVICECONNECTION_GROUP) + scriptLocation: inlineScript + inlineScript: | + workspace_params="--workspace-name $(AMLWORKSPACE_NAME) --resource-group $(RESOURCE_GROUP)" + # Install ML extension + az extension add -n azure-cli-ml + # Get default datastore + datastore=$(az ml datastore show-default $workspace_params --query name -o tsv) + # Upload train files + az ml datastore upload --name $datastore \ + --src-path $(download_files.datapath_train) \ + --target-path $(download_files.datapath_train) \ + --overwrite true \ + $workspace_params + # Upload inference files + az ml datastore upload --name $datastore \ + --src-path $(download_files.datapath_inference) \ + --target-path $(download_files.datapath_inference) \ + --overwrite true \ + $workspace_params + + +- job: register_dataset + displayName: 'Register Dataset' + dependsOn: sample_files + variables: + datapath_train: $[ dependencies.sample_files.outputs['download_files.datapath_train'] ] + datapath_inference: $[ dependencies.sample_files.outputs['download_files.datapath_inference'] ] + aml_datasetname_train: $(AML_DATASET_NAME)_train + aml_datasetname_inference: $(AML_DATASET_NAME)_inference + steps: + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: 3.7 + + - task: AzureCLI@1 + displayName: 'Register dataset' + inputs: + azureSubscription: $(SERVICECONNECTION_GROUP) + scriptLocation: inlineScript + inlineScript: | + # Install dependencies + dependencies="azureml-sdk==$(SDK_VERSION)" + python -m pip install --upgrade pip && python -m pip install $dependencies + # Register train dataset + register_dataset_script=mlops-pipelines/scripts/register_or_update_dataset.py + python $register_dataset_script --path $DATAPATH_TRAIN \ + --name $AML_DATASETNAME_TRAIN \ + --subscription-id $(az account show --query id -o tsv) \ + --resource-group $(RESOURCE_GROUP) \ + --workspace-name $(AMLWORKSPACE_NAME) + # Register inference dataset + python $register_dataset_script --path $DATAPATH_INFERENCE \ + --name $AML_DATASETNAME_INFERENCE \ + --subscription-id $(az account show --query id -o tsv) \ + --resource-group $(RESOURCE_GROUP) \ + --workspace-name $(AMLWORKSPACE_NAME) diff --git a/mlops-pipelines/1-setup/deploy-infra.template.yml b/mlops-pipelines/1-setup/deploy-infra.template.yml new file mode 100644 index 00000000..ebb74b2b --- /dev/null +++ b/mlops-pipelines/1-setup/deploy-infra.template.yml @@ -0,0 +1,74 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Azure Pipeline Template for ML Workspace Resources Deployment + +jobs: + +- job: iac_build + displayName: 'IaC Build' + steps: + - task: CopyFiles@2 + displayName: 'Copy ARM templates' + inputs: + sourceFolder: 'mlops-pipelines/1-setup/arm-templates' + targetFolder: '$(Build.ArtifactStagingDirectory)' + - publish: '$(Build.ArtifactStagingDirectory)' + artifact: infratemplates + +- deployment: iac_deployment + displayName: 'IaC Deployment' + environment: $(ENVIRONMENT) + strategy: + runOnce: + deploy: + steps: + - download: current + artifact: infratemplates + + - task: AzureResourceGroupDeployment@2 + displayName: 'Deploy Storage Account' + inputs: + azureSubscription: $(SERVICECONNECTION_GROUP) + resourceGroupName: $(RESOURCE_GROUP) + location: $(LOCATION) + csmFile: '$(Pipeline.Workspace)/infratemplates/storage.template.json' + csmParametersFile: '$(Pipeline.Workspace)/infratemplates/storage.parameters.json' + overrideParameters: '-name $(STORAGEACCOUNT_NAME) -location $(LOCATION)' + + - task: AzureResourceGroupDeployment@2 + displayName: 'Deploy Key Vault' + inputs: + azureSubscription: $(SERVICECONNECTION_GROUP) + resourceGroupName: $(RESOURCE_GROUP) + location: $(LOCATION) + csmFile: '$(Pipeline.Workspace)/infratemplates/keyvault.template.json' + csmParametersFile: '$(Pipeline.Workspace)/infratemplates/keyvault.parameters.json' + overrideParameters: '-name $(KEYVAULT_NAME) -location $(LOCATION)' + + - task: AzureResourceGroupDeployment@2 + displayName: 'Deploy Application Insights' + inputs: + azureSubscription: $(SERVICECONNECTION_GROUP) + resourceGroupName: $(RESOURCE_GROUP) + location: $(LOCATION) + csmFile: '$(Pipeline.Workspace)/infratemplates/appinsights.template.json' + overrideParameters: '-name $(APPINSIGHTS_NAME) -location $(LOCATION)' + + - task: AzureResourceGroupDeployment@2 + displayName: 'Deploy Container Registry' + inputs: + azureSubscription: $(SERVICECONNECTION_GROUP) + resourceGroupName: $(RESOURCE_GROUP) + location: $(LOCATION) + csmFile: '$(Pipeline.Workspace)/infratemplates/containerregistry.template.json' + overrideParameters: '-name $(CONTAINERREGISTRY_NAME) -location $(LOCATION)' + + - task: AzureResourceGroupDeployment@2 + displayName: 'Deploy AML Workspace' + inputs: + azureSubscription: $(SERVICECONNECTION_GROUP) + resourceGroupName: $(RESOURCE_GROUP) + location: $(LOCATION) + csmFile: '$(Pipeline.Workspace)/infratemplates/mlworkspace.template.json' + overrideParameters: '-workspaceName $(AMLWORKSPACE_NAME) -keyVaultName $(KEYVAULT_NAME) -appInsightsName $(APPINSIGHTS_NAME) -containerRegistryName $(CONTAINERREGISTRY_NAME) -storageAccountName $(STORAGEACCOUNT_NAME)' diff --git a/mlops-pipelines/1-setup/environment-setup.template.yml b/mlops-pipelines/1-setup/environment-setup.template.yml new file mode 100644 index 00000000..efa7e3f8 --- /dev/null +++ b/mlops-pipelines/1-setup/environment-setup.template.yml @@ -0,0 +1,42 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Azure Pipeline Template for ML Workspace Setup + +jobs: + +- job: aml_compute + displayName: 'Deploy AML Compute' + steps: + - task: AzureResourceGroupDeployment@2 + displayName: 'Deploy AML Compute' + inputs: + azureSubscription: $(SERVICECONNECTION_GROUP) + resourceGroupName: $(RESOURCE_GROUP) + location: $(LOCATION) + csmFile: '$(Build.SourcesDirectory)/mlops-pipelines/1-setup/arm-templates/mlcompute.template.json' + csmParametersFile: '$(Build.SourcesDirectory)/mlops-pipelines/1-setup/arm-templates/mlcompute.parameters.json' + overrideParameters: '-workspaceName $(AMLWORKSPACE_NAME) -clusterName $(AML_COMPUTE_NAME)' + + +- job: aml_aks + condition: and( variables['AKS_NAME'], variables['AKS_RESOURCE_GROUP'] ) + displayName: 'Attach AKS Target to AML' + steps: + - task: AzureCLI@1 + displayName: 'Attach AKS Target to AML' + inputs: + azureSubscription: $(SERVICECONNECTION_GROUP) + scriptLocation: inlineScript + inlineScript: | + workspace_params="--workspace-name $(AMLWORKSPACE_NAME) --resource-group $(RESOURCE_GROUP)" + # Install ML extension + az extension add -n azure-cli-ml + # Check if AKS target is already attached and attach if not + az ml computetarget show --name $(AML_AKS_NAME) $workspace_params + if [ $? -eq 1 ]; then + # Get AKS resource id + aks_id=$(az resource list -n $(AKS_NAME) -g $(AKS_RESOURCE_GROUP) --query "[0].id" -o tsv) + # Attach AKS + az ml computetarget attach aks --name $(AML_AKS_NAME) --compute-resource-id $aks_id $workspace_params + fi diff --git a/mlops-pipelines/1-setup/pipeline-setup.yml b/mlops-pipelines/1-setup/pipeline-setup.yml new file mode 100644 index 00000000..24844b0a --- /dev/null +++ b/mlops-pipelines/1-setup/pipeline-setup.yml @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Pipeline that configures the environment for this solution accelerator + +# trigger: +# branches: +# include: +# - v2-preview +# paths: +# include: +# - mlops-pipelines/1-setup/* +# - mlops-pipelines/scripts/download_data.py +# - mlops-pipelines/scripts/register_or_update_dataset.py + +variables: +- template: ../configuration/many-models-variables.yml +- group: manymodels-vg + +pool: + vmImage: ubuntu-latest + +stages: +- stage: deploy_infrastructure + displayName: 'Deploy Infrastructure' + jobs: + - template: deploy-infra.template.yml + +- stage: environment_setup + displayName: 'Environment Setup' + dependsOn: deploy_infrastructure + jobs: + - template: environment-setup.template.yml + +- stage: data_preparation + displayName: 'Data Preparation' + dependsOn: deploy_infrastructure + jobs: + - template: data-prep.template.yml diff --git a/mlops-pipelines/2-modeling/Details.md b/mlops-pipelines/2-modeling/Details.md new file mode 100644 index 00000000..c4b2088c --- /dev/null +++ b/mlops-pipelines/2-modeling/Details.md @@ -0,0 +1,173 @@ +# 2 - Modeling Pipeline - Details + +## Data Update + +The data update stage is there for demonstration purposes only, as the Orange Juice dataset is not going to change. +But in a real scenario the training dataset would be updated before training to incorporate the latest observations. + +If you change the `DATASET_MAXFILES` variable in the [variable group](../#3-create-variable-group) after running the data preparation step in the previous pipeline, this step will update the training dataset with the new number of files. + +## Training + +During the training stage, three main tasks are performed: + +- Create an Azure Machine Learning Pipeline that will train many models in parallel and publish it into the AML workspace. +- Trigger the many models training by invoking the training AML Pipeline previously published. +- Register the models into the AML workspace. + +The training will be performed using the `train.py` script in the corresponding [scripts folder](../../scripts/). +If you are using the Custom Script version, you should modify that script to meet your needs. + +In both versions, AutoML and Custom Script, script settings are read from the `script_settings.json` file in the corresponding [configuration folder](../configuration/). These settings are right now based on the orange juice dataset. You can modify them if you want to use a different dataset. + +#### Customizing tags in the model registry + +When registered in the AML workspace, models are tagged with the model type ("lr" or "automl"), and the ID columns ("Store" and "Brand" in our example). + +But if you have additional columns in you dataset that categorize you model (for example, a "City" column that applies to each of the stores you have), this can be specified in the [script settings](../configuration/#customizing-script-settings) so this tag is also added to the model. To do this, we would add the following line to the `script_settings.json` file: + +``` +{ + ... + "tag_columns": ["City"], + ... +} +``` + +We do not have a "City" tag in our example dataset but have created three syntetic tags "StoreGroup10", "StoreGroup100" and "StoreGroup1000" that we use to showcase this functionality. + +#### Dropping columns from dataset before training + +Some columns in the dataset might be used for identifying or tagging the model, but shouldn't be considered in training. +If that's the case, there are three different settings in the [settings file](../configuration/#customizing-script-settings) you can use: + +- `drop_id`: to drop the ID columns before training +- `drop_tags`: to drop the tag columns before training +- `drop_columns`: for any other additional column you want to drop before training + +For example, setting these values to: + +``` +{ + ... + "drop_id": true, + "drop_tags": true, + "drop_columns": ["Revenue"], + ... +} +``` + +will drop all ID columns, all tag columns, and also the "Revenue" column. + +#### Customize ParallelRunStep configuration + +You can also modify the configuration for the ParallelRunStep, more details [here](../configuration/#customizing-parallelrunstep-config). + +## Deployment [Optional] + +Three tasks are involved in the deployment stage: + +- Group the registered models according to specified tags and maximum container size (500 by default). +- Deploy each group into a different webservice hosted in ACI and/or AKS. We call these "*model webservices*". +- [Optional] Deploy the entry point that will route the requests to the corresponding model webservice. We call this "*router webservice*". + +All *model webservices* will use the same `model_webservice.py` script in the corresponding [scripts folder](../../scripts/). + +The *router webservice* will use the `routing_webservice.py` script in the corresponding [scripts folder](../../scripts/). + +#### Enabling deployment + +Deployment of models for real-time forecasting is optional and disabled by default. +To enable it, you must add a `DEPLOY_ACI` or/and `DEPLOY_AKS` variable to the [variable group](../#3-create-variable-group) +and set them to `true`. This will trigger the corresponding deployment stage. + +Deploying in ACI (Azure Container Instances) is only recommended for development or testing pruposes, in a system in production you should use AKS (Azure Kubernetes Service) instead. In that case, you must also set variables to identify the AKS cluster you will be using. + +These are the variables you have to set if you want to enable deployment of models: + +| Variable Name | Short description | +| --------------------------- | ----------------- | +| DEPLOY_ACI | Whether to deploy in ACI (`true`/`false`, default false) | +| DEPLOY_AKS | Whether to deploy in AKS (`true`/`false`, default false) | +| AKS_NAME | [Optional, only if DEPLOY_AKS is `true`] Name of the AKS resource you'll use for deploying the models | +| AKS_RESOURCE_GROUP | [Optional, only if DEPLOY_AKS is `true`] Name of the resource group where the AKS resource is located | + +When you enable AKS deployment, make sure you run the "Environment Setup" stage of the [setup pipeline](../1-setup/) before launching this pipeline, as the AKS cluster needs to be attached to the AML workspace first. + +#### Changing the container size + +By default, models are packed in groups of 500 and deployed in a single webservice container that will pick the correct model to generate predictions based on the information provided in the request body. + +But you can change this behaviour by adding the following variable to the [variable group](../#3-create-variable-group): + +| Variable Name | Short description | +| --------------------------- | ----------------- | +| MAX_CONTAINER_SIZE | Maximum number of models to fit into one webservice container | + +Decreasing the maximum contaniner size will increase the number of webservices deployed, while increasing the size will make the models fit into less webservices. The maximum size supported for the moment is 1000. + +#### Disabling the router + +Since there is a limit in the number of models that can be deployed in the same webservice (see section above), when deploying many models we will most likely end up with several endpoints, each of them able to make predictions for different models. + +By default, a router webservice is deployed to act as an entrypoint for all the requests, forward them to the appropiate model endpoint, and return the response back to the user. Sending a GET request to the router will return the mapping table it's using. + +But you can disable the deployment of the router webservice by adding the following variable to the [variable group](../#3-create-variable-group): + +| Variable Name | Short description | +| --------------------------- | ----------------- | +| DEPLOY_ROUTING_WEBSERVICE | Set to `false` to avoid deploying the routing webservice | + +#### Customizing container configuration + +The parameters used to configure the containers (AKS or ACI) are read from files placed in the corresponding [configuration/ folder](../configuration). + +The files for the model container are: + +- AKS: `model-deployment-config-aks.yml` +- ACI: `model-deployment-config-aci.yml` + +The files for the router container are: + +- AKS: `routing-deployment-config-aks.yml` +- ACI: `routing-deployment-config-aci.yml` + +For more details on how to customize container configuration check [here](../configuration/#customizing-parallelrunstep-config). + +#### Customizing the update behaviour + +In the third stage of the pipeline, models are deployed in a *smart way*, meaning that only models that have changed since last deployment are updated. New models are deployed, new versions are updated, models that have been deleted are removed, and the rest of them remain unchanged. + +This means that if you change the deployment configuration file, these changes will only be applied in the webservices that contain new/updated models. If you want the new configuration to be applied to all webservices, you can do so by setting the `UPDATE_DEPLOYMENT` variable in the [variable group](../#3-create-variable-group). + +If you want to resize the containers to allow a higher o lower number of models per webservice, you need to reset the deployment via the `RESET_DEPLOYMENT` variable. Please notice that webservices will not be available during this redeployment operation. + +To sum up, the two variables that you can add to customize update behaviour are: + +| Variable Name | Short description | +| --------------------------- | ----------------- | +| RESET_DEPLOYMENT | Set to `true` to reset existing webservices (to resize containers) | +| UPDATE_DEPLOYMENT | Set to `true` to update all existing webservices (for config changes to apply) | + +#### Customizing grouping + +By default, models are packed in groups of fixed size following the order they have in the model registry. + +If you want to **sort them before splitting in groups** of fixed size, you can set the order criteria in the `AML_MODEL_SORTING_TAGS` variable defined in the [`many-models-variables.yml`](../configuration/many-models-variables.yml) file. +For example, adding the following: + +``` + - name: AML_MODEL_SORTING_TAGS + - value: Store +``` + +would sort the models by store before splitting, ensuring that in most cases, the three models for each store (corresponding to the three different brands) would fall into the same webservice. + +If you want to **split them by any particular criteria** you can specify that as well in the [`many-models-variables.yml`](../configuration/many-models-variables.yml) file, using the `AML_MODEL_SPLITTING_TAGS` variable. For example, if you had a "City" tag, adding the following: + +``` + - name: AML_MODEL_SORTING_TAGS + - value: City +``` + +would make each webservice contain models belonging to one specific city. We do not have a "City" tag in our example dataset but have created three syntetic tags "StoreGroup10", "StoreGroup100" and "StoreGroup1000" that group stores in groups of 10, 100 and 1000 that you can use if you want to test this functionality. diff --git a/mlops-pipelines/2-modeling/README.md b/mlops-pipelines/2-modeling/README.md new file mode 100644 index 00000000..934df409 --- /dev/null +++ b/mlops-pipelines/2-modeling/README.md @@ -0,0 +1,58 @@ +# 2 - Modeling Pipeline + +The modeling pipeline will: + +- Update the training dataset with the latest version of data. +- Train and register the models. +- [Optional] Deploy the models into webservices ready to do real-time forecasting upon request. + +## Instructions + +Before creating the Azure DevOps pipeline: + +1. Make sure the [AML extension](https://marketplace.visualstudio.com/items?itemName=ms-air-aiagility.vss-services-azureml) is installed in the Azure DevOps organization. + +2. Create an **Azure Resource Manager** [service connection](https://docs.microsoft.com/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection) to access the Machine Learning Workspace that was created when you ran the [setup pipeline](../1-setup/). As you did [when you created a service connection before](../#1-create-service-connection), mark the option to grant access permission to all pipelines, and copy the name as you'll need it in the next step. + +3. Modify the **``manymodels-vg``** [variable group you created before](../#3-create-variable-group), and add two new variables: + +| Variable Name | Short description | +| --------------------------- | ----------------- | +| SERVICECONNECTION_WORKSPACE | Name of the connection to the AML Workspace you have just created | +| TRAINING_METHOD | "automl" or "customscript" depending which method you want to use for training the models | + +Then, create the pipeline as you did in the [setup pipeline](../1-setup/), selecting branch **``v2-preview``** and setting the path to [`/mlops-pipelines/2-modeling/pipeline-modeling.yml`](pipeline-modeling.yml). + +Run the pipeline and wait for it to finish. + +## Result + +The pipeline run should look like this (deployment stages will have run or not depending on your configuration, read more [here](Details.md#enabling-deployment)): + +Stages and jobs as described below + +Containing the following stages and jobs: + +- Update Data for Training + - Download New Sample Files + - Update Registered Training Dataset +- Run Model Training + - Check Training Method + - Publish Training AML Pipeline + - Get Training Pipeline ID + - Run Training +- Deploy Models to ACI [Optional] + - Deploy Models + - Register Routing Model [Optional] + - Deploy Routing Webservice [Optional] +- Deploy Models to AKS [Optional] + - Deploy Models + - Register Routing Model [Optional] + - Deploy Routing Webservice [Optional] + +## Details + +If you want to learn about the details and ways to customize the modeling pipeline please read the [details page](Details.md). diff --git a/Custom_Script/mlops-pipelines/3-modeling/deploy-models/deploy-models.template.yml b/mlops-pipelines/2-modeling/deploy-models.template.yml similarity index 60% rename from Custom_Script/mlops-pipelines/3-modeling/deploy-models/deploy-models.template.yml rename to mlops-pipelines/2-modeling/deploy-models.template.yml index d94a6a98..702b0817 100644 --- a/Custom_Script/mlops-pipelines/3-modeling/deploy-models/deploy-models.template.yml +++ b/mlops-pipelines/2-modeling/deploy-models.template.yml @@ -4,34 +4,14 @@ # Azure Pipeline Template for deploying models to ACI parameters: -- name: sdkVersion - type: string -- name: serviceConnection - type: string -- name: resourceGroup - type: string -- name: amlWorkspaceName - type: string - name: deploymentType type: string - name: amlAksName type: string -- name: sortingTags - type: string -- name: splittingTags - type: string - name: webservicePrefix type: string -- name: containerSize - type: string -- name: resetContainers - type: string - name: routingModelName type: string -- name: routingModelTagName - type: string -- name: routingModelTagValue - type: string - name: routingServiceName type: string @@ -51,28 +31,31 @@ jobs: - task: AzureCLI@1 displayName: 'Deploy Models in Groups' inputs: - azureSubscription: ${{parameters.serviceConnection}} + azureSubscription: $(SERVICECONNECTION_WORKSPACE) scriptLocation: inlineScript inlineScript: | # Install dependencies - python -m pip install --upgrade pip && python -m pip install pyyaml azureml-sdk==${{parameters.sdkVersion}} + dependencies="pyyaml azureml-sdk==$(SDK_VERSION)" + python -m pip install --upgrade pip && python -m pip install $dependencies # Deploy/update models in groups in webservices - deployment_config_path="Custom_Script/mlops-pipelines/3-modeling/deploy-models/" - deployment_config_file="$deployment_config_path/forecasting-deployment-config-${{parameters.deploymentType}}.yml" - deploy_script="Custom_Script/mlops-pipelines/scripts/deploy_or_update_models.py" - if [ "${{parameters.resetContainers}}" == "true" ]; then reset="--reset"; fi - python $deploy_script --splitting-tags '${{parameters.splittingTags}}' \ - --sorting-tags '${{parameters.sortingTags}}' \ + deployment_config_path="mlops-pipelines/configuration/$(TRAINING_METHOD)/" + deployment_config_file="$deployment_config_path/model-deployment-config-${{parameters.deploymentType}}.yml" + deploy_script="mlops-pipelines/scripts/deploy_or_update_models.py" + if [ "$(RESET_DEPLOYMENT)" == "true" ]; then reset="--reset"; fi + if [ "$(UPDATE_DEPLOYMENT)" == "true" ]; then update="--update"; fi + python $deploy_script --splitting-tags '$(AML_MODEL_SPLITTING_TAGS)' \ + --sorting-tags '$(AML_MODEL_SORTING_TAGS)' \ + --scripts-dir 'scripts/$(TRAINING_METHOD)/' \ --routing-model-name '${{parameters.routingModelName}}' \ --output 'models_deployed.json' \ --deploy-config-file $deployment_config_file \ --aks-target '${{parameters.amlAksName}}' \ --service-prefix '${{parameters.webservicePrefix}}' \ - --container-size '${{parameters.containerSize}}' \ - $reset \ + --container-size '$(MAX_CONTAINER_SIZE)' \ + $reset $update \ --subscription-id $(az account show --query id -o tsv) \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} + --resource-group $(RESOURCE_GROUP) \ + --workspace-name $(AMLWORKSPACE_NAME) - publish: 'models_deployed.json' artifact: modelsdeployed-${{parameters.deploymentType}} @@ -82,6 +65,7 @@ jobs: - job: routing_model displayName: 'Register Routing Model' dependsOn: deploy_models + condition: and(succeeded(), in(variables['DEPLOY_ROUTING_WEBSERVICE'], '', 'true')) steps: - download: current @@ -90,7 +74,7 @@ jobs: - task: AzureCLI@1 displayName: 'Register Routing Model' inputs: - azureSubscription: ${{parameters.serviceConnection}} + azureSubscription: $(SERVICECONNECTION_WORKSPACE) scriptLocation: inlineScript inlineScript: | # Install ML extension @@ -98,14 +82,14 @@ jobs: # Register model az ml model register --name ${{parameters.routingModelName}} \ --model-path '$(Pipeline.Workspace)/modelsdeployed-${{parameters.deploymentType}}/models_deployed.json' \ - --tag '${{parameters.routingModelTagName}}=${{parameters.routingModelTagValue}}' \ + --tag '$(AML_ROUTING_MODEL_TAG_NAME)=$(AML_ROUTING_MODEL_TAG_VALUE)' \ --output-metadata-file routing_model.json \ - --workspace-name ${{parameters.amlWorkspaceName}} \ - --resource-group ${{parameters.resourceGroup}} + --workspace-name $(AMLWORKSPACE_NAME) \ + --resource-group $(RESOURCE_GROUP) - publish: 'routing_model.json' artifact: routingmodel-${{parameters.deploymentType}} - displayName: 'Publish Routing Model Artifact' + displayName: 'Artifact: Routing Model' - job: routing_webservice @@ -121,19 +105,21 @@ jobs: - task: AzureCLI@1 displayName: 'Deploy Routing Webservice' inputs: - azureSubscription: ${{parameters.serviceConnection}} + azureSubscription: $(SERVICECONNECTION_WORKSPACE) scriptLocation: inlineScript inlineScript: | # Install dependencies - python -m pip install --upgrade pip && python -m pip install pyyaml azureml-sdk==${{parameters.sdkVersion}} + dependencies="pyyaml azureml-sdk==$(SDK_VERSION)" + python -m pip install --upgrade pip && python -m pip install $dependencies # Deploy/update routing webservice - deployment_config_path="Custom_Script/mlops-pipelines/3-modeling/deploy-models/" + deployment_config_path="mlops-pipelines/configuration/$(TRAINING_METHOD)/" deployment_config_file="$deployment_config_path/routing-deployment-config-${{parameters.deploymentType}}.yml" - deploy_script="Custom_Script/mlops-pipelines/scripts/deploy_or_update_router.py" + deploy_script="mlops-pipelines/scripts/deploy_or_update_router.py" python $deploy_script --model-name '${{parameters.routingModelName}}' \ --service-name '${{parameters.routingServiceName}}' \ + --scripts-dir 'scripts/$(TRAINING_METHOD)/' \ --deploy-config-file $deployment_config_file \ --aks-target '${{parameters.amlAksName}}' \ --subscription-id $(az account show --query id -o tsv) \ - --resource-group ${{parameters.resourceGroup}} \ - --workspace-name ${{parameters.amlWorkspaceName}} + --resource-group $(RESOURCE_GROUP) \ + --workspace-name $(AMLWORKSPACE_NAME) diff --git a/mlops-pipelines/2-modeling/pipeline-modeling.yml b/mlops-pipelines/2-modeling/pipeline-modeling.yml new file mode 100644 index 00000000..eb4010b8 --- /dev/null +++ b/mlops-pipelines/2-modeling/pipeline-modeling.yml @@ -0,0 +1,76 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Pipeline that runs AML Pipeline for model training + +# trigger: +# branches: +# include: +# - v2-preview +# paths: +# include: +# - mlops-pipelines/2-modeling/* +# - mlops-pipelines/scripts/download_data.py +# - mlops-pipelines/scripts/register_or_update_dataset.py +# - mlops-pipelines/scripts/create_training_pipeline.py +# - mlops-pipelines/scripts/deploy_or_update_models.py +# - scripts/*/train.py +# - scripts/*/train.conda.yml +# - scripts/*/model_webservice.py +# - scripts/*/model_webservice.conda.yml +# - scripts/*/routing_webservice.py +# - scripts/*/routing_webservice.conda.yml + +# schedules: +# - cron: "0 0 * * 1" +# displayName: 'Weekly training at midnight on Mondays' +# branches: +# include: +# - v2-preview +# always: true + +variables: +- template: ../configuration/many-models-variables.yml +- group: manymodels-vg + +pool: + vmImage: ubuntu-latest + +stages: + +- stage: update_data + displayName: 'Update Data for Training/Retraining' + jobs: + - template: update-data.template.yml + +- stage: run_training + displayName: 'Run Model Training' + dependsOn: update_data + jobs: + - template: run-training.template.yml + +- stage: deploy_models_aci + displayName: 'Deploy Models to ACI' + dependsOn: run_training + condition: and(succeeded(), eq(variables['DEPLOY_ACI'], 'true')) + jobs: + - template: deploy-models.template.yml + parameters: + deploymentType: aci + amlAksName: '' + webservicePrefix: test-'$(AML_WEBSERVICE_PREFIX)' + routingModelName: test-'$(AML_ROUTING_MODEL_NAME)' + routingServiceName: test-'$(AML_ROUTING_WEBSERVICE)' + +- stage: deploy_models_aks + displayName: 'Deploy Models to AKS' + dependsOn: run_training + condition: and(succeeded(), eq(variables['DEPLOY_AKS'], 'true'), variables['AML_AKS_NAME']) + jobs: + - template: deploy-models.template.yml + parameters: + deploymentType: aks + amlAksName: '$(AML_AKS_NAME)' + webservicePrefix: '$(AML_WEBSERVICE_PREFIX)' + routingModelName: '$(AML_ROUTING_MODEL_NAME)' + routingServiceName: '$(AML_ROUTING_WEBSERVICE)' diff --git a/mlops-pipelines/2-modeling/run-training.template.yml b/mlops-pipelines/2-modeling/run-training.template.yml new file mode 100644 index 00000000..63f6c2ab --- /dev/null +++ b/mlops-pipelines/2-modeling/run-training.template.yml @@ -0,0 +1,89 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Azure Pipeline Template for running the training pipeline + +jobs: + +- job: check_training_method + displayName: 'Check Training Method' + steps: + - bash: | + if [ "$(TRAINING_METHOD)" != "automl" ] && [ "$(TRAINING_METHOD)" != "customscript" ]; then + >&2 echo "Error: variable TRAINING_METHOD must be set to 'automl' or 'customscript'" + fi + displayName: 'Check Training Method' + failOnStderr: true + + +- job: publish_training_pipeline + displayName: 'Publish Training AML Pipeline' + dependsOn: check_training_method + steps: + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: 3.7 + + - task: AzureCLI@1 + displayName: 'Publish Training AML Pipeline' + inputs: + azureSubscription: $(SERVICECONNECTION_WORKSPACE) + scriptLocation: inlineScript + inlineScript: | + # Install dependencies + dependencies="pyyaml azureml-sdk==$(SDK_VERSION)" + python -m pip install --upgrade pip && python -m pip install $dependencies + # Create/update training pipeline + trainingbuild_script=mlops-pipelines/scripts/create_training_pipeline.py + python $trainingbuild_script --name $(AML_TRAINING_PIPELINE_NAME) \ + --version $(Build.BuildId) \ + --scripts-dir 'scripts/$(TRAINING_METHOD)/' \ + --scripts-settings 'mlops-pipelines/configuration/$(TRAINING_METHOD)/script_settings.json' \ + --prs-config 'mlops-pipelines/configuration/$(TRAINING_METHOD)/train-parallelrunstep-config.yml' \ + --dataset $(AML_DATASET_NAME)_train \ + --compute $(AML_COMPUTE_NAME) \ + --artifact 'training_pipeline_id.txt' \ + --subscription-id $(az account show --query id -o tsv) \ + --resource-group $(RESOURCE_GROUP) \ + --workspace-name $(AMLWORKSPACE_NAME) + + - publish: 'training_pipeline_id.txt' + artifact: training_pipeline_id + displayName: 'Publish Training Pipeline ID' + + +- job: pipeline_id + displayName: 'Get Training Pipeline ID' + dependsOn: publish_training_pipeline + steps: + + - download: current + artifact: training_pipeline_id + + - bash: | + # Get pipeline ID + artifact='$(Pipeline.Workspace)/training_pipeline_id/training_pipeline_id.txt' + pipeline_id=$(cat $artifact) + echo "##vso[task.setvariable variable=pipeline_id;isOutput=true;]$pipeline_id" + name: get_pipeline_id + displayName: 'Get Training Pipeline ID' + failOnStderr: true + + +- job: run_training + displayName: 'Run Training' + pool: server + timeoutInMinutes: 0 + dependsOn: pipeline_id + variables: + pipeline_id: $[ dependencies.pipeline_id.outputs['get_pipeline_id.pipeline_id'] ] + steps: + + - task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0 + displayName: 'Invoke AML Training Pipeline' + inputs: + azureSubscription: '$(SERVICECONNECTION_WORKSPACE)' + PipelineId: '$(PIPELINE_ID)' + ExperimentName: '$(AML_TRAINING_PIPELINE_NAME)' diff --git a/mlops-pipelines/2-modeling/update-data.template.yml b/mlops-pipelines/2-modeling/update-data.template.yml new file mode 100644 index 00000000..e91d5a8b --- /dev/null +++ b/mlops-pipelines/2-modeling/update-data.template.yml @@ -0,0 +1,80 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Azure Pipeline Template for updating training dataset +# The dataset in this example is updated for illustration purposes only, the data downloaded is the same + +jobs: + +- job: download_new_data + displayName: 'Download New Sample Files' + steps: + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: 3.7 + + - bash: | + # Install dependencies + dependencies="azureml-opendatasets==$(SDK_VERSION)" + python -m pip install --upgrade pip && python -m pip install $dependencies + # Download sample files + data_script="mlops-pipelines/scripts/download_data.py" + python $data_script --maxfiles $(DATASET_MAXFILES) --train-path "new-sample-data-train" --inference-path "new-sample-data-inference" + + name: download_files + displayName: 'Download Sample Files' + failOnStderr: true + + - task: AzureCLI@1 + name: upload_files + displayName: 'Upload trainimg files to AML datastore' + inputs: + azureSubscription: $(SERVICECONNECTION_WORKSPACE) + scriptLocation: inlineScript + inlineScript: | + workspace_params="--workspace-name $(AMLWORKSPACE_NAME) --resource-group $(RESOURCE_GROUP)" + # Install ML extension + az extension add -n azure-cli-ml + # Get default datastore + datastore=$(az ml datastore show-default $workspace_params --query name -o tsv) + # Upload train files + datapath_train=new-sample-data-$(date +"%Y%m%d-%H%M%S")-train + az ml datastore upload --name $datastore \ + --src-path "new-sample-data-train" \ + --target-path $datapath_train \ + --overwrite true \ + $workspace_params + echo "##vso[task.setvariable variable=datapath_train;isOutput=true;]$datapath_train" + + +- job: update_dataset + displayName: 'Update Registered Training Dataset' + dependsOn: download_new_data + variables: + datapath_train: $[ dependencies.download_new_data.outputs['upload_files.datapath_train'] ] + aml_datasetname_train: $(AML_DATASET_NAME)_train + steps: + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: 3.7 + + - task: AzureCLI@1 + displayName: 'Update training dataset' + inputs: + azureSubscription: $(SERVICECONNECTION_WORKSPACE) + scriptLocation: inlineScript + inlineScript: | + # Install dependencies + dependencies="azureml-sdk==$(SDK_VERSION)" + python -m pip install --upgrade pip && python -m pip install $dependencies + # Update train dataset + update_dataset_script=mlops-pipelines/scripts/register_or_update_dataset.py + python $update_dataset_script --path $DATAPATH_TRAIN \ + --name $AML_DATASETNAME_TRAIN \ + --subscription-id $(az account show --query id -o tsv) \ + --resource-group $(RESOURCE_GROUP) \ + --workspace-name $(AMLWORKSPACE_NAME) diff --git a/mlops-pipelines/3-batch-forecasting/Details.md b/mlops-pipelines/3-batch-forecasting/Details.md new file mode 100644 index 00000000..b66d27b7 --- /dev/null +++ b/mlops-pipelines/3-batch-forecasting/Details.md @@ -0,0 +1,36 @@ +# 3 - Batch Forecasting Pipeline - Details + +## Data Update + +The data update stage is there for demonstration purposes only, as the Orange Juice dataset is not going to change. +But in a real scenario the dataset would be updated before launching batch forecasting to incorporate the latest observations and make predictions into the future. + +If you change the `DATASET_MAXFILES` variable in the [variable group](../#3-create-variable-group) after running the data preparation step in the previous pipeline, this step will update the inference dataset with the new number of files, but will fail to make forecasts for some models if no training has been run for these new files. + +## Batch Forecasting + +During the forecasting stage, three main tasks are performed: + +- Create an Azure Machine Learning Pipeline that will generate batch forecasts for all the models in parallel, and publish it into the AML workspace. +- Trigger the many models batch forecasting by invoking the batch forecasting AML Pipeline previously published. +- Store the predictions in a separate container. + +The predictions will be generated using the `batch_forecasting.py` script in the corresponding [scripts folder](../../scripts/). +If you are using the Custom Script version, you should modify that script to meet your needs. + +In both versions, AutoML and Custom Script, script settings are read from the `script_settings.json` file in the corresponding [configuration folder](../configuration/). These settings are right now based on the orange juice dataset. You can modify them if you want to use a different dataset. + +#### Customizing name of the predictions container + +The name of the container where the predictions will be stored is defined in the [`many-models-variables.yml`](../configuration/many-models-variables.yml) file and it's set to "predictions" right now: + +``` + - name: PREDICTIONS_CONTAINER_NAME + value: predictions +``` + +You can modify that file if you want a different name for that container. + +#### Customize ParallelRunStep configuration + +You can also modify the configuration for the ParallelRunStep, more details [here](../configuration/#customizing-parallelrunstep-config). diff --git a/mlops-pipelines/3-batch-forecasting/README.md b/mlops-pipelines/3-batch-forecasting/README.md new file mode 100644 index 00000000..a512232e --- /dev/null +++ b/mlops-pipelines/3-batch-forecasting/README.md @@ -0,0 +1,36 @@ +# 3 - Batch Forecasting Pipeline + +The batch forecasting pipeline will: + +- Update the inference dataset with the latest version of data. +- Launch batch forecasting and create batch predictions for all the models. + +## Instructions + +Just create the pipeline as you did before in the [setup pipeline](../1-setup/) and [modeling pipeline](../2-modeling/), selecting branch **``v2-preview``** and setting the path to [`/mlops-pipelines/3-batch-forecasting/pipeline-batch-forecasting.yml`](pipeline-batch-forecasting.yml). + +Then, run the pipeline and wait for it to finish. + +## Result + +The pipeline run should look like this: + +Stages and jobs as described below + +Containing the following stages and jobs: + +- Update Data for Batch Forecasting + - Download New Sample Files + - Update Registered Inference Dataset +- Run Model Forecasting + - Check Training Method + - Publish Forecasting AML Pipeline + - Get Forecasting Pipeline ID + - Run Forecasting + +## Details + +If you want to learn about the details and ways to customize the batch forecasting pipeline please read the [details page](Details.md). diff --git a/mlops-pipelines/3-batch-forecasting/pipeline-batch-forecasting.yml b/mlops-pipelines/3-batch-forecasting/pipeline-batch-forecasting.yml new file mode 100644 index 00000000..0ce705db --- /dev/null +++ b/mlops-pipelines/3-batch-forecasting/pipeline-batch-forecasting.yml @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Pipeline that runs AML Pipeline for model training + +# trigger: +# branches: +# include: +# - v2-preview +# paths: +# include: +# - mlops-pipelines/3-batch-forecasting/* +# - mlops-pipelines/scripts/download_data.py +# - mlops-pipelines/scripts/register_or_update_dataset.py +# - mlops-pipelines/scripts/create_forecasting_pipeline.py +# - scripts/*/batch_forecasting.py +# - scripts/*/forecast.conda.yml +# - scripts/*/copy_predictions.py + +# schedules: +# - cron: "30 7 * * *" +# displayName: 'Daily forecasting at 07:30 AM' +# branches: +# include: +# - v2-preview +# always: true + +variables: +- template: ../configuration/many-models-variables.yml +- group: manymodels-vg + +pool: + vmImage: ubuntu-latest + +stages: + +- stage: update_data + displayName: 'Update Data for Batch Forecasting' + jobs: + - template: update-data.template.yml + +- stage: run_forecasting + displayName: 'Run Model Forecasting' + jobs: + - template: run-forecasting.template.yml diff --git a/mlops-pipelines/3-batch-forecasting/run-forecasting.template.yml b/mlops-pipelines/3-batch-forecasting/run-forecasting.template.yml new file mode 100644 index 00000000..409ae8bf --- /dev/null +++ b/mlops-pipelines/3-batch-forecasting/run-forecasting.template.yml @@ -0,0 +1,90 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Azure Pipeline Template for running the forecasting pipeline + +jobs: + +- job: check_training_method + displayName: 'Check Training Method' + steps: + - bash: | + if [ "$(TRAINING_METHOD)" != "automl" ] && [ "$(TRAINING_METHOD)" != "customscript" ]; then + >&2 echo "Error: variable TRAINING_METHOD must be set to 'automl' or 'customscript'" + fi + displayName: 'Check Training Method' + failOnStderr: true + + +- job: publish_forecasting_pipeline + displayName: 'Publish Forecasting AML Pipeline' + dependsOn: check_training_method + steps: + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: 3.7 + + - task: AzureCLI@1 + displayName: 'Publish Forecasting AML Pipeline' + inputs: + azureSubscription: $(SERVICECONNECTION_WORKSPACE) + scriptLocation: inlineScript + inlineScript: | + # Install dependencies + dependencies="pyyaml azureml-sdk==$(SDK_VERSION)" + python -m pip install --upgrade pip && python -m pip install $dependencies + # Create/update forecasting pipeline + forecastingbuild_script=mlops-pipelines/scripts/create_forecasting_pipeline.py + python $forecastingbuild_script --name $(AML_FORECASTING_PIPELINE_NAME) \ + --version $(Build.BuildId) \ + --scripts-dir 'scripts/$(TRAINING_METHOD)/' \ + --scripts-settings 'mlops-pipelines/configuration/$(TRAINING_METHOD)/script_settings.json' \ + --prs-config 'mlops-pipelines/configuration/$(TRAINING_METHOD)/forecast-parallelrunstep-config.yml' \ + --dataset $(AML_DATASET_NAME)_inference \ + --output $(PREDICTIONS_CONTAINER_NAME) \ + --compute $(AML_COMPUTE_NAME) \ + --artifact 'forecasting_pipeline_id.txt' \ + --subscription-id $(az account show --query id -o tsv) \ + --resource-group $(RESOURCE_GROUP) \ + --workspace-name $(AMLWORKSPACE_NAME) + + - publish: 'forecasting_pipeline_id.txt' + artifact: forecasting_pipeline_id + displayName: 'Publish Forecasting Pipeline ID' + + +- job: pipeline_id + displayName: 'Get Forecasting Pipeline ID' + dependsOn: publish_forecasting_pipeline + steps: + + - download: current + artifact: forecasting_pipeline_id + + - bash: | + # Get pipeline ID + artifact='$(Pipeline.Workspace)/forecasting_pipeline_id/forecasting_pipeline_id.txt' + pipeline_id=$(cat $artifact) + echo "##vso[task.setvariable variable=pipeline_id;isOutput=true;]$pipeline_id" + name: get_pipeline_id + displayName: 'Get Forecasting Pipeline ID' + failOnStderr: true + + +- job: run_forecasting + displayName: 'Run Forecasting' + pool: server + timeoutInMinutes: 0 + dependsOn: pipeline_id + variables: + pipeline_id: $[ dependencies.pipeline_id.outputs['get_pipeline_id.pipeline_id'] ] + steps: + + - task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0 + displayName: 'Invoke AML Forecasting Pipeline' + inputs: + azureSubscription: '$(SERVICECONNECTION_WORKSPACE)' + PipelineId: '$(PIPELINE_ID)' + ExperimentName: '$(AML_FORECASTING_PIPELINE_NAME)' diff --git a/mlops-pipelines/3-batch-forecasting/update-data.template.yml b/mlops-pipelines/3-batch-forecasting/update-data.template.yml new file mode 100644 index 00000000..3ac33cf4 --- /dev/null +++ b/mlops-pipelines/3-batch-forecasting/update-data.template.yml @@ -0,0 +1,81 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Azure Pipeline Template for updating training dataset +# The dataset in this example is updated for illustration purposes only, the data downloaded is the same + + +jobs: + +- job: download_new_data + displayName: 'Download New Sample Files' + steps: + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: 3.7 + + - bash: | + # Install dependencies + dependencies="azureml-opendatasets==$(SDK_VERSION)" + python -m pip install --upgrade pip && python -m pip install $dependencies + # Download sample files + data_script="mlops-pipelines/scripts/download_data.py" + python $data_script --maxfiles $(DATASET_MAXFILES) --train-path "new-sample-data-train" --inference-path "new-sample-data-inference" + + name: download_files + displayName: 'Download Sample Files' + failOnStderr: true + + - task: AzureCLI@1 + name: upload_files + displayName: 'Upload inference files to AML datastore' + inputs: + azureSubscription: $(SERVICECONNECTION_WORKSPACE) + scriptLocation: inlineScript + inlineScript: | + workspace_params="--workspace-name $(AMLWORKSPACE_NAME) --resource-group $(RESOURCE_GROUP)" + # Install ML extension + az extension add -n azure-cli-ml + # Get default datastore + datastore=$(az ml datastore show-default $workspace_params --query name -o tsv) + # Upload inference files + datapath_inference=new-sample-data-$(date +"%Y%m%d-%H%M%S")-inference + az ml datastore upload --name $datastore \ + --src-path "new-sample-data-inference" \ + --target-path $datapath_inference \ + --overwrite true \ + $workspace_params + echo "##vso[task.setvariable variable=datapath_inference;isOutput=true;]$datapath_inference" + + +- job: update_dataset + displayName: 'Update Registered Inference Dataset' + dependsOn: download_new_data + variables: + datapath_inference: $[ dependencies.download_new_data.outputs['upload_files.datapath_inference'] ] + aml_datasetname_inference: $(AML_DATASET_NAME)_inference + steps: + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.7' + inputs: + versionSpec: 3.7 + + - task: AzureCLI@1 + displayName: 'Update inference dataset' + inputs: + azureSubscription: $(SERVICECONNECTION_WORKSPACE) + scriptLocation: inlineScript + inlineScript: | + # Install dependencies + dependencies="azureml-sdk==$(SDK_VERSION)" + python -m pip install --upgrade pip && python -m pip install $dependencies + # Update inference dataset + update_dataset_script=mlops-pipelines/scripts/register_or_update_dataset.py + python $update_dataset_script --path $DATAPATH_INFERENCE \ + --name $AML_DATASETNAME_INFERENCE \ + --subscription-id $(az account show --query id -o tsv) \ + --resource-group $(RESOURCE_GROUP) \ + --workspace-name $(AMLWORKSPACE_NAME) diff --git a/mlops-pipelines/Details.md b/mlops-pipelines/Details.md new file mode 100644 index 00000000..1ce9e935 --- /dev/null +++ b/mlops-pipelines/Details.md @@ -0,0 +1,40 @@ +# Many Models MLOps Pipelines - Details + +There are several things that can be customized in the Many Models Solution Accelerator. + +### Modifying training / forecasting code + +You may want to modify the training and forecasting scripts, specially if you are training using the Custom Script option. + +All these scripts are located in [`scripts/`](../scripts/) folder in the repository root. + +### Configure settings + +If you want to configure the compute targets, specify the specifics of your dataset or customize the names of the AML artifacts, +you should go to the [`configuration/`](configuration/) folder inside the MLOps Pipelines section. + +### Configure Azure resources + +If you want to make changes to the Azure resources deployed, check [`arm-templates/`](1-setup/arm-templates). + +### Variable group + +There are many additional variables that can be added to the variable group. All of them are properly explained in the specific pipelines' folders, but below you can find a summary of all the variables supported: + +| Variable Name | Short description | +| --------------------------- | ----------------- | +| RESOURCE_GROUP | Name of the Azure Resource Group that you'll be using | +| SERVICECONNECTION_GROUP | Name of the connection to the resource group | +| LOCATION | [Azure location](https://azure.microsoft.com/global-infrastructure/locations/), no spaces | +| NAMESPACE | Unique naming prefix for created resources, to avoid name collisions | +| DATASET_MAXFILES | Number of sample files to use (1 file = 1 model) | +| SERVICECONNECTION_WORKSPACE | Name of the connection to the AML Workspace | +| TRAINING_METHOD | "automl" or "customscript" depending which method you want to use for training the models | +| DEPLOY_ACI | [Optional] Whether to deploy in ACI (`true`/`false`, default false) | +| DEPLOY_AKS | [Optional] Whether to deploy in AKS (`true`/`false`, default false) | +| AKS_NAME | [Optional, only if DEPLOY_AKS is `true`] Name of the AKS resource you'll use for deploying the models | +| AKS_RESOURCE_GROUP | [Optional, only if DEPLOY_AKS is `true`] Name of the resource group where the AKS resource is located | +| MAX_CONTAINER_SIZE | [Optional] Maximum number of models to fit into one webservice container (default 500) | +| DEPLOY_ROUTING_WEBSERVICE | [Optional] Set to `false` to avoid deploying the routing webservice | +| RESET_DEPLOYMENT | [Optional] Set to `true` to reset existing webservices (to resize containers) | +| UPDATE_DEPLOYMENT | [Optional] Set to `true` to update all existing webservices (for config changes to apply) | diff --git a/mlops-pipelines/README.md b/mlops-pipelines/README.md new file mode 100644 index 00000000..ec85cd8a --- /dev/null +++ b/mlops-pipelines/README.md @@ -0,0 +1,45 @@ +# Many Models MLOps Pipelines + +You'll use Azure DevOps for running the MLOps pipelines. Follow the steps below to set them up. + +## 1. Create Many Models project in Azure DevOps + +Create an [organization](https://docs.microsoft.com/azure/devops/organizations/accounts/create-organization?view=azure-devops#create-an-organization) and a [project](https://docs.microsoft.com/azure/devops/organizations/projects/create-project?view=azure-devops&tabs=preview-page#create-a-project) for the Many Models Solution Accelerator. + +## 2. Create service connection + +Next, create an **Azure Resource Manager** [service connection](https://docs.microsoft.com/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#create-a-service-connection) to access the subscription and resource group where you plan to deploy the Many Models solution. The resource group should have already been created. + +Mark the option to grant access permission to all pipelines. + +Choose any name you want and copy it, as you'll need it in the next step. + +## 3. Create variable group + +The last step you need to do before creating the pipelines is creating a variable group in Azure DevOps. Instructions on how to create a variable group are [here](https://docs.microsoft.com/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=classic#create-a-variable-group). + +Call it **``manymodels-vg``**, and add the following variables: + +| Variable Name | Short description | +| --------------------------- | ----------------- | +| RESOURCE_GROUP | Name of the Azure Resource Group that you'll be using (should be already created) | +| SERVICECONNECTION_GROUP | Name of the connection you created in the [last step](#1-create-service-connection) | +| LOCATION | [Azure location](https://azure.microsoft.com/global-infrastructure/locations/), no spaces | +| NAMESPACE | Unique naming prefix for created resources, to avoid name collisions | +| DATASET_MAXFILES | Number of sample files to use (1 file = 1 model) | + +## 3. Create pipelines + +There are three pipelines in the Many Models Solution Accelerator: + +1. [Setup Pipeline](1-setup/): set up workspace and data +2. [Modeling Pipeline](2-modeling/): training & optionally deploying all the models +3. [Batch Forecasting Pipeline](3-batch-forecasting/): generate batch predictions for all the models + +Each pipeline is a YAML file that contains all the steps necessary to achieve each mision. The process for creating a pipeline from a YAML file in Azure DevOps is explained [here](https://github.com/microsoft/MLOpsPython/blob/master/docs/getting_started.md#create-the-iac-pipeline). + +Navigate into each pipeline's folder for specific instructions on how to set them up. + +## [Optional] Customizing the pipelines + +There are several things that can be customized in the Many Models Solution Accelerator. Please read the [details page](Details.md) if you want to find out more. diff --git a/mlops-pipelines/configuration/README.md b/mlops-pipelines/configuration/README.md new file mode 100644 index 00000000..f0f1ac87 --- /dev/null +++ b/mlops-pipelines/configuration/README.md @@ -0,0 +1,73 @@ +# MLOps Pipelines Configuration + +This folder contains all the configuration files that are used inside the MLOps pipelines. + +To customize the names of the AML artifacts, check `many-models-variables.yml`(many-models-variables.yml). + +The rest of the parameters depend on the training method you are using: Automated ML or Custom Script. If you navigate to the corresponding folder you will find the following files: + +- `script_settings.json`: settings for the training/forecasting scripts in the many models [scripts/](../../scripts/) folder. +- `train-parallelrunstep-config.yml`: configuration for the training ParallelRunStep. +- `forecast-parallelrunstep-config.yml`: configuration for the forecasting ParallelRunStep. +- `model-deployment-config-aks.yml`: configuration for the model webservice in AKS. +- `model-deployment-config-aci.yml`: configuration for the model webservice in ACI. +- `routing-deployment-config-aks.yml`: configuration for the routing webservice in AKS. +- `routing-deployment-config-aci.yml`: configuration for the routing webservice in ACI. + +## Customizing script settings + +The training and forecasting scripts many models (located in the corresponding [scripts/](../../scripts/) folder) receive a settings file as a parameter that contain all the relevant settings related to the dataset format and the training configuration. + +This file is called `script_settings.json`, and the contents are different depending on the training method you are using: Automated ML or Custom Script. + +## Customizing ParallelRunStep config + +All parameters defined in the `-parallelrunstep-config.yml` files are passed directly to the [`ParallelRunConfig`](https://docs.microsoft.com/python/api/azureml-pipeline-steps/azureml.pipeline.steps.parallelrunconfig) build during the creation of the AML Pipeline. + +There you can specify parameters like: + +- `mini_batch_size` +- `run_invocation_timeout` +- `error_threshold` +- `process_count_per_node` +- `node_count` + +## Customizing container configuration + +All four `-deployment-config-` files follow the same pattern. There are three main sections in the YAML file: + +- `computeType`: can be `AKS` or `ACI`. +- `containerResourceRequirements`: parameters that will be used to configure the compute. +- `environmentVariables`: additional environment variables that will be set in the container. These can be used to enable concurrency in the container. Read [the next section](#enabling-concurrency-in-container) for more information about this. + +All parameters defined under `containerResourceRequirements` are passed directly to the corresponding `deploy_configuration()` method of the azureml SDK. + +- AKS: [`AksWebservice.deploy_configuration()`](https://docs.microsoft.com/python/api/azureml-core/azureml.core.webservice.akswebservice?view=azure-ml-py#deploy-configuration-autoscale-enabled-none--autoscale-min-replicas-none--autoscale-max-replicas-none--autoscale-refresh-seconds-none--autoscale-target-utilization-none--collect-model-data-none--auth-enabled-none--cpu-cores-none--memory-gb-none--enable-app-insights-none--scoring-timeout-ms-none--replica-max-concurrent-requests-none--max-request-wait-time-none--num-replicas-none--primary-key-none--secondary-key-none--tags-none--properties-none--description-none--gpu-cores-none--period-seconds-none--initial-delay-seconds-none--timeout-seconds-none--success-threshold-none--failure-threshold-none--namespace-none--token-auth-enabled-none--compute-target-name-none--cpu-cores-limit-none--memory-gb-limit-none-) + +- ACI: [`AciWebservice.deploy_configuration()`](https://docs.microsoft.com/python/api/azureml-core/azureml.core.webservice.aciwebservice?view=azure-ml-py#deploy-configuration-cpu-cores-none--memory-gb-none--tags-none--properties-none--description-none--location-none--auth-enabled-none--ssl-enabled-none--enable-app-insights-none--ssl-cert-pem-file-none--ssl-key-pem-file-none--ssl-cname-none--dns-name-label-none--primary-key-none--secondary-key-none--collect-model-data-none--cmk-vault-base-url-none--cmk-key-name-none--cmk-key-version-none--vnet-name-none--subnet-name-none-) + +For example: + +- `cpu_cores` +- `memory_gb` +- `autoscale_enabled` +- `autoscale_min_replicas` +- `autoscale_max_replicas` +- `scoring_timeout_ms` +- `max_request_wait_time` + +## Enabling concurrency in container + +As stated in the [documentation](https://docs.microsoft.com/python/api/azureml-core/azureml.core.webservice.akswebservice?view=azure-ml-py#deploy-configuration-autoscale-enabled-none--autoscale-min-replicas-none--autoscale-max-replicas-none--autoscale-refresh-seconds-none--autoscale-target-utilization-none--collect-model-data-none--auth-enabled-none--cpu-cores-none--memory-gb-none--enable-app-insights-none--scoring-timeout-ms-none--replica-max-concurrent-requests-none--max-request-wait-time-none--num-replicas-none--primary-key-none--secondary-key-none--tags-none--properties-none--description-none--gpu-cores-none--period-seconds-none--initial-delay-seconds-none--timeout-seconds-none--success-threshold-none--failure-threshold-none--namespace-none--token-auth-enabled-none--compute-target-name-none--cpu-cores-limit-none--memory-gb-limit-none-), the `replica_max_concurrent_requests` parameter should be left unchanged to the default value of 1. + +However, if you want to enable concurrency in the containers you can follow this instructions. This can be useful to remove possible bottlenecks in the routing webservice for heavy workloads. **Please notice this is a preview feature, use it at your discretion**. + +1. Under `containerResourceRequirements`, set `replica_max_concurrent_requests` to the number of worker processes you want to enable in the container. + +2. Add the following environment variables to the `environmentVariables` section in the config file: + +- `WORKER_COUNT`: set to the same as `replica_max_concurrent_requests`. This is the number of worker processes (gunicorn+flask) that will be spun up. +- `MKL_NUM_THREADS`, `OMP_NUM_THREADS`: number of threads per worker. These parameters are designed to protect neighbour containers, as by default they are set to the number of CPU cores on the machine. You need to check what is relevant for your ML framework. +- `WORKER_PRELOAD`: `true` flag can be set to enable shared memory (might cause issues with some models, specially Tensorflow/Keras). + +If you have a use case that could benefit from this and need any further guidance please contact Azure Machine Learning Product Group. diff --git a/mlops-pipelines/configuration/automl/forecast-parallelrunstep-config.yml b/mlops-pipelines/configuration/automl/forecast-parallelrunstep-config.yml new file mode 100644 index 00000000..9283fcd4 --- /dev/null +++ b/mlops-pipelines/configuration/automl/forecast-parallelrunstep-config.yml @@ -0,0 +1,5 @@ +mini_batch_size: "1" +run_invocation_timeout: 300 +error_threshold: -1 +process_count_per_node: 8 +node_count: 3 diff --git a/Custom_Script/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aci.yml b/mlops-pipelines/configuration/automl/model-deployment-config-aci.yml similarity index 89% rename from Custom_Script/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aci.yml rename to mlops-pipelines/configuration/automl/model-deployment-config-aci.yml index d46e6f3d..438a84dd 100644 --- a/Custom_Script/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aci.yml +++ b/mlops-pipelines/configuration/automl/model-deployment-config-aci.yml @@ -3,7 +3,7 @@ containerResourceRequirements: cpu_cores: 1 memory_gb: 1 # environmentVariables: - # WORKER_COUNT: 4 + # WORKER_COUNT: 1 # MKL_NUM_THREADS: 4 # OMP_NUM_THREADS: 4 # WORKER_PRELOAD: False diff --git a/Custom_Script/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aks.yml b/mlops-pipelines/configuration/automl/model-deployment-config-aks.yml similarity index 89% rename from Custom_Script/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aks.yml rename to mlops-pipelines/configuration/automl/model-deployment-config-aks.yml index ca69174e..8a3631c3 100644 --- a/Custom_Script/mlops-pipelines/3-modeling/deploy-models/forecasting-deployment-config-aks.yml +++ b/mlops-pipelines/configuration/automl/model-deployment-config-aks.yml @@ -3,7 +3,7 @@ containerResourceRequirements: cpu_cores: 1 memory_gb: 1 # environmentVariables: - # WORKER_COUNT: 4 + # WORKER_COUNT: 1 # MKL_NUM_THREADS: 4 # OMP_NUM_THREADS: 4 # WORKER_PRELOAD: False diff --git a/mlops-pipelines/configuration/automl/routing-deployment-config-aci.yml b/mlops-pipelines/configuration/automl/routing-deployment-config-aci.yml new file mode 100644 index 00000000..5fb8c474 --- /dev/null +++ b/mlops-pipelines/configuration/automl/routing-deployment-config-aci.yml @@ -0,0 +1,9 @@ +computeType: ACI +containerResourceRequirements: + cpu_cores: 0.1 + memory_gb: 0.5 +# environmentVariables: + # WORKER_COUNT: 1 + # MKL_NUM_THREADS: 1 + # OMP_NUM_THREADS: 1 + # WORKER_PRELOAD: False diff --git a/mlops-pipelines/configuration/automl/routing-deployment-config-aks.yml b/mlops-pipelines/configuration/automl/routing-deployment-config-aks.yml new file mode 100644 index 00000000..20395928 --- /dev/null +++ b/mlops-pipelines/configuration/automl/routing-deployment-config-aks.yml @@ -0,0 +1,9 @@ +computeType: AKS +containerResourceRequirements: + cpu_cores: 0.1 + memory_gb: 0.5 +# environmentVariables: +# WORKER_COUNT: 8 +# MKL_NUM_THREADS: 1 +# OMP_NUM_THREADS: 1 +# WORKER_PRELOAD: False diff --git a/mlops-pipelines/configuration/automl/script_settings.json b/mlops-pipelines/configuration/automl/script_settings.json new file mode 100644 index 00000000..579cbea5 --- /dev/null +++ b/mlops-pipelines/configuration/automl/script_settings.json @@ -0,0 +1,18 @@ +{ + "task": "forecasting", + "primary_metric": "normalized_root_mean_squared_error", + "iteration_timeout_minutes": 10, + "iterations": 15, + "experiment_timeout_hours": 1, + "label_column_name": "Quantity", + "n_cross_validations": 3, + "verbosity": 20, + "debug_log": "automl_oj_sales_debug.txt", + "time_column_name": "WeekStarting", + "max_horizon": 20, + "group_column_names": ["Store", "Brand"], + "grain_column_names": ["Store", "Brand"], + "drop_column_names": ["Revenue"], + "process_count_per_node": 1, + "retrain_failed_models": false +} \ No newline at end of file diff --git a/mlops-pipelines/configuration/automl/train-parallelrunstep-config.yml b/mlops-pipelines/configuration/automl/train-parallelrunstep-config.yml new file mode 100644 index 00000000..9283fcd4 --- /dev/null +++ b/mlops-pipelines/configuration/automl/train-parallelrunstep-config.yml @@ -0,0 +1,5 @@ +mini_batch_size: "1" +run_invocation_timeout: 300 +error_threshold: -1 +process_count_per_node: 8 +node_count: 3 diff --git a/mlops-pipelines/configuration/customscript/forecast-parallelrunstep-config.yml b/mlops-pipelines/configuration/customscript/forecast-parallelrunstep-config.yml new file mode 100644 index 00000000..9283fcd4 --- /dev/null +++ b/mlops-pipelines/configuration/customscript/forecast-parallelrunstep-config.yml @@ -0,0 +1,5 @@ +mini_batch_size: "1" +run_invocation_timeout: 300 +error_threshold: -1 +process_count_per_node: 8 +node_count: 3 diff --git a/Custom_Script/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aci.yml b/mlops-pipelines/configuration/customscript/model-deployment-config-aci.yml similarity index 72% rename from Custom_Script/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aci.yml rename to mlops-pipelines/configuration/customscript/model-deployment-config-aci.yml index d2d27de5..438a84dd 100644 --- a/Custom_Script/mlops-pipelines/3-modeling/deploy-models/routing-deployment-config-aci.yml +++ b/mlops-pipelines/configuration/customscript/model-deployment-config-aci.yml @@ -1,9 +1,9 @@ computeType: ACI containerResourceRequirements: - cpu_cores: 0.1 - memory_gb: 0.5 + cpu_cores: 1 + memory_gb: 1 # environmentVariables: - # WORKER_COUNT: 4 + # WORKER_COUNT: 1 # MKL_NUM_THREADS: 4 # OMP_NUM_THREADS: 4 # WORKER_PRELOAD: False diff --git a/mlops-pipelines/configuration/customscript/model-deployment-config-aks.yml b/mlops-pipelines/configuration/customscript/model-deployment-config-aks.yml new file mode 100644 index 00000000..8a3631c3 --- /dev/null +++ b/mlops-pipelines/configuration/customscript/model-deployment-config-aks.yml @@ -0,0 +1,9 @@ +computeType: AKS +containerResourceRequirements: + cpu_cores: 1 + memory_gb: 1 +# environmentVariables: + # WORKER_COUNT: 1 + # MKL_NUM_THREADS: 4 + # OMP_NUM_THREADS: 4 + # WORKER_PRELOAD: False diff --git a/mlops-pipelines/configuration/customscript/routing-deployment-config-aci.yml b/mlops-pipelines/configuration/customscript/routing-deployment-config-aci.yml new file mode 100644 index 00000000..5fb8c474 --- /dev/null +++ b/mlops-pipelines/configuration/customscript/routing-deployment-config-aci.yml @@ -0,0 +1,9 @@ +computeType: ACI +containerResourceRequirements: + cpu_cores: 0.1 + memory_gb: 0.5 +# environmentVariables: + # WORKER_COUNT: 1 + # MKL_NUM_THREADS: 1 + # OMP_NUM_THREADS: 1 + # WORKER_PRELOAD: False diff --git a/mlops-pipelines/configuration/customscript/routing-deployment-config-aks.yml b/mlops-pipelines/configuration/customscript/routing-deployment-config-aks.yml new file mode 100644 index 00000000..20395928 --- /dev/null +++ b/mlops-pipelines/configuration/customscript/routing-deployment-config-aks.yml @@ -0,0 +1,9 @@ +computeType: AKS +containerResourceRequirements: + cpu_cores: 0.1 + memory_gb: 0.5 +# environmentVariables: +# WORKER_COUNT: 8 +# MKL_NUM_THREADS: 1 +# OMP_NUM_THREADS: 1 +# WORKER_PRELOAD: False diff --git a/mlops-pipelines/configuration/customscript/script_settings.json b/mlops-pipelines/configuration/customscript/script_settings.json new file mode 100644 index 00000000..51ad59ca --- /dev/null +++ b/mlops-pipelines/configuration/customscript/script_settings.json @@ -0,0 +1,11 @@ +{ + "id_columns": ["Store", "Brand"], + "target_column": "Quantity", + "timestamp_column": "WeekStarting", + "tag_columns": ["StoreGroup10", "StoreGroup100", "StoreGroup1000"], + "drop_id": true, + "drop_tags": true, + "drop_columns": ["Revenue"], + "model_type": "lr", + "test_size": 20 +} diff --git a/mlops-pipelines/configuration/customscript/train-parallelrunstep-config.yml b/mlops-pipelines/configuration/customscript/train-parallelrunstep-config.yml new file mode 100644 index 00000000..9283fcd4 --- /dev/null +++ b/mlops-pipelines/configuration/customscript/train-parallelrunstep-config.yml @@ -0,0 +1,5 @@ +mini_batch_size: "1" +run_invocation_timeout: 300 +error_threshold: -1 +process_count_per_node: 8 +node_count: 3 diff --git a/Custom_Script/mlops-pipelines/many-models-variables.yml b/mlops-pipelines/configuration/many-models-variables.yml similarity index 97% rename from Custom_Script/mlops-pipelines/many-models-variables.yml rename to mlops-pipelines/configuration/many-models-variables.yml index 1342a76f..a431d17c 100644 --- a/Custom_Script/mlops-pipelines/many-models-variables.yml +++ b/mlops-pipelines/configuration/many-models-variables.yml @@ -9,7 +9,7 @@ variables: - name: ENVIRONMENT value: DEV - name: SDK_VERSION - value: 1.10.0 + value: 1.14.0 # Infra - name: AMLWORKSPACE_NAME @@ -37,9 +37,9 @@ variables: - name: AML_AKS_NAME value: manymodels-aks - name: AML_MODEL_SPLITTING_TAGS - value: Store + value: - name: AML_MODEL_SORTING_TAGS - value: Brand + value: Store - name: AML_WEBSERVICE_PREFIX value: manymodels- - name: AML_ROUTING_MODEL_NAME diff --git a/mlops-pipelines/scripts/create_forecasting_pipeline.py b/mlops-pipelines/scripts/create_forecasting_pipeline.py new file mode 100644 index 00000000..cdf9c586 --- /dev/null +++ b/mlops-pipelines/scripts/create_forecasting_pipeline.py @@ -0,0 +1,133 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import argparse +import shutil + +from azureml.core import Workspace, Datastore +from azureml.core.compute import AmlCompute +from azureml.data.data_reference import DataReference +from azureml.pipeline.steps import PythonScriptStep + +from utils.pipelines import create_parallelrunstep, publish_pipeline + + +def main(ws, pipeline_name, pipeline_version, dataset_name, output_name, compute_name, + scripts_dir, scripts_settings_file, config_file): + + # Get compute target + compute = AmlCompute(ws, compute_name) + + # Get datastores + datastore_default = ws.get_default_datastore() + datastore_predictions = Datastore.register_azure_blob_container( + workspace=ws, + datastore_name=output_name, + container_name=output_name, + account_name=datastore_default.account_name, + account_key=datastore_default.account_key, + create_if_not_exists=True + ) + + # Setup settings file to be read in script + settings_filename = os.path.basename(scripts_settings_file) + shutil.copyfile(scripts_settings_file, os.path.join(scripts_dir, settings_filename)) + + # Create the pipeline step for parallel batch forecasting + parallel_run_step = create_parallelrunstep( + ws, + name='many-models-parallel-forecasting', + compute=compute, + datastore=datastore_default, + input_dataset=dataset_name, + output_dir='forecasting_output', + script_dir=scripts_dir, + script_file='batch_forecasting.py', + environment_file=os.path.join(scripts_dir, 'forecast.conda.yml'), + config_file=config_file, + arguments=[ + '--settings-file', settings_filename + ] + ) + + # Create the pipeline step to copy predictions + prev_step_output = parallel_run_step._output[0] + predictions_dref = DataReference(datastore_predictions) + upload_predictions_step = PythonScriptStep( + name='many-models-copy-predictions', + source_directory=scripts_dir, + script_name='copy_predictions.py', + compute_target=compute, + inputs=[predictions_dref, prev_step_output], + allow_reuse=False, + arguments=[ + '--parallel_run_step_output', prev_step_output, + '--output_dir', predictions_dref, + '--settings-file', settings_filename + ] + ) + + # Publish the pipeline + forecasting_pipeline = publish_pipeline( + ws, + name=pipeline_name, + steps=[parallel_run_step, upload_predictions_step], + description='Many Models forecasting pipeline', + version=pipeline_version + ) + + return forecasting_pipeline.id + + +def disable_old_pipelines(ws, pipeline_name): + for pipeline in PublishedPipeline.list(ws): + if pipeline.name == pipeline_name: + pipeline.disable() + + +def parse_args(args=None): + parser = argparse.ArgumentParser() + parser.add_argument('--subscription-id', required=True, type=str) + parser.add_argument('--resource-group', required=True, type=str) + parser.add_argument('--workspace-name', required=True, type=str) + parser.add_argument('--name', required=True, type=str) + parser.add_argument('--version', required=True, type=str) + parser.add_argument('--scripts-dir', required=True, type=str) + parser.add_argument('--scripts-settings', required=True, type=str) + parser.add_argument('--prs-config', required=True, type=str) + parser.add_argument('--dataset', type=str, default='oj_sales_data_inference') + parser.add_argument('--output', type=str, default='predictions') + parser.add_argument('--compute', type=str, default='cpu-compute') + parser.add_argument('--artifact', type=str) + args_parsed = parser.parse_args(args) + return args_parsed + + +if __name__ == "__main__": + args = parse_args() + + # Connect to workspace + ws = Workspace.get( + name=args.workspace_name, + subscription_id=args.subscription_id, + resource_group=args.resource_group + ) + + pipeline_id = main( + ws, + pipeline_name=args.name, + pipeline_version=args.version, + dataset_name=args.dataset, + output_name=args.output, + compute_name=args.compute, + scripts_dir=args.scripts_dir, + scripts_settings_file=args.scripts_settings, + config_file=args.prs_config + ) + + if args.artifact: + with open(args.artifact, 'w') as f: + f.write(pipeline_id) + + print('Forecasting pipeline {} version {} published with ID {}'.format(args.name, args.version, pipeline_id)) diff --git a/mlops-pipelines/scripts/create_training_pipeline.py b/mlops-pipelines/scripts/create_training_pipeline.py new file mode 100644 index 00000000..1653281f --- /dev/null +++ b/mlops-pipelines/scripts/create_training_pipeline.py @@ -0,0 +1,100 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import argparse +import shutil + +from azureml.core import Workspace +from azureml.core.compute import AmlCompute + +from utils.pipelines import create_parallelrunstep, publish_pipeline + + +def main(ws, pipeline_name, pipeline_version, dataset_name, compute_name, + scripts_dir, scripts_settings_file, config_file): + + # Get the compute target + compute = AmlCompute(ws, compute_name) + + # Get datastore + datastore_default = ws.get_default_datastore() + + # Setup settings file to be read in script + print(f"Settings origin: {scripts_settings_file}") + print(f"Settings dest dir: {scripts_dir}") + settings_filename = os.path.basename(scripts_settings_file) + shutil.copyfile(scripts_settings_file, os.path.join(scripts_dir, settings_filename)) + + # Create the pipeline step for parallel training + parallel_run_step = create_parallelrunstep( + ws, + name='many-models-parallel-training', + compute=compute, + datastore=datastore_default, + input_dataset=dataset_name, + output_dir='training_output', + script_dir=scripts_dir, + script_file='train.py', + environment_file=os.path.join(scripts_dir, 'train.conda.yml'), + config_file=config_file, + arguments=[ + '--settings-file', settings_filename + ] + ) + + # Publish the pipeline + train_pipeline = publish_pipeline( + ws, + name=pipeline_name, + steps=[parallel_run_step], + description='Many Models training/retraining pipeline', + version=pipeline_version + ) + + return train_pipeline.id + + +def parse_args(args=None): + parser = argparse.ArgumentParser() + parser.add_argument('--subscription-id', required=True, type=str) + parser.add_argument('--resource-group', required=True, type=str) + parser.add_argument('--workspace-name', required=True, type=str) + parser.add_argument('--name', required=True, type=str) + parser.add_argument('--version', required=True, type=str) + parser.add_argument('--scripts-dir', required=True, type=str) + parser.add_argument('--scripts-settings', required=True, type=str) + parser.add_argument('--prs-config', required=True, type=str) + parser.add_argument('--dataset', type=str, default='oj_sales_data_train') + parser.add_argument('--compute', type=str, default='cpu-compute') + parser.add_argument('--artifact', type=str) + args_parsed = parser.parse_args(args) + return args_parsed + + +if __name__ == "__main__": + args = parse_args() + + # Connect to workspace + ws = Workspace.get( + name=args.workspace_name, + subscription_id=args.subscription_id, + resource_group=args.resource_group + ) + + pipeline_id = main( + ws, + pipeline_name=args.name, + pipeline_version=args.version, + dataset_name=args.dataset, + compute_name=args.compute, + scripts_dir=args.scripts_dir, + scripts_settings_file=args.scripts_settings, + config_file=args.prs_config + ) + + if args.artifact: + with open(args.artifact, 'w') as f: + f.write(pipeline_id) + + print('Training pipeline {} version {} published with ID {}'.format(args.name, args.version, pipeline_id)) diff --git a/Custom_Script/mlops-pipelines/scripts/deploy_or_update_models.py b/mlops-pipelines/scripts/deploy_or_update_models.py similarity index 94% rename from Custom_Script/mlops-pipelines/scripts/deploy_or_update_models.py rename to mlops-pipelines/scripts/deploy_or_update_models.py index 57a04ba3..8b4bfe97 100644 --- a/Custom_Script/mlops-pipelines/scripts/deploy_or_update_models.py +++ b/mlops-pipelines/scripts/deploy_or_update_models.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import os import argparse import warnings import json @@ -12,16 +13,16 @@ from utils.deployment import build_deployment_config, launch_deployment -def main(ws, config_file, routing_model_name, +def main(ws, scripts_dir, config_file, routing_model_name, sorting_tags=[], splitting_tags=[], container_size=500, - aks_target=None, service_prefix='manymodels-', reset=False): + aks_target=None, service_prefix='manymodels-', reset=False, update=False): # Deployment configuration deployment_config = build_deployment_config( ws, - script_dir='Custom_Script/scripts/', - script_file='forecast_webservice.py', - environment_file='Custom_Script/scripts/forecast_webservice.conda.yml', + script_dir=scripts_dir, + script_file='model_webservice.py', + environment_file=os.path.join(scripts_dir, 'model_webservice.conda.yml'), config_file=config_file, aks_target=aks_target ) @@ -34,6 +35,7 @@ def main(ws, config_file, routing_model_name, models_deployed = {} for service in existing_services.values(): service.delete() + existing_services = {} # Get models registered models_registered = get_models_registered(ws, exclude_names=[routing_model_name]) @@ -45,6 +47,11 @@ def main(ws, config_file, routing_model_name, container_size=container_size ) + # Force update in all webservices (when deployment config changes for example) + if update: + groups_update.update(groups_unchanged) + groups_unchanged = {} + print(f'{len(groups_delete)} services to be deleted,', f'{len(groups_new)} new groups to be deployed,', f'{len(groups_update)} groups to be updated,', @@ -52,7 +59,8 @@ def main(ws, config_file, routing_model_name, # Delete old services for group_name in groups_delete: - existing_services[group_name].delete() + service_name = get_service_name(group_name, service_prefix) + existing_services[service_name].delete() deployments = [] @@ -188,7 +196,7 @@ def create_deployment_groups(models_registered, models_deployed, names_all_subgroups_delete += [get_subgroup_name(group_name, i) for i in subgroups_deployed if i not in subgroups_registered] - container_sizes = [len(container) for container in all_subgroups_registered] + container_sizes = [len(container) for container in all_subgroups_registered.values()] print(f'Grouped models in {len(all_subgroups_registered)} groups.', f'Min size: {min(container_sizes)}. Max size: {max(container_sizes)}.') @@ -322,6 +330,7 @@ def parse_args(args=None): parser.add_argument('--subscription-id', required=True, type=str) parser.add_argument('--resource-group', required=True, type=str) parser.add_argument('--workspace-name', required=True, type=str) + parser.add_argument('--scripts-dir', required=True, type=str) parser.add_argument('--deploy-config-file', required=True, type=str) parser.add_argument('--splitting-tags', default='', type=lambda str: [t for t in str.split(',') if t]) parser.add_argument('--sorting-tags', default='', type=lambda str: [t for t in str.split(',') if t]) @@ -331,6 +340,7 @@ def parse_args(args=None): parser.add_argument('--service-prefix', type=str, default=None) parser.add_argument('--container-size', type=int, default=500) parser.add_argument('--reset', action='store_true') + parser.add_argument('--update', action='store_true') args_parsed = parser.parse_args(args) if args_parsed.service_prefix is None: @@ -351,6 +361,7 @@ def parse_args(args=None): models_deployed = main( ws, + scripts_dir=args.scripts_dir, config_file=args.deploy_config_file, routing_model_name=args.routing_model_name, splitting_tags=args.splitting_tags, @@ -358,7 +369,8 @@ def parse_args(args=None): aks_target=args.aks_target, service_prefix=args.service_prefix, container_size=args.container_size, - reset=args.reset + reset=args.reset, + update=args.update ) with open(args.output, 'w') as f: diff --git a/Custom_Script/mlops-pipelines/scripts/deploy_or_update_router.py b/mlops-pipelines/scripts/deploy_or_update_router.py similarity index 87% rename from Custom_Script/mlops-pipelines/scripts/deploy_or_update_router.py rename to mlops-pipelines/scripts/deploy_or_update_router.py index f013163e..d3a70b3b 100644 --- a/Custom_Script/mlops-pipelines/scripts/deploy_or_update_router.py +++ b/mlops-pipelines/scripts/deploy_or_update_router.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import os import argparse import warnings import json @@ -11,13 +12,13 @@ from utils.deployment import build_deployment_config, launch_deployment -def main(ws, config_file, model_name, service_name, aks_target=None): +def main(ws, scripts_dir, config_file, model_name, service_name, aks_target=None): deployment_config = build_deployment_config( ws, - script_dir='Custom_Script/scripts/', + script_dir=scripts_dir, script_file='routing_webservice.py', - environment_file='Custom_Script/scripts/routing_webservice.conda.yml', + environment_file=os.path.join(scripts_dir, 'routing_webservice.conda.yml'), config_file=config_file, aks_target=aks_target ) @@ -39,6 +40,7 @@ def parse_args(args=None): parser.add_argument('--subscription-id', required=True, type=str) parser.add_argument('--resource-group', required=True, type=str) parser.add_argument('--workspace-name', required=True, type=str) + parser.add_argument('--scripts-dir', required=True, type=str) parser.add_argument('--deploy-config-file', required=True, type=str) parser.add_argument('--model-name', type=str, default=None) parser.add_argument('--service-name', type=str, default=None) @@ -66,6 +68,7 @@ def parse_args(args=None): main( ws, + scripts_dir=args.scripts_dir, config_file=args.deploy_config_file, model_name=args.model_name, service_name=args.service_name, diff --git a/Custom_Script/mlops-pipelines/scripts/download_data.py b/mlops-pipelines/scripts/download_data.py similarity index 100% rename from Custom_Script/mlops-pipelines/scripts/download_data.py rename to mlops-pipelines/scripts/download_data.py diff --git a/Custom_Script/mlops-pipelines/scripts/register_or_update_dataset.py b/mlops-pipelines/scripts/register_or_update_dataset.py similarity index 100% rename from Custom_Script/mlops-pipelines/scripts/register_or_update_dataset.py rename to mlops-pipelines/scripts/register_or_update_dataset.py diff --git a/Custom_Script/mlops-pipelines/scripts/tests/__init__.py b/mlops-pipelines/scripts/tests/__init__.py similarity index 100% rename from Custom_Script/mlops-pipelines/scripts/tests/__init__.py rename to mlops-pipelines/scripts/tests/__init__.py diff --git a/Custom_Script/mlops-pipelines/scripts/tests/test_deploy.py b/mlops-pipelines/scripts/tests/test_deploy.py similarity index 100% rename from Custom_Script/mlops-pipelines/scripts/tests/test_deploy.py rename to mlops-pipelines/scripts/tests/test_deploy.py diff --git a/Custom_Script/mlops-pipelines/scripts/tests/utils.py b/mlops-pipelines/scripts/tests/utils.py similarity index 100% rename from Custom_Script/mlops-pipelines/scripts/tests/utils.py rename to mlops-pipelines/scripts/tests/utils.py diff --git a/mlops-pipelines/scripts/utils/common.py b/mlops-pipelines/scripts/utils/common.py new file mode 100644 index 00000000..3884ee72 --- /dev/null +++ b/mlops-pipelines/scripts/utils/common.py @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import yaml + +from azureml.core import Environment + + +def create_environment_conda(conda_file, environment_variables=None, name='many_models_environment'): + + environment = Environment.from_conda_specification( + name=name, + file_path=conda_file + ) + if environment_variables: + environment.environment_variables = environment_variables + + return environment + + +def read_config_file(config_file): + + with open(config_file) as f: + config = yaml.load(f, Loader=yaml.FullLoader) + + return config diff --git a/Custom_Script/mlops-pipelines/scripts/utils/deployment.py b/mlops-pipelines/scripts/utils/deployment.py similarity index 79% rename from Custom_Script/mlops-pipelines/scripts/utils/deployment.py rename to mlops-pipelines/scripts/utils/deployment.py index 6568786b..8581ec3f 100644 --- a/Custom_Script/mlops-pipelines/scripts/utils/deployment.py +++ b/mlops-pipelines/scripts/utils/deployment.py @@ -2,13 +2,16 @@ # Licensed under the MIT License. import yaml +import inspect -from azureml.core import Environment, Model, Webservice +from azureml.core import Model, Webservice from azureml.core.model import InferenceConfig from azureml.core.compute import AksCompute from azureml.core.webservice import AciWebservice, AksWebservice from azureml.exceptions import WebserviceException +from utils.common import read_config_file, create_environment_conda + def build_deployment_config(ws, script_dir, script_file, environment_file, config_file, aks_target=None): @@ -24,12 +27,7 @@ def build_deployment_config(ws, script_dir, script_file, environment_file, confi deployment_target = AksCompute(ws, aks_target) if compute_type == 'aks' else None # Inference environment - environment = Environment.from_conda_specification( - name='many_models_environment', - file_path=environment_file - ) - if environment_config: - environment.environment_variables = environment_config + environment = create_environment_conda(environment_file, environment_config) # Inference configuration inference_config = InferenceConfig( @@ -61,9 +59,7 @@ def build_deployment_config(ws, script_dir, script_file, environment_file, confi def read_config(config_file): - with open(config_file) as f: - config = yaml.load(f, Loader=yaml.FullLoader) - + config = read_config_file(config_file) compute_type = config['computeType'].lower() webservice_config = config['containerResourceRequirements'] environment_config = config.get('environmentVariables', {}) @@ -85,9 +81,10 @@ def launch_deployment(ws, service_name, models, deployment_config, existing_serv service = existing_services.get(service_name) if service: print(f'Launching updating of service {service.name}...') + update_params = build_update_params(service, deployment_config) service.update( models=models, - inference_config=deployment_config['inference_config'] + **update_params ) print(f'Updating of {service.name} started') else: @@ -102,3 +99,19 @@ def launch_deployment(ws, service_name, models, deployment_config, existing_serv print(f'Deployment of {service.name} started') return service + + +def build_update_params(service, deployment_config): + + update_args = inspect.getfullargspec(service.update)[0] + + params = { + **{ + # Params of the configuration object that are defined as arguments in the update function + k:v for k,v in inspect.getmembers(deployment_config['deployment_config']) + if k in update_args and v is not None + }, + 'inference_config': deployment_config['inference_config'] + } + + return params diff --git a/mlops-pipelines/scripts/utils/pipelines.py b/mlops-pipelines/scripts/utils/pipelines.py new file mode 100644 index 00000000..024e8749 --- /dev/null +++ b/mlops-pipelines/scripts/utils/pipelines.py @@ -0,0 +1,112 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from azureml.core import Environment, Dataset +from azureml.pipeline.core import Pipeline, PipelineData, PublishedPipeline +from azureml.pipeline.steps import ParallelRunConfig, ParallelRunStep + +from utils.common import read_config_file, create_environment_conda + + +def create_parallelrunstep(ws, name, compute, datastore, input_dataset, output_dir, + script_dir, script_file, environment_file, config_file, arguments): + + config_params = read_config_file(config_file) + + # Get input dataset + dataset = Dataset.get_by_name(ws, name=input_dataset) + pipeline_input = dataset.as_named_input(input_dataset) + + # Set output + pipeline_output = PipelineData(name=output_dir, datastore=datastore) + + # Create ParallelRunConfig + parallel_run_config = create_parallelrunconfig( + ws, compute, script_dir, script_file, environment_file, config_params) + + parallel_run_step = ParallelRunStep( + name=name, + parallel_run_config=parallel_run_config, + inputs=[pipeline_input], + output=pipeline_output, + allow_reuse=False, + arguments=arguments + ) + + return parallel_run_step + + +def create_parallelrunconfig(ws, compute, script_dir, script_file, environment_file, config_params): + + # Configure environment for ParallelRunStep + train_env = create_environment_conda(environment_file) + + # Set up ParallelRunStep configuration + + default_params = { + 'mini_batch_size': '1', + 'run_invocation_timeout': 300, + 'error_threshold': -1, + 'process_count_per_node': 8, + 'node_count': 3 + } + for param, default_value in default_params.items(): + if not param in config_params: + config_params[param] = default_value + + parallel_run_config = ParallelRunConfig( + source_directory=script_dir, + entry_script=script_file, + environment=train_env, + compute_target=compute, + output_action='append_row', + **config_params + ) + + validate_parallel_run_config(parallel_run_config) + + return parallel_run_config + + +def validate_parallel_run_config(parallel_run_config): + errors = False + + if parallel_run_config.mini_batch_size != 1: + errors = True + print('Error: mini_batch_size should be set to 1') + + if 'automl' in parallel_run_config.source_directory: + max_concurrency = 20 + curr_concurrency = parallel_run_config.process_count_per_node * parallel_run_config.node_count + if curr_concurrency > max_concurrency: + errors = True + print(f'Error: node_count*process_count_per_node must be between 1 and max_concurrency {max_concurrency}.', + f'Please decrease concurrency from current {curr_concurrency} to maximum of {max_concurrency}', + 'as currently AutoML does not support it.') + + if not errors: + print('Validation successful') + + +def publish_pipeline(ws, name, steps, description=None, version=None): + + # Create the pipeline + pipeline = Pipeline(workspace=ws, steps=steps) + pipeline.validate() + + # Publish it replacing old pipeline + disable_old_pipelines(ws, name) + published_pipeline = pipeline.publish( + name=name, + description=description, + version=version, + continue_on_step_failure=False + ) + + return published_pipeline + + +def disable_old_pipelines(ws, name): + for pipeline in PublishedPipeline.list(ws): + if pipeline.name == name: + pipeline.disable() diff --git a/00_Setup_AML_Workspace.ipynb b/notebooks/00_Setup_AML_Workspace.ipynb similarity index 100% rename from 00_Setup_AML_Workspace.ipynb rename to notebooks/00_Setup_AML_Workspace.ipynb diff --git a/01_Data_Preparation.ipynb b/notebooks/01_Data_Preparation.ipynb similarity index 97% rename from 01_Data_Preparation.ipynb rename to notebooks/01_Data_Preparation.ipynb index 931549ba..f2a47bef 100644 --- a/01_Data_Preparation.ipynb +++ b/notebooks/01_Data_Preparation.ipynb @@ -54,7 +54,7 @@ }, "outputs": [], "source": [ - "%pip install azureml-opendatasets==1.10.0" + "%pip install azureml-opendatasets" ] }, { @@ -94,7 +94,8 @@ "os.makedirs(target_path, exist_ok=True)\n", "\n", "# Download the data\n", - "oj_sales_files.download(target_path, overwrite=True)" + "files_paths = oj_sales_files.download(target_path, overwrite=True)\n", + "print(f'{len(files_paths)} files downloaded. First one is in:\\n{files_paths[0]}')" ] }, { @@ -114,7 +115,7 @@ "metadata": {}, "outputs": [], "source": [ - "from scripts.helper import split_data\n", + "from helpers.dataprep import split_data\n", "\n", "# Provide name of timestamp column in the data and date from which to split into the inference dataset\n", "timestamp_column = 'WeekStarting'\n", diff --git a/Automated_ML/02_AutoML_Training_Pipeline/02_AutoML_Training_Pipeline.ipynb b/notebooks/Automated_ML/02_AutoML_Training_Pipeline/02_AutoML_Training_Pipeline.ipynb similarity index 100% rename from Automated_ML/02_AutoML_Training_Pipeline/02_AutoML_Training_Pipeline.ipynb rename to notebooks/Automated_ML/02_AutoML_Training_Pipeline/02_AutoML_Training_Pipeline.ipynb diff --git a/Automated_ML/02_AutoML_Training_Pipeline/scripts/helper.py b/notebooks/Automated_ML/02_AutoML_Training_Pipeline/scripts/helper.py similarity index 100% rename from Automated_ML/02_AutoML_Training_Pipeline/scripts/helper.py rename to notebooks/Automated_ML/02_AutoML_Training_Pipeline/scripts/helper.py diff --git a/Automated_ML/02_AutoML_Training_Pipeline/scripts/log.py b/notebooks/Automated_ML/02_AutoML_Training_Pipeline/scripts/log.py similarity index 100% rename from Automated_ML/02_AutoML_Training_Pipeline/scripts/log.py rename to notebooks/Automated_ML/02_AutoML_Training_Pipeline/scripts/log.py diff --git a/Automated_ML/03_AutoML_Forecasting_Pipeline/03_AutoML_Forecasting_Pipeline.ipynb b/notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/03_AutoML_Forecasting_Pipeline.ipynb similarity index 100% rename from Automated_ML/03_AutoML_Forecasting_Pipeline/03_AutoML_Forecasting_Pipeline.ipynb rename to notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/03_AutoML_Forecasting_Pipeline.ipynb diff --git a/Automated_ML/03_AutoML_Forecasting_Pipeline/03_AutoML_Realtime_Forecasting_Deployment.ipynb b/notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/03_AutoML_Realtime_Forecasting_Deployment.ipynb similarity index 100% rename from Automated_ML/03_AutoML_Forecasting_Pipeline/03_AutoML_Realtime_Forecasting_Deployment.ipynb rename to notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/03_AutoML_Realtime_Forecasting_Deployment.ipynb diff --git a/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/forecast.py b/notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/forecast.py similarity index 100% rename from Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/forecast.py rename to notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/forecast.py diff --git a/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/forecast_webservice.py b/notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/forecast_webservice.py similarity index 100% rename from Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/forecast_webservice.py rename to notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/forecast_webservice.py diff --git a/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/helper.py b/notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/helper.py similarity index 100% rename from Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/helper.py rename to notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/helper.py diff --git a/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/routing_webservice.conda.yml b/notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/routing_webservice.conda.yml similarity index 100% rename from Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/routing_webservice.conda.yml rename to notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/routing_webservice.conda.yml diff --git a/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/routing_webservice.py b/notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/routing_webservice.py similarity index 100% rename from Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/routing_webservice.py rename to notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/routing_webservice.py diff --git a/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/utils/forecast_helper.py b/notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/utils/forecast_helper.py similarity index 100% rename from Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/utils/forecast_helper.py rename to notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/utils/forecast_helper.py diff --git a/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/utils/webservices.py b/notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/utils/webservices.py similarity index 100% rename from Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/utils/webservices.py rename to notebooks/Automated_ML/03_AutoML_Forecasting_Pipeline/scripts/utils/webservices.py diff --git a/Automated_ML/common/scripts/helper.py b/notebooks/Automated_ML/common/scripts/helper.py similarity index 100% rename from Automated_ML/common/scripts/helper.py rename to notebooks/Automated_ML/common/scripts/helper.py diff --git a/Automated_ML/requirements.txt b/notebooks/Automated_ML/requirements.txt similarity index 100% rename from Automated_ML/requirements.txt rename to notebooks/Automated_ML/requirements.txt diff --git a/Custom_Script/02_CustomScript_Training_Pipeline.ipynb b/notebooks/Custom_Script/02_CustomScript_Training_Pipeline.ipynb similarity index 85% rename from Custom_Script/02_CustomScript_Training_Pipeline.ipynb rename to notebooks/Custom_Script/02_CustomScript_Training_Pipeline.ipynb index 453e828f..708ae5d5 100644 --- a/Custom_Script/02_CustomScript_Training_Pipeline.ipynb +++ b/notebooks/Custom_Script/02_CustomScript_Training_Pipeline.ipynb @@ -20,29 +20,15 @@ "\n", "This notebook demonstrates how to create a pipeline that trains and registers many models using a custom script. We utilize the [ParallelRunStep](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-use-parallel-run-step) to parallelize the process of training the models to make the process more efficient. For this solution accelerator we are using the [OJ Sales Dataset](https://azure.microsoft.com/en-us/services/open-datasets/catalog/sample-oj-sales-simulated/) to train individual models that predict sales for each store and brand of orange juice.\n", "\n", - "The model we use here is a simple, regression-based forecaster built on scikit-learn and pandas utilities. See the [training script](scripts/train.py) to see how the forecaster is constructed. This forecaster is intended for demonstration purposes, so it does not handle the large variety of special cases that one encounters in time-series modeling. For instance, the model here assumes that all time-series are comprised of regularly sampled observations on a contiguous interval with no missing values. The model does not include any handling of categorical variables. For a more general-use forecaster that handles missing data, advanced featurization, and automatic model selection, see the [AutoML Forecasting task](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-auto-train-forecast). Also, see the notebooks demonstrating [AutoML forecasting in a many models scenario](../Automated_ML).\n", + "The model we use here is a simple, regression-based forecaster built on scikit-learn and pandas utilities. See the [training script](../../scripts/customscript/train.py) to see how the forecaster is constructed. This forecaster is intended for demonstration purposes, so it does not handle the large variety of special cases that one encounters in time-series modeling. For instance, the model here assumes that all time-series are comprised of regularly sampled observations on a contiguous interval with no missing values. The model does not include any handling of categorical variables. For a more general-use forecaster that handles missing data, advanced featurization, and automatic model selection, see the [AutoML Forecasting task](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-auto-train-forecast). Also, see the notebooks demonstrating [AutoML forecasting in a many models scenario](../Automated_ML).\n", "\n", "### Prerequisites\n", "At this point, you should have already:\n", "\n", "1. Created your AML Workspace using the [00_Setup_AML_Workspace notebook](../00_Setup_AML_Workspace.ipynb)\n", - "2. Run [01_Data_Preparation.ipynb](../01_Data_Preparation.ipynb) to setup your compute and create the dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Please ensure you have the latest version of the Azure ML SDK and also install Pipeline Steps Package" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#!pip install --upgrade azureml-sdk" + "2. Run [01_Data_Preparation.ipynb](../01_Data_Preparation.ipynb) to setup your compute and create the dataset\n", + "\n", + "Also please ensure you have the latest version of the Azure ML SDK and also install Pipeline Steps Package" ] }, { @@ -51,7 +37,7 @@ "metadata": {}, "outputs": [], "source": [ - "# !pip install azureml-pipeline-steps" + "%pip install --upgrade azureml-sdk azureml-pipeline-steps" ] }, { @@ -101,7 +87,7 @@ "source": [ "from azureml.core import Experiment\n", "\n", - "experiment = Experiment(ws, 'oj_training_pipeline')\n", + "experiment = Experiment(ws, 'oj_training_customscript_notebook')\n", "\n", "print('Experiment name: ' + experiment.name)" ] @@ -209,7 +195,67 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 4.3 Set up ParallelRunConfig\n", + "### 4.3 Specify training script settings\n", + "\n", + "Many Models training will be performed by executing a custom training script in the compute target we just chose.\n", + "That script is located under the [`/scripts/customscript`](../../scripts/customscript/) directory in the repository. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train_script_dir = '../../scripts/customscript/'\n", + "train_script_name = 'train.py'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The training script uses a settings file that contains the schema for the timeseries data - i.e. the names of target, timestamp, and id columns - as well as columns that should be dropped prior to modeling, a string identifying the model type, and the number of observations we want to leave aside for testing. \n", + "\n", + "We will create that file now and place it in the scripts directory." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "customscript_settings = {\n", + " \"id_columns\": [\"Store\", \"Brand\"], # columns to use for uniquely identifying the model\n", + " \"target_column\": \"Quantity\",\n", + " \"timestamp_column\": \"WeekStarting\",\n", + " \"drop_id\": True, # whether to drop the ID columns from the dataset for training\n", + " \"drop_columns\": [\"Revenue\"], # additional columns to drop from the dataset for training\n", + " \"model_type\": \"lr\",\n", + " \"test_size\": 20\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.append('..')\n", + "from helpers.settings import dump_settings\n", + "\n", + "settings_file = 'customscript_settings.json'\n", + "dump_settings(customscript_settings, train_script_dir, settings_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.4 Set up ParallelRunConfig\n", "\n", "[ParallelRunConfig](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-steps/azureml.pipeline.steps.parallel_run_config.parallelrunconfig?view=azure-ml-py) provides the configuration for the ParallelRunStep we'll be creating next. Here we specify the environment and compute target we created above along with the entry script that will be for each batch.\n", "\n", @@ -239,8 +285,8 @@ "timeout = 180\n", "\n", "parallel_run_config = ParallelRunConfig(\n", - " source_directory='./scripts',\n", - " entry_script='train.py',\n", + " source_directory=train_script_dir,\n", + " entry_script=train_script_name,\n", " mini_batch_size=\"1\",\n", " run_invocation_timeout=timeout,\n", " error_threshold=10,\n", @@ -255,7 +301,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 4.4 Set up ParallelRunStep\n", + "### 4.5 Set up ParallelRunStep\n", "\n", "This [ParallelRunStep](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-steps/azureml.pipeline.steps.parallel_run_step.parallelrunstep?view=azure-ml-py) is the main step in our training pipeline. \n", "\n", @@ -285,7 +331,7 @@ "\n", "- **output**: A PipelineData object that corresponds to the output directory. We'll use the output directory we just defined. \n", "\n", - "- **arguments**: A list of arguments required for the train.py entry script. Here, we provide the schema for the timeseries data - i.e. the names of target, timestamp, and id columns - as well as columns that should be dropped prior to modeling, a string identifying the model type, and the number of observations we want to leave aside for testing." + "- **arguments**: A list of arguments required for the train.py entry script. Here we provide the settings file." ] }, { @@ -304,12 +350,7 @@ " inputs=[dataset_input],\n", " output=output_dir,\n", " allow_reuse=False,\n", - " arguments=['--id_columns', 'Store', 'Brand',\n", - " '--target_column', 'Quantity', \n", - " '--timestamp_column', 'WeekStarting', \n", - " '--drop_columns', 'Revenue', 'Store', 'Brand',\n", - " '--model_type', 'lr',\n", - " '--test_size', 20]\n", + " arguments=['--settings-file', settings_file]\n", ")" ] }, diff --git a/Custom_Script/03_CustomScript_Forecasting_Pipeline.ipynb b/notebooks/Custom_Script/03_CustomScript_Forecasting_Pipeline.ipynb similarity index 89% rename from Custom_Script/03_CustomScript_Forecasting_Pipeline.ipynb rename to notebooks/Custom_Script/03_CustomScript_Forecasting_Pipeline.ipynb index 915bb99c..8abc9284 100644 --- a/Custom_Script/03_CustomScript_Forecasting_Pipeline.ipynb +++ b/notebooks/Custom_Script/03_CustomScript_Forecasting_Pipeline.ipynb @@ -23,23 +23,9 @@ "\n", "1. Created your AML Workspace using the [00_Setup_AML_Workspace notebook](../00_Setup_AML_Workspace.ipynb)\n", "2. Run [01_Data_Preparation.ipynb](../01_Data_Preparation.ipynb) to setup your compute and create the dataset\n", - "3. Run [02_CustomScript_Training_Pipeline.ipynb](02_CustomScript_Training_Pipeline.ipynb) to train the models" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Please ensure you have the latest version of the Azure ML SDK and also install Pipeline Steps Package" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#!pip install --upgrade azureml-sdk" + "3. Run [02_CustomScript_Training_Pipeline.ipynb](02_CustomScript_Training_Pipeline.ipynb) to train the models\n", + "\n", + "Also please ensure you have the latest version of the Azure ML SDK and also install Pipeline Steps Package:" ] }, { @@ -48,7 +34,7 @@ "metadata": {}, "outputs": [], "source": [ - "# !pip install azureml-pipeline-steps" + "%pip install --upgrade azureml-sdk azureml-pipeline-steps" ] }, { @@ -97,7 +83,7 @@ "source": [ "from azureml.core import Experiment\n", "\n", - "experiment = Experiment(ws, 'forecasting_pipeline')" + "experiment = Experiment(ws, 'oj_forecasting_customscript_notebook')" ] }, { @@ -139,7 +125,7 @@ "metadata": {}, "source": [ "## 4.0 Create ParallelRunStep for the forecasting pipeline\n", - "As we did with the training pipeline, we'll create a ParallelRunStep to parallelize our forecasting process. You'll notice this code is essentially the same as the last step except that we'll be parallelizing [**forecast.py**](scripts/forecast.py) rather than train.py. Note that we still need to pass the timeseries schema (timestamp column name, timeseries ID column names, etc) to the forecasting script.\n", + "As we did with the training pipeline, we'll create a ParallelRunStep to parallelize our forecasting process. You'll notice this code is essentially the same as the last step except that we'll be parallelizing [**batch_forecasting.py**](../../scripts/customscript/batch_forecasting.py) rather than train.py. Note that we still need to pass the timeseries schema (timestamp column name, timeseries ID column names, etc) to the forecasting script.\n", "\n", "Unlike the training script, however, the name of target column is not required for the forecasting script. In a true forecasting scenario the actual values of the target are not available, of course, so the forecasting pipeline would just return predictions. However, the forecasting pipeline can also return the actuals if they are present in the inference dataset.\n", "\n", @@ -196,6 +182,42 @@ "compute = AmlCompute(ws, cpu_cluster_name)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.3 Specify forecasting script settings\n", + "\n", + "Many Models batch forecasting will be performed by executing a custom forecasting script in the compute target we just chose.\n", + "That script is also located under the [`/scripts/customscript`](../../scripts/customscript/) directory in the repository, together with the training script we saw in the previous step and some others." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "batchforecast_script_dir = '../../scripts/customscript/'\n", + "batchforecast_script_name = 'batch_forecasting.py'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The batch forecasting script uses a settings file that contains the names of timestamp and id columns, as well as a string identifying the model type. We will reuse the settings file we created during training, as the schema and contents are the same." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "settings_file = 'customscript_settings.json'" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -218,8 +240,8 @@ "timeout = 180\n", "\n", "parallel_run_config = ParallelRunConfig(\n", - " source_directory='./scripts',\n", - " entry_script='forecast.py',\n", + " source_directory=batchforecast_script_dir,\n", + " entry_script=batchforecast_script_name,\n", " mini_batch_size='1',\n", " run_invocation_timeout=timeout, \n", " error_threshold=10,\n", @@ -255,9 +277,7 @@ " inputs=[dataset_input],\n", " output=output_dir,\n", " allow_reuse=False,\n", - " arguments=['--id_columns', 'Store', 'Brand',\n", - " '--timestamp_column', 'WeekStarting',\n", - " '--model_type', 'lr']\n", + " arguments=['--settings-file', settings_file]\n", ")" ] }, @@ -300,7 +320,7 @@ "metadata": {}, "source": [ "### 5.2 Create PythonScriptStep\n", - "Next, we define the [PythonScriptStep](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-steps/azureml.pipeline.steps.python_script_step.pythonscriptstep?view=azure-ml-py) and give it our newly create datastore as well as the location of the *parallel_run_step.txt*. Note that the copy script also uses the timeseries schema; the reason is that the copy script creates a header row for the prediction data and, thus, needs to know the column names. The target column is passed here since it was present in the data used for inferencing." + "Next, we define the [PythonScriptStep](https://docs.microsoft.com/en-us/python/api/azureml-pipeline-steps/azureml.pipeline.steps.python_script_step.pythonscriptstep?view=azure-ml-py) and give it our newly create datastore as well as the location of the *parallel_run_step.txt*. Note that the copy script also uses the settings file; the reason is that the copy script creates a header row for the prediction data and, thus, needs to know the column names." ] }, { @@ -315,14 +335,12 @@ " name=\"copy_predictions\",\n", " script_name=\"copy_predictions.py\",\n", " compute_target=compute,\n", - " source_directory='./scripts',\n", + " source_directory=batchforecast_script_dir,\n", " inputs=[output_dref, output_dir],\n", " allow_reuse=False,\n", " arguments=['--parallel_run_step_output', output_dir,\n", " '--output_dir', output_dref,\n", - " '--id_columns', 'Store', 'Brand',\n", - " '--timestamp_column', 'WeekStarting',\n", - " '--target_column', 'Quantity']\n", + " '--settings-file', settings_file]\n", ")" ] }, diff --git a/Custom_Script/03_CustomScript_Realtime_Forecasting_Deployment.ipynb b/notebooks/Custom_Script/03_CustomScript_Realtime_Forecasting_Deployment.ipynb similarity index 90% rename from Custom_Script/03_CustomScript_Realtime_Forecasting_Deployment.ipynb rename to notebooks/Custom_Script/03_CustomScript_Realtime_Forecasting_Deployment.ipynb index 853e23b9..b5b317f6 100644 --- a/Custom_Script/03_CustomScript_Realtime_Forecasting_Deployment.ipynb +++ b/notebooks/Custom_Script/03_CustomScript_Realtime_Forecasting_Deployment.ipynb @@ -87,11 +87,11 @@ "source": [ "### 2.2 Group models by store\n", "\n", - "We will group the models by store. Therefore, each group will contain three models, one for each of the orange juice brands, and all of them corresponding to the same store.\n", + "We will create groups of models splitting by store. Therefore, each group will contain three models, one for each of the orange juice brands, and all of them corresponding to the same store.\n", "\n", - "You can change the grouping strategy by modifying the `grouping_tags` variable below and specifying the names of the tags you want to use for grouping.\n", + "You can change the grouping strategy by modifying the `splitting_tags` variable below and specifying the names of the tags you want to use for splitting. If you leave it empty all the models will be deployed into a single webservice.\n", "\n", - "To create custom tags, include them in the dataset, add their names as part of the `id_columns` parameter of the [training script](scripts/train.py) and run the training again." + "To create custom tags, include them in the dataset, add their names as part of the `tags_columns` setting in the settings file of the [training script](../../scripts/customscript/train.py) and run the training again." ] }, { @@ -100,7 +100,7 @@ "metadata": {}, "outputs": [], "source": [ - "grouping_tags = ['Store']" + "splitting_tags = ['Store']" ] }, { @@ -115,9 +115,11 @@ " if m.tags['ModelType'] == '_meta_':\n", " continue\n", " \n", - " group_name = '/'.join([m.tags[t] for t in grouping_tags])\n", + " group_name = '/'.join([m.tags[t] for t in splitting_tags]) if splitting_tags else 'allmodels'\n", " group = grouped_models.setdefault(group_name, [])\n", - " group.append(m)" + " group.append(m)\n", + " \n", + "print(f'{len(grouped_models)} group(s) created. Names: {list(grouped_models.keys())}')" ] }, { @@ -168,8 +170,8 @@ "from azureml.core.model import InferenceConfig\n", "\n", "inference_config = InferenceConfig(\n", - " entry_script='forecast_webservice.py',\n", - " source_directory='./scripts',\n", + " source_directory='../../scripts/customscript/',\n", + " entry_script='model_webservice.py',\n", " environment=forecast_env\n", ")" ] @@ -292,7 +294,9 @@ "source": [ "## 4.0 Deploy the models\n", "\n", - "We will now deploy one webservice for each of the groups of models. Deployment takes some minutes to complete, so we'll request all of them and then wait for them to finish." + "We will now deploy one webservice for each of the groups of models. Deployment takes some minutes to complete, so we'll request all of them and then wait for them to finish.\n", + "\n", + "We will store the information on a python dictionary that we'll use later on to find the corresponding webservice for a given model." ] }, { @@ -331,6 +335,7 @@ "cell_type": "code", "execution_count": null, "metadata": { + "scrolled": true, "tags": [] }, "outputs": [], @@ -434,21 +439,46 @@ "]" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Get webservice endpoint and key:" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "import requests\n", + "import sys\n", + "sys.path.append('../../scripts/customscript')\n", + "from utils.models import get_model_name\n", "\n", - "model_name = f'lr_{store1}_{brand1}'\n", + "model_name = get_model_name('lr', {'Store': store1, 'Brand': brand1})\n", "\n", "try:\n", " url = models_deployed[model_name]['endpoint']\n", " key = models_deployed[model_name]['key']\n", "except KeyError as e:\n", - " raise ValueError(f'Model for store {store1} and brand {brand1} has not been deployed')\n", + " raise ValueError(f'Model for store {store1} and brand {brand1} has not been deployed')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Send request to model webservice to get forecasts:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", "\n", "request_headers = {'Content-Type': 'application/json'}\n", "if key:\n", @@ -495,12 +525,13 @@ "source": [ "import json\n", "\n", - "with open 'models_deployed.json' as f:\n", + "artifact_path = 'models_deployed.json'\n", + "with open(artifact_path, 'w') as f:\n", " json.dump(models_deployed, f, indent=4)\n", "\n", "dep_model = Model.register(\n", " workspace=ws, \n", - " model_path ='models_deployed.json',\n", + " model_path=artifact_path,\n", " model_name='deployed_models_info',\n", " tags={'ModelType': '_meta_'},\n", " description='Dictionary of the service endpoint where each model is deployed'\n", @@ -523,12 +554,12 @@ "outputs": [], "source": [ "routing_env = Environment(name=\"many_models_routing_environment\")\n", - "routing_env_deps = CondaDependencies.create(pip_packages=['azureml-defaults', 'json'])\n", + "routing_env_deps = CondaDependencies.create(pip_packages=['azureml-defaults'])\n", "routing_env.python.conda_dependencies = routing_env_deps\n", "\n", "routing_infconfig = InferenceConfig(\n", + " source_directory='../../scripts/customscript/',\n", " entry_script='routing_webservice.py',\n", - " source_directory='./scripts',\n", " environment=routing_env\n", ")\n", "\n", diff --git a/Custom_Script/requirements.txt b/notebooks/Custom_Script/requirements.txt similarity index 100% rename from Custom_Script/requirements.txt rename to notebooks/Custom_Script/requirements.txt diff --git a/EnvironmentSetup.md b/notebooks/EnvironmentSetup.md similarity index 83% rename from EnvironmentSetup.md rename to notebooks/EnvironmentSetup.md index 380e9e9d..2f169563 100644 --- a/EnvironmentSetup.md +++ b/notebooks/EnvironmentSetup.md @@ -7,7 +7,8 @@ To start with, we will create a Azure ML Compute Instance. The Compute Instance 1. Open [Azure Machine Learning Studio](https://ml.azure.com/). 2. Navigate to 'Compute Instances' tab in Compute and click on 'New'. 3. Choose some sufficiently unique name, keep the default VM type (STANDARD_DS3V2 -- a fairly inexpensive machine type costing about $0.27/hour) and click 'Create': -![](./images/create_notebook_vm.png) + +![New Compute Instance View](../.images/create_notebook_vm.png) See [here](https://docs.microsoft.com/en-us/azure/machine-learning/concept-compute-instance) for details on creating AzureML Compute Instances. @@ -18,13 +19,17 @@ See [here](https://docs.microsoft.com/en-us/azure/machine-learning/concept-compu To clone this git repository onto the workspace, follow the steps below: 1. To get started, first navigate to the JupyterLab instance running on the Compute Instance by clicking on the JupyterLab link shown below: -![](./images/computes_view.png) -1. After going through authentication, you will see the JupyterLab frontend. As you authenticate, make sure to use the same user to log in as was used to create the Compute Instance, or else your access will be denied. Next open an Terminal (either by File/New/Terminal, or by just clicking on Terminal in the Launcher Window). -![](./images/terminal.png) +![JupyterLab link under Application URI in Compute section in AML](../.images/computes_view.png) + +2. After going through authentication, you will see the JupyterLab frontend. As you authenticate, make sure to use the same user to log in as was used to create the Compute Instance, or else your access will be denied. Next open an Terminal (either by File/New/Terminal, or by just clicking on Terminal in the Launcher Window). + +![Terminal in menu](../.images/terminal.png) + +3. In the terminal window clone this repository by typing: -1. In the terminal window clone this repository by typing: ``` git clone https://github.com/microsoft/solution-accelerator-many-models.git ./manymodels ``` + 4. You will be prompted to provide your github username and for your password you will need to provide a personal access token. Please follow the steps here to [create a personal access token.](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) diff --git a/notebooks/README.md b/notebooks/README.md new file mode 100644 index 00000000..84518c0e --- /dev/null +++ b/notebooks/README.md @@ -0,0 +1,57 @@ +# Many Models Notebooks + +This folder contains the notebooks for running the Many Models Solution Accelerator. The functionality is broken into different notebooks designed to be run sequentially. + +## Setup + +### 1. Deploy Resources + +Start by deploying the resources to Azure. The button below will deploy Azure Machine Learning and its related resources: + + + + + +### 2. Configure Development Environment + +Next you'll need to configure your [development environment](https://docs.microsoft.com/azure/machine-learning/how-to-configure-environment) for Azure Machine Learning. We recommend using a [Notebook VM](https://docs.microsoft.com/azure/machine-learning/how-to-configure-environment#compute-instance) as it's the fastest way to get up and running. Follow the steps in [EnvironmentSetup.md](./EnvironmentSetup.md) to create a Notebook VM and clone the repo onto it. + +## Running the notebooks + +Once your development environment is set up, run through the Jupyter Notebooks sequentially following the steps outlined. By the end, you'll know how to train, score, and make predictions using the many models pattern on Azure Machine Learning. + +There are two ways to train many models: + +1. Using a custom training script +2. Using Automated ML + +However, the steps needed to set the workspace up and prepare the datasets are the same no matter which option you choose. + +![Sequence of Notebooks](../.images/mmsa-overview.png) + +### Before training the models + +| Notebook | Description | +|----------------|--------------------------------------------| +| [`00_Setup_AML_Workspace.ipynb`](00_Setup_AML_Workspace.ipynb) | Creates and configures the AML Workspace, including deploying a compute cluster for training. | +| [`01_Data_Preparation.ipynb`](01_Data_Preparation.ipynb) | Prepares the datasets that will be used during training and forecasting. | + +### Using a custom training script to train the models: + +The following notebooks are located under the [`Custom_Script/`](Custom_Script/) folder. + +| Notebook | Description | +|----------------|--------------------------------------------| +| [`02_CustomScript_Training_Pipeline.ipynb`](Custom_Script/02_CustomScript_Training_Pipeline.ipynb) | Creates a pipeline to train a model for each store and orange juice brand in the dataset using a custom script. | +| [`03_CustomScript_Forecasting_Pipeline.ipynb`](Custom_Script/03_CustomScript_Forecasting_Pipeline.ipynb) | Creates a pipeline to forecast future orange juice sales using the models trained in the previous step. | +| [`03_CustomScript_Realtime_Forecasting_Deployment.ipynb`](Custom_Script/03_CustomScript_Realtime_Forecasting_Deployment.ipynb) | Deploys the models into webservices to do real-time forecasting of future orange juice sales upon request. | + +### Using Automated ML to train the models: + +The following notebooks are located under the [`Automated_ML/`](Automated_ML/) folder. + +| Notebook | Description | +|----------------|--------------------------------------------| +| [`02_AutoML_Training_Pipeline.ipynb`](Automated_ML/02_AutoML_Training_Pipeline/02_AutoML_Training_Pipeline.ipynb) | Creates a pipeline to train a model for each store and orange juice brand in the dataset using Automated ML. | +| [`03_AutoML_Forecasting_Pipeline.ipynb`](Automated_ML/03_AutoML_Forecasting_Pipeline/03_AutoML_Forecasting_Pipeline.ipynb) | Creates a pipeline to forecast future orange juice sales using the models trained in the previous step. | +| [`03_CustomScript_Realtime_Forecasting_Deployment.ipynb`](Custom_Script/03_AutoML_Forecasting_Pipeline/03_CustomScript_Realtime_Forecasting_Deployment.ipynb) | Deploys the models into webservices to do real-time forecasting of future orange juice sales upon request. | diff --git a/scripts/helper.py b/notebooks/helpers/dataprep.py similarity index 100% rename from scripts/helper.py rename to notebooks/helpers/dataprep.py diff --git a/notebooks/helpers/settings.py b/notebooks/helpers/settings.py new file mode 100644 index 00000000..f2a88efd --- /dev/null +++ b/notebooks/helpers/settings.py @@ -0,0 +1,10 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import json + + +def dump_settings(settings, scripts_dir, file_name): + with open(os.path.join(scripts_dir, file_name), 'w', encoding='utf-8') as f: + json.dump(settings, f, ensure_ascii=False, indent=4) diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..b6fe8855 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,6 @@ +# Many Models Scripts + +This folder contains all the scripts that are used for training or forecasting many models. + +The scripts are different depending on the training method you are using: Automated ML or Custom Script. +If you are using Custom Script, you should modify these scripts to fit you specific needs. diff --git a/Automated_ML/02_AutoML_Training_Pipeline/scripts/train_automl.py b/scripts/automl/train.py similarity index 88% rename from Automated_ML/02_AutoML_Training_Pipeline/scripts/train_automl.py rename to scripts/automl/train.py index ad589688..0c0ac936 100644 --- a/Automated_ML/02_AutoML_Training_Pipeline/scripts/train_automl.py +++ b/scripts/automl/train.py @@ -33,29 +33,21 @@ parser = argparse.ArgumentParser("split") -parser.add_argument("--process_count_per_node", default=1, type=int, help="number of processes per node") -parser.add_argument("--retrain_failed_models", default=False, type=str2bool, help="retrain failed models only") - +parser.add_argument("--settings-file", type=str, required=True, help="file containing the script settings") args, _ = parser.parse_known_args() -def read_from_json(): - full_path = Path(__file__).absolute().parent - with open(str(full_path) + "/automlconfig.json") as json_file: - return json.load(json_file) - - -automl_settings = read_from_json() -# ''"{\"task\": \"forecasting\", \"iteration_timeout_minutes\": 10, \"iterations\": 10, \"n_cross_validations\": 3, -# \"primary_metric\": \"accuracy\", \"preprocess\": false, \"verbosity\": 20, \"label_column_name\": \"Quantity\", -# \"debug_log\": \"automl_oj_sales_errors.log\", \"time_column_name\": \"WeekStarting\", \"max_horizon\": 6, -# \"drop_column_names\": [\"logQuantity\"], \"group_column_names\": [\"Store\", \"Brand\"]}"'' +base_path = Path(__file__).absolute().parent +with open(os.path.join(base_path, args.settings_file), 'r') as f: + automl_settings = json.load(f) timestamp_column = automl_settings.get('time_column_name', None) grain_column_names = automl_settings.get('grain_column_names', []) group_column_names = automl_settings.get('group_column_names', []) max_horizon = automl_settings.get('max_horizon', 0) target_column = automl_settings.get('label_column_name', None) +process_count_per_node = automl_settings.get('process_count_per_node', 1) +retrain_failed_models = automl_settings.get('retrain_failed_models', False) print("max_horizon: {}".format(max_horizon)) @@ -63,7 +55,7 @@ def read_from_json(): print("timestamp_column: {}".format(timestamp_column)) print("group_column_names: {}".format(group_column_names)) print("grain_column_names: {}".format(grain_column_names)) -print("retrain_failed_models: {}".format(args.retrain_failed_models)) +print("retrain_failed_models: {}".format(retrain_failed_models)) def init(): @@ -77,7 +69,7 @@ def init(): t_log_dir = Path(log_dir) t_log_dir.mkdir(parents=True, exist_ok=True) automl_settings['many_models'] = True - automl_settings['many_models_process_count_per_node'] = args.process_count_per_node + automl_settings['many_models_process_count_per_node'] = process_count_per_node debug_log = automl_settings.get('debug_log', None) if debug_log is not None: @@ -151,7 +143,7 @@ def run(input_data): tags_dict.update(group_columns_dict) - if args.retrain_failed_models: + if retrain_failed_models: logger.info('querying for existing models') try: tags = [[k, v] for k, v in tags_dict.items()] diff --git a/Automated_ML/02_AutoML_Training_Pipeline/scripts/train_automl_helper.py b/scripts/automl/train_automl_helper.py similarity index 100% rename from Automated_ML/02_AutoML_Training_Pipeline/scripts/train_automl_helper.py rename to scripts/automl/train_automl_helper.py diff --git a/Custom_Script/scripts/train.conda.yml b/scripts/customscript/batch_forecasting.conda.yml similarity index 54% rename from Custom_Script/scripts/train.conda.yml rename to scripts/customscript/batch_forecasting.conda.yml index 33cb7095..53762c75 100644 --- a/Custom_Script/scripts/train.conda.yml +++ b/scripts/customscript/batch_forecasting.conda.yml @@ -4,4 +4,8 @@ dependencies: - pip - pip: - joblib + - pandas - sklearn + - azureml-core + - azureml-defaults + - azureml-dataset-runtime[fuse] diff --git a/Custom_Script/scripts/forecast.py b/scripts/customscript/batch_forecasting.py similarity index 58% rename from Custom_Script/scripts/forecast.py rename to scripts/customscript/batch_forecasting.py index 891f62cd..d0fb5fcf 100644 --- a/Custom_Script/scripts/forecast.py +++ b/scripts/customscript/batch_forecasting.py @@ -1,28 +1,37 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import os +import json import argparse import joblib -import pandas as pd +from pathlib import Path +import pandas as pd from azureml.core.model import Model from azureml.core.run import Run +from utils.models import get_model_name -# 0.0 Parse input arguments -parser = argparse.ArgumentParser("split") -parser.add_argument("--id_columns", type=str, nargs='*', required=True, help="input columns identifying the model entity") -parser.add_argument("--timestamp_column", type=str, help="timestamp column from data", required=True) -parser.add_argument("--model_type", type=str, help="model type", required=True) +# 0.0 Parse input arguments +parser = argparse.ArgumentParser() +parser.add_argument("--settings-file", type=str, required=True, help="file containing the script settings") args, _ = parser.parse_known_args() -current_run = None +base_path = Path(__file__).absolute().parent +with open(os.path.join(base_path, args.settings_file), 'r') as f: + customscript_settings = json.load(f) + +id_columns = customscript_settings['id_columns'] +timestamp_column = customscript_settings['timestamp_column'] +model_type = customscript_settings['model_type'] def init(): - global current_run + global ws current_run = Run.get_context() + ws = current_run.experiment.workspace def run(input_data): @@ -33,18 +42,14 @@ def run(input_data): for csv_file_path in input_data: # 3.0 Set up data to predict on - data = (pd.read_csv(csv_file_path, parse_dates=[args.timestamp_column], header=0) - .set_index(args.timestamp_column)) + data = (pd.read_csv(csv_file_path, parse_dates=[timestamp_column], header=0) + .set_index(timestamp_column)) # 4.0 Load registered model from Workspace - id_dict = {id_col: str(data[id_col].iloc[0]) for id_col in args.id_columns} - tag_list = [list(kv) for kv in id_dict.items()] - tag_list.append(['ModelType', args.model_type]) - ws = Run.get_context().experiment.workspace - models = Model.list(ws, tags=tag_list, latest=True) - if len(models) > 1: - raise ValueError(f'More than one models encountered for given timeseries ID: {[m.name for m in models]}') - model_path = models[0].download() + id_dict = {id_col: str(data[id_col].iloc[0]) for id_col in id_columns} + model_name = get_model_name(model_type, id_dict) + model = Model(ws, model_name) + model_path = model.download() forecaster = joblib.load(model_path) # 5.0 Make predictions diff --git a/Custom_Script/scripts/copy_predictions.py b/scripts/customscript/copy_predictions.py similarity index 56% rename from Custom_Script/scripts/copy_predictions.py rename to scripts/customscript/copy_predictions.py index b5e021e0..4363746b 100644 --- a/Custom_Script/scripts/copy_predictions.py +++ b/scripts/customscript/copy_predictions.py @@ -1,32 +1,36 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import pandas as pd import os +import json import datetime import argparse +from pathlib import Path +import pandas as pd # Parse input arguments -parser = argparse.ArgumentParser("parallel run step results directory") -parser.add_argument("--parallel_run_step_output", type=str, help="output directory from parallel run step", - required=True) -parser.add_argument("--output_dir", type=str, help="output directory", required=True) -parser.add_argument("--id_columns", type=str, nargs='*', required=True, help="input columns identifying the model entity") -parser.add_argument("--target_column", type=str, help="column with actual values", default=None) -parser.add_argument("--timestamp_column", type=str, help="timestamp column from data", required=True) -# add list for the columns to pull ? - +parser = argparse.ArgumentParser() +parser.add_argument("--parallel_run_step_output", type=str, required=True, help="output directory from parallel run step") +parser.add_argument("--output_dir", type=str, required=True, help="output directory") +parser.add_argument("--settings-file", type=str, required=True, help="file containing the script settings") args, _ = parser.parse_known_args() -result_file = os.path.join(args.parallel_run_step_output, 'parallel_run_step.txt') +base_path = Path(__file__).absolute().parent +with open(os.path.join(base_path, args.settings_file), 'r') as f: + customscript_settings = json.load(f) + +id_columns = customscript_settings['id_columns'] +target_column = customscript_settings['target_column'] +timestamp_column = customscript_settings['timestamp_column'] # Read the log file and set the column names from the input timeseries schema # The parallel run step log does not have a header row, so add it for easier downstream processing +result_file = os.path.join(args.parallel_run_step_output, 'parallel_run_step.txt') df_predictions = pd.read_csv(result_file, delimiter=" ", header=None) -pred_column_names = [args.timestamp_column, 'Prediction'] -if args.target_column is not None: - pred_column_names.append(args.target_column) -pred_column_names.extend(args.id_columns) +pred_column_names = [timestamp_column, 'Prediction'] +if target_column is not None: + pred_column_names.append(target_column) +pred_column_names.extend(id_columns) print('Using column names: {}'.format(pred_column_names)) assert len(df_predictions.columns) == len(pred_column_names), \ 'Number of columns in prediction data does not match given timeseries schema.' diff --git a/Custom_Script/scripts/forecast_webservice.conda.yml b/scripts/customscript/model_webservice.conda.yml similarity index 100% rename from Custom_Script/scripts/forecast_webservice.conda.yml rename to scripts/customscript/model_webservice.conda.yml diff --git a/Custom_Script/scripts/forecast_webservice.py b/scripts/customscript/model_webservice.py similarity index 69% rename from Custom_Script/scripts/forecast_webservice.py rename to scripts/customscript/model_webservice.py index 0f3c4767..6a95453b 100644 --- a/Custom_Script/scripts/forecast_webservice.py +++ b/scripts/customscript/model_webservice.py @@ -4,6 +4,8 @@ import os import joblib from pathlib import Path + +from azureml.contrib.services.aml_request import AMLRequest, rawhttp from azureml.contrib.services.aml_response import AMLResponse from utils.webservices import read_input, format_output_record @@ -18,7 +20,18 @@ def init(): print('Models loaded:', model_dict.keys()) -def run(rawdata): +@rawhttp +def run(request): + if request.method == 'GET': + return list(model_dict.keys()) + elif request.method == 'POST': + rawdata = request.get_data(cache=False, as_text=True) + return serve_forecasting_request(rawdata) + else: + return AMLResponse("bad request", 500) + + +def serve_forecasting_request(rawdata): batch = read_input(rawdata, format=True) @@ -27,7 +40,7 @@ def run(rawdata): metadata = model_record['metadata'] data_historical, data_future = model_record['data'] - print(f'Received request for: {metadata}') + # print(f'Received request for: {metadata}') # Load model try: @@ -63,15 +76,18 @@ def load_all_models(models_root_path): ''' Load all the models from the models root dir. It returns a dict with key as model name and value as desealized model ready for scoring ''' - p = Path(models_root_path) - models_path_list = [x for x in p.iterdir() if x.is_dir()] - model_dict = {} - for model_dir_path in models_path_list: - model_name = model_dir_path.name - last_version_dir = max(v for v in model_dir_path.iterdir() if v.is_dir()) - model_dict[model_name] = load_model_via_joblib(last_version_dir, model_name, file_extn='') + models_root_dir = Path(models_root_path) + models_path_list = [x for x in models_root_dir.iterdir() if x.is_dir()] + if models_path_list: # Multiple models deployed + for model_dir_path in models_path_list: + model_name = model_dir_path.name + last_version_dir = max(v for v in model_dir_path.iterdir() if v.is_dir()) + model_dict[model_name] = load_model_via_joblib(last_version_dir, model_name, file_extn='') + else: # Single model deployed + model_name = models_root_dir.parent.name + model_dict[model_name] = load_model_via_joblib(models_root_dir, model_name, file_extn='') return model_dict diff --git a/Custom_Script/scripts/routing_webservice.conda.yml b/scripts/customscript/routing_webservice.conda.yml similarity index 100% rename from Custom_Script/scripts/routing_webservice.conda.yml rename to scripts/customscript/routing_webservice.conda.yml diff --git a/Custom_Script/scripts/routing_webservice.py b/scripts/customscript/routing_webservice.py similarity index 95% rename from Custom_Script/scripts/routing_webservice.py rename to scripts/customscript/routing_webservice.py index 9dbe9997..5b6465f7 100644 --- a/Custom_Script/scripts/routing_webservice.py +++ b/scripts/customscript/routing_webservice.py @@ -36,7 +36,6 @@ def run(request): return routing_model_artifact elif request.method == 'POST': rawdata = request.get_data(cache=False, as_text=True) - print(rawdata) return route_forecasting_requests(rawdata) else: return AMLResponse("bad request", 500) @@ -55,7 +54,7 @@ def route_forecasting_requests(rawdata): model_name = get_model_name(model_data['model_type'], model_data['id']) model_service = service_mapping[model_name] except KeyError: - return AMLResponse(f"Model not found of type {model_data['model_type']} for ID {model_data['id']}", 400) + return AMLResponse(f"Router: Model not found of type {model_data['model_type']} for ID {model_data['id']}", 400) # Append data to service minibatch services_tocall[model_service].append(model_data) diff --git a/Custom_Script/scripts/forecast.conda.yml b/scripts/customscript/train.conda.yml similarity index 69% rename from Custom_Script/scripts/forecast.conda.yml rename to scripts/customscript/train.conda.yml index dbba36c9..672f696b 100644 --- a/Custom_Script/scripts/forecast.conda.yml +++ b/scripts/customscript/train.conda.yml @@ -6,3 +6,5 @@ dependencies: - joblib - pandas - sklearn + - azureml-core + - azureml-dataset-runtime[fuse] diff --git a/Custom_Script/scripts/train.py b/scripts/customscript/train.py similarity index 66% rename from Custom_Script/scripts/train.py rename to scripts/customscript/train.py index dfa91883..4e711c46 100644 --- a/Custom_Script/scripts/train.py +++ b/scripts/customscript/train.py @@ -1,13 +1,16 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from azureml.core import Run -import pandas as pd -import numpy as np import os +import json import argparse import datetime import joblib +from pathlib import Path + +import numpy as np +import pandas as pd +from azureml.core import Run from sklearn.metrics import mean_squared_error, mean_absolute_error from sklearn.linear_model import LinearRegression @@ -16,20 +19,25 @@ from utils.timeseries import ColumnDropper, SimpleLagger, SimpleCalendarFeaturizer, SimpleForecaster -# 0.0 Parse input arguments -parser = argparse.ArgumentParser("split") -parser.add_argument("--id_columns", type=str, nargs='*', required=True, help="input columns identifying the model entity") -parser.add_argument("--target_column", type=str, required=True, help="input target column") -parser.add_argument("--timestamp_column", type=str, required=True, help="input timestamp column") -parser.add_argument("--tag_columns", type=str, nargs='*', default=[], help="input columns to set as tags for the model") -parser.add_argument("--drop_id", action='store_true', help="flag to drop columns used as ID") -parser.add_argument("--drop_tags", action='store_true', help="flag to drop columns used as tags") -parser.add_argument("--drop_columns", type=str, nargs='*', default=[], help="list of columns to drop prior to modeling") -parser.add_argument("--model_type", type=str, required=True, help="input model type") -parser.add_argument("--test_size", type=int, required=True, help="number of observations to be used for testing") - +# 0.0 Parse input arguments and read settings +parser = argparse.ArgumentParser() +parser.add_argument("--settings-file", type=str, required=True, help="file containing the script settings") args, _ = parser.parse_known_args() +base_path = Path(__file__).absolute().parent +with open(os.path.join(base_path, args.settings_file), 'r') as f: + customscript_settings = json.load(f) + +id_columns = customscript_settings['id_columns'] +target_column = customscript_settings['target_column'] +timestamp_column = customscript_settings['timestamp_column'] +model_type = customscript_settings['model_type'] +tag_columns = customscript_settings.get('tag_columns', []) +drop_id = customscript_settings.get('drop_id', False) +drop_tags = customscript_settings.get('drop_tags', False) +drop_columns = customscript_settings.get('drop_columns', []) +test_size = customscript_settings.get('test_size', 10) + def init(): global current_run @@ -49,32 +57,32 @@ def run(input_data): start_datetime = datetime.datetime.now() # 1.0 Read the data from CSV - parse timestamps as datetime type and put the time in the index - data = (pd.read_csv(csv_file_path, parse_dates=[args.timestamp_column], header=0) - .set_index(args.timestamp_column) + data = (pd.read_csv(csv_file_path, parse_dates=[timestamp_column], header=0) + .set_index(timestamp_column) .sort_index(ascending=True)) # ID and tags: uses the values from the first row of data - id_dict = {id_col: str(data[id_col].iloc[0]) for id_col in args.id_columns} - model_name = get_model_name(args.model_type, id_dict) - tags_dict = {tag_col: str(data[tag_col].iloc[0]) for tag_col in args.tag_columns} - cols_todrop = args.drop_columns + \ - (args.id_columns if args.drop_id else []) + \ - (args.tag_columns if args.drop_tags else []) + id_dict = {id_col: str(data[id_col].iloc[0]) for id_col in id_columns} + model_name = get_model_name(model_type, id_dict) + tags_dict = {tag_col: str(data[tag_col].iloc[0]) for tag_col in tag_columns} + cols_todrop = drop_columns + \ + (id_columns if drop_id else []) + \ + (tag_columns if drop_tags else []) print(f'Model name: "{model_name}"') print(f'ID tags: {id_dict}') print(f'Extra tags: {tags_dict}') print(f'Columns to drop: {cols_todrop}') # 2.0 Split the data into train and test sets - train = data[:-args.test_size] - test = data[-args.test_size:] + train = data[:-test_size] + test = data[-test_size:] # 3.0 Create and fit the forecasting pipeline # The pipeline will drop unhelpful features, make a calendar feature, and make lag features - lagger = SimpleLagger(args.target_column, lag_orders=[1, 2, 3, 4]) + lagger = SimpleLagger(target_column, lag_orders=[1, 2, 3, 4]) transform_steps = [('column_dropper', ColumnDropper(cols_todrop)), ('calendar_featurizer', SimpleCalendarFeaturizer()), ('lagger', lagger)] - forecaster = SimpleForecaster(transform_steps, LinearRegression(), args.target_column, args.timestamp_column) + forecaster = SimpleForecaster(transform_steps, LinearRegression(), target_column, timestamp_column) forecaster.fit(train) print('Featurized data example:') print(forecaster.transform(train).head()) @@ -84,10 +92,10 @@ def run(input_data): compare_data = test.assign(forecasts=forecasts).dropna() # 5.0 Calculate accuracy metrics for the fit - mse = mean_squared_error(compare_data[args.target_column], compare_data['forecasts']) + mse = mean_squared_error(compare_data[target_column], compare_data['forecasts']) rmse = np.sqrt(mse) - mae = mean_absolute_error(compare_data[args.target_column], compare_data['forecasts']) - actuals = compare_data[args.target_column].values + mae = mean_absolute_error(compare_data[target_column], compare_data['forecasts']) + actuals = compare_data[target_column].values preds = compare_data['forecasts'].values mape = np.mean(np.abs((actuals - preds) / actuals) * 100) @@ -106,14 +114,14 @@ def run(input_data): current_run.upload_file(model_name, model_path) # 9.0 Register the model to the workspace to be used in forecasting - all_tags = {'ModelType': args.model_type, **id_dict, **tags_dict} + all_tags = {'ModelType': model_type, **id_dict, **tags_dict} current_run.register_model(model_path=model_name, model_name=model_name, - model_framework=args.model_type, tags=all_tags) + model_framework=model_type, tags=all_tags) # 10.0 Add data to output end_datetime = datetime.datetime.now() result.update(id_dict) - result['model_type'] = args.model_type + result['model_type'] = model_type result['file_name'] = csv_file_path result['model_name'] = model_name result['start_date'] = str(start_datetime) diff --git a/Custom_Script/scripts/utils/forecasting.py b/scripts/customscript/utils/forecasting.py similarity index 100% rename from Custom_Script/scripts/utils/forecasting.py rename to scripts/customscript/utils/forecasting.py diff --git a/Custom_Script/scripts/utils/models.py b/scripts/customscript/utils/models.py similarity index 100% rename from Custom_Script/scripts/utils/models.py rename to scripts/customscript/utils/models.py diff --git a/Custom_Script/scripts/utils/timeseries.py b/scripts/customscript/utils/timeseries.py similarity index 100% rename from Custom_Script/scripts/utils/timeseries.py rename to scripts/customscript/utils/timeseries.py diff --git a/Custom_Script/scripts/utils/webservices.py b/scripts/customscript/utils/webservices.py similarity index 100% rename from Custom_Script/scripts/utils/webservices.py rename to scripts/customscript/utils/webservices.py