diff --git a/examples/k8s-upgrade/v1.7.9-hybrid.json.env b/examples/k8s-upgrade/v1.7.9-hybrid.json.env index 7ec290ed25..45f41d2632 100644 --- a/examples/k8s-upgrade/v1.7.9-hybrid.json.env +++ b/examples/k8s-upgrade/v1.7.9-hybrid.json.env @@ -1,2 +1,2 @@ ACSE_POSTDEPLOY=examples/k8s-upgrade/k8s-upgrade.sh -EXPECTED_ORCHESTRATOR_VERSION=1.8.9 \ No newline at end of file +EXPECTED_ORCHESTRATOR_VERSION=1.8.9 diff --git a/examples/kubernetes-aci.json b/examples/kubernetes-aci.json new file mode 100644 index 0000000000..ee975bcb46 --- /dev/null +++ b/examples/kubernetes-aci.json @@ -0,0 +1,90 @@ +{ + "apiVersion": "vlabs", + "plan": {}, + "properties": { + "provisioningState": "", + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorVersion": "1.8.2", + "kubernetesConfig": { + "networkPolicy": "none", + "kubeletConfig": { + "--cloud-provider": "", + "--cloud-config": "", + "--azure-container-registry-config": "" + }, + "addons": [ + { + "name": "tiller", + "enabled" : false + }, + { + "name": "kubernetes-dashboard", + "enabled" : false + } + ] + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "caas-test-eastus-linux-03", + "vmSize": "Standard_D2_v2", + "firstConsecutiveStaticIP": "10.240.255.5" + }, + "agentPoolProfiles": [ + { + "name": "system", + "count": 2, + "vmSize": "Standard_F1", + "availabilityProfile": "AvailabilitySet", + "storageProfile": "StorageAccount", + "osType": "Linux" + }, + { + "name": "agentpool1", + "count": 2, + "vmSize": "Standard_F2", + "availabilityProfile": "AvailabilitySet", + "storageProfile": "StorageAccount", + "osType": "Linux" + }, + { + "name": "agentpool2", + "count": 3, + "vmSize": "Standard_F1", + "availabilityProfile": "AvailabilitySet", + "storageProfile": "StorageAccount", + "osType": "Linux", + "osDiskSizeGB": 50 + }, + { + "name": "agentpool3", + "count": 3, + "vmSize": "Standard_F1", + "availabilityProfile": "AvailabilitySet", + "storageProfile": "StorageAccount", + "osType": "Linux", + "osDiskSizeGB": 50 + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "windowsProfile": { + "adminUsername": "", + "adminPassword": "" + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + }, + "certificateProfile": {} + } +} diff --git a/parts/agentoutputs.t b/parts/agentoutputs.t index f809141e28..e286720f78 100644 --- a/parts/agentoutputs.t +++ b/parts/agentoutputs.t @@ -9,14 +9,14 @@ {{if and .IsAvailabilitySets .IsStorageAccount}} "{{.Name}}StorageAccountOffset": { "type": "int", - "value": "[variables('{{.Name}}StorageAccountOffset')]" + "value": "[variables('{{.Name}}Variables').StorageAccountOffset]" }, "{{.Name}}StorageAccountCount": { "type": "int", - "value": "[variables('{{.Name}}StorageAccountsCount')]" + "value": "[variables('{{.Name}}Variables').StorageAccountsCount]" }, "{{.Name}}SubnetName": { "type": "string", - "value": "[variables('{{.Name}}SubnetName')]" + "value": "[variables('{{.Name}}Variables').SubnetName]" }, {{end}} \ No newline at end of file diff --git a/parts/k8s/addons/kubernetesmasteraddons-kube-dns-deployment.yaml b/parts/k8s/addons/kubernetesmasteraddons-kube-dns-deployment.yaml index 42ac79a934..e37c837c26 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-kube-dns-deployment.yaml +++ b/parts/k8s/addons/kubernetesmasteraddons-kube-dns-deployment.yaml @@ -75,6 +75,11 @@ spec: selector: matchLabels: k8s-app: kube-dns + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate template: metadata: labels: @@ -210,4 +215,5 @@ spec: dnsPolicy: Default serviceAccountName: kube-dns nodeSelector: - beta.kubernetes.io/os: linux \ No newline at end of file + beta.kubernetes.io/os: linux + agentpool: system diff --git a/parts/k8s/addons/kubernetesmasteraddons-kube-proxy-daemonset.yaml b/parts/k8s/addons/kubernetesmasteraddons-kube-proxy-daemonset.yaml index 9b5bce2847..9ae072488f 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-kube-proxy-daemonset.yaml +++ b/parts/k8s/addons/kubernetesmasteraddons-kube-proxy-daemonset.yaml @@ -29,9 +29,6 @@ spec: image: imagePullPolicy: IfNotPresent name: kube-proxy - resources: - requests: - cpu: 100m securityContext: privileged: true volumeMounts: diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml index 209ebc7a04..6f2dd6dcc5 100644 --- a/parts/k8s/kubernetesagentcustomdata.yml +++ b/parts/k8s/kubernetesagentcustomdata.yml @@ -98,6 +98,14 @@ write_files: RemainAfterExit=yes ExecStart=/usr/local/bin/health-monitor.sh container-runtime +- path: /etc/apt/preferences + permissions: "0644" + owner: root + content: | + Package: walinuxagent + Pin: version 2.2.32.2 + Pin-Priority: 550 + {{if .KubernetesConfig.RequiresDocker}} {{if not .IsCoreOS}} - path: /etc/systemd/system/docker.service.d/clear_mount_propagation_flags.conf @@ -268,6 +276,5 @@ coreos: runcmd: - set -x - . /opt/azure/containers/provision_source.sh -- timeout 10 apt-mark hold walinuxagent{{GetKubernetesAgentPreprovisionYaml .}} -- timeout 10 apt-mark unhold walinuxagent +- aptmarkWALinuxAgent hold{{GetKubernetesAgentPreprovisionYaml .}} {{end}} diff --git a/parts/k8s/kubernetesagentresourcesvmas.t b/parts/k8s/kubernetesagentresourcesvmas.t index 72aa01bb15..0a2dfd8791 100644 --- a/parts/k8s/kubernetesagentresourcesvmas.t +++ b/parts/k8s/kubernetesagentresourcesvmas.t @@ -1,7 +1,7 @@ { "apiVersion": "[variables('apiVersionNetwork')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "loop" }, "dependsOn": [ @@ -26,7 +26,7 @@ {{end}} ], "location": "[variables('location')]", - "name": "[concat(variables('{{.Name}}VMNamePrefix'), 'nic-', copyIndex(variables('{{.Name}}Offset')))]", + "name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, 'nic-', copyIndex(variables('{{.Name}}Variables').Offset))]", "properties": { "enableAcceleratedNetworking" : {{.AcceleratedNetworkingEnabled}}, {{if not IsOpenShift}} @@ -50,11 +50,24 @@ "name": "ipconfig{{$seq}}", "properties": { {{if eq $seq 1}} - "primary": true, + "primary": true, + {{if eq $.Name "system"}} + "privateIPAddress": "[concat(variables('aciSystemNodeAddrPrefix'), copyIndex(add(50, int(variables('aciPrimaryIPOctet4')))))]", + {{else if eq $.Name "agentpool1"}} + "privateIPAddress": "[concat(variables('aciSystemNodeAddrPrefix'), copyIndex(add(100, int(variables('aciPrimaryIPOctet4')))))]", + {{else}} + "privateIPAddress": "[concat(variables('aciCustomerNodeAddrPrefix'), copyIndex(mul(25, sub(int(variables('{{$.Name}}Index')), 2))), '.', variables('aciPrimaryIPOctet4'))]", + {{end}} + {{else if eq $.Name "system"}} + "privateIPAddress": "[concat(variables('aciSystemPodAddrPrefix'), copyIndex(add(50, int(variables('aciPrimaryIPOctet4')))), '.', add(sub({{$seq}}, 1), int(variables('aciPrimaryIPOctet4'))))]", + {{else if eq $.Name "agentpool1"}} + "privateIPAddress": "[concat(variables('aciSystemPodAddrPrefix'), copyIndex(add(100, int(variables('aciPrimaryIPOctet4')))), '.', add(sub({{$seq}}, 1), int(variables('aciPrimaryIPOctet4'))))]", + {{else}} + "privateIPAddress": "[concat(variables('aciCustomerPodAddrPrefix'), copyIndex(mul(25, sub(int(variables('{{$.Name}}Index')), 2))), '.', add(sub({{$seq}}, 1), int(variables('aciPrimaryIPOctet4'))))]", {{end}} - "privateIPAllocationMethod": "Dynamic", + "privateIPAllocationMethod": "Static", "subnet": { - "id": "[variables('{{$.Name}}VnetSubnetID')]" + "id": "[variables('{{$.Name}}Variables').VnetSubnetID]" } {{if eq $.Role "infra"}} , @@ -79,12 +92,13 @@ {{if .IsManagedDisks}} { "location": "[variables('location')]", - "name": "[variables('{{.Name}}AvailabilitySet')]", - "apiVersion": "[variables('apiVersionCompute')]", + "name": "[variables('{{.Name}}Variables').AvailabilitySet]", + "apiVersion": "[variables('apiVersionStorageManagedDisks')]", "properties": { "platformFaultDomainCount": 2, "platformUpdateDomainCount": 3 + "managed" : "true" }, "sku": { "name": "Aligned" @@ -95,7 +109,7 @@ { "apiVersion": "[variables('apiVersionStorage')]", "copy": { - "count": "[variables('{{.Name}}StorageAccountsCount')]", + "count": "[variables('{{.Name}}Variables').StorageAccountsCount]", "name": "loop" }, {{if not IsHostedMaster}} @@ -105,10 +119,25 @@ ], {{end}} {{end}} + "kind": "Storage", "location": "[variables('location')]", - "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName'))]", + "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName'))]", + "properties": { + "encryption": { + "keySource": "Microsoft.Storage", + "services": { + "blob": { + "enabled": true + }, + "file": { + "enabled": true + } + } + }, + "supportsHttpsTrafficOnly": true + }, "sku": { - "name": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" + "name": "[variables('vmSizesMap')[variables('{{.Name}}Variables').VMSize].storageAccountType]" }, "type": "Microsoft.Storage/storageAccounts" }, @@ -116,7 +145,7 @@ { "apiVersion": "[variables('apiVersionStorage')]", "copy": { - "count": "[variables('{{.Name}}StorageAccountsCount')]", + "count": "[variables('{{.Name}}Variables').StorageAccountsCount]", "name": "datadiskLoop" }, {{if not IsHostedMaster}} @@ -126,17 +155,32 @@ ], {{end}} {{end}} + "kind": "Storage", "location": "[variables('location')]", - "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}DataAccountName'))]", + "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('{{.Name}}DataAccountName'))]", + "properties": { + "encryption": { + "keySource": "Microsoft.Storage", + "services": { + "blob": { + "enabled": true + }, + "file": { + "enabled": true + } + } + }, + "supportsHttpsTrafficOnly": true + }, "sku": { - "name": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" + "name": "[variables('vmSizesMap')[variables('{{.Name}}Variables').VMSize].storageAccountType]" }, "type": "Microsoft.Storage/storageAccounts" }, {{end}} { "location": "[variables('location')]", - "name": "[variables('{{.Name}}AvailabilitySet')]", + "name": "[variables('{{.Name}}Variables').AvailabilitySet]", "apiVersion": "[variables('apiVersionCompute')]", "properties": {}, "type": "Microsoft.Compute/availabilitySets" @@ -145,30 +189,30 @@ { "apiVersion": "[variables('apiVersionCompute')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "vmLoopNode" }, "dependsOn": [ {{if .IsStorageAccount}} - "[concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName'))]", + "[concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName'))]", {{if .HasDisks}} - "[concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('dataStorageAccountPrefixSeed')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('dataStorageAccountPrefixSeed')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}DataAccountName'))]", + "[concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('dataStorageAccountPrefixSeed')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('dataStorageAccountPrefixSeed')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}DataAccountName'))]", {{end}} {{end}} - "[concat('Microsoft.Network/networkInterfaces/', variables('{{.Name}}VMNamePrefix'), 'nic-', copyIndex(variables('{{.Name}}Offset')))]", - "[concat('Microsoft.Compute/availabilitySets/', variables('{{.Name}}AvailabilitySet'))]" + "[concat('Microsoft.Network/networkInterfaces/', variables('{{.Name}}Variables').VMNamePrefix, 'nic-', copyIndex(variables('{{.Name}}Variables').Offset))]", + "[concat('Microsoft.Compute/availabilitySets/', variables('{{.Name}}Variables').AvailabilitySet)]" ], "tags": { - "creationSource" : "[concat(parameters('generatorCode'), '-', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", + "creationSource" : "[concat(parameters('generatorCode'), '-', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]", "resourceNameSuffix" : "[parameters('nameSuffix')]", "orchestrator" : "[variables('orchestratorNameVersionTag')]", "acsengineVersion" : "[parameters('acsengineVersion')]", "poolName" : "{{.Name}}" }, "location": "[variables('location')]", - "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", + "name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]", {{if UseManagedIdentity}} {{if UserAssignedIDEnabled}} "identity": { @@ -192,21 +236,21 @@ {{end}} "properties": { "availabilitySet": { - "id": "[resourceId('Microsoft.Compute/availabilitySets',variables('{{.Name}}AvailabilitySet'))]" + "id": "[resourceId('Microsoft.Compute/availabilitySets',variables('{{.Name}}Variables').AvailabilitySet)]" }, "hardwareProfile": { - "vmSize": "[variables('{{.Name}}VMSize')]" + "vmSize": "[variables('{{.Name}}Variables').VMSize]" }, "networkProfile": { "networkInterfaces": [ { - "id": "[resourceId('Microsoft.Network/networkInterfaces',concat(variables('{{.Name}}VMNamePrefix'), 'nic-', copyIndex(variables('{{.Name}}Offset'))))]" + "id": "[resourceId('Microsoft.Network/networkInterfaces',concat(variables('{{.Name}}Variables').VMNamePrefix, 'nic-', copyIndex(variables('{{.Name}}Variables').Offset)))]" } ] }, "osProfile": { "adminUsername": "[parameters('linuxAdminUsername')]", - "computername": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", + "computername": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]", {{if not IsOpenShift}} {{GetKubernetesAgentCustomData .}} {{end}} @@ -232,21 +276,21 @@ {{end}} "imageReference": { {{if UseAgentCustomImage .}} - "id": "[resourceId(variables('{{.Name}}osImageResourceGroup'), 'Microsoft.Compute/images', variables('{{.Name}}osImageName'))]" + "id": "[resourceId(variables('{{.Name}}Variables').osImageResourceGroup, 'Microsoft.Compute/images', variables('{{.Name}}Variables').osImageName)]" {{else}} - "offer": "[variables('{{.Name}}osImageOffer')]", - "publisher": "[variables('{{.Name}}osImagePublisher')]", - "sku": "[variables('{{.Name}}osImageSKU')]", - "version": "[variables('{{.Name}}osImageVersion')]" + "offer": "[variables('{{.Name}}Variables').osImageOffer]", + "publisher": "[variables('{{.Name}}Variables').osImagePublisher]", + "sku": "[variables('{{.Name}}Variables').osImageSKU]", + "version": "[variables('{{.Name}}Variables').osImageVersion]" {{end}} }, "osDisk": { "createOption": "FromImage" ,"caching": "ReadWrite" {{if .IsStorageAccount}} - ,"name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')),'-osdisk')]" + ,"name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset),'-osdisk')]" ,"vhd": { - "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName')),variables('apiVersionStorage')).primaryEndpoints.blob,'osdisk/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), '-osdisk.vhd')]" + "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName')),variables('apiVersionStorage')).primaryEndpoints.blob,'osdisk/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), '-osdisk.vhd')]" } {{end}} {{if ne .OSDiskSizeGB 0}} @@ -262,35 +306,35 @@ { "apiVersion": "[variables('apiVersionAuthorization')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "vmLoopNode" }, - "name": "[guid(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), 'vmidentity'))]", + "name": "[guid(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), 'vmidentity'))]", "type": "Microsoft.Authorization/roleAssignments", "properties": { "roleDefinitionId": "[variables('readerRoleDefinitionId')]", - "principalId": "[reference(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset'))), '2017-03-30', 'Full').identity.principalId]" + "principalId": "[reference(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset)), '2017-03-30', 'Full').identity.principalId]" } }, {{end}} { "type": "Microsoft.Compute/virtualMachines/extensions", - "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), '/ManagedIdentityExtension')]", + "name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), '/ManagedIdentityExtension')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "vmLoopNode" }, "apiVersion": "[variables('apiVersionCompute')]", "location": "[resourceGroup().location]", {{if UserAssignedIDEnabled}} "dependsOn": [ - "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", + "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]", "[concat('Microsoft.Authorization/roleAssignments/',guid(concat(variables('userAssignedID'), 'roleAssignment', resourceGroup().id)))]" ], {{else}} "dependsOn": [ - "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", - "[concat('Microsoft.Authorization/roleAssignments/', guid(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), 'vmidentity')))]" + "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]", + "[concat('Microsoft.Authorization/roleAssignments/', guid(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), 'vmidentity')))]" ], {{end}} "properties": { @@ -308,19 +352,19 @@ { "apiVersion": "[variables('apiVersionCompute')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "vmLoopNode" }, "dependsOn": [ {{if UseManagedIdentity}} - "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), '/extensions/ManagedIdentityExtension')]" + "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), '/extensions/ManagedIdentityExtension')]" {{else}} - "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]" + "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]" {{end}} ], "location": "[variables('location')]", "type": "Microsoft.Compute/virtualMachines/extensions", - "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')),'/cse', '-agent-', copyIndex(variables('{{.Name}}Offset')))]", + "name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset),'/cse', '-agent-', copyIndex(variables('{{.Name}}Variables').Offset))]", "properties": { "publisher": "Microsoft.Azure.Extensions", "type": "CustomScript", @@ -328,26 +372,28 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - {{if IsOpenShift }} - "script": "{{ Base64 (OpenShiftGetNodeSh .) }}" - {{else}} - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not (IsFeatureEnabled "BlockOutboundInternet")}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if IsFeatureEnabled "CSERunInBackground" }} &{{end}}\"')]" - {{end}} + {{if eq $.Name "system"}} + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not (IsFeatureEnabled "BlockOutboundInternet")}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' VNET_CNI_PLUGINS_URL=', parameters('vnetCniLinuxPluginsURL'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if IsFeatureEnabled "CSERunInBackground" }} &{{end}}\"')]" + {{else if eq $.Name "agentpool1"}} + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not (IsFeatureEnabled "BlockOutboundInternet")}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' VNET_CNI_PLUGINS_URL=', parameters('vnetCniLinuxPluginsURL'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if IsFeatureEnabled "CSERunInBackground" }} &{{end}}\"')]" + {{else}} + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not (IsFeatureEnabled "BlockOutboundInternet")}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' VNET_CNI_PLUGINS_URL=', parameters('vnetCniMultitenancyLinuxPluginsURL'),' GPU_NODE={{IsNSeriesSKU .}} /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1{{if IsFeatureEnabled "CSERunInBackground" }} &{{end}}\"')]" + {{end}} } } } {{if UseAksExtension}} ,{ "type": "Microsoft.Compute/virtualMachines/extensions", - "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), '/computeAksLinuxBilling')]", + "name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), '/computeAksLinuxBilling')]", "apiVersion": "[variables('apiVersionCompute')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "vmLoopNode" }, "location": "[variables('location')]", "dependsOn": [ - "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]" + "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]" ], "properties": { "publisher": "Microsoft.AKS", diff --git a/parts/k8s/kubernetesagentvars.t b/parts/k8s/kubernetesagentvars.t index 3212f9f807..f8801f6e7f 100644 --- a/parts/k8s/kubernetesagentvars.t +++ b/parts/k8s/kubernetesagentvars.t @@ -1,33 +1,38 @@ -{{if .IsStorageAccount}} - "{{.Name}}StorageAccountOffset": "[mul(variables('maxStorageAccountsPerAgent'),variables('{{.Name}}Index'))]", - "{{.Name}}StorageAccountsCount": "[add(div(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')), mod(add(mod(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')),2), add(mod(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')),1)))]", -{{end}} - "{{.Name}}Count": "[parameters('{{.Name}}Count')]", - "{{.Name}}VMNamePrefix": "{{GetAgentVMPrefix .}}", {{if .IsWindows}} "winResourceNamePrefix" : "[substring(parameters('nameSuffix'), 0, 5)]", {{end}} -{{if .IsAvailabilitySets}} - "{{.Name}}Offset": "[parameters('{{.Name}}Offset')]", - "{{.Name}}AvailabilitySet": "[concat('{{.Name}}-availabilitySet-', parameters('nameSuffix'))]", -{{else}} - {{if .IsLowPriorityScaleSet}} - "{{.Name}}ScaleSetPriority": "[parameters('{{.Name}}ScaleSetPriority')]", - "{{.Name}}ScaleSetEvictionPolicy": "[parameters('{{.Name}}ScaleSetEvictionPolicy')]", + +"{{.Name}}Count": "[parameters('{{.Name}}Count')]", +"{{.Name}}Variables": +{ + {{if .IsStorageAccount}} + "StorageAccountOffset": "[mul(variables('maxStorageAccountsPerAgent'),variables('{{.Name}}Index'))]", + "StorageAccountsCount": "[add(div(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')), mod(add(mod(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')),2), add(mod(variables('{{.Name}}Count'), variables('maxVMsPerStorageAccount')),1)))]", {{end}} -{{end}} - "{{.Name}}VMSize": "[parameters('{{.Name}}VMSize')]", -{{if .IsCustomVNET}} - "{{.Name}}VnetSubnetID": "[parameters('{{.Name}}VnetSubnetID')]", - "{{.Name}}SubnetName": "[parameters('{{.Name}}VnetSubnetID')]", - "{{.Name}}VnetParts": "[split(parameters('{{.Name}}VnetSubnetID'),'/subnets/')]", -{{else}} - "{{.Name}}VnetSubnetID": "[variables('vnetSubnetID')]", - "{{.Name}}SubnetName": "[variables('subnetName')]", -{{end}} - "{{.Name}}osImageOffer": "[parameters('{{.Name}}osImageOffer')]", - "{{.Name}}osImageSKU": "[parameters('{{.Name}}osImageSKU')]", - "{{.Name}}osImagePublisher": "[parameters('{{.Name}}osImagePublisher')]", - "{{.Name}}osImageVersion": "[parameters('{{.Name}}osImageVersion')]", - "{{.Name}}osImageName": "[parameters('{{.Name}}osImageName')]", - "{{.Name}}osImageResourceGroup": "[parameters('{{.Name}}osImageResourceGroup')]", + {{if .IsAvailabilitySets}} + "Offset": "[parameters('{{.Name}}Offset')]", + "AvailabilitySet": "[concat('{{.Name}}-availabilitySet-', parameters('nameSuffix'))]", + {{else}} + {{if .IsLowPriorityScaleSet}} + "ScaleSetPriority": "[parameters('{{.Name}}ScaleSetPriority')]", + "ScaleSetEvictionPolicy": "[parameters('{{.Name}}ScaleSetEvictionPolicy')]", + {{end}} + {{end}} + "VMNamePrefix": "{{GetAgentVMPrefix .}}", + "VMSize": "[parameters('{{.Name}}VMSize')]", + {{if .IsCustomVNET}} + "VnetSubnetID": "[parameters('{{.Name}}VnetSubnetID')]", + "SubnetName": "[parameters('{{.Name}}VnetSubnetID')]", + "VnetParts": "[split(parameters('{{.Name}}VnetSubnetID'),'/subnets/')]", + {{else}} + "VnetSubnetID": "[variables('vnetSubnetID')]", + "SubnetName": "[variables('subnetName')]", + {{end}} + "osImageOffer": "[parameters('{{.Name}}osImageOffer')]", + "osImageSKU": "[parameters('{{.Name}}osImageSKU')]", + "osImagePublisher": "[parameters('{{.Name}}osImagePublisher')]", + "osImageVersion": "[parameters('{{.Name}}osImageVersion')]", + "osImageName": "[parameters('{{.Name}}osImageName')]", + "osImageResourceGroup": "[parameters('{{.Name}}osImageResourceGroup')]" +}, + diff --git a/parts/k8s/kubernetesconfigs.sh b/parts/k8s/kubernetesconfigs.sh index 46615b0856..e8de233e81 100755 --- a/parts/k8s/kubernetesconfigs.sh +++ b/parts/k8s/kubernetesconfigs.sh @@ -105,11 +105,6 @@ ensureRPC() { systemctlEnableAndStart rpc-statd || exit $ERR_SYSTEMCTL_START_FAIL } -runAptDaily() { - wait_for_apt_locks - /usr/lib/apt/apt.systemd.daily -} - generateAggregatedAPICerts() { AGGREGATED_API_CERTS_SETUP_FILE=/etc/kubernetes/generate-proxy-certs.sh wait_for_file 1200 1 $AGGREGATED_API_CERTS_SETUP_FILE || exit $ERR_FILE_WATCH_TIMEOUT @@ -127,57 +122,64 @@ configureK8s() { chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}" chown root:root "${APISERVER_PUBLIC_KEY_PATH}" - AZURE_JSON_PATH="/etc/kubernetes/azure.json" - touch "${AZURE_JSON_PATH}" - chmod 0600 "${AZURE_JSON_PATH}" - chown root:root "${AZURE_JSON_PATH}" - set +x echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}" echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}" # Perform the required JSON escaping for special characters " and \ SERVICE_PRINCIPAL_CLIENT_SECRET=$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | sed "s|\\\\|\\\\\\\|g") SERVICE_PRINCIPAL_CLIENT_SECRET=$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | sed 's|"|\\"|g') - cat << EOF > "${AZURE_JSON_PATH}" + + if [[ ! -z "${MASTER_NODE}" ]]; then + echo "MASTER_NODE is non-empty, master node, configure azure json." + + AZURE_JSON_PATH="/etc/kubernetes/azure.json" + touch "${AZURE_JSON_PATH}" + chmod 0600 "${AZURE_JSON_PATH}" + chown root:root "${AZURE_JSON_PATH}" + cat << EOF > "${AZURE_JSON_PATH}" { - "cloud":"${TARGET_ENVIRONMENT}", - "tenantId": "${TENANT_ID}", - "subscriptionId": "${SUBSCRIPTION_ID}", - "aadClientId": "${SERVICE_PRINCIPAL_CLIENT_ID}", - "aadClientSecret": "${SERVICE_PRINCIPAL_CLIENT_SECRET}", - "resourceGroup": "${RESOURCE_GROUP}", - "location": "${LOCATION}", - "vmType": "${VM_TYPE}", - "subnetName": "${SUBNET}", - "securityGroupName": "${NETWORK_SECURITY_GROUP}", - "vnetName": "${VIRTUAL_NETWORK}", - "vnetResourceGroup": "${VIRTUAL_NETWORK_RESOURCE_GROUP}", - "routeTableName": "${ROUTE_TABLE}", - "primaryAvailabilitySetName": "${PRIMARY_AVAILABILITY_SET}", - "primaryScaleSetName": "${PRIMARY_SCALE_SET}", - "cloudProviderBackoff": ${CLOUDPROVIDER_BACKOFF}, - "cloudProviderBackoffRetries": ${CLOUDPROVIDER_BACKOFF_RETRIES}, - "cloudProviderBackoffExponent": ${CLOUDPROVIDER_BACKOFF_EXPONENT}, - "cloudProviderBackoffDuration": ${CLOUDPROVIDER_BACKOFF_DURATION}, - "cloudProviderBackoffJitter": ${CLOUDPROVIDER_BACKOFF_JITTER}, - "cloudProviderRatelimit": ${CLOUDPROVIDER_RATELIMIT}, - "cloudProviderRateLimitQPS": ${CLOUDPROVIDER_RATELIMIT_QPS}, - "cloudProviderRateLimitBucket": ${CLOUDPROVIDER_RATELIMIT_BUCKET}, - "useManagedIdentityExtension": ${USE_MANAGED_IDENTITY_EXTENSION}, - "userAssignedIdentityID": "${USER_ASSIGNED_IDENTITY_ID}", - "useInstanceMetadata": ${USE_INSTANCE_METADATA}, - "loadBalancerSku": "${LOAD_BALANCER_SKU}", - "excludeMasterFromStandardLB": ${EXCLUDE_MASTER_FROM_STANDARD_LB}, - "providerVaultName": "${KMS_PROVIDER_VAULT_NAME}", - "providerKeyName": "k8s", - "providerKeyVersion": "" + "cloud":"${TARGET_ENVIRONMENT}", + "tenantId": "${TENANT_ID}", + "subscriptionId": "${SUBSCRIPTION_ID}", + "aadClientId": "${SERVICE_PRINCIPAL_CLIENT_ID}", + "aadClientSecret": "${SERVICE_PRINCIPAL_CLIENT_SECRET}", + "resourceGroup": "${RESOURCE_GROUP}", + "location": "${LOCATION}", + "vmType": "${VM_TYPE}", + "subnetName": "${SUBNET}", + "securityGroupName": "${NETWORK_SECURITY_GROUP}", + "vnetName": "${VIRTUAL_NETWORK}", + "vnetResourceGroup": "${VIRTUAL_NETWORK_RESOURCE_GROUP}", + "routeTableName": "${ROUTE_TABLE}", + "primaryAvailabilitySetName": "${PRIMARY_AVAILABILITY_SET}", + "primaryScaleSetName": "${PRIMARY_SCALE_SET}", + "cloudProviderBackoff": ${CLOUDPROVIDER_BACKOFF}, + "cloudProviderBackoffRetries": ${CLOUDPROVIDER_BACKOFF_RETRIES}, + "cloudProviderBackoffExponent": ${CLOUDPROVIDER_BACKOFF_EXPONENT}, + "cloudProviderBackoffDuration": ${CLOUDPROVIDER_BACKOFF_DURATION}, + "cloudProviderBackoffJitter": ${CLOUDPROVIDER_BACKOFF_JITTER}, + "cloudProviderRatelimit": ${CLOUDPROVIDER_RATELIMIT}, + "cloudProviderRateLimitQPS": ${CLOUDPROVIDER_RATELIMIT_QPS}, + "cloudProviderRateLimitBucket": ${CLOUDPROVIDER_RATELIMIT_BUCKET}, + "useManagedIdentityExtension": ${USE_MANAGED_IDENTITY_EXTENSION}, + "userAssignedIdentityID": "${USER_ASSIGNED_IDENTITY_ID}", + "useInstanceMetadata": ${USE_INSTANCE_METADATA}, + "loadBalancerSku": "${LOAD_BALANCER_SKU}", + "excludeMasterFromStandardLB": ${EXCLUDE_MASTER_FROM_STANDARD_LB}, + "providerVaultName": "${KMS_PROVIDER_VAULT_NAME}", + "providerKeyName": "k8s", + "providerKeyVersion": "" } EOF - set -x - if [[ ! -z "${MASTER_NODE}" ]]; then - if [[ "${ENABLE_AGGREGATED_APIS}" = True ]]; then - generateAggregatedAPICerts + set -x + if [[ ! -z "${MASTER_NODE}" ]]; then + if [[ "${ENABLE_AGGREGATED_APIS}" = True ]]; then + generateAggregatedAPICerts + fi fi + else + set -x + echo "MASTER_NODE is empty, worker node, skip azure json." fi } diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index 4ec1cd0697..fefeacf741 100755 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -36,11 +36,7 @@ if [[ $OS == $COREOS_OS_NAME ]]; then KUBECTL=/opt/kubectl fi -if [ -f /var/run/reboot-required ]; then - REBOOTREQUIRED=true -else - REBOOTREQUIRED=false -fi +REBOOTREQUIRED=false if [ -f /var/log/azure/golden-image-install.complete ]; then echo "detected golden image pre-install" @@ -51,20 +47,11 @@ else FULL_INSTALL_REQUIRED=true fi -holdWALinuxAgent() { - if [[ $OS == $UBUNTU_OS_NAME ]]; then - wait_for_apt_locks - retrycmd_if_failure 120 5 25 apt-mark hold walinuxagent || exit $ERR_HOLD_WALINUXAGENT - wait_for_apt_locks - fi -} - if [[ ! -z "${MASTER_NODE}" ]]; then installEtcd fi if $FULL_INSTALL_REQUIRED; then - holdWALinuxAgent installDeps else echo "Golden image; skipping dependencies installation" @@ -133,10 +120,6 @@ if $FULL_INSTALL_REQUIRED; then # mitigation for bug https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1676635 echo 2dd1ce17-079e-403c-b352-a1921ee207ee > /sys/bus/vmbus/drivers/hv_util/unbind sed -i "13i\echo 2dd1ce17-079e-403c-b352-a1921ee207ee > /sys/bus/vmbus/drivers/hv_util/unbind\n" /etc/rc.local - - wait_for_apt_locks - retrycmd_if_failure 120 5 25 apt-mark unhold walinuxagent || exit $ERR_RELEASE_HOLD_WALINUXAGENT - wait_for_apt_locks fi fi @@ -149,6 +132,12 @@ ps auxfww > /opt/azure/provision-ps.log & if $REBOOTREQUIRED; then echo 'reboot required, rebooting node in 1 minute' /bin/bash -c "shutdown -r 1 &" + if [[ $OS == $UBUNTU_OS_NAME ]]; then + aptmarkWALinuxAgent unhold & + fi else - runAptDaily & + if [[ $OS == $UBUNTU_OS_NAME ]]; then + /usr/lib/apt/apt.systemd.daily & + aptmarkWALinuxAgent unhold & + fi fi \ No newline at end of file diff --git a/parts/k8s/kubernetesinstalls.sh b/parts/k8s/kubernetesinstalls.sh index 8cb76ff2b0..f8270b0321 100755 --- a/parts/k8s/kubernetesinstalls.sh +++ b/parts/k8s/kubernetesinstalls.sh @@ -25,7 +25,7 @@ installDeps() { retrycmd_if_failure_no_stats 120 5 25 curl -fsSL https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT retrycmd_if_failure 60 5 10 dpkg -i /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_PKG_ADD_FAIL apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - apt_get_install 30 1 600 apt-transport-https blobfuse ca-certificates ceph-common cgroup-lite cifs-utils conntrack ebtables ethtool fuse git glusterfs-client init-system-helpers iproute2 ipset iptables jq mount nfs-common pigz socat util-linux xz-utils zip || exit $ERR_APT_INSTALL_TIMEOUT + apt_get_install 30 1 600 apt-transport-https blobfuse ca-certificates ceph-common cgroup-lite cifs-utils conntrack ebtables ethtool fuse git glusterfs-client init-system-helpers iproute2 ipset iptables jq mount nfs-common pigz socat util-linux xz-utils zip openvswitch-switch || exit $ERR_APT_INSTALL_TIMEOUT } installGPUDrivers() { diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index c2a2b91d1b..ed7c135b1c 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -104,6 +104,14 @@ write_files: RemainAfterExit=yes ExecStart=/usr/local/bin/health-monitor.sh container-runtime +- path: /etc/apt/preferences + permissions: "0644" + owner: root + content: | + Package: walinuxagent + Pin: version 2.2.32.2 + Pin-Priority: 550 + {{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}} {{if not .MasterProfile.IsCoreOS}} - path: /etc/systemd/system/docker.service.d/clear_mount_propagation_flags.conf @@ -299,7 +307,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{else}} sed -i "s||{{WrapAsParameter "kubernetesKubeDNSSpec"}}|g; s||{{WrapAsParameter "kubernetesDNSMasqSpec"}}|g; s||{{WrapAsParameter "kubernetesDNSSidecarSpec"}}|g; s||{{WrapAsParameter "kubernetesKubeletClusterDomain"}}|g; s||{{WrapAsParameter "kubeDNSServiceIP"}}|g" $KUBEDNS {{end}} - sed -i "s||{{WrapAsParameter "kubernetesHeapsterSpec"}}|g; s||{{WrapAsParameter "kubernetesAddonResizerSpec"}}|g" /etc/kubernetes/addons/kube-heapster-deployment.yaml + # sed -i "s||{{WrapAsParameter "kubernetesHeapsterSpec"}}|g; s||{{WrapAsParameter "kubernetesAddonResizerSpec"}}|g" /etc/kubernetes/addons/kube-heapster-deployment.yaml {{if AdminGroupID }} sed -i "s||{{WrapAsParameter "aadAdminGroupId"}}|g" "/etc/kubernetes/addons/aad-default-admin-group-rbac.yaml" @@ -469,6 +477,6 @@ coreos: {{else}} runcmd: - set -x -- timeout 10 apt-mark hold walinuxagent{{GetKubernetesMasterPreprovisionYaml}} -- timeout 10 apt-mark unhold walinuxagent +- . /opt/azure/containers/provision_source.sh +- aptmarkWALinuxAgent hold{{GetKubernetesMasterPreprovisionYaml}} {{end}} diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t index 78a64adb74..7119d9598c 100644 --- a/parts/k8s/kubernetesmasterresources.t +++ b/parts/k8s/kubernetesmasterresources.t @@ -6,8 +6,8 @@ "name": "[variables('masterAvailabilitySet')]", "properties": { - "platformFaultDomainCount": 2, - "platformUpdateDomainCount": 3 + "platformFaultDomainCount": 1, + "platformUpdateDomainCount": 1 }, "sku": { "name": "Aligned" @@ -32,8 +32,23 @@ "[concat('Microsoft.Network/publicIPAddresses/', variables('masterPublicIPAddressName'))]" ], {{end}} + "kind": "Storage", "location": "[variables('location')]", "name": "[variables('masterStorageAccountName')]", + "properties": { + "encryption": { + "keySource": "Microsoft.Storage", + "services": { + "blob": { + "enabled": true + }, + "file": { + "enabled": true + } + } + }, + "supportsHttpsTrafficOnly": true + }, "sku": { "name": "[variables('vmSizesMap')[parameters('masterVMSize')].storageAccountType]" }, @@ -89,36 +104,6 @@ "name": "[variables('nsgName')]", "properties": { "securityRules": [ - {{if .HasWindows}} - { - "name": "allow_rdp", - "properties": { - "access": "Allow", - "description": "Allow RDP traffic to master", - "destinationAddressPrefix": "*", - "destinationPortRange": "3389-3389", - "direction": "Inbound", - "priority": 102, - "protocol": "Tcp", - "sourceAddressPrefix": "*", - "sourcePortRange": "*" - } - }, - {{end}} - { - "name": "allow_ssh", - "properties": { - "access": "Allow", - "description": "Allow SSH traffic to master", - "destinationAddressPrefix": "*", - "destinationPortRange": "22-22", - "direction": "Inbound", - "priority": 101, - "protocol": "Tcp", - "sourceAddressPrefix": "*", - "sourcePortRange": "*" - } - }, { "name": "allow_kube_tls", "properties": { @@ -271,7 +256,10 @@ "id": "[variables('masterLbIPConfigID')]" }, "frontendPort": "[variables('sshNatPorts')[copyIndex(variables('masterOffset'))]]", - "protocol": "Tcp" + "protocol": "Tcp", + "backendIPConfiguration": { + "id": "[concat(resourceId('Microsoft.Network/networkInterfaces',concat(variables('masterVMNamePrefix'),'nic-', copyIndex(variables('masterOffset')))), '/ipConfigurations/ipconfig1')]" + } }, "type": "Microsoft.Network/loadBalancers/inboundNatRules" }, @@ -322,7 +310,7 @@ "id": "[concat(variables('masterLbID'),'/inboundNatRules/SSH-',variables('masterVMNamePrefix'),copyIndex(variables('masterOffset')))]" } ], - "privateIPAddress": "[variables('masterPrivateIpAddrs')[copyIndex(variables('masterOffset'))]]", + "privateIPAddress": "[concat(variables('aciSystemNodeAddrPrefix'), copyIndex(int(variables('aciPrimaryIPOctet4'))))]", "primary": true, "privateIPAllocationMethod": "Static", "subnet": { @@ -337,7 +325,8 @@ "name": "ipconfig{{$seq}}", "properties": { "primary": false, - "privateIPAllocationMethod": "Dynamic", + "privateIPAddress": "[concat(variables('aciSystemPodAddrPrefix'), copyIndex(int(variables('aciPrimaryIPOctet4'))), '.', add(sub({{$seq}}, 1), int(variables('aciPrimaryIPOctet4'))))]", + "privateIPAllocationMethod": "Static", "subnet": { "id": "[variables('vnetSubnetID')]" } @@ -979,7 +968,7 @@ {{if IsOpenShift}} "script": "{{ Base64 OpenShiftGetMasterSh }}" {{else}} - "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not (IsFeatureEnabled "BlockOutboundInternet")}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" + "commandToExecute": "[concat('retrycmd_if_failure() { r=$1; w=$2; t=$3; shift && shift && shift; for i in $(seq 1 $r); do timeout $t ${@}; [ $? -eq 0 ] && break || if [ $i -eq $r ]; then return 1; else sleep $w; fi; done };{{if not (IsFeatureEnabled "BlockOutboundInternet")}} ERR_OUTBOUND_CONN_FAIL=50; retrycmd_if_failure 50 1 3 nc -vz {{if IsMooncake}}gcr.azk8s.cn 80{{else}}k8s.gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz gcr.io 443 && retrycmd_if_failure 50 1 3 nc -vz docker.io 443{{end}} || exit $ERR_OUTBOUND_CONN_FAIL;{{end}} for i in $(seq 1 1200); do if [ -f /opt/azure/containers/provision.sh ]; then break; fi; if [ $i -eq 1200 ]; then exit 100; else sleep 1; fi; done; ', variables('provisionScriptParametersCommon'),' ',variables('provisionScriptParametersMaster'),' VNET_CNI_PLUGINS_URL=', parameters('vnetCniLinuxPluginsURL'), ' /usr/bin/nohup /bin/bash -c \"/bin/bash /opt/azure/containers/provision.sh >> /var/log/azure/cluster-provision.log 2>&1\"')]" {{end}} } } diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 4b876f7fb9..fe56b7a934 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -141,12 +141,12 @@ "sshdConfig": "{{GetB64sshdConfig}}", "systemConf": "{{GetB64systemConf}}", {{if not IsOpenShift}} - "provisionScriptParametersCommon": "[concat('ADMINUSER=',parameters('linuxAdminUsername'),' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' DOCKER_ENGINE_REPO=',parameters('dockerEngineDownloadRepo'),' TENANT_ID=',variables('tenantID'),' KUBERNETES_VERSION={{.OrchestratorProfile.OrchestratorVersion}} HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' VNET_CNI_PLUGINS_URL=',parameters('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' IS_HOSTED_MASTER={{IsHostedMaster}}')]", + "provisionScriptParametersCommon": "[concat(' ETCD_DOWNLOAD_URL=',parameters('etcdDownloadURLBase'),' ETCD_VERSION=',parameters('etcdVersion'),' DOCKER_ENGINE_REPO=',parameters('dockerEngineDownloadRepo'),' KUBERNETES_VERSION={{.OrchestratorProfile.OrchestratorVersion}} HYPERKUBE_URL=',parameters('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',parameters('apiserverCertificate'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' NETWORK_PLUGIN=',parameters('networkPlugin'),' NETWORK_POLICY=',parameters('networkPolicy'),' CNI_PLUGINS_URL=',parameters('cniPluginsURL'),' CONTAINER_RUNTIME=',parameters('containerRuntime'),' CONTAINERD_DOWNLOAD_URL_BASE=',parameters('containerdDownloadURLBase'),' POD_INFRA_CONTAINER_SPEC=',parameters('kubernetesPodInfraContainerSpec'),' IS_HOSTED_MASTER={{IsHostedMaster}}')]", {{if not IsHostedMaster}} {{if IsMasterVirtualMachineScaleSets}} - "provisionScriptParametersMaster": "[concat('MASTER_NODE=true NO_OUTBOUND={{IsFeatureEnabled "BlockOutboundInternet"}} CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", + "provisionScriptParametersMaster": "[concat('MASTER_NODE=true NO_OUTBOUND={{IsFeatureEnabled "BlockOutboundInternet"}} CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ADMINUSER=',parameters('linuxAdminUsername'),' TENANT_ID=',variables('tenantID'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", {{else}} - "provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true NO_OUTBOUND={{IsFeatureEnabled "BlockOutboundInternet"}} CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", + "provisionScriptParametersMaster": "[concat('MASTER_VM_NAME=',variables('masterVMNames')[variables('masterOffset')],' ETCD_PEER_URL=',variables('masterEtcdPeerURLs')[variables('masterOffset')],' ETCD_CLIENT_URL=',variables('masterEtcdClientURLs')[variables('masterOffset')],' MASTER_NODE=true NO_OUTBOUND={{IsFeatureEnabled "BlockOutboundInternet"}} CLUSTER_AUTOSCALER_ADDON=',parameters('kubernetesClusterAutoscalerEnabled'),' ADMINUSER=',parameters('linuxAdminUsername'),' TENANT_ID=',variables('tenantID'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' VM_TYPE=',variables('vmType'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' PRIMARY_SCALE_SET=',variables('primaryScaleSetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' TARGET_ENVIRONMENT=',parameters('targetEnvironment'),' CLOUDPROVIDER_BACKOFF=',toLower(string(parameters('cloudproviderConfig').cloudProviderBackoff)),' CLOUDPROVIDER_BACKOFF_RETRIES=',parameters('cloudproviderConfig').cloudProviderBackoffRetries,' CLOUDPROVIDER_BACKOFF_EXPONENT=',parameters('cloudproviderConfig').cloudProviderBackoffExponent,' CLOUDPROVIDER_BACKOFF_DURATION=',parameters('cloudproviderConfig').cloudProviderBackoffDuration,' CLOUDPROVIDER_BACKOFF_JITTER=',parameters('cloudproviderConfig').cloudProviderBackoffJitter,' CLOUDPROVIDER_RATELIMIT=',toLower(string(parameters('cloudproviderConfig').cloudProviderRatelimit)),' CLOUDPROVIDER_RATELIMIT_QPS=',parameters('cloudproviderConfig').cloudProviderRatelimitQPS,' CLOUDPROVIDER_RATELIMIT_BUCKET=',parameters('cloudproviderConfig').cloudProviderRatelimitBucket,' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USER_ASSIGNED_IDENTITY_ID=',variables('userAssignedClientID'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' LOAD_BALANCER_SKU=',variables('loadBalancerSku'),' EXCLUDE_MASTER_FROM_STANDARD_LB=',variables('excludeMasterFromStandardLB'),' ACI_CONNECTOR_ADDON=',parameters('kubernetesACIConnectorEnabled'),' APISERVER_PRIVATE_KEY=',parameters('apiServerPrivateKey'),' CA_CERTIFICATE=',parameters('caCertificate'),' CA_PRIVATE_KEY=',parameters('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',parameters('kubeConfigCertificate'),' KUBECONFIG_KEY=',parameters('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',parameters('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',parameters('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',parameters('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',parameters('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ENABLE_AGGREGATED_APIS=',string(parameters('enableAggregatedAPIs')),' KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyVaultName'),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", {{end}} {{end}} {{end}} @@ -210,7 +210,7 @@ "primaryAvailabilitySetName": "", "vmType": "vmss", {{else}} - "primaryAvailabilitySetName": "[concat('{{ (index .AgentPoolProfiles 0).Name }}-availabilitySet-',parameters('nameSuffix'))]", + "primaryAvailabilitySetName": "[concat('{{ (index .AgentPoolProfiles 1).Name }}-availabilitySet-',parameters('nameSuffix'))]", "primaryScaleSetName": "", "vmType": "standard", {{end}} @@ -262,6 +262,11 @@ "masterFirstAddrPrefix": "[concat(variables('masterFirstAddrOctets')[0],'.',variables('masterFirstAddrOctets')[1],'.',variables('masterFirstAddrOctets')[2],'.')]", "masterEtcdServerPort": {{GetMasterEtcdServerPort}}, "masterEtcdClientPort": {{GetMasterEtcdClientPort}}, + "aciCustomerNodeAddrPrefix": "10.240.", + "aciCustomerPodAddrPrefix": "10.244.", + "aciSystemNodeAddrPrefix": "10.240.255.", + "aciSystemPodAddrPrefix": "10.250.", + "aciPrimaryIPOctet4": "5", {{if IsMasterVirtualMachineScaleSets}} "masterVMNamePrefix": "[concat(parameters('orchestratorName'), '-master-', parameters('nameSuffix'), '-')]", {{else}} diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index fd7bf0ec8f..133d3b0456 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -389,10 +389,18 @@ "defaultValue": "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-linux-amd64-latest.tgz", "type": "string" }, + "vnetCniMultitenancyLinuxPluginsURL": { + "defaultValue": "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-multitenancy-linux-amd64-latest.tgz", + "type": "string" + }, "vnetCniWindowsPluginsURL": { "defaultValue": "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-windows-amd64-latest.zip", "type": "string" }, + "vnetCnsWindowsPluginsURL": { + "defaultValue": "https://github.com/Azure/azure-container-networking/releases/download/v1.0.26/azure-cns-windows-amd64-v1.0.26.zip", + "type": "string" + }, "maxPods": { "defaultValue": 30, "metadata": { diff --git a/parts/k8s/kubernetesprovisionsource.sh b/parts/k8s/kubernetesprovisionsource.sh index eaaaf3ee29..94a7f2ca67 100755 --- a/parts/k8s/kubernetesprovisionsource.sh +++ b/parts/k8s/kubernetesprovisionsource.sh @@ -56,6 +56,16 @@ NVIDIA_DOCKER_VERSION=2.0.3 DOCKER_VERSION=1.13.1-1 NVIDIA_CONTAINER_RUNTIME_VERSION=2.0.0 +aptmarkWALinuxAgent() { + wait_for_apt_locks + retrycmd_if_failure 120 5 25 apt-mark $1 walinuxagent || \ + if [[ "$1" == "hold" ]]; then + exit $ERR_HOLD_WALINUXAGENT + elif [[ "$1" == "unhold" ]]; then + exit $ERR_RELEASE_HOLD_WALINUXAGENT + fi +} + retrycmd_if_failure() { retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift for i in $(seq 1 $retries); do diff --git a/parts/k8s/kuberneteswinagentresourcesvmas.t b/parts/k8s/kuberneteswinagentresourcesvmas.t index b22079948b..8699581574 100644 --- a/parts/k8s/kuberneteswinagentresourcesvmas.t +++ b/parts/k8s/kuberneteswinagentresourcesvmas.t @@ -18,7 +18,7 @@ { "apiVersion": "[variables('apiVersionNetwork')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "loop" }, "dependsOn": [ @@ -29,7 +29,7 @@ {{end}} ], "location": "[variables('location')]", - "name": "[concat(variables('{{.Name}}VMNamePrefix'), 'nic-', copyIndex(variables('{{.Name}}Offset')))]", + "name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, 'nic-', copyIndex(variables('{{.Name}}Variables').Offset))]", "properties": { "enableAcceleratedNetworking" : "{{.AcceleratedNetworkingEnabledWindows}}", {{if .IsCustomVNET}} @@ -47,7 +47,7 @@ {{end}} "privateIPAllocationMethod": "Dynamic", "subnet": { - "id": "[variables('{{$.Name}}VnetSubnetID')]" + "id": "[variables('{{$.Name}}Variables').VnetSubnetID]" } } } @@ -64,7 +64,7 @@ {{if .IsManagedDisks}} { "location": "[variables('location')]", - "name": "[variables('{{.Name}}AvailabilitySet')]", + "name": "[variables('{{.Name}}Variables').AvailabilitySet]", "apiVersion": "[variables('apiVersionCompute')]", "properties": { @@ -80,7 +80,7 @@ { "apiVersion": "[variables('apiVersionStorage')]", "copy": { - "count": "[variables('{{.Name}}StorageAccountsCount')]", + "count": "[variables('{{.Name}}Variables').StorageAccountsCount]", "name": "loop" }, {{if not IsHostedMaster}} @@ -90,10 +90,25 @@ ], {{end}} {{end}} + "kind": "Storage", "location": "[variables('location')]", - "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName'))]", + "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName'))]", + "properties": { + "encryption": { + "keySource": "Microsoft.Storage", + "services": { + "blob": { + "enabled": true + }, + "file": { + "enabled": true + } + } + }, + "supportsHttpsTrafficOnly": true + }, "sku": { - "name": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" + "name": "[variables('vmSizesMap')[variables('{{.Name}}Variables').VMSize].storageAccountType]" }, "type": "Microsoft.Storage/storageAccounts" }, @@ -101,7 +116,7 @@ { "apiVersion": "[variables('apiVersionStorage')]", "copy": { - "count": "[variables('{{.Name}}StorageAccountsCount')]", + "count": "[variables('{{.Name}}Variables').StorageAccountsCount]", "name": "datadiskLoop" }, {{if not IsHostedMaster}} @@ -111,17 +126,32 @@ ], {{end}} {{end}} + "kind": "Storage", "location": "[variables('location')]", - "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}DataAccountName'))]", + "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('{{.Name}}DataAccountName'))]", + "properties": { + "encryption": { + "keySource": "Microsoft.Storage", + "services": { + "blob": { + "enabled": true + }, + "file": { + "enabled": true + } + } + }, + "supportsHttpsTrafficOnly": true + }, "sku": { - "name": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]" + "name": "[variables('vmSizesMap')[variables('{{.Name}}Variables').VMSize].storageAccountType]" }, "type": "Microsoft.Storage/storageAccounts" }, {{end}} { "location": "[variables('location')]", - "name": "[variables('{{.Name}}AvailabilitySet')]", + "name": "[variables('{{.Name}}Variables').AvailabilitySet]", "apiVersion": "[variables('apiVersionCompute')]", "properties": {}, "type": "Microsoft.Compute/availabilitySets" @@ -130,29 +160,29 @@ { "apiVersion": "[variables('apiVersionCompute')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "vmLoopNode" }, "dependsOn": [ {{if .IsStorageAccount}} - "[concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName'))]", + "[concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName'))]", {{if .HasDisks}} - "[concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('dataStorageAccountPrefixSeed')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('dataStorageAccountPrefixSeed')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}DataAccountName'))]", + "[concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('dataStorageAccountPrefixSeed')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('dataStorageAccountPrefixSeed')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}DataAccountName'))]", {{end}} {{end}} - "[concat('Microsoft.Network/networkInterfaces/', variables('{{.Name}}VMNamePrefix'), 'nic-', copyIndex(variables('{{.Name}}Offset')))]", - "[concat('Microsoft.Compute/availabilitySets/', variables('{{.Name}}AvailabilitySet'))]" + "[concat('Microsoft.Network/networkInterfaces/', variables('{{.Name}}Variables').VMNamePrefix, 'nic-', copyIndex(variables('{{.Name}}Variables').Offset))]", + "[concat('Microsoft.Compute/availabilitySets/', variables('{{.Name}}Variables').AvailabilitySet)]" ], "tags": { - "creationSource" : "[concat(parameters('generatorCode'), '-', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", + "creationSource" : "[concat(parameters('generatorCode'), '-', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]", "resourceNameSuffix" : "[variables('winResourceNamePrefix')]", "orchestrator" : "[variables('orchestratorNameVersionTag')]", "acsengineVersion" : "[parameters('acsengineVersion')]", "poolName" : "{{.Name}}" }, "location": "[variables('location')]", - "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", + "name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]", {{if UseManagedIdentity}} "identity": { "type": "systemAssigned" @@ -160,23 +190,26 @@ {{end}} "properties": { "availabilitySet": { - "id": "[resourceId('Microsoft.Compute/availabilitySets',variables('{{.Name}}AvailabilitySet'))]" + "id": "[resourceId('Microsoft.Compute/availabilitySets',variables('{{.Name}}Variables').AvailabilitySet)]" }, "hardwareProfile": { - "vmSize": "[variables('{{.Name}}VMSize')]" + "vmSize": "[variables('{{.Name}}Variables').VMSize]" }, "networkProfile": { "networkInterfaces": [ { - "id": "[resourceId('Microsoft.Network/networkInterfaces',concat(variables('{{.Name}}VMNamePrefix'), 'nic-', copyIndex(variables('{{.Name}}Offset'))))]" + "id": "[resourceId('Microsoft.Network/networkInterfaces',concat(variables('{{.Name}}Variables').VMNamePrefix, 'nic-', copyIndex(variables('{{.Name}}Variables').Offset)))]" } ] }, "osProfile": { - "computername": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", + "computername": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]", {{GetKubernetesWindowsAgentCustomData .}} "adminUsername": "[parameters('windowsAdminUsername')]", - "adminPassword": "[parameters('windowsAdminPassword')]" + "adminPassword": "[parameters('windowsAdminPassword')]", + "windowsConfiguration": { + "enableAutomaticUpdates": false + } }, "storageProfile": { {{GetDataDisks .}} @@ -194,9 +227,9 @@ "createOption": "FromImage" ,"caching": "ReadWrite" {{if .IsStorageAccount}} - ,"name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')),'-osdisk')]" + ,"name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset),'-osdisk')]" ,"vhd": { - "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(div(copyIndex(variables('{{.Name}}Offset')),variables('maxVMsPerStorageAccount')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName')),variables('apiVersionStorage')).primaryEndpoints.blob,'osdisk/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), '-osdisk.vhd')]" + "uri": "[concat(reference(concat('Microsoft.Storage/storageAccounts/',variables('storageAccountPrefixes')[mod(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(div(copyIndex(variables('{{.Name}}Variables').Offset),variables('maxVMsPerStorageAccount')),variables('{{.Name}}Variables').StorageAccountOffset),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName')),variables('apiVersionStorage')).primaryEndpoints.blob,'osdisk/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), '-osdisk.vhd')]" } {{end}} {{if ne .OSDiskSizeGB 0}} @@ -211,28 +244,28 @@ { "apiVersion": "[variables('apiVersionAuthorization')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "vmLoopNode" }, - "name": "[guid(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')),'vmidentity'))]", + "name": "[guid(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset),'vmidentity'))]", "type": "Microsoft.Authorization/roleAssignments", "properties": { "roleDefinitionId": "[variables('readerRoleDefinitionId')]", - "principalId": "[reference(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset'))), '2017-03-30', 'Full').identity.principalId]" + "principalId": "[reference(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset)), '2017-03-30', 'Full').identity.principalId]" } }, { "type": "Microsoft.Compute/virtualMachines/extensions", - "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), '/ManagedIdentityExtension')]", + "name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), '/ManagedIdentityExtension')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "vmLoopNode" }, "apiVersion": "[variables('apiVersionCompute')]", "location": "[resourceGroup().location]", "dependsOn": [ - "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", - "[concat('Microsoft.Authorization/roleAssignments/', guid(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), 'vmidentity')))]" + "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]", + "[concat('Microsoft.Authorization/roleAssignments/', guid(concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), 'vmidentity')))]" ], "properties": { "publisher": "Microsoft.ManagedIdentity", @@ -249,19 +282,19 @@ { "apiVersion": "[variables('apiVersionCompute')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "vmLoopNode" }, "dependsOn": [ {{if UseManagedIdentity}} - "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), '/extensions/ManagedIdentityExtension')]" + "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), '/extensions/ManagedIdentityExtension')]" {{else}} - "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]" + "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]" {{end}} ], "location": "[variables('location')]", "type": "Microsoft.Compute/virtualMachines/extensions", - "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')),'/cse', '-agent-', copyIndex(variables('{{.Name}}Offset')))]", + "name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), '/cse', '-agent-', copyIndex(variables('{{.Name}}Variables').Offset))]", "properties": { "publisher": "Microsoft.Compute", "type": "CustomScriptExtension", @@ -269,22 +302,22 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "commandToExecute": "[concat('powershell.exe -ExecutionPolicy Unrestricted -command \"', '$arguments = ', variables('singleQuote'),'-MasterIP ',variables('kubernetesAPIServerIP'),' -KubeDnsServiceIp ',parameters('kubeDnsServiceIp'),' -MasterFQDNPrefix ',variables('masterFqdnPrefix'),' -Location ',variables('location'),' -AgentKey ',parameters('clientPrivateKey'),' -AADClientId ',variables('servicePrincipalClientId'),' -AADClientSecret ',variables('servicePrincipalClientSecret'),variables('singleQuote'), ' ; ', variables('windowsCustomScriptSuffix'), '\" > %SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.log 2>&1')]" + "commandToExecute": "[concat('powershell.exe -ExecutionPolicy Unrestricted -command \"', '$arguments = ', variables('singleQuote'),'-MasterIP ',variables('kubernetesAPIServerIP'),' -KubeDnsServiceIp ',parameters('kubeDnsServiceIp'),' -MasterFQDNPrefix ',variables('masterFqdnPrefix'),' -Location ',variables('location'),' -AgentKey ',parameters('clientPrivateKey'),variables('singleQuote'), ' ; ', variables('windowsCustomScriptSuffix'), '\" > %SYSTEMDRIVE%\\AzureData\\CustomDataSetupScript.log 2>&1')]" } } } {{if UseAksExtension}} ,{ "type": "Microsoft.Compute/virtualMachines/extensions", - "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')), '/computeAksLinuxBilling')]", + "name": "[concat(variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset), '/computeAksLinuxBilling')]", "apiVersion": "[variables('apiVersionCompute')]", "copy": { - "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Offset'))]", + "count": "[sub(variables('{{.Name}}Count'), variables('{{.Name}}Variables').Offset)]", "name": "vmLoopNode" }, "location": "[variables('location')]", "dependsOn": [ - "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]" + "[concat('Microsoft.Compute/virtualMachines/', variables('{{.Name}}Variables').VMNamePrefix, copyIndex(variables('{{.Name}}Variables').Offset))]" ], "properties": { "publisher": "Microsoft.AKS", diff --git a/parts/k8s/kuberneteswindowssetup.ps1 b/parts/k8s/kuberneteswindowssetup.ps1 index 54b9c32cee..21b7375de3 100644 --- a/parts/k8s/kuberneteswindowssetup.ps1 +++ b/parts/k8s/kuberneteswindowssetup.ps1 @@ -32,15 +32,7 @@ param( [parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - $AgentKey, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientId, - - [parameter(Mandatory=$true)] - [ValidateNotNullOrEmpty()] - $AADClientSecret + $AgentKey ) @@ -107,6 +99,7 @@ $global:AzureCNIConfDir = [Io.path]::Combine("$global:AzureCNIDir", "netconf") # $global:NetworkPolicy = "{{WrapAsParameter "networkPolicy"}}" # BUG: unused $global:NetworkPlugin = "{{WrapAsParameter "networkPlugin"}}" $global:VNetCNIPluginsURL = "{{WrapAsParameter "vnetCniWindowsPluginsURL"}}" +$global:VNetCNSPluginsURL = "{{WrapAsParameter "vnetCnsWindowsPluginsURL"}}" # Base64 representation of ZIP archive $zippedFiles = "{{ GetKubernetesWindowsAgentFunctions }}" @@ -137,7 +130,11 @@ try # to the windows machine, and run the script manually to watch # the output. if ($true) { + Write-Log "Disable automatic Windows update" + reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\WindowsUpdate\Auto Update" /v AUOptions /t REG_DWORD /d 1 /f + Write-Log "Provisioning $global:DockerServiceName... with IP $MasterIP" + net start Docker Write-Log "Apply telemetry data setting" Set-TelemetrySetting -WindowsTelemetryGUID $global:WindowsTelemetryGUID @@ -163,29 +160,6 @@ try Get-KubeBinaries -KubeBinariesURL $global:WindowsKubeBinariesURL } - - Write-Log "Write Azure cloud provider config" - Write-AzureConfig ` - -KubeDir $global:KubeDir ` - -AADClientId $AADClientId ` - -AADClientSecret $AADClientSecret ` - -TenantId $global:TenantId ` - -SubscriptionId $global:SubscriptionId ` - -ResourceGroup $global:ResourceGroup ` - -Location $Location ` - -VmType $global:VmType ` - -SubnetName $global:SubnetName ` - -SecurityGroupName $global:SecurityGroupName ` - -VNetName $global:VNetName ` - -RouteTableName $global:RouteTableName ` - -PrimaryAvailabilitySetName $global:PrimaryAvailabilitySetName ` - -PrimaryScaleSetName $global:PrimaryScaleSetName ` - -UseManagedIdentityExtension $global:UseManagedIdentityExtension ` - -UserAssignedClientID $global:UserAssignedClientID ` - -UseInstanceMetadata $global:UseInstanceMetadata ` - -LoadBalancerSku $global:LoadBalancerSku ` - -ExcludeMasterFromStandardLB $global:ExcludeMasterFromStandardLB - Write-Log "Write ca root" Write-CACert -CACertificate $global:CACertificate ` -KubeDir $global:KubeDir @@ -214,6 +188,12 @@ try -KubeClusterCIDR $global:KubeClusterCIDR ` -MasterSubnet $global:MasterSubnet ` -KubeServiceCIDR $global:KubeServiceCIDR + + #testing changes + DeployCNSServiceAndLinkToKubelet -KubeDir $global:KubeDir ` + -AzureCNIBinDir $global:AzureCNIBinDir ` + -VNetCNSPluginsURL $global:VNetCNSPluginsURL + } elseif ($global:NetworkPlugin -eq "kubenet") { Update-WinCNI -CNIPath $global:CNIPath Get-HnsPsm1 -HNSModule $global:HNSModule @@ -257,7 +237,7 @@ try else { # keep for debugging purposes - Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AgentKey $AgentKey -AADClientId $AADClientId -AADClientSecret $AADClientSecret" + Write-Log ".\CustomDataSetupScript.ps1 -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp -MasterFQDNPrefix $MasterFQDNPrefix -Location $Location -AgentKey $AgentKey" } } catch diff --git a/parts/k8s/windowsazurecnifunc.ps1 b/parts/k8s/windowsazurecnifunc.ps1 index d117f8c9a3..5a4208f51e 100644 --- a/parts/k8s/windowsazurecnifunc.ps1 +++ b/parts/k8s/windowsazurecnifunc.ps1 @@ -78,3 +78,48 @@ Set-AzureCNIConfig $configJson | ConvertTo-Json -depth 20 | Out-File -encoding ASCII -filepath $fileName } + + +function DeployCNSServiceAndLinkToKubelet() +{ + Param( + [Parameter(Mandatory=$true)][string] + $KubeDir, + [Parameter(Mandatory=$true)][string] + $AzureCNIBinDir, + [Parameter(Mandatory=$true)][string] + $VNetCNSPluginsURL + ) + $output = "$PSScriptRoot\azure-vnet-cns.zip" + + Invoke-WebRequest -Uri $VNetCNSPluginsURL -OutFile $output + Expand-Archive -Path $output -DestinationPath $AzureCNIBinDir + + $AzureCNS = [Io.path]::Combine("$AzureCNIBinDir", "azure-cns.exe") + $AzureCNSStartFile = [Io.path]::Combine("$AzureCNIBinDir", "startazurecns.ps1") + + $azureCNSstartStr = "$AzureCNS -c tcp://0.0.0.0:10090" + $azureCNSstartStr | Out-File -encoding ASCII -filepath $AzureCNSStartFile + + & "$KubeDir\nssm.exe" install CNSService C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe + & "$KubeDir\nssm.exe" set CNSService AppDirectory $AzureCNIBinDir + & "$KubeDir\nssm.exe" set CNSService AppParameters $AzureCNSStartFile + & "$KubeDir\nssm.exe" set CNSService DisplayName CNSService + & "$KubeDir\nssm.exe" set CNSService Description CNSService + & "$KubeDir\nssm.exe" set CNSService Start SERVICE_AUTO_START + & "$KubeDir\nssm.exe" set CNSService ObjectName LocalSystem + & "$KubeDir\nssm.exe" set CNSService Type SERVICE_WIN32_OWN_PROCESS + & "$KubeDir\nssm.exe" set CNSService AppThrottle 1500 + & "$KubeDir\nssm.exe" set CNSService AppStdoutCreationDisposition 4 + & "$KubeDir\nssm.exe" set CNSService AppStderrCreationDisposition 4 + & "$KubeDir\nssm.exe" set CNSService AppRotateFiles 1 + & "$KubeDir\nssm.exe" set CNSService AppRotateOnline 1 + & "$KubeDir\nssm.exe" set CNSService AppRotateSeconds 86400 + & "$KubeDir\nssm.exe" set CNSService AppRotateBytes 1048576 + + sc.exe failure "CNSService" actions= restart/60000/restart/60000/restart/60000 reset= 900 + + $value = Get-ItemProperty -Path "Registry::HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\kubelet" | Select-Object -ExpandProperty "DependOnService" -ErrorAction Stop + Set-ItemProperty -Path "Registry::HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\kubelet" -Name "DependOnService" -Value "$value\0CNSService" +} + diff --git a/parts/k8s/windowskubeletfunc.ps1 b/parts/k8s/windowskubeletfunc.ps1 index 89dfe00fe8..79132db83d 100644 --- a/parts/k8s/windowskubeletfunc.ps1 +++ b/parts/k8s/windowskubeletfunc.ps1 @@ -559,27 +559,30 @@ try # Turn off Firewall to enable pods to talk to service endpoints. (Kubelet should eventually do this) netsh advfirewall set allprofiles state off - # startup the service - `$hnsNetwork = Get-HnsNetwork | ? Name -EQ `$global:NetworkMode.ToLower() - - if (`$hnsNetwork) + if (Get-Module -Name HostNetworkingService) { - # Kubelet has been restarted with existing network. - # Cleanup all containers - docker ps -q | foreach {docker rm `$_ -f} - # cleanup network - Write-Host "Cleaning up old HNS network found" - Remove-HnsNetwork `$hnsNetwork - Start-Sleep 10 + # startup the service + `$hnsNetwork = Get-HnsNetwork | ? Name -EQ `$global:NetworkMode.ToLower() + + if (`$hnsNetwork) + { + # Kubelet has been restarted with existing network. + # Cleanup all containers + docker ps -q | foreach {docker rm `$_ -f} + # cleanup network + Write-Host "Cleaning up old HNS network found" + Remove-HnsNetwork `$hnsNetwork + Start-Sleep 10 + } + + Write-Host "Creating a new hns Network" + ipmo `$global:HNSModule + + `$hnsNetwork = New-HNSNetwork -Type `$global:NetworkMode -AddressPrefix `$podCIDR -Gateway `$masterSubnetGW -Name `$global:NetworkMode.ToLower() -Verbose + # New network has been created, Kubeproxy service has to be restarted + Restart-Service Kubeproxy } - Write-Host "Creating a new hns Network" - ipmo `$global:HNSModule - - `$hnsNetwork = New-HNSNetwork -Type `$global:NetworkMode -AddressPrefix `$podCIDR -Gateway `$masterSubnetGW -Name `$global:NetworkMode.ToLower() -Verbose - # New network has been created, Kubeproxy service has to be restarted - Restart-Service Kubeproxy - Start-Sleep 10 # Add route to all other POD networks Update-CNIConfig `$podCIDR `$masterSubnetGW @@ -601,20 +604,23 @@ catch `$env:KUBE_NETWORK = "$KubeNetwork" `$global:NetworkMode = "$NetworkMode" `$global:HNSModule = "$HNSModule" -`$hnsNetwork = Get-HnsNetwork | ? Name -EQ $KubeNetwork -while (!`$hnsNetwork) +if (Get-Module -Name HostNetworkingService) { - Write-Host "Waiting for Network [$KubeNetwork] to be created . . ." - Start-Sleep 10 - `$hnsNetwork = Get-HnsNetwork | ? Name -EQ $KubeNetwork + `$hnsNetwork = Get-HnsNetwork | ? Name -EQ $KubeNetwork + while (!`$hnsNetwork) + { + Write-Host "Waiting for Network [$KubeNetwork] to be created . . ." + Start-Sleep 10 + `$hnsNetwork = Get-HnsNetwork | ? Name -EQ $KubeNetwork + } + + # + # cleanup the persisted policy lists + # + ipmo `$global:HNSModule + Get-HnsPolicyList | Remove-HnsPolicyList } -# -# cleanup the persisted policy lists -# -ipmo `$global:HNSModule -Get-HnsPolicyList | Remove-HnsPolicyList - $KubeDir\kube-proxy.exe --v=3 --proxy-mode=kernelspace --hostname-override=$env:computername --kubeconfig=$KubeDir\config "@ diff --git a/pkg/acsengine/artifacts.go b/pkg/acsengine/artifacts.go index 8e89cfe514..b30130c284 100644 --- a/pkg/acsengine/artifacts.go +++ b/pkg/acsengine/artifacts.go @@ -123,7 +123,7 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesFeatureSet "kubernetesmasteraddons-heapster-deployment.yaml", "kube-heapster-deployment.yaml", - true, + false, profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultKubeHeapsterDeploymentAddonName), }, { diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go index a2ba7b9e1a..b08129ac5b 100644 --- a/pkg/acsengine/params_k8s.go +++ b/pkg/acsengine/params_k8s.go @@ -107,6 +107,7 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params addValue(parametersMap, "containerdDownloadURLBase", cloudSpecConfig.KubernetesSpecConfig.ContainerdDownloadURLBase) addValue(parametersMap, "cniPluginsURL", cloudSpecConfig.KubernetesSpecConfig.CNIPluginsDownloadURL) addValue(parametersMap, "vnetCniLinuxPluginsURL", kubernetesConfig.GetAzureCNIURLLinux(cloudSpecConfig)) + addValue(parametersMap, "vnetCniMultitenancyLinuxPluginsURL", cloudSpecConfig.KubernetesSpecConfig.VnetCNIMultitenancyLinuxPluginsDownloadURL) addValue(parametersMap, "vnetCniWindowsPluginsURL", kubernetesConfig.GetAzureCNIURLWindows(cloudSpecConfig)) addValue(parametersMap, "gchighthreshold", kubernetesConfig.GCHighThreshold) addValue(parametersMap, "gclowthreshold", kubernetesConfig.GCLowThreshold) diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go index 0b2f48c8d9..c6b016a93c 100644 --- a/pkg/acsengine/template_generator.go +++ b/pkg/acsengine/template_generator.go @@ -377,8 +377,10 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat return "" }, "UseAksExtension": func() bool { - cloudSpecConfig := cs.GetCloudSpecConfig() - return cloudSpecConfig.CloudName == api.AzurePublicCloud + //cloudSpecConfig := cs.GetCloudSpecConfig() + //return cloudSpecConfig.CloudName == api.AzurePublicCloud + // ACI: This extension puts us over the ARM resource limit + return false }, "IsMooncake": func() bool { cloudSpecConfig := cs.GetCloudSpecConfig() diff --git a/pkg/acsengine/types.go b/pkg/acsengine/types.go index fd1da1d69a..45e75fb43e 100644 --- a/pkg/acsengine/types.go +++ b/pkg/acsengine/types.go @@ -47,18 +47,19 @@ type DCOSSpecConfig struct { //KubernetesSpecConfig is the kubernetes container images used. type KubernetesSpecConfig struct { - KubernetesImageBase string - TillerImageBase string - ACIConnectorImageBase string - NVIDIAImageBase string - AzureCNIImageBase string - EtcdDownloadURLBase string - KubeBinariesSASURLBase string - WindowsTelemetryGUID string - CNIPluginsDownloadURL string - VnetCNILinuxPluginsDownloadURL string - VnetCNIWindowsPluginsDownloadURL string - ContainerdDownloadURLBase string + KubernetesImageBase string + TillerImageBase string + ACIConnectorImageBase string + NVIDIAImageBase string + AzureCNIImageBase string + EtcdDownloadURLBase string + KubeBinariesSASURLBase string + WindowsTelemetryGUID string + CNIPluginsDownloadURL string + VnetCNILinuxPluginsDownloadURL string + VnetCNIMultitenancyLinuxPluginsDownloadURL string + VnetCNIWindowsPluginsDownloadURL string + ContainerdDownloadURLBase string } //AzureEndpointConfig describes an Azure endpoint diff --git a/pkg/api/azenvtypes.go b/pkg/api/azenvtypes.go index b1409940d6..0f8b640a05 100644 --- a/pkg/api/azenvtypes.go +++ b/pkg/api/azenvtypes.go @@ -36,18 +36,19 @@ type DCOSSpecConfig struct { //KubernetesSpecConfig is the kubernetes container images used. type KubernetesSpecConfig struct { - KubernetesImageBase string - TillerImageBase string - ACIConnectorImageBase string - NVIDIAImageBase string - AzureCNIImageBase string - EtcdDownloadURLBase string - KubeBinariesSASURLBase string - WindowsTelemetryGUID string - CNIPluginsDownloadURL string - VnetCNILinuxPluginsDownloadURL string - VnetCNIWindowsPluginsDownloadURL string - ContainerdDownloadURLBase string + KubernetesImageBase string + TillerImageBase string + ACIConnectorImageBase string + NVIDIAImageBase string + AzureCNIImageBase string + EtcdDownloadURLBase string + KubeBinariesSASURLBase string + WindowsTelemetryGUID string + CNIPluginsDownloadURL string + VnetCNILinuxPluginsDownloadURL string + VnetCNIMultitenancyLinuxPluginsDownloadURL string + VnetCNIWindowsPluginsDownloadURL string + ContainerdDownloadURLBase string } //AzureEndpointConfig describes an Azure endpoint @@ -66,18 +67,19 @@ type AzureOSImageConfig struct { var ( //DefaultKubernetesSpecConfig is the default Docker image source of Kubernetes DefaultKubernetesSpecConfig = KubernetesSpecConfig{ - KubernetesImageBase: "k8s.gcr.io/", - TillerImageBase: "gcr.io/kubernetes-helm/", - ACIConnectorImageBase: "microsoft/", - NVIDIAImageBase: "nvidia/", - AzureCNIImageBase: "containernetworking/", - EtcdDownloadURLBase: "https://acs-mirror.azureedge.net/github-coreos", - KubeBinariesSASURLBase: "https://acs-mirror.azureedge.net/wink8s/", - WindowsTelemetryGUID: "fb801154-36b9-41bc-89c2-f4d4f05472b0", - CNIPluginsDownloadURL: "https://acs-mirror.azureedge.net/cni/cni-plugins-amd64-" + CNIPluginVer + ".tgz", - VnetCNILinuxPluginsDownloadURL: "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-linux-amd64-" + AzureCniPluginVerLinux + ".tgz", - VnetCNIWindowsPluginsDownloadURL: "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-windows-amd64-" + AzureCniPluginVerWindows + ".zip", - ContainerdDownloadURLBase: "https://storage.googleapis.com/cri-containerd-release/", + KubernetesImageBase: "k8s.gcr.io/", + TillerImageBase: "gcr.io/kubernetes-helm/", + ACIConnectorImageBase: "microsoft/", + NVIDIAImageBase: "nvidia/", + AzureCNIImageBase: "containernetworking/", + EtcdDownloadURLBase: "https://acs-mirror.azureedge.net/github-coreos", + KubeBinariesSASURLBase: "https://acs-mirror.azureedge.net/wink8s/", + WindowsTelemetryGUID: "fb801154-36b9-41bc-89c2-f4d4f05472b0", + CNIPluginsDownloadURL: "https://acs-mirror.azureedge.net/cni/cni-plugins-amd64-" + CNIPluginVer + ".tgz", + VnetCNILinuxPluginsDownloadURL: "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-linux-amd64-" + AzureCniPluginVerLinux + ".tgz", + VnetCNIMultitenancyLinuxPluginsDownloadURL: "https://github.com/Azure/azure-container-networking/releases/download/" + AzureCniPluginVerLinux + "/azure-vnet-cni-multitenancy-linux-amd64-" + AzureCniPluginVerLinux + ".tgz", + VnetCNIWindowsPluginsDownloadURL: "https://acs-mirror.azureedge.net/cni/azure-vnet-cni-windows-amd64-" + AzureCniPluginVerWindows + ".zip", + ContainerdDownloadURLBase: "https://storage.googleapis.com/cri-containerd-release/", } //DefaultDCOSSpecConfig is the default DC/OS binary download URL. diff --git a/pkg/api/defaults-kubelet.go b/pkg/api/defaults-kubelet.go index 959dde77db..c696e864e7 100644 --- a/pkg/api/defaults-kubelet.go +++ b/pkg/api/defaults-kubelet.go @@ -105,6 +105,7 @@ func (cs *ContainerService) setKubeletConfig() { for _, key := range []string{"--anonymous-auth", "--client-ca-file"} { delete(o.KubernetesConfig.KubeletConfig, key) } + o.KubernetesConfig.KubeletConfig["--authorization-mode"] = "AlwaysAllow" } removeKubeletFlags(o.KubernetesConfig.KubeletConfig, o.OrchestratorVersion) @@ -130,6 +131,13 @@ func (cs *ContainerService) setKubeletConfig() { for key, val := range staticWindowsKubeletConfig { profile.KubernetesConfig.KubeletConfig[key] = val } + // Remove secure kubelet flags, if configured + if !helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableSecureKubelet) { + for _, key := range []string{"--anonymous-auth", "--client-ca-file"} { + delete(profile.KubernetesConfig.KubeletConfig, key) + } + profile.KubernetesConfig.KubeletConfig["--authorization-mode"] = "AlwaysAllow" + } } } setMissingKubeletValues(profile.KubernetesConfig, o.KubernetesConfig.KubeletConfig) diff --git a/pkg/api/defaults.go b/pkg/api/defaults.go index f13e8b2ca4..3a1334f97e 100644 --- a/pkg/api/defaults.go +++ b/pkg/api/defaults.go @@ -305,7 +305,7 @@ func (p *Properties) setMasterProfileDefaults(isUpgrade bool) { p.MasterProfile.Subnet = DefaultKubernetesMasterSubnet p.MasterProfile.AgentSubnet = DefaultKubernetesAgentSubnetVMSS } else { - p.MasterProfile.FirstConsecutiveStaticIP = p.MasterProfile.GetFirstConsecutiveStaticIPAddress(p.MasterProfile.Subnet) + p.MasterProfile.FirstConsecutiveStaticIP = DefaultFirstConsecutiveKubernetesStaticIP } } } else {