From 9c74400a9a5ed1a80e3b48217754625d6308ace5 Mon Sep 17 00:00:00 2001 From: Qi Ke Date: Fri, 13 Mar 2026 14:13:38 -0400 Subject: [PATCH 1/2] fix node bootstrap script for held kube packages --- cli/internal/config/nodebootstrap/assets/script.sh.tmpl | 2 +- cli/internal/config/nodebootstrap/script_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/internal/config/nodebootstrap/assets/script.sh.tmpl b/cli/internal/config/nodebootstrap/assets/script.sh.tmpl index e4eeea9..b75b657 100644 --- a/cli/internal/config/nodebootstrap/assets/script.sh.tmpl +++ b/cli/internal/config/nodebootstrap/assets/script.sh.tmpl @@ -25,7 +25,7 @@ apt-get update -y apt-get upgrade -y {{- end }} {{- if .Packages }} -apt-get install -y {{ join .Packages " " }} +apt-get install -y --allow-change-held-packages {{ join .Packages " " }} {{- end }} {{- end }} {{- if .FQDN }} diff --git a/cli/internal/config/nodebootstrap/script_test.go b/cli/internal/config/nodebootstrap/script_test.go index 3166125..22743c4 100644 --- a/cli/internal/config/nodebootstrap/script_test.go +++ b/cli/internal/config/nodebootstrap/script_test.go @@ -36,7 +36,7 @@ func Test_marshalScript_basic(t *testing.T) { "#!/bin/bash", "set -euo pipefail", "apt-get update -y", - "apt-get install -y curl", + "apt-get install -y --allow-change-held-packages curl", "mkdir -p '/tmp'", `cat <<'EOF' > '/tmp/config.json'`, `{"key":"value"}`, From 9fe58b28f381931be726079e5a23fc6036818833 Mon Sep 17 00:00:00 2001 From: Qi Ke Date: Sat, 14 Mar 2026 13:16:33 -0400 Subject: [PATCH 2/2] fix Azure profile-aware AKS Flex bootstrap --- cli/internal/aks/deploy/cilium.go | 70 ++++++++++++++++++- cli/internal/aks/deploy/cilium_test.go | 42 +++++++++++ cli/internal/config/configcmd/defaults.go | 42 ++++++++++- .../config/configcmd/defaults_test.go | 24 +++++++ plugin/pkg/util/config/config.go | 7 +- plugin/pkg/util/config/config_test.go | 27 +++++++ 6 files changed, 207 insertions(+), 5 deletions(-) create mode 100644 cli/internal/aks/deploy/cilium_test.go create mode 100644 cli/internal/config/configcmd/defaults_test.go create mode 100644 plugin/pkg/util/config/config_test.go diff --git a/cli/internal/aks/deploy/cilium.go b/cli/internal/aks/deploy/cilium.go index 46ca558..38fb546 100644 --- a/cli/internal/aks/deploy/cilium.go +++ b/cli/internal/aks/deploy/cilium.go @@ -3,10 +3,14 @@ package deploy import ( "context" "errors" + "fmt" + "log" + "net/url" "os" "os/exec" "github.com/Azure/aks-flex/plugin/pkg/util/config" + "k8s.io/client-go/tools/clientcmd" ) var ciliumInstallInstruction = errors.New( @@ -28,18 +32,78 @@ func deployCilium( kubeconfigFile string, cfg *config.Config, ) error { + k8sServiceHost, k8sServicePort, err := kubeconfigAPIServer(kubeconfigFile) + if err != nil { + return err + } + cmd := exec.CommandContext( ctx, "cilium", "install", - "--set", "azure.resourceGroup="+cfg.ResourceGroupName, + "--kubeconfig", kubeconfigFile, + "--context", cfg.ClusterName+"-admin", + "--namespace", "kube-system", + "--datapath-mode", "aks-byocni", + "--helm-set", "aksbyocni.enabled=true", + "--helm-set", "cluster.name="+cfg.ClusterName, + "--helm-set", "operator.replicas=1", + "--helm-set", "kubeProxyReplacement=true", + "--helm-set", "k8sServiceHost="+k8sServiceHost, + "--helm-set", "k8sServicePort="+k8sServicePort, ) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Env = append( - cmd.Env, + cmd.Environ(), "KUBECONFIG="+kubeconfigFile, - "PATH="+os.Getenv("PATH"), // so cilium can find other tools + "PATH="+os.Getenv("PATH"), ) + log.Printf("Running: cilium install --kubeconfig %s --context %s --namespace kube-system --datapath-mode aks-byocni --helm-set aksbyocni.enabled=true --helm-set cluster.name=%s --helm-set operator.replicas=1 --helm-set kubeProxyReplacement=true --helm-set k8sServiceHost=%s --helm-set k8sServicePort=%s", kubeconfigFile, cfg.ClusterName+"-admin", cfg.ClusterName, k8sServiceHost, k8sServicePort) return cmd.Run() } + +func kubeconfigAPIServer(kubeconfigFile string) (string, string, error) { + kcfg, err := clientcmd.LoadFromFile(kubeconfigFile) + if err != nil { + return "", "", fmt.Errorf("loading kubeconfig for cilium install: %w", err) + } + + ctxName := kcfg.CurrentContext + if ctxName == "" { + return "", "", errors.New("kubeconfig missing current context") + } + + ctxCfg, ok := kcfg.Contexts[ctxName] + if !ok || ctxCfg == nil { + return "", "", fmt.Errorf("kubeconfig missing context %q", ctxName) + } + + clusterCfg, ok := kcfg.Clusters[ctxCfg.Cluster] + if !ok || clusterCfg == nil { + return "", "", fmt.Errorf("kubeconfig missing cluster %q", ctxCfg.Cluster) + } + + u, err := url.Parse(clusterCfg.Server) + if err != nil { + return "", "", fmt.Errorf("parsing API server URL %q: %w", clusterCfg.Server, err) + } + + hostname := u.Hostname() + port := u.Port() + if hostname == "" { + return "", "", fmt.Errorf("API server URL missing hostname: %q", clusterCfg.Server) + } + if port == "" { + switch u.Scheme { + case "https": + port = "443" + case "http": + port = "80" + default: + return "", "", fmt.Errorf("API server URL missing port and unsupported scheme %q", u.Scheme) + } + } + + return hostname, port, nil +} diff --git a/cli/internal/aks/deploy/cilium_test.go b/cli/internal/aks/deploy/cilium_test.go new file mode 100644 index 0000000..c5ec79e --- /dev/null +++ b/cli/internal/aks/deploy/cilium_test.go @@ -0,0 +1,42 @@ +package deploy + +import ( + "os" + "path/filepath" + "testing" +) + +func TestKubeconfigAPIServer(t *testing.T) { + dir := t.TempDir() + kubeconfig := filepath.Join(dir, "config") + if err := os.WriteFile(kubeconfig, []byte(`apiVersion: v1 +kind: Config +current-context: test +clusters: +- name: test-cluster + cluster: + server: https://example.hcp.eastus2.azmk8s.io:443 +contexts: +- name: test + context: + cluster: test-cluster + user: test-user +users: +- name: test-user + user: + token: test +`), 0o600); err != nil { + t.Fatalf("write kubeconfig: %v", err) + } + + host, port, err := kubeconfigAPIServer(kubeconfig) + if err != nil { + t.Fatalf("kubeconfigAPIServer returned error: %v", err) + } + if host != "example.hcp.eastus2.azmk8s.io" { + t.Fatalf("unexpected host %q", host) + } + if port != "443" { + t.Fatalf("unexpected port %q", port) + } +} diff --git a/cli/internal/config/configcmd/defaults.go b/cli/internal/config/configcmd/defaults.go index c9dc067..a3f310b 100644 --- a/cli/internal/config/configcmd/defaults.go +++ b/cli/internal/config/configcmd/defaults.go @@ -2,8 +2,10 @@ package configcmd import ( "context" + "encoding/json" "fmt" "os" + "path/filepath" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" @@ -42,7 +44,11 @@ func OrPlaceholder(val string) string { // reachable or the required environment variables are not set, it falls back // to placeholder values that the user must replace manually. func DefaultKubeadmConfig(ctx context.Context) *kubeadm.Config { - credentials, err := azidentity.NewAzureCLICredential(nil) + credOptions := &azidentity.AzureCLICredentialOptions{} + if tenantID := azureConfigTenantID(); tenantID != "" { + credOptions.TenantID = tenantID + } + credentials, err := azidentity.NewAzureCLICredential(credOptions) if err != nil { fmt.Fprintf(os.Stderr, "Warning: could not obtain Azure CLI credentials: %v\n", err) fmt.Fprintln(os.Stderr, "Using placeholder values — edit the output before applying.") @@ -64,3 +70,37 @@ func DefaultKubeadmConfig(ctx context.Context) *kubeadm.Config { } return cfg } + +func azureConfigTenantID() string { + azureConfigDir := os.Getenv("AZURE_CONFIG_DIR") + if azureConfigDir == "" { + azureConfigDir = filepath.Join(os.Getenv("HOME"), ".azure") + } + + b, err := os.ReadFile(filepath.Join(azureConfigDir, "azureProfile.json")) + if err != nil { + return "" + } + + var profile struct { + Subscriptions []struct { + IsDefault bool `json:"isDefault"` + TenantID string `json:"tenantId"` + } `json:"subscriptions"` + } + if err := json.Unmarshal(b, &profile); err != nil { + return "" + } + + for _, sub := range profile.Subscriptions { + if sub.IsDefault && sub.TenantID != "" { + return sub.TenantID + } + } + + if len(profile.Subscriptions) == 1 { + return profile.Subscriptions[0].TenantID + } + + return "" +} diff --git a/cli/internal/config/configcmd/defaults_test.go b/cli/internal/config/configcmd/defaults_test.go new file mode 100644 index 0000000..1fd7af8 --- /dev/null +++ b/cli/internal/config/configcmd/defaults_test.go @@ -0,0 +1,24 @@ +package configcmd + +import ( + "os" + "path/filepath" + "testing" +) + +func TestAzureConfigTenantIDUsesAzureConfigDirProfile(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + azureConfigDir := filepath.Join(t.TempDir(), "azure-profile") + if err := os.MkdirAll(azureConfigDir, 0o755); err != nil { + t.Fatalf("mkdir azure config dir: %v", err) + } + if err := os.WriteFile(filepath.Join(azureConfigDir, "azureProfile.json"), []byte(`{"subscriptions":[{"id":"sub","isDefault":true,"tenantId":"tenant-123"}]}`), 0o600); err != nil { + t.Fatalf("write azureProfile.json: %v", err) + } + t.Setenv("AZURE_CONFIG_DIR", azureConfigDir) + + if got := azureConfigTenantID(); got != "tenant-123" { + t.Fatalf("unexpected tenant id %q", got) + } +} diff --git a/plugin/pkg/util/config/config.go b/plugin/pkg/util/config/config.go index 7da7d0b..23c5bab 100644 --- a/plugin/pkg/util/config/config.go +++ b/plugin/pkg/util/config/config.go @@ -107,7 +107,12 @@ func defaultSubscriptionID() string { return subscriptionID } - b, err := os.ReadFile(filepath.Join(os.Getenv("HOME"), ".azure/clouds.config")) + azureConfigDir := os.Getenv("AZURE_CONFIG_DIR") + if azureConfigDir == "" { + azureConfigDir = filepath.Join(os.Getenv("HOME"), ".azure") + } + + b, err := os.ReadFile(filepath.Join(azureConfigDir, "clouds.config")) if err != nil { return "" } diff --git a/plugin/pkg/util/config/config_test.go b/plugin/pkg/util/config/config_test.go new file mode 100644 index 0000000..1710aa7 --- /dev/null +++ b/plugin/pkg/util/config/config_test.go @@ -0,0 +1,27 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDefaultSubscriptionIDHonorsAzureConfigDir(t *testing.T) { + t.Setenv("AZURE_SUBSCRIPTION_ID", "") + home := t.TempDir() + t.Setenv("HOME", home) + + azureConfigDir := filepath.Join(t.TempDir(), "azure-custom") + if err := os.MkdirAll(azureConfigDir, 0o755); err != nil { + t.Fatalf("mkdir azureConfigDir: %v", err) + } + if err := os.WriteFile(filepath.Join(azureConfigDir, "clouds.config"), []byte("[AzureCloud]\nsubscription = 11111111-2222-3333-4444-555555555555\n"), 0o600); err != nil { + t.Fatalf("write clouds.config: %v", err) + } + t.Setenv("AZURE_CONFIG_DIR", azureConfigDir) + + got := defaultSubscriptionID() + if got != "11111111-2222-3333-4444-555555555555" { + t.Fatalf("unexpected subscription id %q", got) + } +}