diff --git a/Makefile b/Makefile index aac4d160..d4ff5082 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,9 @@ all: build-cli build-service build-cli: cd cli && go build -v && mv cli ../oms-cli +build-cli-linux: + GOOS=linux GOARCH=amd64 go build -C cli -o ../oms-cli + build-service: cd service && go build -v && mv service ../oms-service diff --git a/cli/cmd/install_k0s.go b/cli/cmd/install_k0s.go index a5139330..71d7de90 100644 --- a/cli/cmd/install_k0s.go +++ b/cli/cmd/install_k0s.go @@ -5,12 +5,17 @@ package cmd import ( "fmt" + "log" + "os" + "path/filepath" packageio "github.com/codesphere-cloud/cs-go/pkg/io" "github.com/spf13/cobra" "github.com/codesphere-cloud/oms/internal/env" "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/node" "github.com/codesphere-cloud/oms/internal/portal" "github.com/codesphere-cloud/oms/internal/util" ) @@ -25,10 +30,13 @@ type InstallK0sCmd struct { type InstallK0sOpts struct { *GlobalOptions - Version string - Package string - Config string - Force bool + Version string + Package string + InstallConfig string + SSHKeyPath string + RemoteHost string + RemoteUser string + Force bool } func (c *InstallK0sCmd) RunE(_ *cobra.Command, args []string) error { @@ -37,12 +45,7 @@ func (c *InstallK0sCmd) RunE(_ *cobra.Command, args []string) error { pm := installer.NewPackage(env.GetOmsWorkdir(), c.Opts.Package) k0s := installer.NewK0s(hw, env, c.FileWriter) - err := c.InstallK0s(pm, k0s) - if err != nil { - return fmt.Errorf("failed to install k0s: %w", err) - } - - return nil + return c.InstallK0s(pm, k0s) } func AddInstallK0sCmd(install *cobra.Command, opts *GlobalOptions) { @@ -54,12 +57,16 @@ func AddInstallK0sCmd(install *cobra.Command, opts *GlobalOptions) { This will either download the k0s binary directly to the OMS workdir, if not already present, and install it or load the k0s binary from the provided package file and install it. If no version is specified, the latest version will be downloaded. - If no install config is provided, k0s will be installed with the '--single' flag.`), + + You must provide a Codesphere install-config file, which will: + - Generate a k0s configuration from the install-config + - Optionally install k0s on remote nodes via SSH`), Example: formatExamplesWithBinary("install k0s", []packageio.Example{ - {Cmd: "", Desc: "Install k0s using the Go-native implementation"}, + {Cmd: "--install-config ", Desc: "Path to Codesphere install-config file to generate k0s config from"}, {Cmd: "--version ", Desc: "Version of k0s to install"}, {Cmd: "--package ", Desc: "Package file (e.g. codesphere-v1.2.3-installer.tar.gz) to load k0s from"}, - {Cmd: "--k0s-config ", Desc: "Path to k0s configuration file, if not set k0s will be installed with the '--single' flag"}, + {Cmd: "--remote-host ", Desc: "Remote host IP to install k0s on (requires --ssh-key-path)"}, + {Cmd: "--ssh-key-path ", Desc: "SSH private key path for remote installation"}, {Cmd: "--force", Desc: "Force new download and installation even if k0s binary exists or is already installed"}, }, "oms-cli"), }, @@ -69,21 +76,79 @@ func AddInstallK0sCmd(install *cobra.Command, opts *GlobalOptions) { } k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", "", "Version of k0s to install") k0s.cmd.Flags().StringVarP(&k0s.Opts.Package, "package", "p", "", "Package file (e.g. codesphere-v1.2.3-installer.tar.gz) to load k0s from") - k0s.cmd.Flags().StringVar(&k0s.Opts.Config, "k0s-config", "", "Path to k0s configuration file") + k0s.cmd.Flags().StringVar(&k0s.Opts.InstallConfig, "install-config", "", "Path to Codesphere install-config file (required)") + k0s.cmd.Flags().StringVar(&k0s.Opts.SSHKeyPath, "ssh-key-path", "", "SSH private key path for remote installation") + k0s.cmd.Flags().StringVar(&k0s.Opts.RemoteHost, "remote-host", "", "Remote host IP to install k0s on") + k0s.cmd.Flags().StringVar(&k0s.Opts.RemoteUser, "remote-user", "root", "Remote user for SSH connection") k0s.cmd.Flags().BoolVarP(&k0s.Opts.Force, "force", "f", false, "Force new download and installation") + _ = k0s.cmd.MarkFlagRequired("install-config") + k0s.cmd.MarkFlagsRequiredTogether("remote-host", "ssh-key-path") + install.AddCommand(k0s.cmd) k0s.cmd.RunE = k0s.RunE } -const defaultK0sPath = "kubernetes/files/k0s" +const ( + defaultK0sPath = "kubernetes/files/k0s" + k0sConfigDir = "/etc/k0s" + k0sConfigFile = "k0s.yaml" +) + +// writeK0sConfigFile writes the k0s config to the specified path +func writeK0sConfigFile(preferredPath string, data []byte) (string, func(), error) { + if err := os.MkdirAll(filepath.Dir(preferredPath), 0755); err != nil { + // Fall back to temp file + tmpPath := filepath.Join(os.TempDir(), "k0s-config.yaml") + if err := os.WriteFile(tmpPath, data, 0644); err != nil { + return "", nil, err + } + log.Printf("Generated k0s configuration at %s (using temp path due to permissions)", tmpPath) + cleanup := func() { _ = os.Remove(tmpPath) } + return tmpPath, cleanup, nil + } + + if err := os.WriteFile(preferredPath, data, 0644); err != nil { + return "", nil, err + } + log.Printf("Generated k0s configuration at %s", preferredPath) + return preferredPath, nil, nil +} func (c *InstallK0sCmd) InstallK0s(pm installer.PackageManager, k0s installer.K0sManager) error { - // Default dependency path for k0s binary within package - k0sPath := pm.GetDependencyPath(defaultK0sPath) + icg := installer.NewInstallConfigManager() + if err := icg.LoadInstallConfigFromFile(c.Opts.InstallConfig); err != nil { + return fmt.Errorf("failed to load install-config: %w", err) + } + + config := icg.GetInstallConfig() + + if !config.Kubernetes.ManagedByCodesphere { + return fmt.Errorf("install-config specifies external Kubernetes, k0s installation is only supported for Codesphere-managed Kubernetes") + } + + log.Println("Generating k0s configuration from install-config...") + k0sConfig, err := installer.GenerateK0sConfig(config) + if err != nil { + return fmt.Errorf("failed to generate k0s config: %w", err) + } + + k0sConfigData, err := k0sConfig.Marshal() + if err != nil { + return fmt.Errorf("failed to marshal k0s config: %w", err) + } - var err error + k0sConfigPath := filepath.Join(k0sConfigDir, k0sConfigFile) + k0sConfigPath, cleanup, err := writeK0sConfigFile(k0sConfigPath, k0sConfigData) + if err != nil { + return fmt.Errorf("failed to write k0s config: %w", err) + } + if cleanup != nil { + defer cleanup() + } + + k0sPath := pm.GetDependencyPath(defaultK0sPath) if c.Opts.Package == "" { k0sPath, err = k0s.Download(c.Opts.Version, c.Opts.Force, false) if err != nil { @@ -91,10 +156,58 @@ func (c *InstallK0sCmd) InstallK0s(pm installer.PackageManager, k0s installer.K0 } } - err = k0s.Install(c.Opts.Config, k0sPath, c.Opts.Force) + if c.Opts.RemoteHost != "" { + return c.InstallK0sRemote(config, k0sPath, k0sConfigPath) + } + + controlPlaneIPs := make([]string, 0, len(config.Kubernetes.ControlPlanes)) + for _, cp := range config.Kubernetes.ControlPlanes { + controlPlaneIPs = append(controlPlaneIPs, cp.IPAddress) + } + nodeIP, err := installer.GetNodeIPAddress(controlPlaneIPs) + if err != nil { + log.Printf("Warning: could not determine node IP: %v. Installing without --kubelet-extra-args", err) + nodeIP = "" + } + + err = k0s.Install(k0sConfigPath, k0sPath, c.Opts.Force, nodeIP) if err != nil { return fmt.Errorf("failed to install k0s: %w", err) } + log.Println("k0s installed successfully using configuration from install-config") + return nil +} + +func (c *InstallK0sCmd) InstallK0sRemote(config *files.RootConfig, k0sBinaryPath string, k0sConfigPath string) error { + log.Printf("Installing k0s on remote host %s", c.Opts.RemoteHost) + + nm := &node.NodeManager{ + FileIO: c.FileWriter, + KeyPath: c.Opts.SSHKeyPath, + } + + remoteNode := &node.Node{ + ExternalIP: c.Opts.RemoteHost, + InternalIP: c.Opts.RemoteHost, + Name: "k0s-node", + User: c.Opts.RemoteUser, + } + + controlPlaneIPs := make([]string, 0, len(config.Kubernetes.ControlPlanes)) + for _, cp := range config.Kubernetes.ControlPlanes { + controlPlaneIPs = append(controlPlaneIPs, cp.IPAddress) + } + nodeIP, err := installer.GetNodeIPAddress(controlPlaneIPs) + if err != nil { + log.Printf("Warning: could not determine node IP from control planes: %v. Using remote host IP: %s", err, c.Opts.RemoteHost) + nodeIP = c.Opts.RemoteHost + } + + if err := remoteNode.InstallK0s(nm, k0sBinaryPath, k0sConfigPath, c.Opts.Force, nodeIP); err != nil { + return fmt.Errorf("failed to install k0s on remote host: %w", err) + } + + log.Printf("k0s successfully installed on remote host %s", c.Opts.RemoteHost) return nil } diff --git a/cli/cmd/install_k0s_integration_test.go b/cli/cmd/install_k0s_integration_test.go new file mode 100644 index 00000000..28a5523c --- /dev/null +++ b/cli/cmd/install_k0s_integration_test.go @@ -0,0 +1,409 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build integration +// +build integration + +package cmd_test + +import ( + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +var _ = Describe("K0s Install-Config Integration", func() { + var ( + tempDir string + configPath string + k0sConfigOut string + ) + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "k0s-integration-test-*") + Expect(err).NotTo(HaveOccurred()) + + configPath = filepath.Join(tempDir, "install-config.yaml") + k0sConfigOut = filepath.Join(tempDir, "k0s-config.yaml") + }) + + AfterEach(func() { + if tempDir != "" { + os.RemoveAll(tempDir) + } + }) + + createBaseConfig := func(name string, ip string) *files.RootConfig { + return &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: name, + City: "Test City", + CountryCode: "US", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: ip}, + }, + APIServerHost: "api.test.example.com", + }, + Codesphere: files.CodesphereConfig{ + Domain: "test.example.com", + PublicIP: ip, + DeployConfig: files.DeployConfig{ + Images: map[string]files.ImageConfig{}, + }, + Plans: files.PlansConfig{ + HostingPlans: map[int]files.HostingPlan{}, + WorkspacePlans: map[int]files.WorkspacePlan{}, + }, + }, + } + } + + Describe("Complete Workflow", func() { + It("should generate valid k0s config from install-config file", func() { + installConfig := createBaseConfig("test-dc", "192.168.1.100") + + // Write and load install-config + configData, err := yaml.Marshal(installConfig) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + icg := installer.NewInstallConfigManager() + err = icg.LoadInstallConfigFromFile(configPath) + Expect(err).NotTo(HaveOccurred()) + + loadedConfig := icg.GetInstallConfig() + Expect(loadedConfig).NotTo(BeNil()) + Expect(loadedConfig.Kubernetes.ManagedByCodesphere).To(BeTrue()) + + // Generate k0s config + k0sConfig, err := installer.GenerateK0sConfig(loadedConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig).NotTo(BeNil()) + + // Verify k0s config structure + Expect(k0sConfig.APIVersion).To(Equal("k0s.k0sproject.io/v1beta1")) + Expect(k0sConfig.Kind).To(Equal("ClusterConfig")) + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-test-dc")) + Expect(k0sConfig.Spec.API.Address).To(Equal("192.168.1.100")) + Expect(k0sConfig.Spec.API.ExternalAddress).To(Equal("api.test.example.com")) + + // Write k0s config to file and verify + k0sData, err := k0sConfig.Marshal() + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(k0sConfigOut, k0sData, 0644) + Expect(err).NotTo(HaveOccurred()) + + Expect(k0sConfigOut).To(BeAnExistingFile()) + data, err := os.ReadFile(k0sConfigOut) + Expect(err).NotTo(HaveOccurred()) + + var verifyConfig installer.K0sConfig + err = yaml.Unmarshal(data, &verifyConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(verifyConfig.APIVersion).To(Equal("k0s.k0sproject.io/v1beta1")) + Expect(verifyConfig.Metadata.Name).To(Equal("codesphere-test-dc")) + }) + }) + + Describe("Configuration Features", func() { + It("should handle multi-control-plane configuration", func() { + installConfig := createBaseConfig("multi-dc", "10.0.0.10") + installConfig.Kubernetes.ControlPlanes = []files.K8sNode{ + {IPAddress: "10.0.0.10"}, + {IPAddress: "10.0.0.11"}, + {IPAddress: "10.0.0.12"}, + } + installConfig.Kubernetes.APIServerHost = "api.cluster.test" + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + // Verify primary IP is used + Expect(k0sConfig.Spec.API.Address).To(Equal("10.0.0.10")) + // Verify all IPs are in SANs + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("10.0.0.10")) + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("10.0.0.11")) + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("10.0.0.12")) + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("api.cluster.test")) + }) + + It("should preserve custom network configuration", func() { + installConfig := createBaseConfig("network-test", "192.168.1.100") + installConfig.Kubernetes.PodCIDR = "10.244.0.0/16" + installConfig.Kubernetes.ServiceCIDR = "10.96.0.0/12" + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + Expect(k0sConfig.Spec.Network).NotTo(BeNil()) + Expect(k0sConfig.Spec.Network.PodCIDR).To(Equal("10.244.0.0/16")) + Expect(k0sConfig.Spec.Network.ServiceCIDR).To(Equal("10.96.0.0/12")) + }) + + It("should configure etcd storage correctly", func() { + installConfig := createBaseConfig("storage-test", "192.168.1.100") + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + Expect(k0sConfig.Spec.Storage).NotTo(BeNil()) + Expect(k0sConfig.Spec.Storage.Type).To(Equal("etcd")) + Expect(k0sConfig.Spec.Storage.Etcd).NotTo(BeNil()) + Expect(k0sConfig.Spec.Storage.Etcd.PeerAddress).To(Equal("192.168.1.100")) + }) + + It("should generate correct cluster name from datacenter", func() { + installConfig := createBaseConfig("prod-us-east", "10.1.2.3") + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-prod-us-east")) + }) + + It("should handle empty control plane list", func() { + installConfig := createBaseConfig("empty-cp", "10.0.0.1") + installConfig.Kubernetes.ControlPlanes = []files.K8sNode{} + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + // Should either handle gracefully or error + if err == nil { + Expect(k0sConfig).NotTo(BeNil()) + } else { + Expect(err).To(HaveOccurred()) + } + }) + + It("should use default network values when not specified", func() { + installConfig := createBaseConfig("defaults-test", "10.0.0.1") + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + // Verify defaults are applied or fields are present + Expect(k0sConfig.Spec.Network).NotTo(BeNil()) + if k0sConfig.Spec.Network.PodCIDR != "" { + Expect(k0sConfig.Spec.Network.PodCIDR).To(MatchRegexp(`^\d+\.\d+\.\d+\.\d+/\d+$`)) + } + if k0sConfig.Spec.Network.ServiceCIDR != "" { + Expect(k0sConfig.Spec.Network.ServiceCIDR).To(MatchRegexp(`^\d+\.\d+\.\d+\.\d+/\d+$`)) + } + }) + + It("should handle special characters in datacenter names", func() { + testCases := []struct { + name string + expected string + }{ + {"test-dc-01", "codesphere-test-dc-01"}, + {"test_dc_02", "codesphere-test_dc_02"}, + {"TestDC03", "codesphere-TestDC03"}, + } + + for _, tc := range testCases { + installConfig := createBaseConfig(tc.name, "10.0.0.1") + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Metadata.Name).To(Equal(tc.expected)) + } + }) + + It("should handle large multi-control-plane setup", func() { + installConfig := createBaseConfig("large-cluster", "10.0.1.1") + controlPlanes := make([]files.K8sNode, 7) + for i := 0; i < 7; i++ { + controlPlanes[i] = files.K8sNode{ + IPAddress: fmt.Sprintf("10.0.1.%d", i+1), + } + } + installConfig.Kubernetes.ControlPlanes = controlPlanes + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.API.Address).To(Equal("10.0.1.1")) + Expect(len(k0sConfig.Spec.API.SANs)).To(BeNumerically(">=", 7)) + }) + + It("should properly configure certificate SANs", func() { + installConfig := createBaseConfig("san-test", "192.168.100.50") + installConfig.Kubernetes.APIServerHost = "k8s.example.com" + installConfig.Codesphere.Domain = "app.example.com" + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("192.168.100.50")) + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("k8s.example.com")) + Expect(len(k0sConfig.Spec.API.SANs)).To(BeNumerically(">=", 2)) + }) + }) + + Describe("Error Handling", func() { + It("should fail when loading non-existent file", func() { + nonExistentPath := filepath.Join(tempDir, "does-not-exist.yaml") + icg := installer.NewInstallConfigManager() + err := icg.LoadInstallConfigFromFile(nonExistentPath) + Expect(err).To(HaveOccurred()) + }) + + It("should fail when generating config from nil", func() { + _, err := installer.GenerateK0sConfig(nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("cannot be nil")) + }) + + It("should fail when loading invalid YAML", func() { + invalidYAML := []byte("invalid: [unclosed bracket") + err := os.WriteFile(configPath, invalidYAML, 0644) + Expect(err).NotTo(HaveOccurred()) + + icg := installer.NewInstallConfigManager() + err = icg.LoadInstallConfigFromFile(configPath) + Expect(err).To(HaveOccurred()) + }) + + It("should handle empty file gracefully", func() { + err := os.WriteFile(configPath, []byte{}, 0644) + Expect(err).NotTo(HaveOccurred()) + + icg := installer.NewInstallConfigManager() + err = icg.LoadInstallConfigFromFile(configPath) + // Empty file loads successfully but returns empty config + Expect(err).NotTo(HaveOccurred()) + config := icg.GetInstallConfig() + Expect(config).NotTo(BeNil()) + }) + + It("should handle external Kubernetes cluster config", func() { + installConfig := createBaseConfig("external-k8s", "10.0.0.1") + installConfig.Kubernetes.ManagedByCodesphere = false + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig).NotTo(BeNil()) + }) + + It("should handle missing APIServerHost", func() { + installConfig := createBaseConfig("missing-host", "10.0.0.1") + installConfig.Kubernetes.APIServerHost = "" + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + if err == nil { + Expect(k0sConfig).NotTo(BeNil()) + Expect(k0sConfig.Spec.API.Address).To(Equal("10.0.0.1")) + } else { + Expect(err).To(HaveOccurred()) + } + }) + + It("should handle missing datacenter name", func() { + installConfig := createBaseConfig("", "10.0.0.1") + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + if err == nil { + Expect(k0sConfig).NotTo(BeNil()) + } else { + Expect(err).To(HaveOccurred()) + } + }) + + It("should fail when writing to read-only directory", func() { + readOnlyDir := filepath.Join(tempDir, "readonly") + err := os.Mkdir(readOnlyDir, 0444) + Expect(err).NotTo(HaveOccurred()) + + readOnlyPath := filepath.Join(readOnlyDir, "config.yaml") + installConfig := createBaseConfig("test", "10.0.0.1") + configData, err := yaml.Marshal(installConfig) + Expect(err).NotTo(HaveOccurred()) + + err = os.WriteFile(readOnlyPath, configData, 0644) + Expect(err).To(HaveOccurred()) + + os.Chmod(readOnlyDir, 0755) + }) + }) + + Describe("YAML Serialization", func() { + It("should marshal and unmarshal k0s config correctly", func() { + installConfig := createBaseConfig("roundtrip-test", "172.16.0.1") + installConfig.Kubernetes.PodCIDR = "10.244.0.0/16" + installConfig.Kubernetes.ServiceCIDR = "10.96.0.0/12" + + original, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + yamlData, err := original.Marshal() + Expect(err).NotTo(HaveOccurred()) + Expect(string(yamlData)).To(ContainSubstring("k0s.k0sproject.io/v1beta1")) + Expect(string(yamlData)).To(ContainSubstring("ClusterConfig")) + + var restored installer.K0sConfig + err = yaml.Unmarshal(yamlData, &restored) + Expect(err).NotTo(HaveOccurred()) + + // Verify critical fields match + Expect(restored.APIVersion).To(Equal(original.APIVersion)) + Expect(restored.Kind).To(Equal(original.Kind)) + Expect(restored.Metadata.Name).To(Equal(original.Metadata.Name)) + Expect(restored.Spec.API.Address).To(Equal(original.Spec.API.Address)) + Expect(restored.Spec.Network.PodCIDR).To(Equal(original.Spec.Network.PodCIDR)) + Expect(restored.Spec.Network.ServiceCIDR).To(Equal(original.Spec.Network.ServiceCIDR)) + }) + }) + + Describe("Config Persistence", func() { + It("should persist and reload config correctly", func() { + originalConfig := createBaseConfig("persist-test", "172.20.30.40") + originalConfig.Kubernetes.PodCIDR = "10.100.0.0/16" + originalConfig.Kubernetes.ServiceCIDR = "10.200.0.0/16" + + // Save install-config + configData, err := yaml.Marshal(originalConfig) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + // Generate and save k0s config + k0sConfig, err := installer.GenerateK0sConfig(originalConfig) + Expect(err).NotTo(HaveOccurred()) + k0sData, err := k0sConfig.Marshal() + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(k0sConfigOut, k0sData, 0644) + Expect(err).NotTo(HaveOccurred()) + + // Reload install-config + icg := installer.NewInstallConfigManager() + err = icg.LoadInstallConfigFromFile(configPath) + Expect(err).NotTo(HaveOccurred()) + reloadedInstallConfig := icg.GetInstallConfig() + + // Reload k0s config + reloadedK0sData, err := os.ReadFile(k0sConfigOut) + Expect(err).NotTo(HaveOccurred()) + var reloadedK0sConfig installer.K0sConfig + err = yaml.Unmarshal(reloadedK0sData, &reloadedK0sConfig) + Expect(err).NotTo(HaveOccurred()) + + // Verify both configs match original + Expect(reloadedInstallConfig.Datacenter.Name).To(Equal(originalConfig.Datacenter.Name)) + Expect(reloadedInstallConfig.Kubernetes.PodCIDR).To(Equal(originalConfig.Kubernetes.PodCIDR)) + Expect(reloadedK0sConfig.Metadata.Name).To(Equal(k0sConfig.Metadata.Name)) + Expect(reloadedK0sConfig.Spec.API.Address).To(Equal(k0sConfig.Spec.API.Address)) + Expect(reloadedK0sConfig.Spec.Network.PodCIDR).To(Equal(k0sConfig.Spec.Network.PodCIDR)) + }) + }) +}) diff --git a/cli/cmd/install_k0s_test.go b/cli/cmd/install_k0s_test.go index 8c59b542..4bbf2e18 100644 --- a/cli/cmd/install_k0s_test.go +++ b/cli/cmd/install_k0s_test.go @@ -4,14 +4,18 @@ package cmd_test import ( - "errors" + "os" + "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + "gopkg.in/yaml.v3" "github.com/codesphere-cloud/oms/cli/cmd" "github.com/codesphere-cloud/oms/internal/env" "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" "github.com/codesphere-cloud/oms/internal/util" ) @@ -32,7 +36,7 @@ var _ = Describe("InstallK0sCmd", func() { GlobalOptions: globalOpts, Version: "", Package: "", - Config: "", + InstallConfig: "", Force: false, } c = cmd.InstallK0sCmd{ @@ -47,63 +51,324 @@ var _ = Describe("InstallK0sCmd", func() { mockFileWriter.AssertExpectations(GinkgoT()) }) + Context("RunE method", func() { + It("fails when install-config is not provided", func() { + c.Opts.InstallConfig = "" + mockEnv.EXPECT().GetOmsWorkdir().Return("/test/workdir") + + err := c.RunE(nil, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("install-config")) + }) + }) + Context("InstallK0s method", func() { - It("fails when package is not specified and k0s download fails", func() { - mockPackageManager := installer.NewMockPackageManager(GinkgoT()) - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) + var ( + mockPM *installer.MockPackageManager + mockK0s *installer.MockK0sManager + tempDir string + ) + + BeforeEach(func() { + mockPM = installer.NewMockPackageManager(GinkgoT()) + mockK0s = installer.NewMockK0sManager(GinkgoT()) + var err error + tempDir, err = os.MkdirTemp("", "install-k0s-test-*") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + mockPM.AssertExpectations(GinkgoT()) + mockK0s.AssertExpectations(GinkgoT()) + if tempDir != "" { + _ = os.RemoveAll(tempDir) + } + }) - c.Opts.Package = "" // No package specified, should download - mockPackageManager.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/workdir/test-package/deps/kubernetes/files/k0s") - mockK0sManager.EXPECT().Download("", false, false).Return("", errors.New("download failed")) + createTestConfig := func(managedByCodesphere bool) *files.RootConfig { + return &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + City: "Test City", + CountryCode: "US", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: managedByCodesphere, + ControlPlanes: []files.K8sNode{ + {IPAddress: "192.168.1.100"}, + }, + APIServerHost: "api.test.example.com", + }, + Codesphere: files.CodesphereConfig{ + Domain: "test.example.com", + PublicIP: "192.168.1.100", + DeployConfig: files.DeployConfig{ + Images: map[string]files.ImageConfig{}, + }, + Plans: files.PlansConfig{ + HostingPlans: map[int]files.HostingPlan{}, + WorkspacePlans: map[int]files.WorkspacePlan{}, + }, + }, + } + } + + It("fails when install-config file does not exist", func() { + c.Opts.InstallConfig = "/nonexistent/install-config.yaml" - err := c.InstallK0s(mockPackageManager, mockK0sManager) + err := c.InstallK0s(mockPM, mockK0s) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to load install-config")) + }) + + It("fails when install-config specifies external Kubernetes", func() { + config := createTestConfig(false) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + c.Opts.InstallConfig = configPath + + err = c.InstallK0s(mockPM, mockK0s) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("external Kubernetes")) + }) + + It("successfully installs k0s locally with valid config", func() { + config := createTestConfig(true) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + c.Opts.InstallConfig = configPath + c.Opts.Package = "test-package.tar.gz" + c.Opts.Force = true + + mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") + mockK0s.EXPECT().Install(mock.Anything, "/test/path/k0s", true, mock.Anything).Return(nil) + + err = c.InstallK0s(mockPM, mockK0s) + Expect(err).NotTo(HaveOccurred()) + }) + + It("downloads k0s when package is not specified", func() { + config := createTestConfig(true) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + c.Opts.InstallConfig = configPath + c.Opts.Package = "" + c.Opts.Version = "v1.29.0+k0s.0" + + mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") + mockK0s.EXPECT().Download("v1.29.0+k0s.0", false, false).Return("/downloaded/k0s", nil) + mockK0s.EXPECT().Install(mock.Anything, "/downloaded/k0s", false, mock.Anything).Return(nil) + + err = c.InstallK0s(mockPM, mockK0s) + Expect(err).NotTo(HaveOccurred()) + }) + + It("fails when k0s download fails", func() { + config := createTestConfig(true) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + c.Opts.InstallConfig = configPath + c.Opts.Package = "" + + mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") + mockK0s.EXPECT().Download("", false, false).Return("", os.ErrNotExist) + + err = c.InstallK0s(mockPM, mockK0s) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("failed to download k0s")) - Expect(err.Error()).To(ContainSubstring("download failed")) }) It("fails when k0s install fails", func() { - mockPackageManager := installer.NewMockPackageManager(GinkgoT()) - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) + config := createTestConfig(true) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) - c.Opts.Package = "" // No package specified, should download - c.Opts.Config = "/path/to/config.yaml" - c.Opts.Force = true - mockPackageManager.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/workdir/test-package/deps/kubernetes/files/k0s") - mockK0sManager.EXPECT().Download("", true, false).Return("/test/workdir/k0s", nil) - mockK0sManager.EXPECT().Install("/path/to/config.yaml", "/test/workdir/k0s", true).Return(errors.New("install failed")) + c.Opts.InstallConfig = configPath + c.Opts.Package = "test-package.tar.gz" - err := c.InstallK0s(mockPackageManager, mockK0sManager) + mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") + mockK0s.EXPECT().Install(mock.Anything, "/test/path/k0s", false, mock.Anything).Return(os.ErrPermission) + + err = c.InstallK0s(mockPM, mockK0s) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("failed to install k0s")) - Expect(err.Error()).To(ContainSubstring("install failed")) }) - It("succeeds when package is not specified and k0s download and install work", func() { - mockPackageManager := installer.NewMockPackageManager(GinkgoT()) - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) + It("handles remote installation when remote-host is specified", func() { + config := createTestConfig(true) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + c.Opts.InstallConfig = configPath + c.Opts.Package = "test-package.tar.gz" + c.Opts.RemoteHost = "192.168.1.50" + c.Opts.SSHKeyPath = "/path/to/key" - c.Opts.Package = "" // No package specified, should download - c.Opts.Config = "" // No config, will use single mode - mockPackageManager.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/workdir/test-package/deps/kubernetes/files/k0s") - mockK0sManager.EXPECT().Download("", false, false).Return("/test/workdir/k0s", nil) - mockK0sManager.EXPECT().Install("", "/test/workdir/k0s", false).Return(nil) + mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") + mockFileWriter.EXPECT().ReadFile("/path/to/key").Return([]byte("invalid-key-data"), nil).Maybe() - err := c.InstallK0s(mockPackageManager, mockK0sManager) - Expect(err).ToNot(HaveOccurred()) + // Remote installation will fail because we can't actually connect, + // but we're testing that it attempts remote installation + err = c.InstallK0s(mockPM, mockK0s) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to install k0s on remote host")) }) + }) + + Context("InstallK0sRemote method", func() { + var ( + config *files.RootConfig + ) - It("succeeds when package is specified and k0s install works", func() { - mockPackageManager := installer.NewMockPackageManager(GinkgoT()) - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) + BeforeEach(func() { + config = &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "192.168.1.100"}, + }, + }, + } + }) + + It("fails when SSH connection cannot be established", func() { + c.Opts.RemoteHost = "192.0.2.1" // TEST-NET-1, should fail to connect + c.Opts.SSHKeyPath = "/tmp/nonexistent-key" + + mockFileWriter.EXPECT().ReadFile("/tmp/nonexistent-key").Return([]byte("invalid-key-data"), nil).Maybe() + + err := c.InstallK0sRemote(config, "/path/to/k0s", "/path/to/config") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to install k0s on remote host")) + }) - c.Opts.Package = "test-package.tar.gz" // Package specified, should use k0s from package - c.Opts.Config = "/path/to/config.yaml" - mockPackageManager.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/workdir/test-package/deps/kubernetes/files/k0s") - mockK0sManager.EXPECT().Install("/path/to/config.yaml", "/test/workdir/test-package/deps/kubernetes/files/k0s", false).Return(nil) + It("fails when SSH key file does not exist", func() { + c.Opts.RemoteHost = "192.168.1.50" + c.Opts.SSHKeyPath = "/nonexistent/ssh/key" - err := c.InstallK0s(mockPackageManager, mockK0sManager) - Expect(err).ToNot(HaveOccurred()) + mockFileWriter.EXPECT().ReadFile("/nonexistent/ssh/key").Return(nil, os.ErrNotExist).Maybe() + + err := c.InstallK0sRemote(config, "/path/to/k0s", "/path/to/config") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to install k0s on remote host")) + }) + + It("fails when SSH key file is invalid", func() { + c.Opts.RemoteHost = "192.168.1.50" + c.Opts.SSHKeyPath = "/path/to/invalid/key" + + mockFileWriter.EXPECT().ReadFile("/path/to/invalid/key").Return([]byte("not-a-valid-key"), nil).Maybe() + + err := c.InstallK0sRemote(config, "/path/to/k0s", "/path/to/config") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to install k0s on remote host")) + }) + + It("uses correct remote host IP for node configuration", func() { + c.Opts.RemoteHost = "10.0.0.50" + c.Opts.SSHKeyPath = "/path/to/key" + + mockFileWriter.EXPECT().ReadFile("/path/to/key").Return([]byte("ssh-key-data"), nil).Maybe() + + err := c.InstallK0sRemote(config, "/path/to/k0s", "/path/to/config") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to install k0s on remote host")) + }) + + It("passes correct paths to InstallK0s", func() { + c.Opts.RemoteHost = "192.168.1.60" + c.Opts.SSHKeyPath = "/custom/ssh/key" + c.Opts.Force = true + + mockFileWriter.EXPECT().ReadFile("/custom/ssh/key").Return([]byte("ssh-key-data"), nil).Maybe() + + err := c.InstallK0sRemote(config, "/custom/k0s/path", "/custom/config/path") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to install k0s on remote host")) + }) + + It("respects the force flag", func() { + c.Opts.RemoteHost = "192.168.1.70" + c.Opts.SSHKeyPath = "/path/to/key" + c.Opts.Force = true + + mockFileWriter.EXPECT().ReadFile("/path/to/key").Return([]byte("ssh-key-data"), nil).Maybe() + + err := c.InstallK0sRemote(config, "/path/to/k0s", "/path/to/config") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to install k0s on remote host")) + }) + + It("uses remote user from options", func() { + c.Opts.RemoteHost = "192.168.1.80" + c.Opts.SSHKeyPath = "/path/to/key" + c.Opts.RemoteUser = "ubuntu" + + mockFileWriter.EXPECT().ReadFile("/path/to/key").Return([]byte("ssh-key-data"), nil).Maybe() + + err := c.InstallK0sRemote(config, "/path/to/k0s", "/path/to/config") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to install k0s on remote host")) + }) + + It("handles empty remote host", func() { + c.Opts.RemoteHost = "" + c.Opts.SSHKeyPath = "/path/to/key" + + mockFileWriter.EXPECT().ReadFile("/path/to/key").Return([]byte("ssh-key-data"), nil).Maybe() + + err := c.InstallK0sRemote(config, "/path/to/k0s", "/path/to/config") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to install k0s on remote host")) + }) + + It("handles timeout during SSH connection", func() { + c.Opts.RemoteHost = "192.0.2.1" // TEST-NET-1 address + c.Opts.SSHKeyPath = "/path/to/key" + + mockFileWriter.EXPECT().ReadFile("/path/to/key").Return([]byte("-----BEGIN RSA PRIVATE KEY-----\ntest\n-----END RSA PRIVATE KEY-----"), nil).Maybe() + + err := c.InstallK0sRemote(config, "/path/to/k0s", "/path/to/config") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to install k0s on remote host")) + }) + + It("wraps errors from InstallK0s with context", func() { + c.Opts.RemoteHost = "10.0.0.100" + c.Opts.SSHKeyPath = "/path/to/key" + + mockFileWriter.EXPECT().ReadFile("/path/to/key").Return([]byte("ssh-key-data"), nil).Maybe() + + err := c.InstallK0sRemote(config, "/path/to/k0s", "/path/to/config") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to install k0s on remote host")) }) }) }) diff --git a/docs/oms-cli_install_k0s.md b/docs/oms-cli_install_k0s.md index e9c7fa06..e0084e7b 100644 --- a/docs/oms-cli_install_k0s.md +++ b/docs/oms-cli_install_k0s.md @@ -8,7 +8,10 @@ Install k0s either from the package or by downloading it. This will either download the k0s binary directly to the OMS workdir, if not already present, and install it or load the k0s binary from the provided package file and install it. If no version is specified, the latest version will be downloaded. -If no install config is provided, k0s will be installed with the '--single' flag. + +You must provide a Codesphere install-config file, which will: +- Generate a k0s configuration from the install-config +- Optionally install k0s on remote nodes via SSH ``` oms-cli install k0s [flags] @@ -17,8 +20,8 @@ oms-cli install k0s [flags] ### Examples ``` -# Install k0s using the Go-native implementation -$ oms-cli install k0s +# Path to Codesphere install-config file to generate k0s config from +$ oms-cli install k0s --install-config # Version of k0s to install $ oms-cli install k0s --version @@ -26,8 +29,11 @@ $ oms-cli install k0s --version # Package file (e.g. codesphere-v1.2.3-installer.tar.gz) to load k0s from $ oms-cli install k0s --package -# Path to k0s configuration file, if not set k0s will be installed with the '--single' flag -$ oms-cli install k0s --k0s-config +# Remote host IP to install k0s on (requires --ssh-key-path) +$ oms-cli install k0s --remote-host + +# SSH private key path for remote installation +$ oms-cli install k0s --ssh-key-path # Force new download and installation even if k0s binary exists or is already installed $ oms-cli install k0s --force @@ -37,11 +43,14 @@ $ oms-cli install k0s --force ### Options ``` - -f, --force Force new download and installation - -h, --help help for k0s - --k0s-config string Path to k0s configuration file - -p, --package string Package file (e.g. codesphere-v1.2.3-installer.tar.gz) to load k0s from - -v, --version string Version of k0s to install + -f, --force Force new download and installation + -h, --help help for k0s + --install-config string Path to Codesphere install-config file (required) + -p, --package string Package file (e.g. codesphere-v1.2.3-installer.tar.gz) to load k0s from + --remote-host string Remote host IP to install k0s on + --remote-user string Remote user for SSH connection (default "root") + --ssh-key-path string SSH private key path for remote installation + -v, --version string Version of k0s to install ``` ### SEE ALSO diff --git a/go.mod b/go.mod index 38dc92f7..df590c25 100644 --- a/go.mod +++ b/go.mod @@ -381,6 +381,7 @@ require ( github.com/pjbgf/sha1cd v0.5.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/sftp v1.13.10 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect github.com/prometheus/client_golang v1.23.2 // indirect diff --git a/hack/lima-oms.yaml b/hack/lima-oms.yaml index 8dabfd2b..67261b71 100644 --- a/hack/lima-oms.yaml +++ b/hack/lima-oms.yaml @@ -18,8 +18,8 @@ disk: "60GiB" # Mount your OMS project mounts: -- location: "." - mountPoint: "/home/user/oms" +- location: "~" + mountPoint: "/home/user/host-home" writable: true # Ports and SSH @@ -61,27 +61,111 @@ provision: set -eux -o pipefail # Install Docker in rootless mode - dockerd-rootless-setuptool.sh install + dockerd-rootless-setuptool.sh install || true - # Set up the OMS project - cd /home/user/oms - export PATH=$PATH:/usr/local/go/bin - go mod download - cd cli && go build -a -buildvcs=false && mv cli ../oms-cli + # Clone OMS repository locally for better build performance + if [ ! -d ~/oms ]; then + git clone https://github.com/codesphere-cloud/oms.git ~/oms + fi + + # Create a test install-config for k0s testing + VM_IP=$(hostname -I | awk '{print $1}') + cat > ~/oms/test-install-config.yaml << 'EOFCONFIG' + datacenter: + id: 1 + name: test-dc + city: Test City + countryCode: US + kubernetes: + managedByCodesphere: true + controlPlanes: + - ipAddress: ${VM_IP} + apiServerHost: api.test.local + codesphere: + domain: test.local + publicIP: ${VM_IP} + deployConfig: + images: {} + plans: + hostingPlans: {} + workspacePlans: {} + EOFCONFIG message: | Your OMS development environment is ready! + VM IP: Run 'hostname -I | awk '{print $1}'' to get your VM's IP address - To access it: + Quick Start: ------ limactl shell lima-oms - cd /home/user/oms - ./oms-cli --help + cd oms + make build-cli ------ - - To install Codesphere eg.: + + Test k0s installation locally: ------ - ./oms-cli install codesphere --package codesphere-v1.66.0-installer --config config.yaml --priv-key ./path-to-private-key + limactl shell lima-oms + cd oms + + # Build the CLI + make build-cli + + # Install k0s (uses test-install-config.yaml which is auto-generated) + sudo ./oms-cli install k0s --install-config test-install-config.yaml --version v1.30.0+k0s.0 --force + + # Check k0s installation + sudo systemctl status k0scontroller + sudo oms-workdir/k0s kubectl get nodes + + # Stop and cleanup k0s + sudo systemctl stop k0scontroller + sudo systemctl disable k0scontroller + sudo oms-workdir/k0s reset + ------ + + Check k0s config that was generated: + ------ + sudo cat /etc/k0s/k0s.yaml + ------ + + Test with package instead of download: + ------ + # First, create a package (run on your host machine with access to real install-config) + ./oms-cli download package --version v1.x.x + + # Then in Lima: + limactl shell lima-oms + cd oms + sudo ./oms-cli install k0s --install-config test-install-config.yaml \ + --package codesphere-vX.X.X-installer.tar.gz --force + ------ + + Run tests: + ------ + limactl shell lima-oms + cd oms + + # Run all tests + go test ./... + + # Run specific k0s tests + go test ./cli/cmd -ginkgo.v -ginkgo.focus "InstallK0s" + go test ./internal/installer -ginkgo.v -ginkgo.focus "k0s" + go test ./internal/installer/node -ginkgo.v -ginkgo.focus "InstallK0s" + ------ + + Build for different platforms: + ------ + # From your Mac (cross-compile to Linux) + make build-cli-linux + + # Inside Lima (native Linux build) + limactl shell lima-oms + cd oms + make build-cli + ------ + + Notes: + - test-install-config.yaml is automatically created in ~/oms with your VM's IP + - Remote installation requires proper SSH setup ------ - - Go 1.24 and Docker are installed and ready to use. diff --git a/internal/installer/k0s.go b/internal/installer/k0s.go index 1c350b37..46e5cbdf 100644 --- a/internal/installer/k0s.go +++ b/internal/installer/k0s.go @@ -6,6 +6,7 @@ package installer import ( "fmt" "log" + "net" "os" "path/filepath" "runtime" @@ -14,12 +15,14 @@ import ( "github.com/codesphere-cloud/oms/internal/env" "github.com/codesphere-cloud/oms/internal/portal" "github.com/codesphere-cloud/oms/internal/util" + "gopkg.in/yaml.v3" ) type K0sManager interface { GetLatestVersion() (string, error) Download(version string, force bool, quiet bool) (string, error) - Install(configPath string, k0sPath string, force bool) error + Install(configPath string, k0sPath string, force bool, nodeIP string) error + Reset(k0sPath string) error } type K0s struct { @@ -30,6 +33,31 @@ type K0s struct { Goarch string } +// valid top-level fields in a k0s ClusterConfig. +// Reference: https://docs.k0sproject.io/stable/configuration/ +var K0sConfigTopLevelKeys = []string{ + "apiVersion", + "kind", + "metadata", + "spec", +} + +// valid fields in the spec section of a k0s ClusterConfig. +// Reference: https://docs.k0sproject.io/stable/configuration/ +var K0sConfigSpecKeys = []string{ + "api", + "controllerManager", + "scheduler", + "extensions", + "network", + "storage", + "telemetry", + "images", + "konnectivity", + "installConfig", + "featureGates", +} + func NewK0s(hw portal.Http, env env.Env, fw util.FileIO) K0sManager { return &K0s{ Env: env, @@ -62,6 +90,12 @@ func (k *K0s) Download(version string, force bool, quiet bool) (string, error) { // Check if k0s binary already exists and create destination file workdir := k.Env.GetOmsWorkdir() + + // Ensure workdir exists + if err := os.MkdirAll(workdir, 0755); err != nil { + return "", fmt.Errorf("failed to create workdir: %w", err) + } + k0sPath := filepath.Join(workdir, "k0s") if k.FileWriter.Exists(k0sPath) && !force { return "", fmt.Errorf("k0s binary already exists at %s. Use --force to overwrite", k0sPath) @@ -93,7 +127,7 @@ func (k *K0s) Download(version string, force bool, quiet bool) (string, error) { return k0sPath, nil } -func (k *K0s) Install(configPath string, k0sPath string, force bool) error { +func (k *K0s) Install(configPath string, k0sPath string, force bool, nodeIP string) error { if k.Goos != "linux" || k.Goarch != "amd64" { return fmt.Errorf("k0s installation is only supported on Linux amd64. Current platform: %s/%s", k.Goos, k.Goarch) } @@ -102,13 +136,35 @@ func (k *K0s) Install(configPath string, k0sPath string, force bool) error { return fmt.Errorf("k0s binary does not exist in '%s', please download first", k0sPath) } + if force { + if err := k.Reset(k0sPath); err != nil { + log.Printf("Warning: failed to reset k0s: %v", err) + } + } + args := []string{k0sPath, "install", "controller"} + + // If config path is provided, filter it to only include k0s-compatible fields if configPath != "" { - args = append(args, "--config", configPath) + filteredConfigPath, err := k.filterConfigForK0s(configPath) + if err != nil { + log.Printf("Warning: failed to filter config, using original: %v", err) + args = append(args, "--config", configPath) + } else { + args = append(args, "--config", filteredConfigPath) + defer func() { _ = os.Remove(filteredConfigPath) }() + } } else { args = append(args, "--single") } + args = append(args, "--enable-worker") + args = append(args, "--no-taints") + + if nodeIP != "" { + args = append(args, "--kubelet-extra-args", fmt.Sprintf("--node-ip=%s", nodeIP)) + } + if force { args = append(args, "--force") } @@ -128,3 +184,112 @@ func (k *K0s) Install(configPath string, k0sPath string, force bool) error { return nil } + +func (k *K0s) filterConfigForK0s(configPath string) (string, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return "", fmt.Errorf("failed to read config: %w", err) + } + + var config map[string]interface{} + if err := yaml.Unmarshal(data, &config); err != nil { + return "", fmt.Errorf("failed to parse config: %w", err) + } + + keysToKeep := make(map[string]bool, len(K0sConfigTopLevelKeys)) + for _, key := range K0sConfigTopLevelKeys { + keysToKeep[key] = true + } + + for key := range config { + if !keysToKeep[key] { + delete(config, key) + } + } + + if spec, ok := config["spec"].(map[string]interface{}); ok { + specKeysToKeep := make(map[string]bool, len(K0sConfigSpecKeys)) + for _, key := range K0sConfigSpecKeys { + specKeysToKeep[key] = true + } + + for key := range spec { + if !specKeysToKeep[key] { + delete(spec, key) + } + } + config["spec"] = spec + } + + filteredData, err := yaml.Marshal(config) + if err != nil { + return "", fmt.Errorf("failed to marshal filtered config: %w", err) + } + + tmpFile, err := os.CreateTemp("", "k0s-config-*.yaml") + if err != nil { + return "", fmt.Errorf("failed to create temp config: %w", err) + } + defer func() { _ = tmpFile.Close() }() + + if _, err := tmpFile.Write(filteredData); err != nil { + return "", fmt.Errorf("failed to write temp config: %w", err) + } + + return tmpFile.Name(), nil +} + +// GetNodeIPAddress finds the IP address of the current node by matching +// against the control plane IPs in the config +func GetNodeIPAddress(controlPlanes []string) (string, error) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", fmt.Errorf("failed to get network interfaces: %w", err) + } + + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + ip := ipnet.IP.String() + for _, cpIP := range controlPlanes { + if ip == cpIP { + return ip, nil + } + } + } + } + } + + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + return ipnet.IP.String(), nil + } + } + } + + return "", fmt.Errorf("no suitable IP address found") +} + +// Reset tears down an existing k0s installation by executing `k0s reset`. +// This command removes all k0s-related resources +func (k *K0s) Reset(k0sPath string) error { + if !k.FileWriter.Exists(k0sPath) { + return nil + } + + log.Println("Resetting existing k0s installation...") + + log.Println("Stopping k0s service if running...") + if err := util.RunCommand("sudo", []string{k0sPath, "stop"}, ""); err != nil { + log.Printf("Note: k0s stop returned error, try to continue the reset: %v", err) + } + + err := util.RunCommand("sudo", []string{k0sPath, "reset"}, "") + if err != nil { + return fmt.Errorf("failed to reset k0s: %w", err) + } + + log.Println("k0s reset completed successfully") + return nil +} diff --git a/internal/installer/k0s_config.go b/internal/installer/k0s_config.go new file mode 100644 index 00000000..a4ac23c2 --- /dev/null +++ b/internal/installer/k0s_config.go @@ -0,0 +1,140 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "fmt" + + "github.com/codesphere-cloud/oms/internal/installer/files" + "gopkg.in/yaml.v3" +) + +type K0sConfig struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata K0sMetadata `yaml:"metadata"` + Spec K0sSpec `yaml:"spec"` +} + +type K0sMetadata struct { + Name string `yaml:"name"` +} + +type K0sSpec struct { + API *K0sAPI `yaml:"api,omitempty"` + Network *K0sNetwork `yaml:"network,omitempty"` + Storage *K0sStorage `yaml:"storage,omitempty"` + Images *K0sImages `yaml:"images,omitempty"` + Telemetry *K0sTelemetry `yaml:"telemetry,omitempty"` +} + +type K0sAPI struct { + Address string `yaml:"address,omitempty"` + ExternalAddress string `yaml:"externalAddress,omitempty"` + SANs []string `yaml:"sans,omitempty"` + Port int `yaml:"port,omitempty"` +} + +type K0sNetwork struct { + PodCIDR string `yaml:"podCIDR,omitempty"` + ServiceCIDR string `yaml:"serviceCIDR,omitempty"` + Provider string `yaml:"provider,omitempty"` +} + +type K0sStorage struct { + Type string `yaml:"type,omitempty"` + Etcd *K0sEtcd `yaml:"etcd,omitempty"` +} + +type K0sEtcd struct { + PeerAddress string `yaml:"peerAddress,omitempty"` +} + +type K0sImages struct { + DefaultPullPolicy string `yaml:"default_pull_policy,omitempty"` +} + +type K0sTelemetry struct { + Enabled bool `yaml:"enabled"` +} + +func GenerateK0sConfig(installConfig *files.RootConfig) (*K0sConfig, error) { + if installConfig == nil { + return nil, fmt.Errorf("installConfig cannot be nil") + } + + k0sConfig := &K0sConfig{ + APIVersion: "k0s.k0sproject.io/v1beta1", + Kind: "ClusterConfig", + Metadata: K0sMetadata{ + Name: fmt.Sprintf("codesphere-%s", installConfig.Datacenter.Name), + }, + Spec: K0sSpec{}, + } + + if installConfig.Kubernetes.ManagedByCodesphere { + if len(installConfig.Kubernetes.ControlPlanes) > 0 { + firstControlPlane := installConfig.Kubernetes.ControlPlanes[0] + k0sConfig.Spec.API = &K0sAPI{ + Address: firstControlPlane.IPAddress, + Port: 6443, + } + + if installConfig.Kubernetes.APIServerHost != "" { + k0sConfig.Spec.API.ExternalAddress = installConfig.Kubernetes.APIServerHost + } + + sans := make([]string, 0, len(installConfig.Kubernetes.ControlPlanes)) + for _, cp := range installConfig.Kubernetes.ControlPlanes { + sans = append(sans, cp.IPAddress) + } + if installConfig.Kubernetes.APIServerHost != "" { + sans = append(sans, installConfig.Kubernetes.APIServerHost) + } + k0sConfig.Spec.API.SANs = sans + } + + k0sConfig.Spec.Network = &K0sNetwork{ + Provider: "calico", + } + + if installConfig.Kubernetes.PodCIDR != "" { + k0sConfig.Spec.Network.PodCIDR = installConfig.Kubernetes.PodCIDR + } else { + k0sConfig.Spec.Network.PodCIDR = "100.96.0.0/11" + } + if installConfig.Kubernetes.ServiceCIDR != "" { + k0sConfig.Spec.Network.ServiceCIDR = installConfig.Kubernetes.ServiceCIDR + } else { + k0sConfig.Spec.Network.ServiceCIDR = "100.64.0.0/13" + } + + k0sConfig.Spec.Images = &K0sImages{ + DefaultPullPolicy: "Never", + } + + k0sConfig.Spec.Telemetry = &K0sTelemetry{ + Enabled: false, + } + + if len(installConfig.Kubernetes.ControlPlanes) > 0 { + k0sConfig.Spec.Storage = &K0sStorage{ + Type: "etcd", + Etcd: &K0sEtcd{ + PeerAddress: installConfig.Kubernetes.ControlPlanes[0].IPAddress, + }, + } + } + } + + return k0sConfig, nil +} + +func (c *K0sConfig) Marshal() ([]byte, error) { + return yaml.Marshal(c) +} + +func (c *K0sConfig) Unmarshal(data []byte) error { + return yaml.Unmarshal(data, c) +} diff --git a/internal/installer/k0s_config_test.go b/internal/installer/k0s_config_test.go new file mode 100644 index 00000000..7bc17128 --- /dev/null +++ b/internal/installer/k0s_config_test.go @@ -0,0 +1,343 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +var _ = Describe("K0sConfig", func() { + Describe("GenerateK0sConfig", func() { + Context("with valid install-config", func() { + It("should generate k0s config with control plane settings", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + APIServerHost: "k8s.example.com", + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + {IPAddress: "10.0.1.11"}, + {IPAddress: "10.0.1.12"}, + }, + PodCIDR: "10.244.0.0/16", + ServiceCIDR: "10.96.0.0/12", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(k0sConfig).ToNot(BeNil()) + + // Check basic structure + Expect(k0sConfig.APIVersion).To(Equal("k0s.k0sproject.io/v1beta1")) + Expect(k0sConfig.Kind).To(Equal("ClusterConfig")) + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-test-dc")) + + // Check API configuration + Expect(k0sConfig.Spec.API).ToNot(BeNil()) + Expect(k0sConfig.Spec.API.Address).To(Equal("10.0.1.10")) + Expect(k0sConfig.Spec.API.ExternalAddress).To(Equal("k8s.example.com")) + Expect(k0sConfig.Spec.API.Port).To(Equal(6443)) + Expect(k0sConfig.Spec.API.SANs).To(ContainElements("10.0.1.10", "10.0.1.11", "10.0.1.12", "k8s.example.com")) + + // Check Network configuration + Expect(k0sConfig.Spec.Network).ToNot(BeNil()) + Expect(k0sConfig.Spec.Network.PodCIDR).To(Equal("10.244.0.0/16")) + Expect(k0sConfig.Spec.Network.ServiceCIDR).To(Equal("10.96.0.0/12")) + Expect(k0sConfig.Spec.Network.Provider).To(Equal("calico")) + Expect(k0sConfig.Spec.Storage.Etcd).ToNot(BeNil()) + Expect(k0sConfig.Spec.Storage.Etcd.PeerAddress).To(Equal("10.0.1.10")) + }) + + It("should handle minimal configuration", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "minimal", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "192.168.1.100"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(k0sConfig).ToNot(BeNil()) + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-minimal")) + }) + + It("should generate valid YAML", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + PodCIDR: "10.244.0.0/16", + ServiceCIDR: "10.96.0.0/12", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).ToNot(HaveOccurred()) + + yamlData, err := k0sConfig.Marshal() + Expect(err).ToNot(HaveOccurred()) + Expect(yamlData).ToNot(BeEmpty()) + + // Verify it can be unmarshalled back + var parsedConfig installer.K0sConfig + err = yaml.Unmarshal(yamlData, &parsedConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(parsedConfig.Metadata.Name).To(Equal("codesphere-test-dc")) + }) + }) + + Context("with invalid input", func() { + It("should return error for nil install-config", func() { + k0sConfig, err := installer.GenerateK0sConfig(nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("installConfig cannot be nil")) + Expect(k0sConfig).To(BeNil()) + }) + }) + + Context("with non-managed Kubernetes", func() { + It("should not configure k0s for external kubernetes", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "external", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: false, + PodCIDR: "10.244.0.0/16", + ServiceCIDR: "10.96.0.0/12", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(k0sConfig).ToNot(BeNil()) + // Should still have basic structure but no specific config + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-external")) + }) + }) + + Context("edge cases and validation", func() { + It("should handle empty datacenter name", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-")) + }) + + It("should handle empty control plane list", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{}, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + // Should have basic structure but no API/Storage config + Expect(k0sConfig.Spec.API).To(BeNil()) + Expect(k0sConfig.Spec.Storage).To(BeNil()) + }) + + It("should handle nil control plane addresses", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: nil, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.API).To(BeNil()) + }) + + It("should handle missing APIServerHost", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + APIServerHost: "", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.API.ExternalAddress).To(BeEmpty()) + Expect(k0sConfig.Spec.API.SANs).To(ConsistOf("10.0.0.1")) + }) + + It("should handle missing network CIDRs", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + PodCIDR: "", + ServiceCIDR: "", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.Network).NotTo(BeNil()) + Expect(k0sConfig.Spec.Network.Provider).To(Equal("calico")) + }) + + It("should use default network provider", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.Network.Provider).To(Equal("calico")) + }) + + It("should generate correct SANs with single control plane", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + APIServerHost: "api.example.com", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.API.SANs).To(HaveLen(2)) + Expect(k0sConfig.Spec.API.SANs).To(ContainElements("10.0.0.1", "api.example.com")) + }) + + It("should handle special characters in datacenter name", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc_01.prod", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-test-dc_01.prod")) + }) + + It("should set correct API port", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.API.Port).To(Equal(6443)) + }) + + It("should configure etcd with first control plane IP", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + {IPAddress: "10.0.0.2"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.Storage.Type).To(Equal("etcd")) + Expect(k0sConfig.Spec.Storage.Etcd.PeerAddress).To(Equal("10.0.0.1")) + }) + }) + }) +}) diff --git a/internal/installer/k0s_internal_test.go b/internal/installer/k0s_internal_test.go new file mode 100644 index 00000000..83ecc537 --- /dev/null +++ b/internal/installer/k0s_internal_test.go @@ -0,0 +1,339 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +var _ = Describe("K0s Internal Methods", func() { + var ( + k0s *K0s + tempConfigDir string + ) + + BeforeEach(func() { + k0s = &K0s{} + var err error + tempConfigDir, err = os.MkdirTemp("", "k0s-config-test-*") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + if tempConfigDir != "" { + _ = os.RemoveAll(tempConfigDir) + } + }) + + createConfigFile := func(content string) string { + configPath := filepath.Join(tempConfigDir, "test-config.yaml") + err := os.WriteFile(configPath, []byte(content), 0644) + Expect(err).NotTo(HaveOccurred()) + return configPath + } + + Describe("filterConfigForK0s", func() { + It("filters out non-k0s top-level fields", func() { + configContent := `apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +metadata: + name: test-cluster +spec: + api: + address: 192.168.1.100 +extraField: should-be-removed +anotherExtra: also-removed +` + configPath := createConfigFile(configContent) + + filteredPath, err := k0s.filterConfigForK0s(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(filteredPath).NotTo(BeEmpty()) + defer func() { _ = os.Remove(filteredPath) }() + + // Read and verify filtered content + data, err := os.ReadFile(filteredPath) + Expect(err).NotTo(HaveOccurred()) + content := string(data) + + Expect(content).To(ContainSubstring("apiVersion")) + Expect(content).To(ContainSubstring("kind")) + Expect(content).To(ContainSubstring("metadata")) + Expect(content).To(ContainSubstring("spec")) + Expect(content).NotTo(ContainSubstring("extraField")) + Expect(content).NotTo(ContainSubstring("anotherExtra")) + }) + + It("preserves all expected k0s fields at top level", func() { + configContent := `apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +metadata: + name: test-cluster +spec: + api: + address: 192.168.1.100 +` + configPath := createConfigFile(configContent) + + filteredPath, err := k0s.filterConfigForK0s(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(filteredPath).NotTo(BeEmpty()) + defer func() { _ = os.Remove(filteredPath) }() + + data, err := os.ReadFile(filteredPath) + Expect(err).NotTo(HaveOccurred()) + content := string(data) + + Expect(content).To(ContainSubstring("apiVersion: k0s.k0sproject.io/v1beta1")) + Expect(content).To(ContainSubstring("kind: ClusterConfig")) + Expect(content).To(ContainSubstring("metadata")) + Expect(content).To(ContainSubstring("spec")) + }) + + It("filters out non-k0s spec fields", func() { + configContent := `apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +spec: + api: + address: 192.168.1.100 + network: + provider: calico + customField: should-be-removed + anotherCustom: also-removed +` + configPath := createConfigFile(configContent) + + filteredPath, err := k0s.filterConfigForK0s(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(filteredPath).NotTo(BeEmpty()) + defer func() { _ = os.Remove(filteredPath) }() + + data, err := os.ReadFile(filteredPath) + Expect(err).NotTo(HaveOccurred()) + content := string(data) + + Expect(content).To(ContainSubstring("api")) + Expect(content).To(ContainSubstring("network")) + Expect(content).NotTo(ContainSubstring("customField")) + Expect(content).NotTo(ContainSubstring("anotherCustom")) + }) + + It("preserves all expected k0s spec fields", func() { + configContent := `apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +spec: + api: + address: 192.168.1.100 + controllerManager: + extraArgs: + - --cluster-cidr=10.244.0.0/16 + scheduler: + extraArgs: + - --bind-address=0.0.0.0 + extensions: + helm: + repositories: + - name: stable + network: + provider: calico + storage: + type: etcd + telemetry: + enabled: false + images: + default_pull_policy: IfNotPresent + konnectivity: + enabled: true +` + configPath := createConfigFile(configContent) + + filteredPath, err := k0s.filterConfigForK0s(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(filteredPath).NotTo(BeEmpty()) + defer func() { _ = os.Remove(filteredPath) }() + + data, err := os.ReadFile(filteredPath) + Expect(err).NotTo(HaveOccurred()) + content := string(data) + + // Verify all expected spec fields are preserved + Expect(content).To(ContainSubstring("api")) + Expect(content).To(ContainSubstring("controllerManager")) + Expect(content).To(ContainSubstring("scheduler")) + Expect(content).To(ContainSubstring("extensions")) + Expect(content).To(ContainSubstring("network")) + Expect(content).To(ContainSubstring("storage")) + Expect(content).To(ContainSubstring("telemetry")) + Expect(content).To(ContainSubstring("images")) + Expect(content).To(ContainSubstring("konnectivity")) + }) + + It("handles config with only required fields", func() { + configContent := `apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +spec: + api: + address: 192.168.1.100 +` + configPath := createConfigFile(configContent) + + filteredPath, err := k0s.filterConfigForK0s(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(filteredPath).NotTo(BeEmpty()) + defer func() { _ = os.Remove(filteredPath) }() + + data, err := os.ReadFile(filteredPath) + Expect(err).NotTo(HaveOccurred()) + + Expect(string(data)).NotTo(BeEmpty()) + }) + + It("creates a temporary file with .yaml extension", func() { + configContent := `apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +spec: + api: + address: 192.168.1.100 +` + configPath := createConfigFile(configContent) + + filteredPath, err := k0s.filterConfigForK0s(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(filteredPath).NotTo(BeEmpty()) + defer func() { _ = os.Remove(filteredPath) }() + + Expect(filteredPath).To(HaveSuffix(".yaml")) + Expect(filteredPath).To(ContainSubstring("k0s-config-")) + + // Verify file exists and is readable + _, err = os.Stat(filteredPath) + Expect(err).NotTo(HaveOccurred()) + }) + + It("fails when config file does not exist", func() { + _, err := k0s.filterConfigForK0s("/nonexistent/config.yaml") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to read config")) + }) + + It("fails when config contains invalid YAML", func() { + configPath := createConfigFile("invalid: yaml: content: [") + + _, err := k0s.filterConfigForK0s(configPath) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to parse config")) + }) + + It("handles complex nested structures correctly", func() { + configContent := `apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +metadata: + name: production-cluster +spec: + api: + address: 192.168.1.100 + port: 6443 + sans: + - api.example.com + - 192.168.1.100 + network: + provider: calico + podCIDR: 10.244.0.0/16 + serviceCIDR: 10.96.0.0/12 + extensions: + helm: + repositories: + - name: stable + url: https://charts.helm.sh/stable + charts: + - name: metrics-server + namespace: kube-system +customTopLevel: should-be-filtered +` + configPath := createConfigFile(configContent) + + filteredPath, err := k0s.filterConfigForK0s(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(filteredPath).NotTo(BeEmpty()) + defer func() { _ = os.Remove(filteredPath) }() + + data, err := os.ReadFile(filteredPath) + Expect(err).NotTo(HaveOccurred()) + content := string(data) + + // Verify nested structures are preserved + Expect(content).To(ContainSubstring("api.example.com")) + Expect(content).To(ContainSubstring("10.244.0.0/16")) + Expect(content).To(ContainSubstring("metrics-server")) + + // Verify custom fields are filtered out + Expect(content).NotTo(ContainSubstring("customTopLevel")) + }) + + It("filters custom fields within spec", func() { + configContent := `apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +spec: + api: + address: 192.168.1.100 + network: + provider: calico + customInSpec: should-be-filtered + anotherCustom: also-filtered +` + configPath := createConfigFile(configContent) + + filteredPath, err := k0s.filterConfigForK0s(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(filteredPath).NotTo(BeEmpty()) + defer func() { _ = os.Remove(filteredPath) }() + + data, err := os.ReadFile(filteredPath) + Expect(err).NotTo(HaveOccurred()) + content := string(data) + + Expect(content).To(ContainSubstring("api")) + Expect(content).To(ContainSubstring("network")) + Expect(content).NotTo(ContainSubstring("customInSpec")) + Expect(content).NotTo(ContainSubstring("anotherCustom")) + }) + + It("returns valid YAML that can be parsed", func() { + configContent := `apiVersion: k0s.k0sproject.io/v1beta1 +kind: ClusterConfig +spec: + api: + address: 192.168.1.100 + network: + provider: calico +extraField: removed +` + configPath := createConfigFile(configContent) + + filteredPath, err := k0s.filterConfigForK0s(configPath) + Expect(err).NotTo(HaveOccurred()) + Expect(filteredPath).NotTo(BeEmpty()) + defer func() { _ = os.Remove(filteredPath) }() + + // Verify the output is valid YAML by parsing it + data, err := os.ReadFile(filteredPath) + Expect(err).NotTo(HaveOccurred()) + + var result map[string]interface{} + err = yaml.Unmarshal(data, &result) + Expect(err).NotTo(HaveOccurred()) + + // Verify expected structure + Expect(result).To(HaveKey("apiVersion")) + Expect(result).To(HaveKey("kind")) + Expect(result).To(HaveKey("spec")) + Expect(result).NotTo(HaveKey("extraField")) + }) + }) +}) diff --git a/internal/installer/k0s_nodeip_test.go b/internal/installer/k0s_nodeip_test.go new file mode 100644 index 00000000..06a3bcd2 --- /dev/null +++ b/internal/installer/k0s_nodeip_test.go @@ -0,0 +1,137 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "net" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("GetNodeIPAddress", func() { + Context("with valid network setup", func() { + It("should return a valid IPv4 address", func() { + ip, err := GetNodeIPAddress([]string{}) + + if err != nil { + Skip("No non-loopback network interfaces available on this system") + } + + Expect(ip).NotTo(BeEmpty()) + parsedIP := net.ParseIP(ip) + Expect(parsedIP).NotTo(BeNil()) + Expect(parsedIP.To4()).NotTo(BeNil()) + }) + + It("should not return loopback address", func() { + ip, err := GetNodeIPAddress([]string{}) + + if err != nil { + Skip("No non-loopback network interfaces available") + } + + Expect(ip).NotTo(Equal("127.0.0.1")) + Expect(ip).NotTo(HavePrefix("127.")) + }) + }) + + Context("with control plane addresses provided", func() { + It("should prioritize control plane IP if available", func() { + interfaces, err := net.Interfaces() + if err != nil { + Skip("Cannot list network interfaces") + } + + var testIP string + for _, iface := range interfaces { + if iface.Flags&net.FlagLoopback != 0 { + continue + } + + addrs, err := iface.Addrs() + if err != nil { + continue + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + if ip == nil || ip.IsLoopback() { + continue + } + + if ip.To4() != nil { + testIP = ip.String() + break + } + } + + if testIP != "" { + break + } + } + + if testIP == "" { + Skip("No suitable test IP found") + } + + result, err := GetNodeIPAddress([]string{testIP, "10.0.0.1"}) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(testIP)) + }) + + It("should fallback if control plane IPs don't match", func() { + result, err := GetNodeIPAddress([]string{"10.254.254.1", "192.168.254.254"}) + + if err == nil { + Expect(result).NotTo(BeEmpty()) + parsedIP := net.ParseIP(result) + Expect(parsedIP).NotTo(BeNil()) + Expect(parsedIP.To4()).NotTo(BeNil()) + } + }) + }) + + Context("with edge cases", func() { + It("should handle empty control plane list", func() { + ip, err := GetNodeIPAddress([]string{}) + + if err == nil { + Expect(ip).NotTo(BeEmpty()) + } else { + Expect(err.Error()).To(ContainSubstring("no suitable")) + } + }) + + It("should handle nil control plane list", func() { + ip, err := GetNodeIPAddress(nil) + + if err == nil { + Expect(ip).NotTo(BeEmpty()) + } else { + Expect(err.Error()).To(ContainSubstring("no suitable")) + } + }) + + It("should return error when no interfaces are available", func() { + ip, err := GetNodeIPAddress([]string{"invalid"}) + + if err != nil { + Expect(err.Error()).To(Or( + ContainSubstring("no suitable"), + ContainSubstring("network"), + )) + } else { + Expect(ip).NotTo(BeEmpty()) + } + }) + }) +}) diff --git a/internal/installer/k0s_test.go b/internal/installer/k0s_test.go index 83b9109f..e7878488 100644 --- a/internal/installer/k0s_test.go +++ b/internal/installer/k0s_test.go @@ -288,7 +288,7 @@ var _ = Describe("K0s", func() { k0sImpl.Goos = "windows" k0sImpl.Goarch = "amd64" - err := k0s.Install("", k0sPath, false) + err := k0s.Install("", k0sPath, false, "") Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("k0s installation is only supported on Linux amd64")) Expect(err.Error()).To(ContainSubstring("windows/amd64")) @@ -298,7 +298,7 @@ var _ = Describe("K0s", func() { k0sImpl.Goos = "linux" k0sImpl.Goarch = "arm64" - err := k0s.Install("", k0sPath, false) + err := k0s.Install("", k0sPath, false, "") Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("k0s installation is only supported on Linux amd64")) Expect(err.Error()).To(ContainSubstring("linux/arm64")) @@ -314,11 +314,39 @@ var _ = Describe("K0s", func() { It("should fail when k0s binary doesn't exist", func() { mockFileWriter.EXPECT().Exists(k0sPath).Return(false) - err := k0s.Install("", k0sPath, false) + err := k0s.Install("", k0sPath, false, "") Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("k0s binary does not exist")) Expect(err.Error()).To(ContainSubstring("please download first")) }) }) }) + + Describe("Reset", func() { + BeforeEach(func() { + k0sImpl.Goos = "linux" + k0sImpl.Goarch = "amd64" + }) + + Context("when k0s binary does not exist", func() { + It("should return nil without attempting reset", func() { + mockFileWriter.EXPECT().Exists(k0sPath).Return(false) + + err := k0s.Reset(k0sPath) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("platform validation", func() { + It("should work regardless of platform for reset", func() { + k0sImpl.Goos = "darwin" + k0sImpl.Goarch = "arm64" + + mockFileWriter.EXPECT().Exists(k0sPath).Return(false) + + err := k0s.Reset(k0sPath) + Expect(err).NotTo(HaveOccurred()) + }) + }) + }) }) diff --git a/internal/installer/mocks.go b/internal/installer/mocks.go index 79d4d7af..8474882b 100644 --- a/internal/installer/mocks.go +++ b/internal/installer/mocks.go @@ -860,16 +860,16 @@ func (_c *MockK0sManager_GetLatestVersion_Call) RunAndReturn(run func() (string, } // Install provides a mock function for the type MockK0sManager -func (_mock *MockK0sManager) Install(configPath string, k0sPath string, force bool) error { - ret := _mock.Called(configPath, k0sPath, force) +func (_mock *MockK0sManager) Install(configPath string, k0sPath string, force bool, nodeIP string) error { + ret := _mock.Called(configPath, k0sPath, force, nodeIP) if len(ret) == 0 { panic("no return value specified for Install") } var r0 error - if returnFunc, ok := ret.Get(0).(func(string, string, bool) error); ok { - r0 = returnFunc(configPath, k0sPath, force) + if returnFunc, ok := ret.Get(0).(func(string, string, bool, string) error); ok { + r0 = returnFunc(configPath, k0sPath, force, nodeIP) } else { r0 = ret.Error(0) } @@ -885,11 +885,12 @@ type MockK0sManager_Install_Call struct { // - configPath string // - k0sPath string // - force bool -func (_e *MockK0sManager_Expecter) Install(configPath interface{}, k0sPath interface{}, force interface{}) *MockK0sManager_Install_Call { - return &MockK0sManager_Install_Call{Call: _e.mock.On("Install", configPath, k0sPath, force)} +// - nodeIP string +func (_e *MockK0sManager_Expecter) Install(configPath interface{}, k0sPath interface{}, force interface{}, nodeIP interface{}) *MockK0sManager_Install_Call { + return &MockK0sManager_Install_Call{Call: _e.mock.On("Install", configPath, k0sPath, force, nodeIP)} } -func (_c *MockK0sManager_Install_Call) Run(run func(configPath string, k0sPath string, force bool)) *MockK0sManager_Install_Call { +func (_c *MockK0sManager_Install_Call) Run(run func(configPath string, k0sPath string, force bool, nodeIP string)) *MockK0sManager_Install_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 string if args[0] != nil { @@ -903,10 +904,15 @@ func (_c *MockK0sManager_Install_Call) Run(run func(configPath string, k0sPath s if args[2] != nil { arg2 = args[2].(bool) } + var arg3 string + if args[3] != nil { + arg3 = args[3].(string) + } run( arg0, arg1, arg2, + arg3, ) }) return _c @@ -917,7 +923,58 @@ func (_c *MockK0sManager_Install_Call) Return(err error) *MockK0sManager_Install return _c } -func (_c *MockK0sManager_Install_Call) RunAndReturn(run func(configPath string, k0sPath string, force bool) error) *MockK0sManager_Install_Call { +func (_c *MockK0sManager_Install_Call) RunAndReturn(run func(configPath string, k0sPath string, force bool, nodeIP string) error) *MockK0sManager_Install_Call { + _c.Call.Return(run) + return _c +} + +// Reset provides a mock function for the type MockK0sManager +func (_mock *MockK0sManager) Reset(k0sPath string) error { + ret := _mock.Called(k0sPath) + + if len(ret) == 0 { + panic("no return value specified for Reset") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(string) error); ok { + r0 = returnFunc(k0sPath) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockK0sManager_Reset_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Reset' +type MockK0sManager_Reset_Call struct { + *mock.Call +} + +// Reset is a helper method to define mock.On call +// - k0sPath string +func (_e *MockK0sManager_Expecter) Reset(k0sPath interface{}) *MockK0sManager_Reset_Call { + return &MockK0sManager_Reset_Call{Call: _e.mock.On("Reset", k0sPath)} +} + +func (_c *MockK0sManager_Reset_Call) Run(run func(k0sPath string)) *MockK0sManager_Reset_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 string + if args[0] != nil { + arg0 = args[0].(string) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *MockK0sManager_Reset_Call) Return(err error) *MockK0sManager_Reset_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockK0sManager_Reset_Call) RunAndReturn(run func(k0sPath string) error) *MockK0sManager_Reset_Call { _c.Call.Return(run) return _c } diff --git a/internal/installer/node/node.go b/internal/installer/node/node.go index 65cc8ddc..333eb56d 100644 --- a/internal/installer/node/node.go +++ b/internal/installer/node/node.go @@ -9,6 +9,7 @@ import ( "net" "os" "path/filepath" + "strings" "syscall" "time" @@ -16,6 +17,7 @@ import ( "github.com/pkg/sftp" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" + "golang.org/x/crypto/ssh/knownhosts" "golang.org/x/term" ) @@ -23,6 +25,7 @@ type Node struct { Name string `json:"name"` ExternalIP string `json:"external_ip"` InternalIP string `json:"internal_ip"` + User string `json:"user,omitempty"` } type NodeManager struct { @@ -30,15 +33,49 @@ type NodeManager struct { KeyPath string } -// getAuthMethods constructs a slice of ssh.AuthMethod, prioritizing the SSH agent. -func (n *NodeManager) getAuthMethods() ([]ssh.AuthMethod, error) { +const ( + jumpboxUser = "ubuntu" + remoteK0sDir = "/usr/local/bin" + remoteK0sConfig = "/etc/k0s/k0s.yaml" + tmpK0sBinary = "/tmp/k0s" + tmpK0sConfig = "/tmp/k0s-config.yaml" +) + +func shellEscape(s string) string { + return strings.ReplaceAll(s, "'", "'\\''") +} + +func (nm *NodeManager) getHostKeyCallback() (ssh.HostKeyCallback, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, fmt.Errorf("failed to get user home directory: %w", err) + } + + knownHostsPath := filepath.Join(homeDir, ".ssh", "known_hosts") + hostKeyCallback, err := knownhosts.New(knownHostsPath) + if err != nil { + sshDir := filepath.Join(homeDir, ".ssh") + if err := os.MkdirAll(sshDir, 0700); err != nil { + return nil, fmt.Errorf("failed to create .ssh directory: %w", err) + } + if _, err := os.Create(knownHostsPath); err != nil { + return nil, fmt.Errorf("failed to create known_hosts file: %w", err) + } + hostKeyCallback, err = knownhosts.New(knownHostsPath) + if err != nil { + return nil, fmt.Errorf("failed to load known_hosts: %w", err) + } + } + + return hostKeyCallback, nil +} + +func (nm *NodeManager) getAuthMethods() ([]ssh.AuthMethod, error) { var authMethods []ssh.AuthMethod if authSocket := os.Getenv("SSH_AUTH_SOCK"); authSocket != "" { - // Connect to the SSH agent's socket conn, err := net.Dial("unix", authSocket) if err == nil { - // Create an Agent client and use it for authentication agentClient := agent.NewClient(conn) authMethods = append(authMethods, ssh.PublicKeysCallback(agentClient.Signers)) return authMethods, nil @@ -46,41 +83,68 @@ func (n *NodeManager) getAuthMethods() ([]ssh.AuthMethod, error) { fmt.Printf("Could not connect to SSH Agent (%s): %v\n", authSocket, err) } - if n.KeyPath != "" { - fmt.Println("Falling back to private key file authentication.") + if nm.KeyPath != "" { + fmt.Printf("Falling back to private key file authentication (key: %s).\n", nm.KeyPath) - // This logic is copied from the previous answer - key, err := n.FileIO.ReadFile(n.KeyPath) + key, err := nm.FileIO.ReadFile(nm.KeyPath) if err != nil { - return nil, fmt.Errorf("failed to read private key file %s: %v", n.KeyPath, err) + return nil, fmt.Errorf("failed to read private key file %s: %v", nm.KeyPath, err) } + log.Printf("Successfully read %d bytes from key file", len(key)) + signer, err := ssh.ParsePrivateKey(key) if err == nil { + log.Printf("Successfully parsed private key (type: %s)", signer.PublicKey().Type()) authMethods = append(authMethods, ssh.PublicKeys(signer)) return authMethods, nil } - if _, ok := err.(*ssh.PassphraseMissingError); ok { - // Key is encrypted, prompt for passphrase - fmt.Printf("Enter passphrase for key '%s': ", n.KeyPath) - passphraseBytes, err := term.ReadPassword(int(syscall.Stdin)) - fmt.Println() - if err != nil { - return nil, fmt.Errorf("failed to read passphrase: %v", err) + log.Printf("Failed to parse private key: %v", err) + if _, ok := err.(*ssh.PassphraseMissingError); ok { + // Check if we're in an interactive terminal + if !term.IsTerminal(int(syscall.Stdin)) { + return nil, fmt.Errorf("passphrase-protected key requires interactive terminal. Use ssh-agent or an unencrypted key for automated scenarios") } - signer, err = ssh.ParsePrivateKeyWithPassphrase(key, passphraseBytes) - // Clear passphrase from memory - for i := range passphraseBytes { - passphraseBytes[i] = 0 + fmt.Printf("Enter passphrase for key '%s': ", nm.KeyPath) + + // Read passphrase with a timeout using a channel + type result struct { + password []byte + err error } + resultChan := make(chan result, 1) + go func() { + passphraseBytes, err := term.ReadPassword(int(syscall.Stdin)) + resultChan <- result{password: passphraseBytes, err: err} + }() + + // Wait for passphrase input with 30 second timeout + select { + case res := <-resultChan: + fmt.Println() + if res.err != nil { + return nil, fmt.Errorf("failed to read passphrase: %v", res.err) + } + + defer func() { + for i := range res.password { + res.password[i] = 0 + } + }() + + signer, err = ssh.ParsePrivateKeyWithPassphrase(key, res.password) + if err != nil { + return nil, fmt.Errorf("failed to parse private key with passphrase: %v", err) + } + authMethods = append(authMethods, ssh.PublicKeys(signer)) + return authMethods, nil - if err != nil { - return nil, fmt.Errorf("failed to parse private key with passphrase: %v", err) + case <-time.After(30 * time.Second): + fmt.Println() + return nil, fmt.Errorf("passphrase input timeout after 30 seconds") } - authMethods = append(authMethods, ssh.PublicKeys(signer)) - return authMethods, nil } return nil, fmt.Errorf("failed to parse private key: %v", err) } @@ -92,18 +156,22 @@ func (n *NodeManager) getAuthMethods() ([]ssh.AuthMethod, error) { return authMethods, nil } -func (n *NodeManager) connectToJumpbox(ip, username string) (*ssh.Client, error) { - authMethods, err := n.getAuthMethods() +func (nm *NodeManager) connectToJumpbox(ip, username string) (*ssh.Client, error) { + authMethods, err := nm.getAuthMethods() if err != nil { return nil, fmt.Errorf("jumpbox authentication setup failed: %v", err) } + hostKeyCallback, err := nm.getHostKeyCallback() + if err != nil { + return nil, fmt.Errorf("failed to get host key callback: %w", err) + } + config := &ssh.ClientConfig{ - User: username, - Auth: authMethods, - Timeout: 10 * time.Second, - // WARNING: Still using InsecureIgnoreHostKey for simplicity. Use known_hosts in production. - HostKeyCallback: ssh.InsecureIgnoreHostKey(), + User: username, + Auth: authMethods, + Timeout: 10 * time.Second, + HostKeyCallback: hostKeyCallback, } addr := fmt.Sprintf("%s:22", ip) @@ -112,27 +180,23 @@ func (n *NodeManager) connectToJumpbox(ip, username string) (*ssh.Client, error) return nil, fmt.Errorf("failed to dial jumpbox %s: %v", addr, err) } - // Enable Agent Forwarding on the jumpbox connection - if err := n.forwardAgent(jumpboxClient, nil); err != nil { + if err := nm.forwardAgent(jumpboxClient, nil); err != nil { fmt.Printf(" Warning: Agent forwarding setup failed on jumpbox: %v\n", err) } return jumpboxClient, nil } -func (n *NodeManager) forwardAgent(client *ssh.Client, session *ssh.Session) error { +func (nm *NodeManager) forwardAgent(client *ssh.Client, session *ssh.Session) error { authSocket := os.Getenv("SSH_AUTH_SOCK") if authSocket == "" { log.Printf("SSH_AUTH_SOCK not set. Cannot perform agent forwarding") } else { - // Connect to the local SSH Agent socket conn, err := net.Dial("unix", authSocket) if err != nil { log.Printf("failed to dial SSH agent socket: %v", err) } else { - // Create an agent client for the local agent ag := agent.NewClient(conn) - // This tells the remote server to proxy authentication requests back to us. if err := agent.ForwardToAgent(client, ag); err != nil { log.Printf("failed to forward agent to remote client: %v", err) } @@ -147,52 +211,51 @@ func (n *NodeManager) forwardAgent(client *ssh.Client, session *ssh.Session) err return nil } -const jumpboxUser = "ubuntu" - -// RunSSHCommand connects to the node, executes a command and streams the output -func (n *NodeManager) RunSSHCommand(jumpboxIp string, ip string, username string, command string) error { - client, err := n.GetClient(jumpboxIp, ip, username) +func (nm *NodeManager) RunSSHCommand(jumpboxIp string, ip string, username string, command string) error { + client, err := nm.GetClient(jumpboxIp, ip, username) if err != nil { return fmt.Errorf("failed to get client: %w", err) } defer util.IgnoreError(client.Close) session, err := client.NewSession() if err != nil { - return fmt.Errorf("failed to create session on jumpbox: %v", err) + return fmt.Errorf("failed to create session on target node (%s): %v", ip, err) } defer util.IgnoreError(session.Close) _ = session.Setenv("OMS_PORTAL_API_KEY", os.Getenv("OMS_PORTAL_API_KEY")) - err = n.forwardAgent(client, session) - - if err != nil { + if err := nm.forwardAgent(client, session); err != nil { fmt.Printf(" Warning: Agent forwarding setup failed on session: %v\n", err) } session.Stdout = os.Stdout session.Stderr = os.Stderr - // Start the command if err := session.Start(command); err != nil { return fmt.Errorf("failed to start command: %v", err) } if err := session.Wait(); err != nil { - // A non-zero exit status from the remote command is also considered an error return fmt.Errorf("command failed: %w", err) } return nil } -func (n *NodeManager) GetClient(jumpboxIp string, ip string, username string) (*ssh.Client, error) { +func (nm *NodeManager) GetClient(jumpboxIp string, ip string, username string) (*ssh.Client, error) { - authMethods, err := n.getAuthMethods() + authMethods, err := nm.getAuthMethods() if err != nil { return nil, fmt.Errorf("failed to get authentication methods: %w", err) } + + hostKeyCallback, err := nm.getHostKeyCallback() + if err != nil { + return nil, fmt.Errorf("failed to get host key callback: %w", err) + } + if jumpboxIp != "" { - jbClient, err := n.connectToJumpbox(jumpboxIp, jumpboxUser) + jbClient, err := nm.connectToJumpbox(jumpboxIp, jumpboxUser) if err != nil { return nil, fmt.Errorf("failed to connect to jumpbox: %v", err) } @@ -201,7 +264,7 @@ func (n *NodeManager) GetClient(jumpboxIp string, ip string, username string) (* User: username, Auth: authMethods, Timeout: 10 * time.Second, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), + HostKeyCallback: hostKeyCallback, } finalAddr := fmt.Sprintf("%s:22", ip) @@ -218,14 +281,10 @@ func (n *NodeManager) GetClient(jumpboxIp string, ip string, username string) (* } config := &ssh.ClientConfig{ - User: username, - Auth: authMethods, - Timeout: 10 * time.Second, - // WARNING: This is INSECURE for production! - // It tells the client to accept any host key. - // For production, you should implement a proper HostKeyCallback - // to verify the remote server's identity. - HostKeyCallback: ssh.InsecureIgnoreHostKey(), + User: username, + Auth: authMethods, + Timeout: 10 * time.Second, + HostKeyCallback: hostKeyCallback, } addr := fmt.Sprintf("%s:22", ip) @@ -236,8 +295,8 @@ func (n *NodeManager) GetClient(jumpboxIp string, ip string, username string) (* return client, nil } -func (n *NodeManager) GetSFTPClient(jumpboxIp string, ip string, username string) (*sftp.Client, error) { - client, err := n.GetClient(jumpboxIp, ip, username) +func (nm *NodeManager) GetSFTPClient(jumpboxIp string, ip string, username string) (*sftp.Client, error) { + client, err := nm.GetClient(jumpboxIp, ip, username) if err != nil { return nil, fmt.Errorf("failed to get SSH client: %v", err) } @@ -248,20 +307,19 @@ func (n *NodeManager) GetSFTPClient(jumpboxIp string, ip string, username string return sftpClient, nil } -// EnsureDirectoryExists creates the directory on the remote node via SSH if it does not exist. func (nm *NodeManager) EnsureDirectoryExists(jumpboxIp string, ip string, username string, dir string) error { - cmd := fmt.Sprintf("mkdir -p '%s'", dir) + cmd := fmt.Sprintf("mkdir -p '%s'", shellEscape(dir)) return nm.RunSSHCommand(jumpboxIp, ip, username, cmd) } -func (n *NodeManager) CopyFile(jumpboxIp string, ip string, username string, src string, dst string) error { - client, err := n.GetSFTPClient(jumpboxIp, ip, username) +func (nm *NodeManager) CopyFile(jumpboxIp string, ip string, username string, src string, dst string) error { + client, err := nm.GetSFTPClient(jumpboxIp, ip, username) if err != nil { return fmt.Errorf("failed to get SSH client: %v", err) } defer util.IgnoreError(client.Close) - srcFile, err := n.FileIO.Open(src) + srcFile, err := nm.FileIO.Open(src) if err != nil { return fmt.Errorf("failed to open source file %s: %v", src, err) } @@ -282,13 +340,9 @@ func (n *NodeManager) CopyFile(jumpboxIp string, ip string, username string, src } func (n *Node) HasCommand(nm *NodeManager, command string) bool { - checkCommand := fmt.Sprintf("command -v %s >/dev/null 2>&1", command) + checkCommand := fmt.Sprintf("command -v '%s' >/dev/null 2>&1", shellEscape(command)) err := nm.RunSSHCommand("", n.ExternalIP, "root", checkCommand) - if err != nil { - // If the command returns a non-zero exit status, it means the command is not found - return false - } - return true + return err == nil } func (n *Node) InstallOms(nm *NodeManager) error { @@ -308,29 +362,29 @@ func (n *Node) InstallOms(nm *NodeManager) error { } func (n *Node) CopyFile(jumpbox *Node, nm *NodeManager, src string, dst string) error { + user := n.User + if user == "" { + user = "root" + } + if jumpbox == nil { - err := nm.EnsureDirectoryExists("", n.ExternalIP, "root", filepath.Dir(dst)) + err := nm.EnsureDirectoryExists("", n.ExternalIP, user, filepath.Dir(dst)) if err != nil { return fmt.Errorf("failed to ensure directory exists: %w", err) } - return nm.CopyFile("", n.ExternalIP, "root", src, dst) + return nm.CopyFile("", n.ExternalIP, user, src, dst) } - err := nm.EnsureDirectoryExists(jumpbox.ExternalIP, n.InternalIP, "root", filepath.Dir(dst)) + err := nm.EnsureDirectoryExists(jumpbox.ExternalIP, n.InternalIP, user, filepath.Dir(dst)) if err != nil { return fmt.Errorf("failed to ensure directory exists: %w", err) } - return nm.CopyFile(jumpbox.ExternalIP, n.InternalIP, "root", src, dst) + return nm.CopyFile(jumpbox.ExternalIP, n.InternalIP, user, src, dst) } -// HasAcceptEnvConfigured checks if AcceptEnv is configured func (n *Node) HasAcceptEnvConfigured(jumpbox *Node, nm *NodeManager) bool { checkCommand := "sudo grep -E '^AcceptEnv OMS_PORTAL_API_KEY' /etc/ssh/sshd_config >/dev/null 2>&1" err := n.RunSSHCommand(jumpbox, nm, "ubuntu", checkCommand) - if err != nil { - // If the command returns a NON-zero exit status, it means AcceptEnv is not configured - return false - } - return true + return err == nil } func (n *Node) ConfigureAcceptEnv(jumpbox *Node, nm *NodeManager) error { @@ -351,26 +405,17 @@ func (n *Node) HasRootLoginEnabled(jumpbox *Node, nm *NodeManager) bool { checkCommandPermit := "sudo grep -E '^PermitRootLogin yes' /etc/ssh/sshd_config >/dev/null 2>&1" err := n.RunSSHCommand(jumpbox, nm, "ubuntu", checkCommandPermit) if err != nil { - // If the command returns a NON-zero exit status, it means root login is not permitted return false } checkCommandAuthorizedKeys := "sudo grep -E '^no-port-forwarding' /root/.ssh/authorized_keys >/dev/null 2>&1" err = n.RunSSHCommand(jumpbox, nm, "ubuntu", checkCommandAuthorizedKeys) - if err == nil { - // If the command returns a ZERO exit status, it means root login is prevented - return false - } - return true + return err != nil } func (n *Node) HasFile(jumpbox *Node, nm *NodeManager, filePath string) bool { - checkCommand := fmt.Sprintf("test -f '%s'", filePath) + checkCommand := fmt.Sprintf("test -f '%s'", shellEscape(filePath)) err := n.RunSSHCommand(jumpbox, nm, "ubuntu", checkCommand) - if err != nil { - // If the command returns a non-zero exit status, it means the file does not exist - return false - } - return true + return err == nil } func (n *Node) RunSSHCommand(jumpbox *Node, nm *NodeManager, username string, command string) error { @@ -420,11 +465,7 @@ func (n *Node) WaitForSSH(jumpbox *Node, nm *NodeManager, timeout time.Duration) func (n *Node) HasInotifyWatchesConfigured(jumpbox *Node, nm *NodeManager) bool { checkCommand := "sudo grep -E '^fs.inotify.max_user_watches=1048576' /etc/sysctl.conf >/dev/null 2>&1" err := n.RunSSHCommand(jumpbox, nm, "root", checkCommand) - if err != nil { - // If the command returns a NON-zero exit status, it means the setting is not configured - return false - } - return true + return err == nil } func (n *Node) ConfigureInotifyWatches(jumpbox *Node, nm *NodeManager) error { @@ -440,3 +481,69 @@ func (n *Node) ConfigureInotifyWatches(jumpbox *Node, nm *NodeManager) error { } return nil } + +func (n *Node) InstallK0s(nm *NodeManager, k0sBinaryPath string, k0sConfigPath string, force bool, nodeIP string) error { + remoteK0sBinary := filepath.Join(remoteK0sDir, "k0s") + remoteConfigPath := remoteK0sConfig + + user := n.User + if user == "" { + user = "root" + } + + // Copy k0s binary to temp location first, then move with sudo + log.Printf("Copying k0s binary to %s:%s", n.ExternalIP, tmpK0sBinary) + if err := nm.CopyFile("", n.ExternalIP, user, k0sBinaryPath, tmpK0sBinary); err != nil { + return fmt.Errorf("failed to copy k0s binary to temp: %w", err) + } + + // Move to final location and make executable with sudo + log.Printf("Moving k0s binary to %s", remoteK0sBinary) + moveCmd := fmt.Sprintf("sudo mv '%s' '%s' && sudo chmod +x '%s'", + shellEscape(tmpK0sBinary), shellEscape(remoteK0sBinary), shellEscape(remoteK0sBinary)) + if err := nm.RunSSHCommand("", n.ExternalIP, user, moveCmd); err != nil { + return fmt.Errorf("failed to move and chmod k0s binary: %w", err) + } + + if k0sConfigPath != "" { + // Copy config to temp location first + log.Printf("Copying k0s config to %s", tmpK0sConfig) + if err := nm.CopyFile("", n.ExternalIP, user, k0sConfigPath, tmpK0sConfig); err != nil { + return fmt.Errorf("failed to copy k0s config to temp: %w", err) + } + + // Create /etc/k0s directory and move config with sudo + log.Printf("Moving k0s config to %s", remoteConfigPath) + setupConfigCmd := fmt.Sprintf("sudo mkdir -p /etc/k0s && sudo mv '%s' '%s' && sudo chmod 644 '%s'", + shellEscape(tmpK0sConfig), shellEscape(remoteConfigPath), shellEscape(remoteConfigPath)) + if err := nm.RunSSHCommand("", n.ExternalIP, user, setupConfigCmd); err != nil { + return fmt.Errorf("failed to setup k0s config: %w", err) + } + } + + installCmd := fmt.Sprintf("sudo '%s' install controller", shellEscape(remoteK0sBinary)) + if k0sConfigPath != "" { + installCmd += fmt.Sprintf(" --config '%s'", shellEscape(remoteConfigPath)) + } else { + installCmd += " --single" + } + + installCmd += " --enable-worker" + installCmd += " --no-taints" + installCmd += fmt.Sprintf(" --kubelet-extra-args='--node-ip=%s'", shellEscape(nodeIP)) + + if force { + installCmd += " --force" + } + + log.Printf("Installing k0s on %s", n.ExternalIP) + if err := nm.RunSSHCommand("", n.ExternalIP, user, installCmd); err != nil { + return fmt.Errorf("failed to install k0s: %w", err) + } + + log.Printf("k0s successfully installed on %s", n.ExternalIP) + log.Printf("You can start it using: ssh %s@%s 'sudo %s start'", user, n.ExternalIP, shellEscape(remoteK0sBinary)) + log.Printf("You can check the status using: ssh %s@%s 'sudo %s status'", user, n.ExternalIP, shellEscape(remoteK0sBinary)) + + return nil +} diff --git a/internal/installer/node/node_test.go b/internal/installer/node/node_test.go new file mode 100644 index 00000000..6200d18b --- /dev/null +++ b/internal/installer/node/node_test.go @@ -0,0 +1,399 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package node_test + +import ( + "errors" + "os" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/internal/installer/node" + "github.com/codesphere-cloud/oms/internal/util" +) + +func TestNode(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Node Suite") +} + +var _ = Describe("Node", func() { + Describe("shellEscape function", func() { + Context("security and injection prevention", func() { + It("should handle single quotes correctly", func() { + testNode := &node.Node{ + ExternalIP: "192.168.1.100", + User: "root", + } + mockFileWriter := util.NewMockFileIO(GinkgoT()) + nm := &node.NodeManager{ + FileIO: mockFileWriter, + KeyPath: "", + } + + result := testNode.HasCommand(nm, "test'; echo 'injected") + Expect(result).To(BeFalse()) + }) + + It("should handle special shell characters safely", func() { + testNode := &node.Node{ + ExternalIP: "192.168.1.100", + User: "root", + } + mockFileWriter := util.NewMockFileIO(GinkgoT()) + nm := &node.NodeManager{ + FileIO: mockFileWriter, + KeyPath: "", + } + + // Test various injection attempts + injectionAttempts := []string{ + "cmd; rm -rf /", + "cmd && malicious", + "cmd | grep password", + "cmd`backdoor`", + "cmd$(malicious)", + "cmd\nrm -rf /", + } + + for _, attempt := range injectionAttempts { + result := testNode.HasCommand(nm, attempt) + Expect(result).To(BeFalse()) + } + }) + + It("should preserve normal commands", func() { + testNode := &node.Node{ + ExternalIP: "192.168.1.100", + User: "root", + } + mockFileWriter := util.NewMockFileIO(GinkgoT()) + nm := &node.NodeManager{ + FileIO: mockFileWriter, + KeyPath: "", + } + + normalCommands := []string{ + "kubectl", + "ls", + "cat /etc/hosts", + "echo test", + } + + for _, cmd := range normalCommands { + result := testNode.HasCommand(nm, cmd) + Expect(result).To(BeFalse()) + } + }) + + It("should handle Unicode characters", func() { + testNode := &node.Node{ + ExternalIP: "192.168.1.100", + User: "root", + } + mockFileWriter := util.NewMockFileIO(GinkgoT()) + nm := &node.NodeManager{ + FileIO: mockFileWriter, + KeyPath: "", + } + + result := testNode.HasCommand(nm, "test-文件-αβγ") + Expect(result).To(BeFalse()) + }) + + It("should handle empty strings", func() { + testNode := &node.Node{ + ExternalIP: "192.168.1.100", + User: "root", + } + mockFileWriter := util.NewMockFileIO(GinkgoT()) + nm := &node.NodeManager{ + FileIO: mockFileWriter, + KeyPath: "", + } + + result := testNode.HasCommand(nm, "") + Expect(result).To(BeFalse()) + }) + + It("should handle nested quotes", func() { + testNode := &node.Node{ + ExternalIP: "192.168.1.100", + User: "root", + } + mockFileWriter := util.NewMockFileIO(GinkgoT()) + nm := &node.NodeManager{ + FileIO: mockFileWriter, + KeyPath: "", + } + + result := testNode.HasCommand(nm, "echo 'test \"nested\" quotes'") + Expect(result).To(BeFalse()) + }) + }) + }) + + Describe("NodeManager", func() { + var ( + nm *node.NodeManager + mockFileWriter *util.MockFileIO + ) + + BeforeEach(func() { + mockFileWriter = util.NewMockFileIO(GinkgoT()) + nm = &node.NodeManager{ + FileIO: mockFileWriter, + KeyPath: "", + } + }) + + AfterEach(func() { + mockFileWriter.AssertExpectations(GinkgoT()) + }) + + Context("authentication methods", func() { + It("should return error when no authentication method is available", func() { + originalAuthSock := os.Getenv("SSH_AUTH_SOCK") + defer func() { + if originalAuthSock != "" { + _ = os.Setenv("SSH_AUTH_SOCK", originalAuthSock) + } else { + _ = os.Unsetenv("SSH_AUTH_SOCK") + } + }() + _ = os.Unsetenv("SSH_AUTH_SOCK") + + nm.KeyPath = "" + + client, err := nm.GetClient("", "10.0.0.1", "root") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no valid authentication methods")) + Expect(client).To(BeNil()) + }) + + It("should return error when key file cannot be read", func() { + originalAuthSock := os.Getenv("SSH_AUTH_SOCK") + defer func() { + if originalAuthSock != "" { + _ = os.Setenv("SSH_AUTH_SOCK", originalAuthSock) + } else { + _ = os.Unsetenv("SSH_AUTH_SOCK") + } + }() + _ = os.Unsetenv("SSH_AUTH_SOCK") + + nm.KeyPath = "/nonexistent/key" + mockFileWriter.EXPECT().ReadFile("/nonexistent/key").Return(nil, errors.New("file not found")) + + client, err := nm.GetClient("", "10.0.0.1", "root") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to read private key file")) + Expect(client).To(BeNil()) + }) + + It("should return error when key file is invalid", func() { + originalAuthSock := os.Getenv("SSH_AUTH_SOCK") + defer func() { + if originalAuthSock != "" { + _ = os.Setenv("SSH_AUTH_SOCK", originalAuthSock) + } else { + _ = os.Unsetenv("SSH_AUTH_SOCK") + } + }() + _ = os.Unsetenv("SSH_AUTH_SOCK") + + invalidKey := []byte("not a valid ssh key") + nm.KeyPath = "/path/to/invalid/key" + mockFileWriter.EXPECT().ReadFile("/path/to/invalid/key").Return(invalidKey, nil) + + client, err := nm.GetClient("", "10.0.0.1", "root") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to parse private key")) + Expect(client).To(BeNil()) + }) + }) + + Context("SSH connection", func() { + It("should fail to connect to invalid host", func() { + privateKey := []byte(`-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACDjKvZvwzXnCdFniXHDZdFPo4LFJ7KJJdBWrJjN1rO1ZQAAAJgNY3PmDWNz +5gAAAAtzc2gtZWQyNTUxOQAAACDjKvZvwzXnCdFniXHDZdFPo4LFJ7KJJdBWrJjN1rO1ZQ +AAAEDcZfnYLBVPEQT3qYDh6e5zMvKjN8x5k4l3n9qYLFJ7MOMq9m/DNecJ0WeJccNl0U+j +gsUnsokl0FasmM3Ws7VlAAAADnRlc3RAZXhhbXBsZS5jb20BAgMEBQ== +-----END OPENSSH PRIVATE KEY-----`) + + nm.KeyPath = "/path/to/key" + mockFileWriter.EXPECT().ReadFile("/path/to/key").Return(privateKey, nil).Maybe() + + client, err := nm.GetClient("", "192.0.2.1", "root") + Expect(err).To(HaveOccurred()) + // Accept either authentication failure or connection failure + Expect(err.Error()).To(Or( + ContainSubstring("failed to dial"), + ContainSubstring("failed to get authentication methods"), + ContainSubstring("failed to parse private key"), + )) + Expect(client).To(BeNil()) + }) + + It("should fail to connect through invalid jumpbox", func() { + privateKey := []byte(`-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACDjKvZvwzXnCdFniXHDZdFPo4LFJ7KJJdBWrJjN1rO1ZQAAAJgNY3PmDWNz +5gAAAAtzc2gtZWQyNTUxOQAAACDjKvZvwzXnCdFniXHDZdFPo4LFJ7KJJdBWrJjN1rO1ZQ +AAAEDcZfnYLBVPEQT3qYDh6e5zMvKjN8x5k4l3n9qYLFJ7MOMq9m/DNecJ0WeJccNl0U+j +gsUnsokl0FasmM3Ws7VlAAAADnRlc3RAZXhhbXBsZS5jb20BAgMEBQ== +-----END OPENSSH PRIVATE KEY-----`) + + nm.KeyPath = "/path/to/key" + mockFileWriter.EXPECT().ReadFile("/path/to/key").Return(privateKey, nil).Maybe() + + client, err := nm.GetClient("192.0.2.1", "192.0.2.2", "root") + Expect(err).To(HaveOccurred()) + // Accept either authentication failure or connection failure + Expect(err.Error()).To(Or( + ContainSubstring("failed to connect to jumpbox"), + ContainSubstring("failed to get authentication methods"), + ContainSubstring("failed to parse private key"), + )) + Expect(client).To(BeNil()) + }) + }) + + Context("file operations", func() { + It("should handle directory creation errors", func() { + err := nm.EnsureDirectoryExists("", "192.0.2.1", "root", "/tmp/test") + Expect(err).To(HaveOccurred()) + }) + + It("should handle copy file errors when source doesn't exist", func() { + mockFileWriter.EXPECT().Open("/nonexistent/file").Return(nil, errors.New("file not found")).Maybe() + + err := nm.CopyFile("", "192.0.2.1", "root", "/nonexistent/file", "/tmp/dest") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to get SSH client")) + }) + }) + }) + + Describe("Node methods", func() { + var ( + n *node.Node + nm *node.NodeManager + mockFileWriter *util.MockFileIO + ) + + BeforeEach(func() { + mockFileWriter = util.NewMockFileIO(GinkgoT()) + nm = &node.NodeManager{ + FileIO: mockFileWriter, + KeyPath: "", + } + n = &node.Node{ + Name: "test-node", + ExternalIP: "10.0.0.1", + InternalIP: "192.168.1.1", + } + }) + + AfterEach(func() { + mockFileWriter.AssertExpectations(GinkgoT()) + }) + + Context("HasCommand", func() { + It("should return false when SSH connection fails", func() { + result := n.HasCommand(nm, "kubectl") + Expect(result).To(BeFalse()) + }) + + It("should handle commands with special characters safely", func() { + result := n.HasCommand(nm, "kubectl'; rm -rf /; echo '") + Expect(result).To(BeFalse()) + }) + }) + + Context("HasFile", func() { + It("should return false when SSH connection fails", func() { + result := n.HasFile(nil, nm, "/etc/k0s/k0s.yaml") + Expect(result).To(BeFalse()) + }) + + It("should handle paths with special characters safely", func() { + result := n.HasFile(nil, nm, "/path'; rm -rf /; echo '/file.txt") + Expect(result).To(BeFalse()) + }) + + It("should support jumpbox connections", func() { + jumpbox := &node.Node{ + ExternalIP: "10.0.0.2", + InternalIP: "10.0.0.2", + } + result := n.HasFile(jumpbox, nm, "/etc/k0s/k0s.yaml") + Expect(result).To(BeFalse()) + }) + }) + + Context("CopyFile", func() { + It("should fail when directory creation fails", func() { + err := n.CopyFile(nil, nm, "/some/file", "/remote/path/dest.txt") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to ensure directory exists")) + }) + }) + + Context("RunSSHCommand", func() { + It("should handle direct connection without jumpbox", func() { + err := n.RunSSHCommand(nil, nm, "root", "echo test") + Expect(err).To(HaveOccurred()) + }) + + It("should handle connection through jumpbox", func() { + jumpbox := &node.Node{ + ExternalIP: "10.0.0.2", + InternalIP: "10.0.0.2", + } + err := n.RunSSHCommand(jumpbox, nm, "ubuntu", "echo test") + Expect(err).To(HaveOccurred()) + }) + }) + + Context("InstallK0s", func() { + It("should handle binary copy failure", func() { + k0sBinaryPath := "/path/to/k0s" + mockFileWriter.EXPECT().Open(k0sBinaryPath).Return(nil, errors.New("file not found")).Maybe() + + err := n.InstallK0s(nm, k0sBinaryPath, "", false, "192.168.1.100") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to copy k0s binary to temp")) + }) + + It("should handle paths with special characters safely", func() { + k0sBinaryPath := "/path/to/k0s'; echo 'injected" + + err := n.InstallK0s(nm, k0sBinaryPath, "", false, "192.168.1.100") + Expect(err).To(HaveOccurred()) + }) + + It("should support force flag parameter", func() { + k0sBinaryPath := "/tmp/k0s" + + err := n.InstallK0s(nm, k0sBinaryPath, "", true, "192.168.1.100") + Expect(err).To(HaveOccurred()) + // Will fail to connect, but tests that force flag is handled + }) + + It("should support config file parameter", func() { + k0sBinaryPath := "/tmp/k0s" + k0sConfigPath := "/tmp/k0s.yaml" + + err := n.InstallK0s(nm, k0sBinaryPath, k0sConfigPath, false, "192.168.1.100") + Expect(err).To(HaveOccurred()) + // Will fail to connect, but tests that config path is handled + }) + }) + }) +})