diff --git a/workspaces/controller/Makefile b/workspaces/controller/Makefile index 6032c8c9..9fc007fc 100644 --- a/workspaces/controller/Makefile +++ b/workspaces/controller/Makefile @@ -64,9 +64,10 @@ test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# Prometheus and CertManager are installed by default; skip with: +# Prometheus, CertManager, and Istio are installed by default; skip with: # - PROMETHEUS_INSTALL_SKIP=true # - CERT_MANAGER_INSTALL_SKIP=true +# - ISTIO_INSTALL_SKIP=true .PHONY: test-e2e test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. @command -v kind >/dev/null 2>&1 || { \ @@ -77,6 +78,7 @@ test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ exit 1; \ } + go test ./test/e2e/ -v -ginkgo.v .PHONY: lint diff --git a/workspaces/controller/go.mod b/workspaces/controller/go.mod index 8e1b5b5b..b5c84804 100644 --- a/workspaces/controller/go.mod +++ b/workspaces/controller/go.mod @@ -6,6 +6,7 @@ require ( github.com/go-logr/logr v1.4.2 github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 + golang.org/x/time v0.3.0 k8s.io/api v0.31.0 k8s.io/apimachinery v0.31.0 k8s.io/client-go v0.31.0 @@ -56,7 +57,6 @@ require ( golang.org/x/sys v0.21.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect - golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.34.2 // indirect diff --git a/workspaces/controller/test/e2e/e2e_suite_test.go b/workspaces/controller/test/e2e/e2e_suite_test.go index 956d1ff0..ca8016f9 100644 --- a/workspaces/controller/test/e2e/e2e_suite_test.go +++ b/workspaces/controller/test/e2e/e2e_suite_test.go @@ -40,6 +40,9 @@ var ( // isPrometheusOperatorAlreadyInstalled will be set true when prometheus CRDs be found on the cluster // isPrometheusOperatorAlreadyInstalled = false + + isIstioctlAlreadyInstalled = false + skipIstioctlInstall = os.Getenv("ISTIO_INSTALL_SKIP") == "true" ) // TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated, @@ -88,6 +91,20 @@ var _ = BeforeSuite(func() { } By("checking that cert manager is running") Expect(utils.WaitCertManagerRunning()).To(Succeed(), "CertManager is not running") + + if !skipIstioctlInstall { + By("checking if istioctl is installed already") + isIstioctlAlreadyInstalled = utils.IsIstioctlInstalled() + if !isIstioctlAlreadyInstalled { + _, _ = fmt.Fprintf(GinkgoWriter, "Installing istioctl...\n") + Expect(utils.InstallIstioctl()).To(Succeed(), "Failed to install istioctl") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: istioctl is already installed. Skipping installation...\n") + } + } + By("checking that istioctl is available") + Expect(utils.WaitIstioAvailable()).To(Succeed(), "istioctl is not available") + }) var _ = AfterSuite(func() { diff --git a/workspaces/controller/test/utils/utils.go b/workspaces/controller/test/utils/utils.go index eba1f02e..baaf73a1 100644 --- a/workspaces/controller/test/utils/utils.go +++ b/workspaces/controller/test/utils/utils.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "os/exec" + "runtime" "strings" . "github.com/onsi/ginkgo/v2" //nolint:golint @@ -27,6 +28,10 @@ import ( const ( + // use LTS version of istioctl + istioctlVersion = "1.27.0" + istioctlURLTemplate = "https://github.com/istio/istio/releases/download/%s/%s" + // use LTS version of prometheus-operator prometheusOperatorVersion = "v0.72.0" prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + @@ -61,6 +66,414 @@ func Run(cmd *exec.Cmd) (string, error) { return string(output), nil } +func UninstallIstioctl() { + // First, uninstall Istio components if they exist + if IsIstioInstalled() { + fmt.Println("Uninstalling Istio components...") + cmd := exec.Command("istioctl", "uninstall", "--purge", "-y") + if _, err := Run(cmd); err != nil { + warnError(fmt.Errorf("failed to uninstall Istio components: %w", err)) + } + + // Delete istio-system namespace + cmd = exec.Command("kubectl", "delete", "namespace", "istio-system", "--ignore-not-found") + if _, err := Run(cmd); err != nil { + warnError(fmt.Errorf("failed to delete istio-system namespace: %w", err)) + } + } + + // Remove istioctl binary from local bin directory + homeDir, err := os.UserHomeDir() + if err != nil { + warnError(fmt.Errorf("failed to get user home directory: %w", err)) + return + } + rmCmd := exec.Command("rm", "-f", homeDir+"/.local/bin/istioctl") + + if _, err := Run(rmCmd); err != nil { + warnError(fmt.Errorf("failed to remove istioctl binary: %w", err)) + } + + fmt.Println("Istioctl uninstalled successfully") +} + +// InstallIstioctl installs the istioctl to be used to manage istio resources. +func InstallIstioctl() error { + osName := runtime.GOOS + archName := runtime.GOARCH + + // Map Go architecture names to Istio release names + switch archName { + case "amd64": + archName = "amd64" + case "arm64": + archName = "arm64" + case "386": + return fmt.Errorf("32-bit architectures are not supported by Istio") + default: + return fmt.Errorf("unsupported architecture: %s", archName) + } + + // Map Go OS names to Istio release names + switch osName { + case "linux": + osName = "linux" + case "darwin": + osName = "osx" + default: + return fmt.Errorf("only Linux and macOS are supported, got: %s", osName) + } + fileExt := "tar.gz" + + // Construct the download URL dynamically + fileName := fmt.Sprintf("istioctl-%s-%s-%s.%s", istioctlVersion, osName, archName, fileExt) + url := fmt.Sprintf(istioctlURLTemplate, istioctlVersion, fileName) + + // Set the binary name based on OS + binaryName := "istioctl" + + // Download the file using curl with wget as fallback + downloadSuccess := false + + // Try curl first + curlCmd := exec.Command("curl", "-L", url, "-o", fileName) + curlCmd.Stdout, curlCmd.Stderr = os.Stdout, os.Stderr + if err := curlCmd.Run(); err == nil { + downloadSuccess = true + } else { + // Try wget as fallback + wgetCmd := exec.Command("wget", "-O", fileName, url) + wgetCmd.Stdout, wgetCmd.Stderr = os.Stdout, os.Stderr + if err := wgetCmd.Run(); err == nil { + downloadSuccess = true + } + } + + if !downloadSuccess { + return fmt.Errorf("failed to download istioctl from %s using both curl and wget", url) + } + + // Extract based on file type + var extractCmd *exec.Cmd + switch fileExt { + case "tar.gz": + extractCmd = exec.Command("tar", "-xzf", fileName) + case "zip": + extractCmd = exec.Command("unzip", "-q", fileName) + default: + return fmt.Errorf("unsupported file extension: %s", fileExt) + } + extractCmd.Stdout, extractCmd.Stderr = os.Stdout, os.Stderr + + if err := extractCmd.Run(); err != nil { + return fmt.Errorf("failed to extract %s: %w", fileName, err) + } + + // Find the extracted binary (it could be in various subdirectories) + findCmd := exec.Command("find", ".", "-name", binaryName, "-type", "f") + output, err := findCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to find istioctl binary after extraction: %w", err) + } + + binaryPath := strings.TrimSpace(string(output)) + if binaryPath == "" { + return fmt.Errorf("istioctl binary not found in extracted files") + } + + // Use the first found binary if multiple exist + if strings.Contains(binaryPath, "\n") { + binaryPath = strings.Split(binaryPath, "\n")[0] + } + + // Copy the binary to current directory with standard name if needed + // Handle case where binaryPath might be "./istioctl" vs "istioctl" + normalizedPath := strings.TrimPrefix(binaryPath, "./") + if normalizedPath != binaryName { + var cpCmd *exec.Cmd + cpCmd = exec.Command("cp", binaryPath, binaryName) + + if err := cpCmd.Run(); err != nil { + return fmt.Errorf("failed to copy istioctl binary: %w", err) + } + } + chmodCmd := exec.Command("chmod", "+x", binaryName) + if err := chmodCmd.Run(); err != nil { + return fmt.Errorf("failed to make istioctl executable: %w", err) + } + // Move to local bin directory + homeDir, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("failed to get user home directory: %w", err) + } + binDir := homeDir + "/.local/bin" + + // Create the bin directory if it doesn't exist + mkdirCmd := exec.Command("mkdir", "-p", binDir) + if err := mkdirCmd.Run(); err != nil { + return fmt.Errorf("failed to create local bin directory: %w", err) + } + + moveCmd := exec.Command("mv", binaryName, binDir+"/istioctl") + + moveCmd.Stdout, moveCmd.Stderr = os.Stdout, os.Stderr + if err := moveCmd.Run(); err != nil { + return fmt.Errorf("failed to move istioctl to bin directory: %w", err) + } + + // Add to PATH notice + fmt.Printf("istioctl installed to %s\n", binDir) + fmt.Printf("Make sure %s is in your PATH by adding this to your shell profile:\n", binDir) + fmt.Printf("export PATH=\"%s:$PATH\"\n", binDir) + + // Clean up downloaded files + cleanupCmd := exec.Command("rm", "-f", fileName) + cleanupCmd.Run() // Ignore cleanup errors + + return nil +} + +// InstallIstioMinimalWithIngress installs Istio with minimal profile and ingressgateway enabled. +func InstallIstioMinimalWithIngress(namespace string) error { + cmd := exec.Command("istioctl", + "install", + "--set", "profile=minimal", + "--set", "values.gateways.istio-ingressgateway.enabled=true", + "--set", fmt.Sprintf("values.global.istioNamespace=%s", namespace), + "-y", + ) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// InstallIstio installs Istio with default configuration. +func InstallIstio() error { + // First ensure istioctl is available + if !IsIstioctlInstalled() { + fmt.Println("istioctl not found, installing istioctl...") + if err := InstallIstioctl(); err != nil { + return fmt.Errorf("failed to install istioctl: %w", err) + } + + // Verify installation + if !IsIstioctlInstalled() { + return fmt.Errorf("istioctl installation failed - binary not available after installation") + } + } + + // Check if Istio is already installed + if IsIstioInstalled() { + fmt.Println("Istio is already installed") + return nil + } + + // Install Istio with default configuration + fmt.Println("Installing Istio...") + cmd := exec.Command("istioctl", "install", + "--set", "values.defaultRevision=default", + "-y") + if _, err := Run(cmd); err != nil { + return fmt.Errorf("failed to install Istio: %w", err) + } + + fmt.Println("Istio installation completed") + return nil +} + +// IsIstioctlInstalled checks if istioctl binary is available and working +func IsIstioctlInstalled() bool { + // Check if istioctl binary exists in PATH + if _, err := exec.LookPath("istioctl"); err != nil { + return false + } + + // Verify istioctl can run and show version + cmd := exec.Command("istioctl", "version", "--short", "--remote=false") + _, err := Run(cmd) + return err == nil +} + +// IsIstioInstalled checks if Istio is installed in the cluster +func IsIstioInstalled() bool { + // Check if istioctl binary is available first + if !IsIstioctlInstalled() { + return false + } + + // Check if istio-system namespace exists + cmd := exec.Command("kubectl", "get", "namespace", "istio-system") + if _, err := Run(cmd); err != nil { + return false + } + + // Check if istiod deployment exists and is available + cmd = exec.Command("kubectl", "get", "deployment", "istiod", "-n", "istio-system") + if _, err := Run(cmd); err != nil { + return false + } + + // Verify istioctl can communicate with the cluster + cmd = exec.Command("istioctl", "version", "--short") + _, err := Run(cmd) + return err == nil +} + +// WaitIstioAvailable waits for Istio to be available and running. +// Returns nil if Istio is ready, or an error if not ready within timeout. +func WaitIstioAvailable() error { + // First check if istio-system namespace exists, if not install Istio + cmd := exec.Command("kubectl", "get", "namespace", "istio-system") + if _, err := Run(cmd); err != nil { + // Namespace doesn't exist, install Istio + fmt.Println("istio-system namespace not found, installing Istio...") + if err := InstallIstio(); err != nil { + return fmt.Errorf("failed to install Istio: %w", err) + } + } + + // Wait for Istio control plane (istiod) pods to be ready + cmd = exec.Command("kubectl", "wait", + "--for=condition=Ready", + "pods", + "-l", "app=istiod", + "-n", "istio-system", + "--timeout=300s") + if _, err := Run(cmd); err != nil { + return fmt.Errorf("istiod pods not ready: %w", err) + } + + // Wait for Istio ingress gateway pods to be ready + cmd = exec.Command("kubectl", "wait", + "--for=condition=Ready", + "pods", + "-l", "app=istio-ingressgateway", + "-n", "istio-system", + "--timeout=300s") + if _, err := Run(cmd); err != nil { + return fmt.Errorf("istio-ingressgateway pods not ready: %w", err) + } + + // Wait for Istio egress gateway pods to be ready (if present) + // Note: egress gateway is optional, so we don't fail if it's not found + cmd = exec.Command("kubectl", "get", + "pods", + "-l", "app=istio-egressgateway", + "-n", "istio-system", + "--no-headers") + output, err := Run(cmd) + if err == nil && len(strings.TrimSpace(output)) > 0 && !strings.Contains(output, "No resources found") { + // Egress gateway exists, wait for it to be ready + cmd = exec.Command("kubectl", "wait", + "--for=condition=Ready", + "pods", + "-l", "app=istio-egressgateway", + "-n", "istio-system", + "--timeout=300s") + if _, err := Run(cmd); err != nil { + return fmt.Errorf("istio-egressgateway pods not ready: %w", err) + } + } + + // Verify istioctl can analyze (optional validation) + cmd = exec.Command("istioctl", "analyze", "--all-namespaces") + if _, err := Run(cmd); err != nil { + return fmt.Errorf("istioctl analyze failed: %w", err) + } + + return nil +} + +// IsIstioIngressGatewayInstalled checks if istio-ingressgateway is installed +func IsIstioIngressGatewayInstalled() bool { + cmd := exec.Command("kubectl", "get", "deployment", + "-n", "istio-system", + "istio-ingressgateway", + "--ignore-not-found") + output, err := Run(cmd) + if err != nil { + return false + } + return len(strings.TrimSpace(output)) > 0 +} + +// InstallIstioIngressGateway installs the istio-ingressgateway +func InstallIstioIngressGateway() error { + // Check if Istio is installed first + if !IsIstioInstalled() { + return fmt.Errorf("istio must be installed before installing ingress gateway") + } + + // Install ingress gateway using istioctl + cmd := exec.Command("istioctl", "install", + "--set", "components.ingressGateways[0].enabled=true", + "--set", "components.ingressGateways[0].name=istio-ingressgateway", + "-y") + + _, err := Run(cmd) + return err +} + +// WaitIstioIngressGatewayReady waits for istio-ingressgateway to be ready +func WaitIstioIngressGatewayReady() error { + // Wait for the deployment to be available + cmd := exec.Command("kubectl", "wait", + "--for=condition=Available", + "deployment/istio-ingressgateway", + "-n", "istio-system", + "--timeout=300s") + + if _, err := Run(cmd); err != nil { + return fmt.Errorf("istio-ingressgateway deployment not available: %w", err) + } + + // Wait for the pods to be ready + cmd = exec.Command("kubectl", "wait", + "--for=condition=Ready", + "pods", + "-l", "app=istio-ingressgateway", + "-n", "istio-system", + "--timeout=300s") + + if _, err := Run(cmd); err != nil { + return fmt.Errorf("istio-ingressgateway pods not ready: %w", err) + } + + // Wait for the service to have external IP (if LoadBalancer type) + cmd = exec.Command("kubectl", "get", "service", + "istio-ingressgateway", + "-n", "istio-system", + "-o", "jsonpath={.status.loadBalancer.ingress[0].ip}") + + // Note: This might not apply for all environments (like kind/minikube) + // so we don't fail if external IP is not assigned + Run(cmd) // Ignore error for external IP check + + return nil +} + +// EnsureIstioIngressGateway checks, installs, and waits for istio-ingressgateway +func EnsureIstioIngressGateway() error { + // Check if already installed + if IsIstioIngressGatewayInstalled() { + fmt.Println("Istio ingress gateway is already installed") + } else { + fmt.Println("Installing Istio ingress gateway...") + if err := InstallIstioIngressGateway(); err != nil { + return fmt.Errorf("failed to install istio-ingressgateway: %w", err) + } + } + + // Wait for it to be ready + fmt.Println("Waiting for Istio ingress gateway to be ready...") + if err := WaitIstioIngressGatewayReady(); err != nil { + return fmt.Errorf("istio-ingressgateway failed to become ready: %w", err) + } + + fmt.Println("Istio ingress gateway is ready!") + return nil +} + // UninstallPrometheusOperator uninstalls the prometheus func UninstallPrometheusOperator() { url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) @@ -175,7 +588,43 @@ func WaitCertManagerRunning() error { "--all-namespaces", "--timeout", "2m", ) - _, err = Run(cmd) + if _, err := Run(cmd); err != nil { + return fmt.Errorf("cert-manager endpoints not ready: %w", err) + } + // First check if cert-manager namespace exists, if not install cert-manager + cmd = exec.Command("kubectl", "get", "namespace", "cert-manager") + if _, err := Run(cmd); err != nil { + // Namespace doesn't exist, install cert-manager + fmt.Println("cert-manager namespace not found, installing cert-manager...") + if err := InstallCertManager(); err != nil { + return fmt.Errorf("failed to install cert-manager: %w", err) + } + } + + // Wait for each CertManager deployment individually by name (most reliable) + deployments := []string{"cert-manager", "cert-manager-cainjector", "cert-manager-webhook"} + + for _, deployment := range deployments { + cmd := exec.Command("kubectl", "wait", "deployment", deployment, + "-n", "cert-manager", + "--for", "condition=Available", + "--timeout", "300s") + + if _, err := Run(cmd); err != nil { + return fmt.Errorf("deployment %s not ready: %w", deployment, err) + } + } + + // Wait for the cert-manager webhook to be ready (critical for functionality) + cmd = exec.Command("kubectl", "wait", "pods", + "-n", "cert-manager", + "-l", "app=webhook", + "--for", "condition=Ready", + "--timeout", "300s") + + if _, err := Run(cmd); err != nil { + return fmt.Errorf("cert-manager webhook pods not ready: %w", err) + } return err }