test: update testing infrastructure to make development easier

Signed-off-by: Jakob Beckmann <f4z3r-github@pm.me>

.dockerignore

@@ -4,3 +4,4 @@ bin/
 target/
 Dockerfile
 Makefile
+.devbox/

Dockerfile.local

@@ -1,12 +1,20 @@
 # syntax=docker/dockerfile:1
-FROM rust:1.77-bookworm as builder
+FROM ghcr.io/rust-cross/cargo-zigbuild:0.20.1 AS builder
+
+ARG TARGETARCH=x86_64
+
+WORKDIR /tmp
+
+RUN rustup target add ${TARGETARCH}-unknown-linux-musl
 
 WORKDIR /app
+
 COPY . .
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/app/target \
-    cargo build --release && cp target/release/wasmcloud-operator .
+    cargo zigbuild --release --target x86_64-unknown-linux-musl --locked && \
+    cp target/${TARGETARCH}-unknown-linux-musl/release/wasmcloud-operator .
 
-FROM gcr.io/distroless/cc-debian12
+FROM scratch
 COPY --from=builder /app/wasmcloud-operator /usr/local/bin/
 ENTRYPOINT ["/usr/local/bin/wasmcloud-operator"]

Makefile

@@ -1,8 +1,9 @@
 repo := ghcr.io/wasmcloud/wasmcloud-operator
+local_repo := localhost:5001/wasmcloud-operator
 version := $(shell git rev-parse --short HEAD)
 platforms := linux/amd64,linux/arm64
 
-.PHONY: build-dev-image build-image buildx-image
+.PHONY: build-dev-image build-image buildx-image setup teardown build-local-dev-image push-local-dev-image deploy-local-dev-image update-local-dev-image
 build-image:
 	docker build -t $(repo):$(version) .
 
@@ -11,3 +12,22 @@ buildx-image:
 
 build-dev-image:
 	docker build -t $(repo):$(version)-dev -f Dockerfile.local .
+
+setup:
+	cd hack && bash ./run-kind-cluster.sh
+
+teardown:
+	cd hack && bash ./teardown-kind-cluster.sh
+
+build-local-dev-image:
+	docker build -t $(local_repo):latest -f Dockerfile.local .
+
+push-local-dev-image:
+	docker push $(local_repo):latest
+
+deploy-local-dev-image:
+	kubectl kustomize deploy/local | kubectl apply -f -
+
+# deleting pod makes it pull a new version thanks to pullPolicy always
+update-local-dev-image:
+	kubectl delete pod -n wasmcloud-operator -l "app=wasmcloud-operator"

README.md

@@ -212,9 +212,61 @@ data:
 
 ## Testing
 
-- Make sure you have a Kubernetes cluster running locally. Some good options
-  include [Kind](https://kind.sigs.k8s.io/) or Docker Desktop.
-- `RUST_LOG=info cargo run`
+### KinD
+
+Make sure you have `kind`, `kubectl`, and `helm` installed. Then you can run `make setup` to create
+a KinD cluster with NATS and wadm installed.
+
+### Deploy Local Code
+
+Once KinD is set up, run the following:
+
+```sh
+# build a docker image from the current code
+make build-local-dev-image
+# push the image to a registry available to kind
+make push-local-dev-image
+# deploy the operator using the pushed image
+make deploy-local-dev-image
+```
+
+### Deploying Test Custom Resources
+
+You can deploy custom resources to test the operator using the following commands:
+
+```sh
+# deploy a host configuration
+kubectl apply -f hack/wasmcloud-hostconfig-sample.yaml
+# once the hosts are available, deploy a sample application
+kubectl apply -f https://raw.githubusercontent.com/wasmCloud/wasmcloud-operator/main/examples/quickstart/hello-world-application.yaml
+```
+
+### Update the Image
+
+After performing local changes to the code, you can update the operator running in the KinD cluster
+using:
+
+```sh
+# re-build a docker image from the current code
+make build-local-dev-image
+# re-push the image to a registry available to kind
+make push-local-dev-image
+# update the existing deployment with the new image
+make update-local-dev-image
+```
+
+### Local Deployment of the Operator
+
+You can locally run the operator outside any Kubernetes cluster using:
+
+```sh
+RUST_LOG=info cargo run --bin wasmcloud-operator
+```
+
+> Note that the operator will try to connect to NATS using the configured URI from the
+> `WasmCloudHostConfig` resources. Since this is likely a DNS entry only resolvable within the
+> Kuberntees cluster, you will need to ensure traffic is correctly routed to NATS in order for this
+> to work (e.g. using an entry in your `/etc/hosts` and a port-forward to the NATS service.
 
 ## Types crate
 

hack/nats-values.yaml

@@ -0,0 +1,13 @@
+config:
+  cluster:
+    enabled: true
+    replicas: 3
+  leafnodes:
+    enabled: true
+  jetstream:
+    enabled: true
+    fileStore:
+      pvc:
+        size: 10Gi
+    merge:
+      domain: default

hack/run-kind-cluster.sh

@@ -18,7 +18,7 @@ fi
 # https://github.com/kubernetes-sigs/kind/issues/2875
 # https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration
 # See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
-cat <<EOF | kind create cluster --image kindest/node:v1.29.4 --config=-
+cat <<EOF | kind create cluster -n wasmcloud-operator-test --image kindest/node:v1.34.0 --config=-
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 containerdConfigPatches:
@@ -36,7 +36,7 @@ EOF
 # We want a consistent name that works from both ends, so we tell containerd to
 # alias localhost:${reg_port} to the registry container when pulling images
 REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
-for node in $(kind get nodes); do
+for node in $(kind get nodes -n wasmcloud-operator-test); do
   docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
   cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
 [host."http://${reg_name}:5000"]
@@ -62,3 +62,15 @@ data:
     host: "localhost:${reg_port}"
     help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
 EOF
+
+# 6. Wait for cluster to come up
+sleep 5s
+kubectl -n kube-system wait --for=condition=Ready --timeout=60s pod -l "component=kube-apiserver"
+
+# 7. Install NATS
+helm repo add nats https://nats-io.github.io/k8s/helm/charts/
+helm upgrade --install -f ./nats-values.yaml nats nats/nats
+kubectl wait --for=condition=Ready --timeout=600s pod -l "app.kubernetes.io/name=nats"
+
+# 8. Install WADM
+helm install wadm -f ./wadm-values.yaml --version 0.2.0 oci://ghcr.io/wasmcloud/charts/wadm

hack/teardown-kind-cluster.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -o errexit
+
+# 1. Delete kind cluster
+kind delete cluster -n wasmcloud-operator-test
+
+# 2. Delete local registry
+docker stop kind-registry
+docker rm kind-registry

hack/wadm-values.yaml

@@ -0,0 +1,4 @@
+wadm:
+  config:
+    nats:
+      server: "nats.default.svc.cluster.local:4222"

hack/wasmcloud-hostconfig-sample.yaml

@@ -0,0 +1,17 @@
+apiVersion: k8s.wasmcloud.dev/v1alpha1
+kind: WasmCloudHostConfig
+metadata:
+  name: my-wasmcloud-cluster
+spec:
+  # The number of wasmCloud host pods to run
+  hostReplicas: 2
+  # The lattice to connect the hosts to
+  lattice: default
+  # Additional labels to apply to the host other than the defaults set in the operator
+  hostLabels:
+    some-label: value
+  # The address to connect to nats
+  natsAddress: nats://nats.default.svc.cluster.local
+  # Which wasmCloud version to use
+  version: 1.0.4
+