ci: move to inbuilt cluster config (#419)

Restructures repository cluster setup and configuration scripts
to be used both for local testing as well as in CI.  Adds a
CI-specific ci.sh script for the differences.  Additionally:
- Upgrades to knative v0.23.0
- Upgrades eventing from v1beta1 API to v1
- Adds binary installation script (linux)
- Serializes Build, Int Test, e2e test steps
- Installs Broker and Channel to support Knative Quickstart tutorial.
- Runs a test echo server during installation
- Adds event-based waits in place of most blind time-based waits
- Adds explicit Kubernetes version for KinD
- Replaces a NodePort patched Koourier with an explicit ingress
- Adds retries while activator webhook becomes available
- Merges allocate and configure
This commit is contained in:
Luke Kingland 2021-07-15 23:45:32 +09:00 committed by GitHub
parent 894f4febda
commit b1d840ad60
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 868 additions and 505 deletions

View File

@ -71,27 +71,26 @@ jobs:
run: make check
integration-test:
needs: build
runs-on: 'ubuntu-latest'
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Go
uses: actions/setup-go@v2
- name: Provision Cluster
uses: container-tools/kind-action@v1 # use ./hack/allocate.sh locally
with:
version: v0.11.1
kubectl_version: v1.21.2
knative_serving: v0.23.0
knative_kourier: v0.23.0
knative_eventing: v0.23.0
config: testdata/cluster.yaml
- name: Configure Cluster
run: ./hack/configure.sh
- name: Install Binaries
run: ./hack/binaries.sh
- name: Allocate Cluster
run: ./hack/allocate.sh
- name: Local Registry Routing
run: ./hack/ci.sh
- name: Verify Configuration
run: ./hack/test.sh
- name: Integration Test
run: make test-integration
e2e-test:
needs: integration-test
runs-on: 'ubuntu-latest'
steps:
- name: Checkout
@ -124,20 +123,14 @@ jobs:
run: make build
env:
PKGER: "./pkger"
- name: Provision Cluster
uses: container-tools/kind-action@v1 # use ./hack/allocate.sh locally
with:
version: v0.10.0
kubectl_version: v1.20.0
knative_serving: v0.22.0
knative_kourier: v0.22.0
knative_eventing: v0.22.0
config: testdata/cluster.yaml
- name: Configure Cluster
run: ./hack/configure.sh
- name: Patch Network Cluster
run: ./test/patch_network.sh
- name: Install Binaries
run: ./hack/binaries.sh
- name: Allocate Cluster
run: ./hack/allocate.sh
- name: Local Registry an Routes (CI)
run: ./hack/ci.sh
- name: Verify Configuration
run: ./hack/test.sh
- name: E2E Test
run: ./test/run_e2e_test.sh # by default runs e2e + 'node' specific tests
#- name: E2E Test Go

30
go.mod
View File

@ -3,29 +3,29 @@ module github.com/boson-project/func
go 1.15
require (
github.com/AlecAivazis/survey/v2 v2.2.12
github.com/buildpacks/pack v0.18.0
github.com/cloudevents/sdk-go/v2 v2.2.0
github.com/containers/image/v5 v5.10.5
github.com/docker/docker v20.10.2+incompatible
github.com/docker/docker-credential-helpers v0.6.3
github.com/AlecAivazis/survey/v2 v2.2.14
github.com/buildpacks/pack v0.19.0
github.com/cloudevents/sdk-go/v2 v2.4.1
github.com/containers/image/v5 v5.13.2
github.com/docker/docker v20.10.7+incompatible
github.com/docker/docker-credential-helpers v0.6.4
github.com/docker/go-connections v0.4.0
github.com/google/go-cmp v0.5.5
github.com/google/go-cmp v0.5.6
github.com/google/uuid v1.2.0
github.com/markbates/pkger v0.17.1
github.com/mitchellh/go-homedir v1.1.0
github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6
github.com/ory/viper v1.7.4
github.com/ory/viper v1.7.5
github.com/pkg/errors v0.9.1
github.com/spf13/cobra v1.1.3
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.19.7
k8s.io/apimachinery v0.19.7
k8s.io/client-go v0.19.7
knative.dev/client v0.22.0
knative.dev/eventing v0.22.0
knative.dev/pkg v0.0.0-20210331065221-952fdd90dbb0
knative.dev/serving v0.22.0
k8s.io/api v0.20.7
k8s.io/apimachinery v0.20.7
k8s.io/client-go v0.20.7
knative.dev/client v0.24.0
knative.dev/eventing v0.24.0
knative.dev/pkg v0.0.0-20210622173328-dd0db4b05c80
knative.dev/serving v0.24.0
)
// knative.dev/serving@v0.21.0 and knative.dev/pkg@v0.0.0-20210331065221-952fdd90dbb0 require different versions of go-openapi/spec

761
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -20,35 +20,39 @@ set -o errexit
set -o nounset
set -o pipefail
export TERM="${TERM:-dumb}"
main() {
local serving_version=v0.22.0
local eventing_version=v0.22.0
local kourier_version=v0.22.0
local kubernetes_version=v1.21.1
local knative_version=v0.23.0
local kourier_version=v0.23.0
local em=$(tput bold)$(tput setaf 2)
local me=$(tput sgr0)
echo "${em}Allocating...${me}"
cluster
kubernetes
serving
dns
eventing
networking
registry
configure
next_steps
echo "${em}DONE${me}"
}
cluster() {
echo "${em}Cluster${me}"
kubernetes() {
echo "${em}Kubernetes${me}"
cat <<EOF | kind create cluster --wait=60s --config=-
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
- role: worker
image: kindest/node:${kubernetes_version}
extraPortMappings:
- containerPort: 30080
hostPort: 80
@ -61,41 +65,123 @@ containerdConfigPatches:
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
endpoint = ["http://kind-registry:5000"]
EOF
sleep 10
kubectl wait pod --for=condition=Ready -l '!job-name' -n kube-system --timeout=5m
}
serving() {
echo "${em}② Knative Serving${me}"
kubectl apply --filename https://github.com/knative/serving/releases/download/$serving_version/serving-crds.yaml
sleep 2
curl -L -s https://github.com/knative/serving/releases/download/$serving_version/serving-core.yaml | yq 'del(.spec.template.spec.containers[]?.resources)' -y | yq 'del(.metadata.annotations."knative.dev/example-checksum")' -y | kubectl apply -f -
kubectl apply --filename https://github.com/knative/serving/releases/download/$knative_version/serving-crds.yaml
sleep 5
kubectl wait --for=condition=Established --all crd --timeout=5m
curl -L -s https://github.com/knative/serving/releases/download/$knative_version/serving-core.yaml | yq 'del(.spec.template.spec.containers[]?.resources)' -y | yq 'del(.metadata.annotations."knative.dev/example-checksum")' -y | kubectl apply -f -
sleep 5
kubectl wait pod --for=condition=Ready -l '!job-name' -n knative-serving --timeout=5m
kubectl get pod -A
}
eventing() {
echo "${em}③ Knative Eventing${me}"
kubectl apply --filename https://github.com/knative/eventing/releases/download/$eventing_version/eventing-crds.yaml
sleep 2
curl -L -s https://github.com/knative/eventing/releases/download/$eventing_version/eventing-core.yaml | yq 'del(.spec.template.spec.containers[]?.resources)' -y | yq 'del(.metadata.annotations."knative.dev/example-checksum")' -y | kubectl apply -f -
curl -L -s https://github.com/knative/eventing/releases/download/$eventing_version/in-memory-channel.yaml | yq 'del(.spec.template.spec.containers[]?.resources)' -y | yq 'del(.metadata.annotations."knative.dev/example-checksum")' -y | kubectl apply -f -
curl -L -s https://github.com/knative/eventing/releases/download/$eventing_version/mt-channel-broker.yaml | yq 'del(.spec.template.spec.containers[]?.resources)' -y | yq 'del(.metadata.annotations."knative.dev/example-checksum")' -y | kubectl apply -f -
dns() {
echo "${em}③ DNS${me}"
i=0; n=3
while :; do
kubectl patch configmap/config-domain \
--namespace knative-serving \
--type merge \
--patch '{"data":{"127.0.0.1.sslip.io":""}}' && break
(( i+=1 ))
if (( i>=n )); then
echo "Unable to set knative domain"
exit 1
fi
echo 'Retrying...'
sleep 2
done
}
networking() {
echo "${em}④ Kourier Networking${me}"
kubectl apply --filename https://github.com/knative-sandbox/net-kourier/releases/download/$kourier_version/kourier.yaml
# Install Eourier
kubectl apply --filename https://github.com/knative/net-kourier/releases/download/$kourier_version/kourier.yaml
sleep 5
kubectl wait pod --for=condition=Ready -l '!job-name' -n kourier-system --timeout=5m
kubectl wait pod --for=condition=Ready -l '!job-name' -n knative-serving --timeout=5m
# Configure Knative to use Kourier
kubectl patch configmap/config-network \
--namespace knative-serving \
--type merge \
--patch '{"data":{"ingress.class":"kourier.ingress.networking.knative.dev"}}'
# Create NodePort ingress for kourier
kubectl apply -f - <<EOF
apiVersion: v1
kind: Service
metadata:
name: kourier-ingress
namespace: kourier-system
labels:
networking.knative.dev/ingress-provider: kourier
spec:
type: NodePort
selector:
app: 3scale-kourier-gateway
ports:
- name: http2
nodePort: 30080
port: 80
targetPort: 8080
- name: https
nodePort: 30443
port: 443
targetPort: 8443
EOF
kubectl wait pod --for=condition=Ready -l '!job-name' -n kourier-system --timeout=5m
kubectl wait pod --for=condition=Ready -l '!job-name' -n knative-serving --timeout=5m
}
eventing() {
echo "${em}⑤ Knative Eventing${me}"
# CRDs
kubectl apply -f https://github.com/knative/eventing/releases/download/$knative_version/eventing-crds.yaml
sleep 5
kubectl wait --for=condition=Established --all crd --timeout=5m
# Core
curl -L -s https://github.com/knative/eventing/releases/download/$knative_version/eventing-core.yaml | yq 'del(.spec.template.spec.containers[]?.resources)' -y | yq 'del(.metadata.annotations."knative.dev/example-checksum")' -y | kubectl apply -f -
sleep 5
kubectl wait pod --for=condition=Ready -l '!job-name' -n knative-eventing --timeout=5m
# Channel
curl -L -s https://github.com/knative/eventing/releases/download/$knative_version/in-memory-channel.yaml | kubectl apply -f -
sleep 5
kubectl wait pod --for=condition=Ready -l '!job-name' -n knative-eventing --timeout=5m
# Broker
curl -L -s https://github.com/knative/eventing/releases/download/$knative_version/mt-channel-broker.yaml | yq 'del(.spec.template.spec.containers[]?.resources)' -y | yq 'del(.metadata.annotations."knative.dev/example-checksum")' -y | kubectl apply -f -
sleep 5
kubectl wait pod --for=condition=Ready -l '!job-name' -n knative-eventing --timeout=5m
}
registry() {
# see https://kind.sigs.k8s.io/docs/user/local-registry/
echo "${em}⑤ Container Registry${me}"
echo "${em} Registry${me}"
docker run -d --restart=always -p "5000:5000" --name "kind-registry" registry:2
docker network connect "kind" "kind-registry"
cat <<EOF | kubectl apply -f -
kubectl apply -f - <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
@ -108,18 +194,63 @@ data:
EOF
}
next_steps() {
local red=$(tput bold)$(tput setaf 1)
configure() {
echo "${em}⑦ Configure Namespace${me}"
echo "${em}⑥ Configure Registry${me}"
echo "Please ${red}manually add 'kind-registry' to your local hosts${me} file:"
echo " echo \"127.0.0.1 kind-registry\" | sudo tee --append /etc/hosts"
# Create Namespace
kubectl create namespace "func"
echo "Please ${red}manually set registry as insecure${me} in the docker daemon config (/etc/docker/daemon.json on linux or ~/.docker/daemon.json on OSX):
{
\"insecure-registries\": [ \"kind-registry:5000\" ],
}"
# Default Broker
kubectl apply -f - <<EOF
apiVersion: eventing.knative.dev/v1
kind: broker
metadata:
name: func-broker
namespace: func
EOF
# Default Channel
kubectl apply -f - << EOF
apiVersion: v1
kind: ConfigMap
metadata:
name: imc-channel
namespace: knative-eventing
data:
channelTemplateSpec: |
apiVersion: messaging.knative.dev/v1
kind: InMemoryChannel
EOF
# Connect Default Broker->Channel
kubectl apply -f - << EOF
apiVersion: v1
kind: ConfigMap
metadata:
name: config-br-defaults
namespace: knative-eventing
data:
default-br-config: |
# This is the cluster-wide default broker channel.
clusterDefault:
brokerClass: MTChannelBasedBroker
apiVersion: v1
kind: ConfigMap
name: imc-channel
namespace: knative-eventing
EOF
}
next_steps() {
local red=$(tput bold)$(tput setaf 1)
echo "${em}Configure Registry${me}"
echo "If not in CI (running ci.sh):
echo " ${red}add 'kind-registry' "to your local hosts${me} file:"
echo " echo \"127.0.0.1 kind-registry\" | sudo tee --append /etc/hosts"
echo " ${red}set registry as insecure${me} in the docker daemon config (/etc/docker/daemon.json on linux or ~/.docker/daemon.json on OSX):
{ \"insecure-registries\": [ \"kind-registry:5000\" ] }"
}
main "$@"

54
hack/binaries.sh Executable file
View File

@ -0,0 +1,54 @@
#!/usr/bin/env bash
#
# Installs binaries on linux systems.
#
# Note that there are multiple 'yq's out there. The one we want is kislyuk/yq,
# which is a thin wrapper around jq.
set -o errexit
set -o nounset
set -o pipefail
export TERM="${TERM:-dumb}"
main() {
local kubectl_version=v1.21.1
local kind_version=v0.11.1
local em=$(tput bold)$(tput setaf 2)
local me=$(tput sgr0)
echo "${em}Fetching Binaries...${me}"
install_kubectl
install_kind
install_yq
echo "${em}DONE${me}"
}
install_kubectl() {
echo 'Installing kubectl...'
curl -sSLO "https://storage.googleapis.com/kubernetes-release/release/$kubectl_version/bin/linux/amd64/kubectl"
chmod +x kubectl
sudo mv kubectl /usr/local/bin/kubectl
kubectl version --client=true
}
install_kind() {
echo 'Installing kind...'
curl -sSLo kind "https://github.com/kubernetes-sigs/kind/releases/download/$kind_version/kind-linux-amd64"
chmod +x kind
sudo mv kind /usr/local/bin/kind
kind version
}
install_yq() {
echo 'Installing yq...'
sudo pip3 install yq
yq --version
}
main "$@"

42
hack/ci.sh Executable file
View File

@ -0,0 +1,42 @@
#!/usr/bin/env bash
#
# CI-specific configuration for linux systems.
# Patches docker to allow the local kind registtry without authentication.
# Adds the registry to the local hosts file
# Restarts the internal webkook (fix).
#
set -o errexit
set -o nounset
set -o pipefail
export TERM="${TERM:-dumb}"
main() {
local em=$(tput bold)$(tput setaf 2)
local me=$(tput sgr0)
echo "${em}Configuring for CI...${me}"
set_registry_insecure
patch_hosts
echo "${em}DONE${me}"
}
set_registry_insecure() {
echo 'Setting registry as trusted local-only'
patch=".\"insecure-registries\" = [\"kind-registry:5000\""]
sudo jq "$patch" /etc/docker/daemon.json > /tmp/daemon.json.tmp && sudo mv /tmp/daemon.json.tmp /etc/docker/daemon.json
sudo service docker restart
}
patch_hosts() {
echo 'Adding registry to hosts'
echo "127.0.0.1 kind-registry" | sudo tee --append /etc/hosts
}
main "$@"

View File

@ -1,167 +0,0 @@
#!/usr/bin/env bash
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Configures the current cluster for use with Functions
#
set -o errexit
set -o nounset
set -o pipefail
DEFAULT_NAMESPACE=func
show_help() {
cat << EOF
Configure a local cluster for use with Functions.
Usage: $(basename "$0") <options>
-h, --help Display help
-n, --namespace The namespace to use for Functions (default: $DEFAULT_NAMESPACE)
EOF
}
main() {
local namespace="$DEFAULT_NAMESPACE"
local em=$(tput bold)$(tput setaf 2)
local me=$(tput sgr0)
parse_command_line "$@"
echo "${em}Configuring...${me}"
namespace
network
kourier_nodeport
default_domain
sleep 10
kubectl --namespace kourier-system get service kourier
echo "${em}DONE${me}"
}
parse_command_line() {
while :; do
case "${1:-}" in
-h|--help)
show_help
exit
;;
-n|--namespace)
if [[ -n "${2:-}" ]]; then
namespace="$2"
shift
else
echo "ERROR: '-n|--namespace' cannot be empty." >&2
show_help
exit 1
fi
;;
*)
break
;;
esac
done
}
namespace() {
echo "${em}① Namespace${me}"
kubectl create namespace "$namespace"
kubectl label namespace "$namespace" knative-eventing-injection=enabled
}
network() {
echo "${em}② Network${me}"
echo "Registering Kourier as ingress"
echo "Enabling subdomains"
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
name: config-network
namespace: knative-serving
data:
# Use Kourier for the networking layer
ingress.class: kourier.ingress.networking.knative.dev
# If there exists an annotation 'func.subdomain' on the service, use it
# instead of the default name.namespace
domainTemplate: |-
{{if index .Annotations "func.subdomain" -}}
{{- index .Annotations "func.subdomain" -}}
{{else -}}
{{- .Name}}.{{.Namespace -}}
{{end -}}
.{{.Domain}}
EOF
}
kourier_nodeport() {
echo "${em}③ Nodeport${me}"
echo 'Setting Kourier service to type NodePort'
# Patch for changing kourier to a NodePort for installations where a
# LoadBalancer is not available (for example local Kind clusters)
# kubectl patch -n kourier-system services/kourier -p "$(cat configure-kourier-nodeport.yaml)"
kubectl patch services/kourier \
--namespace kourier-system \
--type merge \
--patch '{
"spec": {
"ports": [
{
"name": "http2",
"nodePort": 30080,
"port": 80,
"protocol": "TCP",
"targetPort": 8080
},
{
"name": "https",
"nodePort": 30443,
"port": 443,
"protocol": "TCP",
"targetPort": 8443
}
],
"type": "NodePort"
}
}'
}
default_domain() {
echo "${em}④ Default Domains${me}"
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
name: config-domain
namespace: knative-serving
data:
example.com: |
selector:
func.domain: "example.com"
example.org: |
selector:
func.domain: "example.org"
# Default is local only.
svc.cluster.local: ""
EOF
}
main "$@"

View File

@ -3,6 +3,12 @@
# Suitable for use locally during development.
# CI/CD uses the very similar knative-kind action
set -o errexit
set -o nounset
set -o pipefail
export TERM="${TERM:-dumb}"
main() {
local green=$(tput bold)$(tput setaf 2)
local red=$(tput bold)$(tput setaf 2)

61
hack/test.sh Executable file
View File

@ -0,0 +1,61 @@
#!/usr/bin/env bash
#
# Runs a test of the knative serving installation by deploying and then
# invoking an http echoing server.
#
set -o errexit
set -o nounset
set -o pipefail
export TERM="${TERM:-dumb}"
main() {
echo "TERM: $TERM"
local em=$(tput bold)$(tput setaf 2)
local me=$(tput sgr0)
# Drop some debug in the event even the above excessive wait does not work.
echo "${em}Testing...${me}"
echo "${em} Creating echo server${me}"
i=0; n=10
while :; do
cat <<EOF | kubectl apply -f - && break
apiVersion: serving.knative.dev/v1
kind: Service
metadata:
name: echo
namespace: func
spec:
template:
spec:
containers:
- image: docker.io/jmalloc/echo-server
EOF
(( i+=1 ))
if (( i>=n )); then
echo "Unable to create echo service"
exit 1
fi
echo "Retrying..."
sleep 5
done
sleep 60
# wait for the route to become ready
kubectl wait --for=condition=Ready route echo -n func
echo "${em} Invoking echo server${me}"
curl http://echo.func.127.0.0.1.sslip.io/
echo "${em}DONE${me}"
}
main "$@"

View File

@ -4,9 +4,9 @@ import (
"fmt"
"time"
clienteventingv1beta1 "knative.dev/client/pkg/eventing/v1beta1"
clienteventingv1 "knative.dev/client/pkg/eventing/v1"
clientservingv1 "knative.dev/client/pkg/serving/v1"
eventingv1beta1 "knative.dev/eventing/pkg/client/clientset/versioned/typed/eventing/v1beta1"
eventingv1 "knative.dev/eventing/pkg/client/clientset/versioned/typed/eventing/v1"
servingv1 "knative.dev/serving/pkg/client/clientset/versioned/typed/serving/v1"
"github.com/boson-project/func/k8s"
@ -33,19 +33,19 @@ func NewServingClient(namespace string) (clientservingv1.KnServingClient, error)
return client, nil
}
func NewEventingClient(namespace string) (clienteventingv1beta1.KnEventingClient, error) {
func NewEventingClient(namespace string) (clienteventingv1.KnEventingClient, error) {
restConfig, err := k8s.GetClientConfig().ClientConfig()
if err != nil {
return nil, fmt.Errorf("failed to create new serving client: %v", err)
}
eventingClient, err := eventingv1beta1.NewForConfig(restConfig)
eventingClient, err := eventingv1.NewForConfig(restConfig)
if err != nil {
return nil, fmt.Errorf("failed to create new eventing client: %v", err)
}
client := clienteventingv1beta1.NewKnEventingClient(eventingClient, namespace)
client := clienteventingv1.NewKnEventingClient(eventingClient, namespace)
return client, nil
}

View File

@ -4,11 +4,11 @@ import (
"context"
"k8s.io/apimachinery/pkg/api/errors"
v1 "knative.dev/client/pkg/serving/v1"
"knative.dev/eventing/pkg/apis/eventing/v1beta1"
clientservingv1 "knative.dev/client/pkg/serving/v1"
eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1"
fn "github.com/boson-project/func"
"github.com/boson-project/func/k8s"
k8s "github.com/boson-project/func/k8s"
)
type Describer struct {
@ -48,7 +48,7 @@ func (d *Describer) Describe(ctx context.Context, name string) (description fn.D
return
}
routes, err := servingClient.ListRoutes(ctx, v1.WithService(name))
routes, err := servingClient.ListRoutes(ctx, clientservingv1.WithService(name))
if err != nil {
return
}
@ -64,7 +64,7 @@ func (d *Describer) Describe(ctx context.Context, name string) (description fn.D
return
}
triggerMatches := func(t *v1beta1.Trigger) bool {
triggerMatches := func(t *eventingv1.Trigger) bool {
return (t.Spec.Subscriber.Ref != nil && t.Spec.Subscriber.Ref.Name == service.Name) ||
(t.Spec.Subscriber.URI != nil && service.Status.Address != nil && service.Status.Address.URL != nil &&
t.Spec.Subscriber.URI.Path == service.Status.Address.URL.Path)