Add support for configuring environment variables on kube-apiserver

2024-10-25 10:44:14 +01:00 · 2024-10-25 10:44:14 +01:00 · 8b89e826d1
parent daea619a59
commit 8b89e826d1
22 changed files with 1561 additions and 1 deletions
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@ -627,6 +627,17 @@ spec:
    logFormat: json
 ```

+### Environment Variables
+```yaml
+spec:
+  kubeAPIServer:
+    env:
+    - name: GOMEMLIMIT
+      value: "2750MiB"
+    - name: GOGC
+      value: 50
+```
+
 ## externalDns

 This block contains configuration options for your `external-DNS` provider.
--- a/k8s/crds/kops.k8s.io_clusters.yaml
+++ b/k8s/crds/kops.k8s.io_clusters.yaml
@ -1885,6 +1885,129 @@ spec:
                    description: EncryptionProviderConfig enables encryption at rest
                      for secrets.
                    type: string
+                  env:
+                    description: |-
+                      Env allows users to pass in env variables to the apiserver container.
+                      This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver
+                      This also allows the flexibility for adding any other variables for future use cases
+                    items:
+                      description: EnvVar represents an environment variable present
+                        in a Container.
+                      properties:
+                        name:
+                          description: Name of the environment variable. Must be a
+                            C_IDENTIFIER.
+                          type: string
+                        value:
+                          description: |-
+                            Variable references $(VAR_NAME) are expanded
+                            using the previously defined environment variables in the container and
+                            any service environment variables. If a variable cannot be resolved,
+                            the reference in the input string will be unchanged. Double $$ are reduced
+                            to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e.
+                            "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)".
+                            Escaped references will never be expanded, regardless of whether the variable
+                            exists or not.
+                            Defaults to "".
+                          type: string
+                        valueFrom:
+                          description: Source for the environment variable's value.
+                            Cannot be used if value is not empty.
+                          properties:
+                            configMapKeyRef:
+                              description: Selects a key of a ConfigMap.
+                              properties:
+                                key:
+                                  description: The key to select.
+                                  type: string
+                                name:
+                                  default: ""
+                                  description: |-
+                                    Name of the referent.
+                                    This field is effectively required, but due to backwards compatibility is
+                                    allowed to be empty. Instances of this type with an empty value here are
+                                    almost certainly wrong.
+                                    More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                  type: string
+                                optional:
+                                  description: Specify whether the ConfigMap or its
+                                    key must be defined
+                                  type: boolean
+                              required:
+                              - key
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            fieldRef:
+                              description: |-
+                                Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['<KEY>']`, `metadata.annotations['<KEY>']`,
+                                spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.
+                              properties:
+                                apiVersion:
+                                  description: Version of the schema the FieldPath
+                                    is written in terms of, defaults to "v1".
+                                  type: string
+                                fieldPath:
+                                  description: Path of the field to select in the
+                                    specified API version.
+                                  type: string
+                              required:
+                              - fieldPath
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            resourceFieldRef:
+                              description: |-
+                                Selects a resource of the container: only resources limits and requests
+                                (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.
+                              properties:
+                                containerName:
+                                  description: 'Container name: required for volumes,
+                                    optional for env vars'
+                                  type: string
+                                divisor:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  description: Specifies the output format of the
+                                    exposed resources, defaults to "1"
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                resource:
+                                  description: 'Required: resource to select'
+                                  type: string
+                              required:
+                              - resource
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            secretKeyRef:
+                              description: Selects a key of a secret in the pod's
+                                namespace
+                              properties:
+                                key:
+                                  description: The key of the secret to select from.  Must
+                                    be a valid secret key.
+                                  type: string
+                                name:
+                                  default: ""
+                                  description: |-
+                                    Name of the referent.
+                                    This field is effectively required, but due to backwards compatibility is
+                                    allowed to be empty. Instances of this type with an empty value here are
+                                    almost certainly wrong.
+                                    More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                  type: string
+                                optional:
+                                  description: Specify whether the Secret or its key
+                                    must be defined
+                                  type: boolean
+                              required:
+                              - key
+                              type: object
+                              x-kubernetes-map-type: atomic
+                          type: object
+                      required:
+                      - name
+                      type: object
+                    type: array
                  etcdCaFile:
                    description: EtcdCAFile is the path to a ca certificate
                    type: string
--- a/nodeup/pkg/model/kube_apiserver.go
+++ b/nodeup/pkg/model/kube_apiserver.go
@ -699,7 +699,7 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops
 	container := &v1.Container{
 		Name:           "kube-apiserver",
 		Image:          image,
-		Env:            proxy.GetProxyEnvVars(b.NodeupConfig.Networking.EgressProxy),
+		Env:            append(kubeAPIServer.Env, proxy.GetProxyEnvVars(b.NodeupConfig.Networking.EgressProxy)...),
 		LivenessProbe:  livenessProbe,
 		ReadinessProbe: readinessProbe,
 		StartupProbe:   startupProbe,
--- a/nodeup/pkg/model/kube_apiserver_test.go
+++ b/nodeup/pkg/model/kube_apiserver_test.go
@ -197,3 +197,10 @@ func TestKubeAPIServerBuilderARM64(t *testing.T) {
 		return builder.Build(target)
 	})
 }
+
+func TestKubeAPIServerEnvBuilder(t *testing.T) {
+	RunGoldenTest(t, "tests/golden/envvars", "kube-apiserver", func(nodeupModelContext *NodeupModelContext, target *fi.NodeupModelBuilderContext) error {
+		builder := KubeAPIServerBuilder{NodeupModelContext: nodeupModelContext}
+		return builder.Build(target)
+	})
+}
--- a/nodeup/pkg/model/tests/golden/envvars/cluster.yaml
+++ b/nodeup/pkg/model/tests/golden/envvars/cluster.yaml
@ -0,0 +1,72 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  name: minimal.example.com
+spec:
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/minimal.example.com
+  etcdClusters:
+  - cpuRequest: 200m
+    etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    memoryRequest: 100Mi
+    name: main
+    provider: Manager
+    backups:
+      backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main
+  - cpuRequest: 100m
+    etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    memoryRequest: 100Mi
+    name: events
+    provider: Manager
+    backups:
+      backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events
+  iam: {}
+  kubeAPIServer:
+    env:
+      - name: GOMEMLIMIT
+        valueFrom:
+          resourceFieldRef:
+            resource: limits.memory
+            divisor: '1'
+      - name: GOGC
+        value: "50"
+  kubelet:
+    anonymousAuth: false
+  kubernetesVersion: v1.28.0
+  masterPublicName: api.minimal.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+  - 0.0.0.0/0
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a
+    type: Public
+    zone: us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  name: master-us-test-1a
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+spec:
+  associatePublicIp: true
+  image: ami-1234
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a
--- a/nodeup/pkg/model/tests/golden/envvars/tasks-kops-controller.yaml
+++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kops-controller.yaml
@ -0,0 +1,112 @@
+mode: "0755"
+path: /etc/kubernetes/kops-controller
+type: directory
+---
+contents: |
+  kubernetes-ca: "3"
+  service-account: "2"
+mode: "0600"
+owner: kops-controller
+path: /etc/kubernetes/kops-controller/keypair-ids.yaml
+type: file
+---
+contents:
+  task:
+    Name: kops-controller
+    alternateNames:
+    - kops-controller.internal.minimal.example.com
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kops-controller
+    type: server
+mode: "0644"
+owner: kops-controller
+path: /etc/kubernetes/kops-controller/kops-controller.crt
+type: file
+---
+contents:
+  task:
+    Name: kops-controller
+    alternateNames:
+    - kops-controller.internal.minimal.example.com
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kops-controller
+    type: server
+mode: "0600"
+owner: kops-controller
+path: /etc/kubernetes/kops-controller/kops-controller.key
+type: file
+---
+contents: |
+  -----BEGIN CERTIFICATE-----
+  MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw
+  FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy
+  MzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA
+  A4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd
+  XEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY
+  OACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1
+  YYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC
+  An8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA
+  NPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB
+  /wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r
+  hzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO
+  HE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe
+  rN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt
+  O6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy
+  VpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh
+  8yGeRx9AbknHh4Ia
+  -----END CERTIFICATE-----
+mode: "0600"
+owner: kops-controller
+path: /etc/kubernetes/kops-controller/kubernetes-ca.crt
+type: file
+---
+contents: |
+  -----BEGIN RSA PRIVATE KEY-----
+  MIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH
+  AZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA
+  gvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF
+  GHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/
+  P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx
+  9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI
+  Bezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9
+  /Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM
+  0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t
+  vpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd
+  cuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G
+  8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj
+  22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1
+  AsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV
+  99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs
+  z2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg
+  ST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK
+  sfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7
+  KhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i
+  Q4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T
+  t9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q
+  dGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z
+  worz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu
+  BAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq
+  Y/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw==
+  -----END RSA PRIVATE KEY-----
+mode: "0600"
+owner: kops-controller
+path: /etc/kubernetes/kops-controller/kubernetes-ca.key
+type: file
+---
+Name: kops-controller
+alternateNames:
+- kops-controller.internal.minimal.example.com
+keypairID: "3"
+signer: kubernetes-ca
+subject:
+  CommonName: kops-controller
+type: server
+---
+Name: kops-controller
+home: ""
+shell: /sbin/nologin
+uid: 10011
--- a/nodeup/pkg/model/tests/golden/envvars/tasks-kube-apiserver.yaml
+++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-apiserver.yaml
@ -0,0 +1,376 @@
+contents: |
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    annotations:
+      dns.alpha.kubernetes.io/external: api.minimal.example.com
+      dns.alpha.kubernetes.io/internal: api.internal.minimal.example.com
+      kubectl.kubernetes.io/default-container: kube-apiserver
+    creationTimestamp: null
+    labels:
+      k8s-app: kube-apiserver
+    name: kube-apiserver
+    namespace: kube-system
+  spec:
+    containers:
+    - args:
+      - --log-file=/var/log/kube-apiserver.log
+      - --also-stdout
+      - /usr/local/bin/kube-apiserver
+      - --allow-privileged=true
+      - --anonymous-auth=false
+      - --api-audiences=kubernetes.svc.default
+      - --apiserver-count=1
+      - --authorization-mode=AlwaysAllow
+      - --bind-address=0.0.0.0
+      - --client-ca-file=/srv/kubernetes/ca.crt
+      - --cloud-config=/etc/kubernetes/in-tree-cloud.config
+      - --cloud-provider=external
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota
+      - --etcd-cafile=/srv/kubernetes/kube-apiserver/etcd-ca.crt
+      - --etcd-certfile=/srv/kubernetes/kube-apiserver/etcd-client.crt
+      - --etcd-keyfile=/srv/kubernetes/kube-apiserver/etcd-client.key
+      - --etcd-servers-overrides=/events#https://127.0.0.1:4002
+      - --etcd-servers=https://127.0.0.1:4001
+      - --feature-gates=InTreePluginAWSUnregister=true
+      - --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt
+      - --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key
+      - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
+      - --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt
+      - --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key
+      - --requestheader-allowed-names=aggregator
+      - --requestheader-client-ca-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt
+      - --requestheader-extra-headers-prefix=X-Remote-Extra-
+      - --requestheader-group-headers=X-Remote-Group
+      - --requestheader-username-headers=X-Remote-User
+      - --secure-port=443
+      - --service-account-issuer=https://api.internal.minimal.example.com
+      - --service-account-jwks-uri=https://api.internal.minimal.example.com/openid/v1/jwks
+      - --service-account-key-file=/srv/kubernetes/kube-apiserver/service-account.pub
+      - --service-account-signing-key-file=/srv/kubernetes/kube-apiserver/service-account.key
+      - --service-cluster-ip-range=100.64.0.0/13
+      - --storage-backend=etcd3
+      - --tls-cert-file=/srv/kubernetes/kube-apiserver/server.crt
+      - --tls-private-key-file=/srv/kubernetes/kube-apiserver/server.key
+      - --v=2
+      command:
+      - /go-runner
+      env:
+      - name: GOMEMLIMIT
+        valueFrom:
+          resourceFieldRef:
+            divisor: "1"
+            resource: limits.memory
+      - name: GOGC
+        value: "50"
+      image: registry.k8s.io/kube-apiserver:v1.28.0
+      livenessProbe:
+        httpGet:
+          host: 127.0.0.1
+          path: /healthz
+          port: 443
+          scheme: HTTPS
+        initialDelaySeconds: 45
+        timeoutSeconds: 15
+      name: kube-apiserver
+      ports:
+      - containerPort: 443
+        hostPort: 443
+        name: https
+      resources:
+        requests:
+          cpu: 150m
+      volumeMounts:
+      - mountPath: /var/log/kube-apiserver.log
+        name: logfile
+      - mountPath: /etc/ssl
+        name: etcssl
+        readOnly: true
+      - mountPath: /etc/pki/tls
+        name: etcpkitls
+        readOnly: true
+      - mountPath: /etc/pki/ca-trust
+        name: etcpkica-trust
+        readOnly: true
+      - mountPath: /usr/share/ssl
+        name: usrsharessl
+        readOnly: true
+      - mountPath: /usr/ssl
+        name: usrssl
+        readOnly: true
+      - mountPath: /usr/lib/ssl
+        name: usrlibssl
+        readOnly: true
+      - mountPath: /usr/local/openssl
+        name: usrlocalopenssl
+        readOnly: true
+      - mountPath: /var/ssl
+        name: varssl
+        readOnly: true
+      - mountPath: /etc/openssl
+        name: etcopenssl
+        readOnly: true
+      - mountPath: /etc/kubernetes/in-tree-cloud.config
+        name: cloudconfig
+        readOnly: true
+      - mountPath: /srv/kubernetes/ca.crt
+        name: kubernetesca
+        readOnly: true
+      - mountPath: /srv/kubernetes/kube-apiserver
+        name: srvkapi
+        readOnly: true
+      - mountPath: /srv/sshproxy
+        name: srvsshproxy
+        readOnly: true
+    hostNetwork: true
+    priorityClassName: system-cluster-critical
+    tolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+    volumes:
+    - hostPath:
+        path: /var/log/kube-apiserver.log
+      name: logfile
+    - hostPath:
+        path: /etc/ssl
+      name: etcssl
+    - hostPath:
+        path: /etc/pki/tls
+      name: etcpkitls
+    - hostPath:
+        path: /etc/pki/ca-trust
+      name: etcpkica-trust
+    - hostPath:
+        path: /usr/share/ssl
+      name: usrsharessl
+    - hostPath:
+        path: /usr/ssl
+      name: usrssl
+    - hostPath:
+        path: /usr/lib/ssl
+      name: usrlibssl
+    - hostPath:
+        path: /usr/local/openssl
+      name: usrlocalopenssl
+    - hostPath:
+        path: /var/ssl
+      name: varssl
+    - hostPath:
+        path: /etc/openssl
+      name: etcopenssl
+    - hostPath:
+        path: /etc/kubernetes/in-tree-cloud.config
+      name: cloudconfig
+    - hostPath:
+        path: /srv/kubernetes/ca.crt
+      name: kubernetesca
+    - hostPath:
+        path: /srv/kubernetes/kube-apiserver
+      name: srvkapi
+    - hostPath:
+        path: /srv/sshproxy
+      name: srvsshproxy
+  status: {}
+path: /etc/kubernetes/manifests/kube-apiserver.manifest
+type: file
+---
+mode: "0755"
+path: /srv/kubernetes/kube-apiserver
+type: directory
+---
+contents: ""
+mode: "0644"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt
+type: file
+---
+contents:
+  task:
+    Name: apiserver-aggregator
+    keypairID: ""
+    signer: apiserver-aggregator-ca
+    subject:
+      CommonName: aggregator
+    type: client
+mode: "0644"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.crt
+type: file
+---
+contents:
+  task:
+    Name: apiserver-aggregator
+    keypairID: ""
+    signer: apiserver-aggregator-ca
+    subject:
+      CommonName: aggregator
+    type: client
+mode: "0600"
+path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.key
+type: file
+---
+contents: ""
+mode: "0644"
+path: /srv/kubernetes/kube-apiserver/etcd-ca.crt
+type: file
+---
+contents:
+  task:
+    Name: etcd-client
+    keypairID: ""
+    signer: etcd-clients-ca
+    subject:
+      CommonName: kube-apiserver
+    type: client
+mode: "0644"
+path: /srv/kubernetes/kube-apiserver/etcd-client.crt
+type: file
+---
+contents:
+  task:
+    Name: etcd-client
+    keypairID: ""
+    signer: etcd-clients-ca
+    subject:
+      CommonName: kube-apiserver
+    type: client
+mode: "0600"
+path: /srv/kubernetes/kube-apiserver/etcd-client.key
+type: file
+---
+contents:
+  task:
+    Name: kubelet-api
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kubelet-api
+    type: client
+mode: "0644"
+path: /srv/kubernetes/kube-apiserver/kubelet-api.crt
+type: file
+---
+contents:
+  task:
+    Name: kubelet-api
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kubelet-api
+    type: client
+mode: "0600"
+path: /srv/kubernetes/kube-apiserver/kubelet-api.key
+type: file
+---
+contents:
+  task:
+    Name: master
+    alternateNames:
+    - kubernetes
+    - kubernetes.default
+    - kubernetes.default.svc
+    - kubernetes.default.svc.cluster.local
+    - api.minimal.example.com
+    - api.internal.minimal.example.com
+    - 100.64.0.1
+    - 127.0.0.1
+    includeRootCertificate: true
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kubernetes-master
+    type: server
+mode: "0644"
+path: /srv/kubernetes/kube-apiserver/server.crt
+type: file
+---
+contents:
+  task:
+    Name: master
+    alternateNames:
+    - kubernetes
+    - kubernetes.default
+    - kubernetes.default.svc
+    - kubernetes.default.svc.cluster.local
+    - api.minimal.example.com
+    - api.internal.minimal.example.com
+    - 100.64.0.1
+    - 127.0.0.1
+    includeRootCertificate: true
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kubernetes-master
+    type: server
+mode: "0600"
+path: /srv/kubernetes/kube-apiserver/server.key
+type: file
+---
+contents: |
+  -----BEGIN RSA PRIVATE KEY-----
+  MIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4
+  9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R
+  2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo
+  xTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+
+  ZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr
+  Kl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh
+  AOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY
+  -----END RSA PRIVATE KEY-----
+mode: "0600"
+path: /srv/kubernetes/kube-apiserver/service-account.key
+type: file
+---
+contents: |
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
+  XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+  -----BEGIN RSA PUBLIC KEY-----
+  MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
+  Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
+  -----END RSA PUBLIC KEY-----
+mode: "0600"
+path: /srv/kubernetes/kube-apiserver/service-account.pub
+type: file
+---
+contents: ""
+ifNotExists: true
+mode: "0400"
+path: /var/log/kube-apiserver.log
+type: file
+---
+Name: apiserver-aggregator
+keypairID: ""
+signer: apiserver-aggregator-ca
+subject:
+  CommonName: aggregator
+type: client
+---
+Name: etcd-client
+keypairID: ""
+signer: etcd-clients-ca
+subject:
+  CommonName: kube-apiserver
+type: client
+---
+Name: kubelet-api
+keypairID: "3"
+signer: kubernetes-ca
+subject:
+  CommonName: kubelet-api
+type: client
+---
+Name: master
+alternateNames:
+- kubernetes
+- kubernetes.default
+- kubernetes.default.svc
+- kubernetes.default.svc.cluster.local
+- api.minimal.example.com
+- api.internal.minimal.example.com
+- 100.64.0.1
+- 127.0.0.1
+includeRootCertificate: true
+keypairID: "3"
+signer: kubernetes-ca
+subject:
+  CommonName: kubernetes-master
+type: server
--- a/nodeup/pkg/model/tests/golden/envvars/tasks-kube-controller-manager.yaml
+++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-controller-manager.yaml
@ -0,0 +1,331 @@
+contents: |
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    creationTimestamp: null
+    labels:
+      k8s-app: kube-controller-manager
+    name: kube-controller-manager
+    namespace: kube-system
+  spec:
+    containers:
+    - args:
+      - --log-file=/var/log/kube-controller-manager.log
+      - --also-stdout
+      - /usr/local/bin/kube-controller-manager
+      - --allocate-node-cidrs=true
+      - --attach-detach-reconcile-sync-period=1m0s
+      - --authentication-kubeconfig=/var/lib/kube-controller-manager/kubeconfig
+      - --authorization-kubeconfig=/var/lib/kube-controller-manager/kubeconfig
+      - --cloud-config=/etc/kubernetes/in-tree-cloud.config
+      - --cloud-provider=external
+      - --cluster-cidr=100.96.0.0/11
+      - --cluster-name=minimal.example.com
+      - --cluster-signing-cert-file=/srv/kubernetes/kube-controller-manager/ca.crt
+      - --cluster-signing-key-file=/srv/kubernetes/kube-controller-manager/ca.key
+      - --configure-cloud-routes=true
+      - --feature-gates=InTreePluginAWSUnregister=true
+      - --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/
+      - --kubeconfig=/var/lib/kube-controller-manager/kubeconfig
+      - --leader-elect=true
+      - --root-ca-file=/srv/kubernetes/ca.crt
+      - --service-account-private-key-file=/srv/kubernetes/kube-controller-manager/service-account.key
+      - --tls-cert-file=/srv/kubernetes/kube-controller-manager/server.crt
+      - --tls-private-key-file=/srv/kubernetes/kube-controller-manager/server.key
+      - --use-service-account-credentials=true
+      - --v=2
+      command:
+      - /go-runner
+      image: registry.k8s.io/kube-controller-manager:v1.28.0
+      livenessProbe:
+        httpGet:
+          host: 127.0.0.1
+          path: /healthz
+          port: 10257
+          scheme: HTTPS
+        initialDelaySeconds: 15
+        timeoutSeconds: 15
+      name: kube-controller-manager
+      resources:
+        requests:
+          cpu: 100m
+      volumeMounts:
+      - mountPath: /var/log/kube-controller-manager.log
+        name: logfile
+      - mountPath: /etc/ssl
+        name: etcssl
+        readOnly: true
+      - mountPath: /etc/pki/tls
+        name: etcpkitls
+        readOnly: true
+      - mountPath: /etc/pki/ca-trust
+        name: etcpkica-trust
+        readOnly: true
+      - mountPath: /usr/share/ssl
+        name: usrsharessl
+        readOnly: true
+      - mountPath: /usr/ssl
+        name: usrssl
+        readOnly: true
+      - mountPath: /usr/lib/ssl
+        name: usrlibssl
+        readOnly: true
+      - mountPath: /usr/local/openssl
+        name: usrlocalopenssl
+        readOnly: true
+      - mountPath: /var/ssl
+        name: varssl
+        readOnly: true
+      - mountPath: /etc/openssl
+        name: etcopenssl
+        readOnly: true
+      - mountPath: /etc/kubernetes/in-tree-cloud.config
+        name: cloudconfig
+        readOnly: true
+      - mountPath: /srv/kubernetes/ca.crt
+        name: cabundle
+        readOnly: true
+      - mountPath: /srv/kubernetes/kube-controller-manager
+        name: srvkcm
+        readOnly: true
+      - mountPath: /var/lib/kube-controller-manager
+        name: varlibkcm
+        readOnly: true
+      - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/
+        name: volplugins
+    hostNetwork: true
+    priorityClassName: system-cluster-critical
+    tolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+    volumes:
+    - hostPath:
+        path: /var/log/kube-controller-manager.log
+      name: logfile
+    - hostPath:
+        path: /etc/ssl
+      name: etcssl
+    - hostPath:
+        path: /etc/pki/tls
+      name: etcpkitls
+    - hostPath:
+        path: /etc/pki/ca-trust
+      name: etcpkica-trust
+    - hostPath:
+        path: /usr/share/ssl
+      name: usrsharessl
+    - hostPath:
+        path: /usr/ssl
+      name: usrssl
+    - hostPath:
+        path: /usr/lib/ssl
+      name: usrlibssl
+    - hostPath:
+        path: /usr/local/openssl
+      name: usrlocalopenssl
+    - hostPath:
+        path: /var/ssl
+      name: varssl
+    - hostPath:
+        path: /etc/openssl
+      name: etcopenssl
+    - hostPath:
+        path: /etc/kubernetes/in-tree-cloud.config
+      name: cloudconfig
+    - hostPath:
+        path: /srv/kubernetes/ca.crt
+      name: cabundle
+    - hostPath:
+        path: /srv/kubernetes/kube-controller-manager
+      name: srvkcm
+    - hostPath:
+        path: /var/lib/kube-controller-manager
+      name: varlibkcm
+    - hostPath:
+        path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/
+      name: volplugins
+  status: {}
+path: /etc/kubernetes/manifests/kube-controller-manager.manifest
+type: file
+---
+mode: "0755"
+path: /srv/kubernetes/kube-controller-manager
+type: directory
+---
+contents: |
+  -----BEGIN CERTIFICATE-----
+  MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw
+  FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy
+  MzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA
+  A4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd
+  XEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY
+  OACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1
+  YYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC
+  An8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA
+  NPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB
+  /wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r
+  hzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO
+  HE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe
+  rN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt
+  O6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy
+  VpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh
+  8yGeRx9AbknHh4Ia
+  -----END CERTIFICATE-----
+mode: "0600"
+path: /srv/kubernetes/kube-controller-manager/ca.crt
+type: file
+---
+contents: |
+  -----BEGIN RSA PRIVATE KEY-----
+  MIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH
+  AZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA
+  gvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF
+  GHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/
+  P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx
+  9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI
+  Bezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9
+  /Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM
+  0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t
+  vpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd
+  cuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G
+  8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj
+  22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1
+  AsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV
+  99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs
+  z2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg
+  ST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK
+  sfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7
+  KhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i
+  Q4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T
+  t9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q
+  dGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z
+  worz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu
+  BAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq
+  Y/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw==
+  -----END RSA PRIVATE KEY-----
+mode: "0600"
+path: /srv/kubernetes/kube-controller-manager/ca.key
+type: file
+---
+contents:
+  task:
+    Name: kube-controller-manager-server
+    alternateNames:
+    - kube-controller-manager.kube-system.svc.cluster.local
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kube-controller-manager
+    type: server
+mode: "0644"
+path: /srv/kubernetes/kube-controller-manager/server.crt
+type: file
+---
+contents:
+  task:
+    Name: kube-controller-manager-server
+    alternateNames:
+    - kube-controller-manager.kube-system.svc.cluster.local
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kube-controller-manager
+    type: server
+mode: "0600"
+path: /srv/kubernetes/kube-controller-manager/server.key
+type: file
+---
+contents: |
+  -----BEGIN RSA PRIVATE KEY-----
+  MIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4
+  9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R
+  2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo
+  xTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+
+  ZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr
+  Kl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh
+  AOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY
+  -----END RSA PRIVATE KEY-----
+mode: "0600"
+path: /srv/kubernetes/kube-controller-manager/service-account.key
+type: file
+---
+contents:
+  task:
+    CA:
+      task:
+        Name: kube-controller-manager
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: system:kube-controller-manager
+        type: client
+    Cert:
+      task:
+        Name: kube-controller-manager
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: system:kube-controller-manager
+        type: client
+    Key:
+      task:
+        Name: kube-controller-manager
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: system:kube-controller-manager
+        type: client
+    Name: kube-controller-manager
+    ServerURL: https://127.0.0.1
+mode: "0400"
+path: /var/lib/kube-controller-manager/kubeconfig
+type: file
+---
+contents: ""
+ifNotExists: true
+mode: "0400"
+path: /var/log/kube-controller-manager.log
+type: file
+---
+Name: kube-controller-manager
+keypairID: "3"
+signer: kubernetes-ca
+subject:
+  CommonName: system:kube-controller-manager
+type: client
+---
+Name: kube-controller-manager-server
+alternateNames:
+- kube-controller-manager.kube-system.svc.cluster.local
+keypairID: "3"
+signer: kubernetes-ca
+subject:
+  CommonName: kube-controller-manager
+type: server
+---
+CA:
+  task:
+    Name: kube-controller-manager
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: system:kube-controller-manager
+    type: client
+Cert:
+  task:
+    Name: kube-controller-manager
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: system:kube-controller-manager
+    type: client
+Key:
+  task:
+    Name: kube-controller-manager
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: system:kube-controller-manager
+    type: client
+Name: kube-controller-manager
+ServerURL: https://127.0.0.1
--- a/nodeup/pkg/model/tests/golden/envvars/tasks-kube-proxy.yaml
+++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-proxy.yaml
@ -0,0 +1,145 @@
+contents: |
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    creationTimestamp: null
+    labels:
+      k8s-app: kube-proxy
+      kubernetes.io/managed-by: nodeup
+      tier: node
+    name: kube-proxy
+    namespace: kube-system
+  spec:
+    containers:
+    - args:
+      - --log-file=/var/log/kube-proxy.log
+      - --also-stdout
+      - /usr/local/bin/kube-proxy
+      - --cluster-cidr=100.96.0.0/11
+      - --conntrack-max-per-core=131072
+      - --kubeconfig=/var/lib/kube-proxy/kubeconfig
+      - --master=https://127.0.0.1
+      - --oom-score-adj=-998
+      - --v=2
+      command:
+      - /go-runner
+      image: registry.k8s.io/kube-proxy:v1.28.0
+      name: kube-proxy
+      resources:
+        requests:
+          cpu: 100m
+      securityContext:
+        privileged: true
+      volumeMounts:
+      - mountPath: /var/log/kube-proxy.log
+        name: logfile
+      - mountPath: /var/lib/kube-proxy/kubeconfig
+        name: kubeconfig
+        readOnly: true
+      - mountPath: /lib/modules
+        name: modules
+        readOnly: true
+      - mountPath: /etc/ssl/certs
+        name: ssl-certs-hosts
+        readOnly: true
+      - mountPath: /run/xtables.lock
+        name: iptableslock
+    hostNetwork: true
+    priorityClassName: system-node-critical
+    tolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+    volumes:
+    - hostPath:
+        path: /var/log/kube-proxy.log
+      name: logfile
+    - hostPath:
+        path: /var/lib/kube-proxy/kubeconfig
+      name: kubeconfig
+    - hostPath:
+        path: /lib/modules
+      name: modules
+    - hostPath:
+        path: /usr/share/ca-certificates
+      name: ssl-certs-hosts
+    - hostPath:
+        path: /run/xtables.lock
+        type: FileOrCreate
+      name: iptableslock
+  status: {}
+path: /etc/kubernetes/manifests/kube-proxy.manifest
+type: file
+---
+beforeServices:
+- kubelet.service
+contents:
+  task:
+    CA:
+      task:
+        Name: kube-proxy
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: system:kube-proxy
+        type: client
+    Cert:
+      task:
+        Name: kube-proxy
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: system:kube-proxy
+        type: client
+    Key:
+      task:
+        Name: kube-proxy
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: system:kube-proxy
+        type: client
+    Name: kube-proxy
+    ServerURL: https://127.0.0.1
+mode: "0400"
+path: /var/lib/kube-proxy/kubeconfig
+type: file
+---
+contents: ""
+ifNotExists: true
+mode: "0400"
+path: /var/log/kube-proxy.log
+type: file
+---
+Name: kube-proxy
+keypairID: "3"
+signer: kubernetes-ca
+subject:
+  CommonName: system:kube-proxy
+type: client
+---
+CA:
+  task:
+    Name: kube-proxy
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: system:kube-proxy
+    type: client
+Cert:
+  task:
+    Name: kube-proxy
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: system:kube-proxy
+    type: client
+Key:
+  task:
+    Name: kube-proxy
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: system:kube-proxy
+    type: client
+Name: kube-proxy
+ServerURL: https://127.0.0.1
--- a/nodeup/pkg/model/tests/golden/envvars/tasks-kube-scheduler.yaml
+++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kube-scheduler.yaml
@ -0,0 +1,187 @@
+contents: |
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    creationTimestamp: null
+    labels:
+      k8s-app: kube-scheduler
+    name: kube-scheduler
+    namespace: kube-system
+  spec:
+    containers:
+    - args:
+      - --log-file=/var/log/kube-scheduler.log
+      - --also-stdout
+      - /usr/local/bin/kube-scheduler
+      - --authentication-kubeconfig=/var/lib/kube-scheduler/kubeconfig
+      - --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig
+      - --config=/var/lib/kube-scheduler/config.yaml
+      - --feature-gates=InTreePluginAWSUnregister=true
+      - --leader-elect=true
+      - --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt
+      - --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key
+      - --v=2
+      command:
+      - /go-runner
+      image: registry.k8s.io/kube-scheduler:v1.28.0
+      livenessProbe:
+        httpGet:
+          host: 127.0.0.1
+          path: /healthz
+          port: 10259
+          scheme: HTTPS
+        initialDelaySeconds: 15
+        timeoutSeconds: 15
+      name: kube-scheduler
+      resources:
+        requests:
+          cpu: 100m
+      volumeMounts:
+      - mountPath: /var/lib/kube-scheduler
+        name: varlibkubescheduler
+        readOnly: true
+      - mountPath: /srv/kubernetes/kube-scheduler
+        name: srvscheduler
+        readOnly: true
+      - mountPath: /var/log/kube-scheduler.log
+        name: logfile
+    hostNetwork: true
+    priorityClassName: system-cluster-critical
+    tolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+    volumes:
+    - hostPath:
+        path: /var/lib/kube-scheduler
+      name: varlibkubescheduler
+    - hostPath:
+        path: /srv/kubernetes/kube-scheduler
+      name: srvscheduler
+    - hostPath:
+        path: /var/log/kube-scheduler.log
+      name: logfile
+  status: {}
+path: /etc/kubernetes/manifests/kube-scheduler.manifest
+type: file
+---
+mode: "0755"
+path: /srv/kubernetes/kube-scheduler
+type: directory
+---
+contents:
+  task:
+    Name: kube-scheduler-server
+    alternateNames:
+    - kube-scheduler.kube-system.svc.cluster.local
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kube-scheduler
+    type: server
+mode: "0644"
+path: /srv/kubernetes/kube-scheduler/server.crt
+type: file
+---
+contents:
+  task:
+    Name: kube-scheduler-server
+    alternateNames:
+    - kube-scheduler.kube-system.svc.cluster.local
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kube-scheduler
+    type: server
+mode: "0600"
+path: /srv/kubernetes/kube-scheduler/server.key
+type: file
+---
+contents: |
+  apiVersion: kubescheduler.config.k8s.io/v1
+  clientConnection:
+    kubeconfig: /var/lib/kube-scheduler/kubeconfig
+  kind: KubeSchedulerConfiguration
+mode: "0400"
+path: /var/lib/kube-scheduler/config.yaml
+type: file
+---
+contents:
+  task:
+    CA:
+      task:
+        Name: kube-scheduler
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: system:kube-scheduler
+        type: client
+    Cert:
+      task:
+        Name: kube-scheduler
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: system:kube-scheduler
+        type: client
+    Key:
+      task:
+        Name: kube-scheduler
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: system:kube-scheduler
+        type: client
+    Name: kube-scheduler
+    ServerURL: https://127.0.0.1
+mode: "0400"
+path: /var/lib/kube-scheduler/kubeconfig
+type: file
+---
+contents: ""
+ifNotExists: true
+mode: "0400"
+path: /var/log/kube-scheduler.log
+type: file
+---
+Name: kube-scheduler
+keypairID: "3"
+signer: kubernetes-ca
+subject:
+  CommonName: system:kube-scheduler
+type: client
+---
+Name: kube-scheduler-server
+alternateNames:
+- kube-scheduler.kube-system.svc.cluster.local
+keypairID: "3"
+signer: kubernetes-ca
+subject:
+  CommonName: kube-scheduler
+type: server
+---
+CA:
+  task:
+    Name: kube-scheduler
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: system:kube-scheduler
+    type: client
+Cert:
+  task:
+    Name: kube-scheduler
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: system:kube-scheduler
+    type: client
+Key:
+  task:
+    Name: kube-scheduler
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: system:kube-scheduler
+    type: client
+Name: kube-scheduler
+ServerURL: https://127.0.0.1
--- a/nodeup/pkg/model/tests/golden/envvars/tasks-kubectl.yaml
+++ b/nodeup/pkg/model/tests/golden/envvars/tasks-kubectl.yaml
@ -0,0 +1,87 @@
+contents:
+  Asset:
+    AssetPath: /path/to/kubectl/asset
+    Key: kubectl
+mode: "0755"
+path: /opt/kops/bin/kubectl
+type: file
+---
+contents:
+  task:
+    CA:
+      task:
+        Name: kubecfg
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: kubecfg
+          Organization:
+          - system:masters
+        type: client
+    Cert:
+      task:
+        Name: kubecfg
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: kubecfg
+          Organization:
+          - system:masters
+        type: client
+    Key:
+      task:
+        Name: kubecfg
+        keypairID: "3"
+        signer: kubernetes-ca
+        subject:
+          CommonName: kubecfg
+          Organization:
+          - system:masters
+        type: client
+    Name: kubecfg
+    ServerURL: https://127.0.0.1
+mode: "0400"
+path: /var/lib/kubectl/kubeconfig
+type: file
+---
+Name: kubecfg
+keypairID: "3"
+signer: kubernetes-ca
+subject:
+  CommonName: kubecfg
+  Organization:
+  - system:masters
+type: client
+---
+CA:
+  task:
+    Name: kubecfg
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kubecfg
+      Organization:
+      - system:masters
+    type: client
+Cert:
+  task:
+    Name: kubecfg
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kubecfg
+      Organization:
+      - system:masters
+    type: client
+Key:
+  task:
+    Name: kubecfg
+    keypairID: "3"
+    signer: kubernetes-ca
+    subject:
+      CommonName: kubecfg
+      Organization:
+      - system:masters
+    type: client
+Name: kubecfg
+ServerURL: https://127.0.0.1
--- a/nodeup/pkg/model/tests/golden/envvars/tasks-secret.yaml
+++ b/nodeup/pkg/model/tests/golden/envvars/tasks-secret.yaml
@ -0,0 +1,32 @@
+contents: |
+  -----BEGIN CERTIFICATE-----
+  MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw
+  FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy
+  MzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA
+  A4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd
+  XEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY
+  OACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1
+  YYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC
+  An8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA
+  NPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB
+  /wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r
+  hzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO
+  HE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe
+  rN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt
+  O6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy
+  VpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh
+  8yGeRx9AbknHh4Ia
+  -----END CERTIFICATE-----
+  -----BEGIN CERTIFICATE-----
+  MIIBZzCCARGgAwIBAgIBBDANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2
+  aWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMjE3WhcNMzEwNTAyMjAzMjE3WjAaMRgw
+  FgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA
+  o4Tridlsf4Yz3UAiup/scSTiG/OqxkUW3Fz7zGKvVcLeYj9GEIKuzoB1VFk1nboD
+  q4cCuGLfdzaQdCQKPIsDuwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T
+  AQH/BAUwAwEB/zAdBgNVHQ4EFgQUhPbxEmUbwVOCa+fZgxreFhf67UEwDQYJKoZI
+  hvcNAQELBQADQQALMsyK2Q7C/bk27eCvXyZKUfrLvor10hEjwGhv14zsKWDeTj/J
+  A1LPYp7U9VtFfgFOkVbkLE9Rstc0ltNrPqxA
+  -----END CERTIFICATE-----
+mode: "0600"
+path: /srv/kubernetes/ca.crt
+type: file
--- a/pkg/apis/kops/componentconfig.go
+++ b/pkg/apis/kops/componentconfig.go
@ -17,6 +17,7 @@ limitations under the License.
 package kops

 import (
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@ -543,6 +544,11 @@ type KubeAPIServerConfig struct {
 	DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"`
 	// DefaultUnreachableTolerationSeconds indicates the tolerationSeconds of the toleration for unreachable:NoExecute that is added by default to every pod that does not already have such a toleration.
 	DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"`
+
+	// Env allows users to pass in env variables to the apiserver container.
+	// This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver
+	// This also allows the flexibility for adding any other variables for future use cases
+	Env []corev1.EnvVar `json:"env,omitempty"`
 }

 // KubeControllerManagerConfig is the configuration for the controller
--- a/pkg/apis/kops/v1alpha2/componentconfig.go
+++ b/pkg/apis/kops/v1alpha2/componentconfig.go
@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha2

 import (
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@ -550,6 +551,11 @@ type KubeAPIServerConfig struct {
 	DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"`
 	// DefaultUnreachableTolerationSeconds
 	DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"`
+
+	// Env allows users to pass in env variables to the apiserver container.
+	// This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver
+	// This also allows the flexibility for adding any other variables for future use cases
+	Env []corev1.EnvVar `json:"env,omitempty"`
 }

 // KubeControllerManagerConfig is the configuration for the controller
--- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
@ -4972,6 +4972,7 @@ func autoConvert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku
 	out.CorsAllowedOrigins = in.CorsAllowedOrigins
 	out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
 	out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
+	out.Env = in.Env
 	return nil
 }

@ -5087,6 +5088,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha2_KubeAPIServerConfig(in *ko
 	out.CorsAllowedOrigins = in.CorsAllowedOrigins
 	out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
 	out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
+	out.Env = in.Env
 	return nil
 }

--- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
@ -3415,6 +3415,13 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) {
 		*out = new(int64)
 		**out = **in
 	}
+	if in.Env != nil {
+		in, out := &in.Env, &out.Env
+		*out = make([]corev1.EnvVar, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }

--- a/pkg/apis/kops/v1alpha2/zz_generated.defaults.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.defaults.go
@ -36,6 +36,23 @@ func RegisterDefaults(scheme *runtime.Scheme) error {

 func SetObjectDefaults_Cluster(in *Cluster) {
 	SetDefaults_ClusterSpec(&in.Spec)
+	if in.Spec.KubeAPIServer != nil {
+		for i := range in.Spec.KubeAPIServer.Env {
+			a := &in.Spec.KubeAPIServer.Env[i]
+			if a.ValueFrom != nil {
+				if a.ValueFrom.ConfigMapKeyRef != nil {
+					if a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name == "" {
+						a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name = ""
+					}
+				}
+				if a.ValueFrom.SecretKeyRef != nil {
+					if a.ValueFrom.SecretKeyRef.LocalObjectReference.Name == "" {
+						a.ValueFrom.SecretKeyRef.LocalObjectReference.Name = ""
+					}
+				}
+			}
+		}
+	}
 }

 func SetObjectDefaults_ClusterList(in *ClusterList) {
--- a/pkg/apis/kops/v1alpha3/componentconfig.go
+++ b/pkg/apis/kops/v1alpha3/componentconfig.go
@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha3

 import (
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@ -541,6 +542,11 @@ type KubeAPIServerConfig struct {
 	DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"`
 	// DefaultUnreachableTolerationSeconds
 	DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"`
+
+	// Env allows users to pass in env variables to the apiserver container.
+	// This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver
+	// This also allows the flexibility for adding any other variables for future use cases
+	Env []corev1.EnvVar `json:"env,omitempty"`
 }

 // KubeControllerManagerConfig is the configuration for the controller
--- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
@ -5368,6 +5368,7 @@ func autoConvert_v1alpha3_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku
 	out.CorsAllowedOrigins = in.CorsAllowedOrigins
 	out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
 	out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
+	out.Env = in.Env
 	return nil
 }

@ -5483,6 +5484,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha3_KubeAPIServerConfig(in *ko
 	out.CorsAllowedOrigins = in.CorsAllowedOrigins
 	out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
 	out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
+	out.Env = in.Env
 	return nil
 }

--- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
@ -3389,6 +3389,13 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) {
 		*out = new(int64)
 		**out = **in
 	}
+	if in.Env != nil {
+		in, out := &in.Env, &out.Env
+		*out = make([]corev1.EnvVar, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }

--- a/pkg/apis/kops/v1alpha3/zz_generated.defaults.go
+++ b/pkg/apis/kops/v1alpha3/zz_generated.defaults.go
@ -36,6 +36,23 @@ func RegisterDefaults(scheme *runtime.Scheme) error {

 func SetObjectDefaults_Cluster(in *Cluster) {
 	SetDefaults_ClusterSpec(&in.Spec)
+	if in.Spec.KubeAPIServer != nil {
+		for i := range in.Spec.KubeAPIServer.Env {
+			a := &in.Spec.KubeAPIServer.Env[i]
+			if a.ValueFrom != nil {
+				if a.ValueFrom.ConfigMapKeyRef != nil {
+					if a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name == "" {
+						a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name = ""
+					}
+				}
+				if a.ValueFrom.SecretKeyRef != nil {
+					if a.ValueFrom.SecretKeyRef.LocalObjectReference.Name == "" {
+						a.ValueFrom.SecretKeyRef.LocalObjectReference.Name = ""
+					}
+				}
+			}
+		}
+	}
 }

 func SetObjectDefaults_ClusterList(in *ClusterList) {
--- a/pkg/apis/kops/zz_generated.deepcopy.go
+++ b/pkg/apis/kops/zz_generated.deepcopy.go
@ -3492,6 +3492,13 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) {
 		*out = new(int64)
 		**out = **in
 	}
+	if in.Env != nil {
+		in, out := &in.Env, &out.Env
+		*out = make([]corev1.EnvVar, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }