mirror of https://github.com/kubernetes/kops.git
Add support for configuring environment variables on kube-apiserver
This commit is contained in:
parent
daea619a59
commit
8b89e826d1
|
@ -627,6 +627,17 @@ spec:
|
||||||
logFormat: json
|
logFormat: json
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
```yaml
|
||||||
|
spec:
|
||||||
|
kubeAPIServer:
|
||||||
|
env:
|
||||||
|
- name: GOMEMLIMIT
|
||||||
|
value: "2750MiB"
|
||||||
|
- name: GOGC
|
||||||
|
value: 50
|
||||||
|
```
|
||||||
|
|
||||||
## externalDns
|
## externalDns
|
||||||
|
|
||||||
This block contains configuration options for your `external-DNS` provider.
|
This block contains configuration options for your `external-DNS` provider.
|
||||||
|
|
|
@ -1885,6 +1885,129 @@ spec:
|
||||||
description: EncryptionProviderConfig enables encryption at rest
|
description: EncryptionProviderConfig enables encryption at rest
|
||||||
for secrets.
|
for secrets.
|
||||||
type: string
|
type: string
|
||||||
|
env:
|
||||||
|
description: |-
|
||||||
|
Env allows users to pass in env variables to the apiserver container.
|
||||||
|
This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver
|
||||||
|
This also allows the flexibility for adding any other variables for future use cases
|
||||||
|
items:
|
||||||
|
description: EnvVar represents an environment variable present
|
||||||
|
in a Container.
|
||||||
|
properties:
|
||||||
|
name:
|
||||||
|
description: Name of the environment variable. Must be a
|
||||||
|
C_IDENTIFIER.
|
||||||
|
type: string
|
||||||
|
value:
|
||||||
|
description: |-
|
||||||
|
Variable references $(VAR_NAME) are expanded
|
||||||
|
using the previously defined environment variables in the container and
|
||||||
|
any service environment variables. If a variable cannot be resolved,
|
||||||
|
the reference in the input string will be unchanged. Double $$ are reduced
|
||||||
|
to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e.
|
||||||
|
"$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)".
|
||||||
|
Escaped references will never be expanded, regardless of whether the variable
|
||||||
|
exists or not.
|
||||||
|
Defaults to "".
|
||||||
|
type: string
|
||||||
|
valueFrom:
|
||||||
|
description: Source for the environment variable's value.
|
||||||
|
Cannot be used if value is not empty.
|
||||||
|
properties:
|
||||||
|
configMapKeyRef:
|
||||||
|
description: Selects a key of a ConfigMap.
|
||||||
|
properties:
|
||||||
|
key:
|
||||||
|
description: The key to select.
|
||||||
|
type: string
|
||||||
|
name:
|
||||||
|
default: ""
|
||||||
|
description: |-
|
||||||
|
Name of the referent.
|
||||||
|
This field is effectively required, but due to backwards compatibility is
|
||||||
|
allowed to be empty. Instances of this type with an empty value here are
|
||||||
|
almost certainly wrong.
|
||||||
|
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
|
||||||
|
type: string
|
||||||
|
optional:
|
||||||
|
description: Specify whether the ConfigMap or its
|
||||||
|
key must be defined
|
||||||
|
type: boolean
|
||||||
|
required:
|
||||||
|
- key
|
||||||
|
type: object
|
||||||
|
x-kubernetes-map-type: atomic
|
||||||
|
fieldRef:
|
||||||
|
description: |-
|
||||||
|
Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['<KEY>']`, `metadata.annotations['<KEY>']`,
|
||||||
|
spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.
|
||||||
|
properties:
|
||||||
|
apiVersion:
|
||||||
|
description: Version of the schema the FieldPath
|
||||||
|
is written in terms of, defaults to "v1".
|
||||||
|
type: string
|
||||||
|
fieldPath:
|
||||||
|
description: Path of the field to select in the
|
||||||
|
specified API version.
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- fieldPath
|
||||||
|
type: object
|
||||||
|
x-kubernetes-map-type: atomic
|
||||||
|
resourceFieldRef:
|
||||||
|
description: |-
|
||||||
|
Selects a resource of the container: only resources limits and requests
|
||||||
|
(limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.
|
||||||
|
properties:
|
||||||
|
containerName:
|
||||||
|
description: 'Container name: required for volumes,
|
||||||
|
optional for env vars'
|
||||||
|
type: string
|
||||||
|
divisor:
|
||||||
|
anyOf:
|
||||||
|
- type: integer
|
||||||
|
- type: string
|
||||||
|
description: Specifies the output format of the
|
||||||
|
exposed resources, defaults to "1"
|
||||||
|
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
|
||||||
|
x-kubernetes-int-or-string: true
|
||||||
|
resource:
|
||||||
|
description: 'Required: resource to select'
|
||||||
|
type: string
|
||||||
|
required:
|
||||||
|
- resource
|
||||||
|
type: object
|
||||||
|
x-kubernetes-map-type: atomic
|
||||||
|
secretKeyRef:
|
||||||
|
description: Selects a key of a secret in the pod's
|
||||||
|
namespace
|
||||||
|
properties:
|
||||||
|
key:
|
||||||
|
description: The key of the secret to select from. Must
|
||||||
|
be a valid secret key.
|
||||||
|
type: string
|
||||||
|
name:
|
||||||
|
default: ""
|
||||||
|
description: |-
|
||||||
|
Name of the referent.
|
||||||
|
This field is effectively required, but due to backwards compatibility is
|
||||||
|
allowed to be empty. Instances of this type with an empty value here are
|
||||||
|
almost certainly wrong.
|
||||||
|
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
|
||||||
|
type: string
|
||||||
|
optional:
|
||||||
|
description: Specify whether the Secret or its key
|
||||||
|
must be defined
|
||||||
|
type: boolean
|
||||||
|
required:
|
||||||
|
- key
|
||||||
|
type: object
|
||||||
|
x-kubernetes-map-type: atomic
|
||||||
|
type: object
|
||||||
|
required:
|
||||||
|
- name
|
||||||
|
type: object
|
||||||
|
type: array
|
||||||
etcdCaFile:
|
etcdCaFile:
|
||||||
description: EtcdCAFile is the path to a ca certificate
|
description: EtcdCAFile is the path to a ca certificate
|
||||||
type: string
|
type: string
|
||||||
|
|
|
@ -699,7 +699,7 @@ func (b *KubeAPIServerBuilder) buildPod(ctx context.Context, kubeAPIServer *kops
|
||||||
container := &v1.Container{
|
container := &v1.Container{
|
||||||
Name: "kube-apiserver",
|
Name: "kube-apiserver",
|
||||||
Image: image,
|
Image: image,
|
||||||
Env: proxy.GetProxyEnvVars(b.NodeupConfig.Networking.EgressProxy),
|
Env: append(kubeAPIServer.Env, proxy.GetProxyEnvVars(b.NodeupConfig.Networking.EgressProxy)...),
|
||||||
LivenessProbe: livenessProbe,
|
LivenessProbe: livenessProbe,
|
||||||
ReadinessProbe: readinessProbe,
|
ReadinessProbe: readinessProbe,
|
||||||
StartupProbe: startupProbe,
|
StartupProbe: startupProbe,
|
||||||
|
|
|
@ -197,3 +197,10 @@ func TestKubeAPIServerBuilderARM64(t *testing.T) {
|
||||||
return builder.Build(target)
|
return builder.Build(target)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestKubeAPIServerEnvBuilder(t *testing.T) {
|
||||||
|
RunGoldenTest(t, "tests/golden/envvars", "kube-apiserver", func(nodeupModelContext *NodeupModelContext, target *fi.NodeupModelBuilderContext) error {
|
||||||
|
builder := KubeAPIServerBuilder{NodeupModelContext: nodeupModelContext}
|
||||||
|
return builder.Build(target)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
|
@ -0,0 +1,72 @@
|
||||||
|
apiVersion: kops.k8s.io/v1alpha2
|
||||||
|
kind: Cluster
|
||||||
|
metadata:
|
||||||
|
name: minimal.example.com
|
||||||
|
spec:
|
||||||
|
kubernetesApiAccess:
|
||||||
|
- 0.0.0.0/0
|
||||||
|
channel: stable
|
||||||
|
cloudProvider: aws
|
||||||
|
configBase: memfs://clusters.example.com/minimal.example.com
|
||||||
|
etcdClusters:
|
||||||
|
- cpuRequest: 200m
|
||||||
|
etcdMembers:
|
||||||
|
- instanceGroup: master-us-test-1a
|
||||||
|
name: us-test-1a
|
||||||
|
memoryRequest: 100Mi
|
||||||
|
name: main
|
||||||
|
provider: Manager
|
||||||
|
backups:
|
||||||
|
backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-main
|
||||||
|
- cpuRequest: 100m
|
||||||
|
etcdMembers:
|
||||||
|
- instanceGroup: master-us-test-1a
|
||||||
|
name: us-test-1a
|
||||||
|
memoryRequest: 100Mi
|
||||||
|
name: events
|
||||||
|
provider: Manager
|
||||||
|
backups:
|
||||||
|
backupStore: memfs://clusters.example.com/minimal.example.com/backups/etcd-events
|
||||||
|
iam: {}
|
||||||
|
kubeAPIServer:
|
||||||
|
env:
|
||||||
|
- name: GOMEMLIMIT
|
||||||
|
valueFrom:
|
||||||
|
resourceFieldRef:
|
||||||
|
resource: limits.memory
|
||||||
|
divisor: '1'
|
||||||
|
- name: GOGC
|
||||||
|
value: "50"
|
||||||
|
kubelet:
|
||||||
|
anonymousAuth: false
|
||||||
|
kubernetesVersion: v1.28.0
|
||||||
|
masterPublicName: api.minimal.example.com
|
||||||
|
networkCIDR: 172.20.0.0/16
|
||||||
|
networking:
|
||||||
|
kubenet: {}
|
||||||
|
nonMasqueradeCIDR: 100.64.0.0/10
|
||||||
|
sshAccess:
|
||||||
|
- 0.0.0.0/0
|
||||||
|
subnets:
|
||||||
|
- cidr: 172.20.32.0/19
|
||||||
|
name: us-test-1a
|
||||||
|
type: Public
|
||||||
|
zone: us-test-1a
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
apiVersion: kops.k8s.io/v1alpha2
|
||||||
|
kind: InstanceGroup
|
||||||
|
metadata:
|
||||||
|
name: master-us-test-1a
|
||||||
|
labels:
|
||||||
|
kops.k8s.io/cluster: minimal.example.com
|
||||||
|
spec:
|
||||||
|
associatePublicIp: true
|
||||||
|
image: ami-1234
|
||||||
|
machineType: m3.medium
|
||||||
|
maxSize: 1
|
||||||
|
minSize: 1
|
||||||
|
role: Master
|
||||||
|
subnets:
|
||||||
|
- us-test-1a
|
|
@ -0,0 +1,112 @@
|
||||||
|
mode: "0755"
|
||||||
|
path: /etc/kubernetes/kops-controller
|
||||||
|
type: directory
|
||||||
|
---
|
||||||
|
contents: |
|
||||||
|
kubernetes-ca: "3"
|
||||||
|
service-account: "2"
|
||||||
|
mode: "0600"
|
||||||
|
owner: kops-controller
|
||||||
|
path: /etc/kubernetes/kops-controller/keypair-ids.yaml
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: kops-controller
|
||||||
|
alternateNames:
|
||||||
|
- kops-controller.internal.minimal.example.com
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kops-controller
|
||||||
|
type: server
|
||||||
|
mode: "0644"
|
||||||
|
owner: kops-controller
|
||||||
|
path: /etc/kubernetes/kops-controller/kops-controller.crt
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: kops-controller
|
||||||
|
alternateNames:
|
||||||
|
- kops-controller.internal.minimal.example.com
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kops-controller
|
||||||
|
type: server
|
||||||
|
mode: "0600"
|
||||||
|
owner: kops-controller
|
||||||
|
path: /etc/kubernetes/kops-controller/kops-controller.key
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: |
|
||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw
|
||||||
|
FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy
|
||||||
|
MzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA
|
||||||
|
A4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd
|
||||||
|
XEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY
|
||||||
|
OACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1
|
||||||
|
YYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC
|
||||||
|
An8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA
|
||||||
|
NPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB
|
||||||
|
/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r
|
||||||
|
hzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO
|
||||||
|
HE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe
|
||||||
|
rN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt
|
||||||
|
O6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy
|
||||||
|
VpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh
|
||||||
|
8yGeRx9AbknHh4Ia
|
||||||
|
-----END CERTIFICATE-----
|
||||||
|
mode: "0600"
|
||||||
|
owner: kops-controller
|
||||||
|
path: /etc/kubernetes/kops-controller/kubernetes-ca.crt
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: |
|
||||||
|
-----BEGIN RSA PRIVATE KEY-----
|
||||||
|
MIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH
|
||||||
|
AZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA
|
||||||
|
gvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF
|
||||||
|
GHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/
|
||||||
|
P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx
|
||||||
|
9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI
|
||||||
|
Bezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9
|
||||||
|
/Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM
|
||||||
|
0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t
|
||||||
|
vpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd
|
||||||
|
cuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G
|
||||||
|
8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj
|
||||||
|
22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1
|
||||||
|
AsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV
|
||||||
|
99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs
|
||||||
|
z2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg
|
||||||
|
ST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK
|
||||||
|
sfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7
|
||||||
|
KhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i
|
||||||
|
Q4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T
|
||||||
|
t9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q
|
||||||
|
dGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z
|
||||||
|
worz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu
|
||||||
|
BAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq
|
||||||
|
Y/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw==
|
||||||
|
-----END RSA PRIVATE KEY-----
|
||||||
|
mode: "0600"
|
||||||
|
owner: kops-controller
|
||||||
|
path: /etc/kubernetes/kops-controller/kubernetes-ca.key
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
Name: kops-controller
|
||||||
|
alternateNames:
|
||||||
|
- kops-controller.internal.minimal.example.com
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kops-controller
|
||||||
|
type: server
|
||||||
|
---
|
||||||
|
Name: kops-controller
|
||||||
|
home: ""
|
||||||
|
shell: /sbin/nologin
|
||||||
|
uid: 10011
|
|
@ -0,0 +1,376 @@
|
||||||
|
contents: |
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
dns.alpha.kubernetes.io/external: api.minimal.example.com
|
||||||
|
dns.alpha.kubernetes.io/internal: api.internal.minimal.example.com
|
||||||
|
kubectl.kubernetes.io/default-container: kube-apiserver
|
||||||
|
creationTimestamp: null
|
||||||
|
labels:
|
||||||
|
k8s-app: kube-apiserver
|
||||||
|
name: kube-apiserver
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- args:
|
||||||
|
- --log-file=/var/log/kube-apiserver.log
|
||||||
|
- --also-stdout
|
||||||
|
- /usr/local/bin/kube-apiserver
|
||||||
|
- --allow-privileged=true
|
||||||
|
- --anonymous-auth=false
|
||||||
|
- --api-audiences=kubernetes.svc.default
|
||||||
|
- --apiserver-count=1
|
||||||
|
- --authorization-mode=AlwaysAllow
|
||||||
|
- --bind-address=0.0.0.0
|
||||||
|
- --client-ca-file=/srv/kubernetes/ca.crt
|
||||||
|
- --cloud-config=/etc/kubernetes/in-tree-cloud.config
|
||||||
|
- --cloud-provider=external
|
||||||
|
- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,NodeRestriction,ResourceQuota
|
||||||
|
- --etcd-cafile=/srv/kubernetes/kube-apiserver/etcd-ca.crt
|
||||||
|
- --etcd-certfile=/srv/kubernetes/kube-apiserver/etcd-client.crt
|
||||||
|
- --etcd-keyfile=/srv/kubernetes/kube-apiserver/etcd-client.key
|
||||||
|
- --etcd-servers-overrides=/events#https://127.0.0.1:4002
|
||||||
|
- --etcd-servers=https://127.0.0.1:4001
|
||||||
|
- --feature-gates=InTreePluginAWSUnregister=true
|
||||||
|
- --kubelet-client-certificate=/srv/kubernetes/kube-apiserver/kubelet-api.crt
|
||||||
|
- --kubelet-client-key=/srv/kubernetes/kube-apiserver/kubelet-api.key
|
||||||
|
- --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP
|
||||||
|
- --proxy-client-cert-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.crt
|
||||||
|
- --proxy-client-key-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator.key
|
||||||
|
- --requestheader-allowed-names=aggregator
|
||||||
|
- --requestheader-client-ca-file=/srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt
|
||||||
|
- --requestheader-extra-headers-prefix=X-Remote-Extra-
|
||||||
|
- --requestheader-group-headers=X-Remote-Group
|
||||||
|
- --requestheader-username-headers=X-Remote-User
|
||||||
|
- --secure-port=443
|
||||||
|
- --service-account-issuer=https://api.internal.minimal.example.com
|
||||||
|
- --service-account-jwks-uri=https://api.internal.minimal.example.com/openid/v1/jwks
|
||||||
|
- --service-account-key-file=/srv/kubernetes/kube-apiserver/service-account.pub
|
||||||
|
- --service-account-signing-key-file=/srv/kubernetes/kube-apiserver/service-account.key
|
||||||
|
- --service-cluster-ip-range=100.64.0.0/13
|
||||||
|
- --storage-backend=etcd3
|
||||||
|
- --tls-cert-file=/srv/kubernetes/kube-apiserver/server.crt
|
||||||
|
- --tls-private-key-file=/srv/kubernetes/kube-apiserver/server.key
|
||||||
|
- --v=2
|
||||||
|
command:
|
||||||
|
- /go-runner
|
||||||
|
env:
|
||||||
|
- name: GOMEMLIMIT
|
||||||
|
valueFrom:
|
||||||
|
resourceFieldRef:
|
||||||
|
divisor: "1"
|
||||||
|
resource: limits.memory
|
||||||
|
- name: GOGC
|
||||||
|
value: "50"
|
||||||
|
image: registry.k8s.io/kube-apiserver:v1.28.0
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
host: 127.0.0.1
|
||||||
|
path: /healthz
|
||||||
|
port: 443
|
||||||
|
scheme: HTTPS
|
||||||
|
initialDelaySeconds: 45
|
||||||
|
timeoutSeconds: 15
|
||||||
|
name: kube-apiserver
|
||||||
|
ports:
|
||||||
|
- containerPort: 443
|
||||||
|
hostPort: 443
|
||||||
|
name: https
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 150m
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /var/log/kube-apiserver.log
|
||||||
|
name: logfile
|
||||||
|
- mountPath: /etc/ssl
|
||||||
|
name: etcssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /etc/pki/tls
|
||||||
|
name: etcpkitls
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /etc/pki/ca-trust
|
||||||
|
name: etcpkica-trust
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /usr/share/ssl
|
||||||
|
name: usrsharessl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /usr/ssl
|
||||||
|
name: usrssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /usr/lib/ssl
|
||||||
|
name: usrlibssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /usr/local/openssl
|
||||||
|
name: usrlocalopenssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /var/ssl
|
||||||
|
name: varssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /etc/openssl
|
||||||
|
name: etcopenssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /etc/kubernetes/in-tree-cloud.config
|
||||||
|
name: cloudconfig
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /srv/kubernetes/ca.crt
|
||||||
|
name: kubernetesca
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /srv/kubernetes/kube-apiserver
|
||||||
|
name: srvkapi
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /srv/sshproxy
|
||||||
|
name: srvsshproxy
|
||||||
|
readOnly: true
|
||||||
|
hostNetwork: true
|
||||||
|
priorityClassName: system-cluster-critical
|
||||||
|
tolerations:
|
||||||
|
- key: CriticalAddonsOnly
|
||||||
|
operator: Exists
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /var/log/kube-apiserver.log
|
||||||
|
name: logfile
|
||||||
|
- hostPath:
|
||||||
|
path: /etc/ssl
|
||||||
|
name: etcssl
|
||||||
|
- hostPath:
|
||||||
|
path: /etc/pki/tls
|
||||||
|
name: etcpkitls
|
||||||
|
- hostPath:
|
||||||
|
path: /etc/pki/ca-trust
|
||||||
|
name: etcpkica-trust
|
||||||
|
- hostPath:
|
||||||
|
path: /usr/share/ssl
|
||||||
|
name: usrsharessl
|
||||||
|
- hostPath:
|
||||||
|
path: /usr/ssl
|
||||||
|
name: usrssl
|
||||||
|
- hostPath:
|
||||||
|
path: /usr/lib/ssl
|
||||||
|
name: usrlibssl
|
||||||
|
- hostPath:
|
||||||
|
path: /usr/local/openssl
|
||||||
|
name: usrlocalopenssl
|
||||||
|
- hostPath:
|
||||||
|
path: /var/ssl
|
||||||
|
name: varssl
|
||||||
|
- hostPath:
|
||||||
|
path: /etc/openssl
|
||||||
|
name: etcopenssl
|
||||||
|
- hostPath:
|
||||||
|
path: /etc/kubernetes/in-tree-cloud.config
|
||||||
|
name: cloudconfig
|
||||||
|
- hostPath:
|
||||||
|
path: /srv/kubernetes/ca.crt
|
||||||
|
name: kubernetesca
|
||||||
|
- hostPath:
|
||||||
|
path: /srv/kubernetes/kube-apiserver
|
||||||
|
name: srvkapi
|
||||||
|
- hostPath:
|
||||||
|
path: /srv/sshproxy
|
||||||
|
name: srvsshproxy
|
||||||
|
status: {}
|
||||||
|
path: /etc/kubernetes/manifests/kube-apiserver.manifest
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
mode: "0755"
|
||||||
|
path: /srv/kubernetes/kube-apiserver
|
||||||
|
type: directory
|
||||||
|
---
|
||||||
|
contents: ""
|
||||||
|
mode: "0644"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/apiserver-aggregator-ca.crt
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: apiserver-aggregator
|
||||||
|
keypairID: ""
|
||||||
|
signer: apiserver-aggregator-ca
|
||||||
|
subject:
|
||||||
|
CommonName: aggregator
|
||||||
|
type: client
|
||||||
|
mode: "0644"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.crt
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: apiserver-aggregator
|
||||||
|
keypairID: ""
|
||||||
|
signer: apiserver-aggregator-ca
|
||||||
|
subject:
|
||||||
|
CommonName: aggregator
|
||||||
|
type: client
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/apiserver-aggregator.key
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: ""
|
||||||
|
mode: "0644"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/etcd-ca.crt
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: etcd-client
|
||||||
|
keypairID: ""
|
||||||
|
signer: etcd-clients-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kube-apiserver
|
||||||
|
type: client
|
||||||
|
mode: "0644"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/etcd-client.crt
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: etcd-client
|
||||||
|
keypairID: ""
|
||||||
|
signer: etcd-clients-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kube-apiserver
|
||||||
|
type: client
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/etcd-client.key
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: kubelet-api
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubelet-api
|
||||||
|
type: client
|
||||||
|
mode: "0644"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/kubelet-api.crt
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: kubelet-api
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubelet-api
|
||||||
|
type: client
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/kubelet-api.key
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: master
|
||||||
|
alternateNames:
|
||||||
|
- kubernetes
|
||||||
|
- kubernetes.default
|
||||||
|
- kubernetes.default.svc
|
||||||
|
- kubernetes.default.svc.cluster.local
|
||||||
|
- api.minimal.example.com
|
||||||
|
- api.internal.minimal.example.com
|
||||||
|
- 100.64.0.1
|
||||||
|
- 127.0.0.1
|
||||||
|
includeRootCertificate: true
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubernetes-master
|
||||||
|
type: server
|
||||||
|
mode: "0644"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/server.crt
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: master
|
||||||
|
alternateNames:
|
||||||
|
- kubernetes
|
||||||
|
- kubernetes.default
|
||||||
|
- kubernetes.default.svc
|
||||||
|
- kubernetes.default.svc.cluster.local
|
||||||
|
- api.minimal.example.com
|
||||||
|
- api.internal.minimal.example.com
|
||||||
|
- 100.64.0.1
|
||||||
|
- 127.0.0.1
|
||||||
|
includeRootCertificate: true
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubernetes-master
|
||||||
|
type: server
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/server.key
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: |
|
||||||
|
-----BEGIN RSA PRIVATE KEY-----
|
||||||
|
MIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4
|
||||||
|
9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R
|
||||||
|
2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo
|
||||||
|
xTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+
|
||||||
|
ZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr
|
||||||
|
Kl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh
|
||||||
|
AOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY
|
||||||
|
-----END RSA PRIVATE KEY-----
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/service-account.key
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: |
|
||||||
|
-----BEGIN RSA PUBLIC KEY-----
|
||||||
|
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm
|
||||||
|
XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ==
|
||||||
|
-----END RSA PUBLIC KEY-----
|
||||||
|
-----BEGIN RSA PUBLIC KEY-----
|
||||||
|
MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF
|
||||||
|
Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ==
|
||||||
|
-----END RSA PUBLIC KEY-----
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/kube-apiserver/service-account.pub
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: ""
|
||||||
|
ifNotExists: true
|
||||||
|
mode: "0400"
|
||||||
|
path: /var/log/kube-apiserver.log
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
Name: apiserver-aggregator
|
||||||
|
keypairID: ""
|
||||||
|
signer: apiserver-aggregator-ca
|
||||||
|
subject:
|
||||||
|
CommonName: aggregator
|
||||||
|
type: client
|
||||||
|
---
|
||||||
|
Name: etcd-client
|
||||||
|
keypairID: ""
|
||||||
|
signer: etcd-clients-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kube-apiserver
|
||||||
|
type: client
|
||||||
|
---
|
||||||
|
Name: kubelet-api
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubelet-api
|
||||||
|
type: client
|
||||||
|
---
|
||||||
|
Name: master
|
||||||
|
alternateNames:
|
||||||
|
- kubernetes
|
||||||
|
- kubernetes.default
|
||||||
|
- kubernetes.default.svc
|
||||||
|
- kubernetes.default.svc.cluster.local
|
||||||
|
- api.minimal.example.com
|
||||||
|
- api.internal.minimal.example.com
|
||||||
|
- 100.64.0.1
|
||||||
|
- 127.0.0.1
|
||||||
|
includeRootCertificate: true
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubernetes-master
|
||||||
|
type: server
|
|
@ -0,0 +1,331 @@
|
||||||
|
contents: |
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
creationTimestamp: null
|
||||||
|
labels:
|
||||||
|
k8s-app: kube-controller-manager
|
||||||
|
name: kube-controller-manager
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- args:
|
||||||
|
- --log-file=/var/log/kube-controller-manager.log
|
||||||
|
- --also-stdout
|
||||||
|
- /usr/local/bin/kube-controller-manager
|
||||||
|
- --allocate-node-cidrs=true
|
||||||
|
- --attach-detach-reconcile-sync-period=1m0s
|
||||||
|
- --authentication-kubeconfig=/var/lib/kube-controller-manager/kubeconfig
|
||||||
|
- --authorization-kubeconfig=/var/lib/kube-controller-manager/kubeconfig
|
||||||
|
- --cloud-config=/etc/kubernetes/in-tree-cloud.config
|
||||||
|
- --cloud-provider=external
|
||||||
|
- --cluster-cidr=100.96.0.0/11
|
||||||
|
- --cluster-name=minimal.example.com
|
||||||
|
- --cluster-signing-cert-file=/srv/kubernetes/kube-controller-manager/ca.crt
|
||||||
|
- --cluster-signing-key-file=/srv/kubernetes/kube-controller-manager/ca.key
|
||||||
|
- --configure-cloud-routes=true
|
||||||
|
- --feature-gates=InTreePluginAWSUnregister=true
|
||||||
|
- --flex-volume-plugin-dir=/usr/libexec/kubernetes/kubelet-plugins/volume/exec/
|
||||||
|
- --kubeconfig=/var/lib/kube-controller-manager/kubeconfig
|
||||||
|
- --leader-elect=true
|
||||||
|
- --root-ca-file=/srv/kubernetes/ca.crt
|
||||||
|
- --service-account-private-key-file=/srv/kubernetes/kube-controller-manager/service-account.key
|
||||||
|
- --tls-cert-file=/srv/kubernetes/kube-controller-manager/server.crt
|
||||||
|
- --tls-private-key-file=/srv/kubernetes/kube-controller-manager/server.key
|
||||||
|
- --use-service-account-credentials=true
|
||||||
|
- --v=2
|
||||||
|
command:
|
||||||
|
- /go-runner
|
||||||
|
image: registry.k8s.io/kube-controller-manager:v1.28.0
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
host: 127.0.0.1
|
||||||
|
path: /healthz
|
||||||
|
port: 10257
|
||||||
|
scheme: HTTPS
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
timeoutSeconds: 15
|
||||||
|
name: kube-controller-manager
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /var/log/kube-controller-manager.log
|
||||||
|
name: logfile
|
||||||
|
- mountPath: /etc/ssl
|
||||||
|
name: etcssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /etc/pki/tls
|
||||||
|
name: etcpkitls
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /etc/pki/ca-trust
|
||||||
|
name: etcpkica-trust
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /usr/share/ssl
|
||||||
|
name: usrsharessl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /usr/ssl
|
||||||
|
name: usrssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /usr/lib/ssl
|
||||||
|
name: usrlibssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /usr/local/openssl
|
||||||
|
name: usrlocalopenssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /var/ssl
|
||||||
|
name: varssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /etc/openssl
|
||||||
|
name: etcopenssl
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /etc/kubernetes/in-tree-cloud.config
|
||||||
|
name: cloudconfig
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /srv/kubernetes/ca.crt
|
||||||
|
name: cabundle
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /srv/kubernetes/kube-controller-manager
|
||||||
|
name: srvkcm
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /var/lib/kube-controller-manager
|
||||||
|
name: varlibkcm
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/
|
||||||
|
name: volplugins
|
||||||
|
hostNetwork: true
|
||||||
|
priorityClassName: system-cluster-critical
|
||||||
|
tolerations:
|
||||||
|
- key: CriticalAddonsOnly
|
||||||
|
operator: Exists
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /var/log/kube-controller-manager.log
|
||||||
|
name: logfile
|
||||||
|
- hostPath:
|
||||||
|
path: /etc/ssl
|
||||||
|
name: etcssl
|
||||||
|
- hostPath:
|
||||||
|
path: /etc/pki/tls
|
||||||
|
name: etcpkitls
|
||||||
|
- hostPath:
|
||||||
|
path: /etc/pki/ca-trust
|
||||||
|
name: etcpkica-trust
|
||||||
|
- hostPath:
|
||||||
|
path: /usr/share/ssl
|
||||||
|
name: usrsharessl
|
||||||
|
- hostPath:
|
||||||
|
path: /usr/ssl
|
||||||
|
name: usrssl
|
||||||
|
- hostPath:
|
||||||
|
path: /usr/lib/ssl
|
||||||
|
name: usrlibssl
|
||||||
|
- hostPath:
|
||||||
|
path: /usr/local/openssl
|
||||||
|
name: usrlocalopenssl
|
||||||
|
- hostPath:
|
||||||
|
path: /var/ssl
|
||||||
|
name: varssl
|
||||||
|
- hostPath:
|
||||||
|
path: /etc/openssl
|
||||||
|
name: etcopenssl
|
||||||
|
- hostPath:
|
||||||
|
path: /etc/kubernetes/in-tree-cloud.config
|
||||||
|
name: cloudconfig
|
||||||
|
- hostPath:
|
||||||
|
path: /srv/kubernetes/ca.crt
|
||||||
|
name: cabundle
|
||||||
|
- hostPath:
|
||||||
|
path: /srv/kubernetes/kube-controller-manager
|
||||||
|
name: srvkcm
|
||||||
|
- hostPath:
|
||||||
|
path: /var/lib/kube-controller-manager
|
||||||
|
name: varlibkcm
|
||||||
|
- hostPath:
|
||||||
|
path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/
|
||||||
|
name: volplugins
|
||||||
|
status: {}
|
||||||
|
path: /etc/kubernetes/manifests/kube-controller-manager.manifest
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
mode: "0755"
|
||||||
|
path: /srv/kubernetes/kube-controller-manager
|
||||||
|
type: directory
|
||||||
|
---
|
||||||
|
contents: |
|
||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw
|
||||||
|
FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy
|
||||||
|
MzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA
|
||||||
|
A4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd
|
||||||
|
XEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY
|
||||||
|
OACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1
|
||||||
|
YYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC
|
||||||
|
An8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA
|
||||||
|
NPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB
|
||||||
|
/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r
|
||||||
|
hzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO
|
||||||
|
HE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe
|
||||||
|
rN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt
|
||||||
|
O6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy
|
||||||
|
VpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh
|
||||||
|
8yGeRx9AbknHh4Ia
|
||||||
|
-----END CERTIFICATE-----
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/kube-controller-manager/ca.crt
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: |
|
||||||
|
-----BEGIN RSA PRIVATE KEY-----
|
||||||
|
MIIEpAIBAAKCAQEA4JwpEprZ5n8RIEt6jT2lAh+UDgRgx/4px21gjgywQivYHVxH
|
||||||
|
AZexVb/E9pBa9Q2G9B1Q7TCO7YsUVRQy4JMDZVt+McFnWVwexnqBYFNcVjkEmDgA
|
||||||
|
gvCYGE0P9d/RwRL4KuLHo+u6fv7P0jXMN+CpOxyLhYZZNa0ZOZDHsSiJSQSj9WGF
|
||||||
|
GHrbCf0KVDpKieR1uBqHrRO+mLR5zkX2L58m74kjK4dsBhmjeq/7OAoTmiG2QgJ/
|
||||||
|
P2IjyhiA2mRqY+hl55lwEUV/0yHYEkJC8LdGkwwZz2eF77aSPGmi/A2CSKgMwDTx
|
||||||
|
9m+P7jcpWreYw6NG9BueGoDIve/tgFKwvVFF6QIDAQABAoIBAA0ktjaTfyrAxsTI
|
||||||
|
Bezb7Zr5NBW55dvuII299cd6MJo+rI/TRYhvUv48kY8IFXp/hyUjzgeDLunxmIf9
|
||||||
|
/Zgsoic9Ol44/g45mMduhcGYPzAAeCdcJ5OB9rR9VfDCXyjYLlN8H8iU0734tTqM
|
||||||
|
0V13tQ9zdSqkGPZOIcq/kR/pylbOZaQMe97BTlsAnOMSMKDgnftY4122Lq3GYy+t
|
||||||
|
vpr+bKVaQZwvkLoSU3rECCaKaghgwCyX7jft9aEkhdJv+KlwbsGY6WErvxOaLWHd
|
||||||
|
cuMQjGapY1Fa/4UD00mvrA260NyKfzrp6+P46RrVMwEYRJMIQ8YBAk6N6Hh7dc0G
|
||||||
|
8Z6i1m0CgYEA9HeCJR0TSwbIQ1bDXUrzpftHuidG5BnSBtax/ND9qIPhR/FBW5nj
|
||||||
|
22nwLc48KkyirlfIULd0ae4qVXJn7wfYcuX/cJMLDmSVtlM5Dzmi/91xRiFgIzx1
|
||||||
|
AsbBzaFjISP2HpSgL+e9FtSXaaqeZVrflitVhYKUpI/AKV31qGHf04sCgYEA6zTV
|
||||||
|
99Sb49Wdlns5IgsfnXl6ToRttB18lfEKcVfjAM4frnkk06JpFAZeR+9GGKUXZHqs
|
||||||
|
z2qcplw4d/moCC6p3rYPBMLXsrGNEUFZqBlgz72QA6BBq3X0Cg1Bc2ZbK5VIzwkg
|
||||||
|
ST2SSux6ccROfgULmN5ZiLOtdUKNEZpFF3i3qtsCgYADT/s7dYFlatobz3kmMnXK
|
||||||
|
sfTu2MllHdRys0YGHu7Q8biDuQkhrJwhxPW0KS83g4JQym+0aEfzh36bWcl+u6R7
|
||||||
|
KhKj+9oSf9pndgk345gJz35RbPJYh+EuAHNvzdgCAvK6x1jETWeKf6btj5pF1U1i
|
||||||
|
Q4QNIw/QiwIXjWZeubTGsQKBgQCbduLu2rLnlyyAaJZM8DlHZyH2gAXbBZpxqU8T
|
||||||
|
t9mtkJDUS/KRiEoYGFV9CqS0aXrayVMsDfXY6B/S/UuZjO5u7LtklDzqOf1aKG3Q
|
||||||
|
dGXPKibknqqJYH+bnUNjuYYNerETV57lijMGHuSYCf8vwLn3oxBfERRX61M/DU8Z
|
||||||
|
worz/QKBgQDCTJI2+jdXg26XuYUmM4XXfnocfzAXhXBULt1nENcogNf1fcptAVtu
|
||||||
|
BAiz4/HipQKqoWVUYmxfgbbLRKKLK0s0lOWKbYdVjhEm/m2ZU8wtXTagNwkIGoyq
|
||||||
|
Y/C1Lox4f1ROJnCjc/hfcOjcxX5M8A8peecHWlVtUPKTJgxQ7oMKcw==
|
||||||
|
-----END RSA PRIVATE KEY-----
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/kube-controller-manager/ca.key
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: kube-controller-manager-server
|
||||||
|
alternateNames:
|
||||||
|
- kube-controller-manager.kube-system.svc.cluster.local
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kube-controller-manager
|
||||||
|
type: server
|
||||||
|
mode: "0644"
|
||||||
|
path: /srv/kubernetes/kube-controller-manager/server.crt
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: kube-controller-manager-server
|
||||||
|
alternateNames:
|
||||||
|
- kube-controller-manager.kube-system.svc.cluster.local
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kube-controller-manager
|
||||||
|
type: server
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/kube-controller-manager/server.key
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: |
|
||||||
|
-----BEGIN RSA PRIVATE KEY-----
|
||||||
|
MIIBPQIBAAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKmXVSysPKgE80QSU4tZ6m4
|
||||||
|
9pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQJBAKt/gmpHqP3qA3u8RA5R
|
||||||
|
2W6L360Z2Mnza1FmkI/9StCCkJGjuE5yDhxU4JcVnFyX/nMxm2ockEEQDqRSu7Oo
|
||||||
|
xTECIQD2QsUsgFL4FnXWzTclySJ6ajE4Cte3gSDOIvyMNMireQIhAOEnsV8UaSI+
|
||||||
|
ZyL7NMLzMPLCgtsrPnlamr8gdrEHf9ITAiEAxCCLbpTI/4LL2QZZrINTLVGT34Fr
|
||||||
|
Kl/yI5pjrrp/M2kCIQDfOktQyRuzJ8t5kzWsUxCkntS+FxHJn1rtQ3Jp8dV4oQIh
|
||||||
|
AOyiVWDyLZJvg7Y24Ycmp86BZjM9Wk/BfWpBXKnl9iDY
|
||||||
|
-----END RSA PRIVATE KEY-----
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/kube-controller-manager/service-account.key
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
CA:
|
||||||
|
task:
|
||||||
|
Name: kube-controller-manager
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-controller-manager
|
||||||
|
type: client
|
||||||
|
Cert:
|
||||||
|
task:
|
||||||
|
Name: kube-controller-manager
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-controller-manager
|
||||||
|
type: client
|
||||||
|
Key:
|
||||||
|
task:
|
||||||
|
Name: kube-controller-manager
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-controller-manager
|
||||||
|
type: client
|
||||||
|
Name: kube-controller-manager
|
||||||
|
ServerURL: https://127.0.0.1
|
||||||
|
mode: "0400"
|
||||||
|
path: /var/lib/kube-controller-manager/kubeconfig
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: ""
|
||||||
|
ifNotExists: true
|
||||||
|
mode: "0400"
|
||||||
|
path: /var/log/kube-controller-manager.log
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
Name: kube-controller-manager
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-controller-manager
|
||||||
|
type: client
|
||||||
|
---
|
||||||
|
Name: kube-controller-manager-server
|
||||||
|
alternateNames:
|
||||||
|
- kube-controller-manager.kube-system.svc.cluster.local
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kube-controller-manager
|
||||||
|
type: server
|
||||||
|
---
|
||||||
|
CA:
|
||||||
|
task:
|
||||||
|
Name: kube-controller-manager
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-controller-manager
|
||||||
|
type: client
|
||||||
|
Cert:
|
||||||
|
task:
|
||||||
|
Name: kube-controller-manager
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-controller-manager
|
||||||
|
type: client
|
||||||
|
Key:
|
||||||
|
task:
|
||||||
|
Name: kube-controller-manager
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-controller-manager
|
||||||
|
type: client
|
||||||
|
Name: kube-controller-manager
|
||||||
|
ServerURL: https://127.0.0.1
|
|
@ -0,0 +1,145 @@
|
||||||
|
contents: |
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
creationTimestamp: null
|
||||||
|
labels:
|
||||||
|
k8s-app: kube-proxy
|
||||||
|
kubernetes.io/managed-by: nodeup
|
||||||
|
tier: node
|
||||||
|
name: kube-proxy
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- args:
|
||||||
|
- --log-file=/var/log/kube-proxy.log
|
||||||
|
- --also-stdout
|
||||||
|
- /usr/local/bin/kube-proxy
|
||||||
|
- --cluster-cidr=100.96.0.0/11
|
||||||
|
- --conntrack-max-per-core=131072
|
||||||
|
- --kubeconfig=/var/lib/kube-proxy/kubeconfig
|
||||||
|
- --master=https://127.0.0.1
|
||||||
|
- --oom-score-adj=-998
|
||||||
|
- --v=2
|
||||||
|
command:
|
||||||
|
- /go-runner
|
||||||
|
image: registry.k8s.io/kube-proxy:v1.28.0
|
||||||
|
name: kube-proxy
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /var/log/kube-proxy.log
|
||||||
|
name: logfile
|
||||||
|
- mountPath: /var/lib/kube-proxy/kubeconfig
|
||||||
|
name: kubeconfig
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /lib/modules
|
||||||
|
name: modules
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /etc/ssl/certs
|
||||||
|
name: ssl-certs-hosts
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /run/xtables.lock
|
||||||
|
name: iptableslock
|
||||||
|
hostNetwork: true
|
||||||
|
priorityClassName: system-node-critical
|
||||||
|
tolerations:
|
||||||
|
- key: CriticalAddonsOnly
|
||||||
|
operator: Exists
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /var/log/kube-proxy.log
|
||||||
|
name: logfile
|
||||||
|
- hostPath:
|
||||||
|
path: /var/lib/kube-proxy/kubeconfig
|
||||||
|
name: kubeconfig
|
||||||
|
- hostPath:
|
||||||
|
path: /lib/modules
|
||||||
|
name: modules
|
||||||
|
- hostPath:
|
||||||
|
path: /usr/share/ca-certificates
|
||||||
|
name: ssl-certs-hosts
|
||||||
|
- hostPath:
|
||||||
|
path: /run/xtables.lock
|
||||||
|
type: FileOrCreate
|
||||||
|
name: iptableslock
|
||||||
|
status: {}
|
||||||
|
path: /etc/kubernetes/manifests/kube-proxy.manifest
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
beforeServices:
|
||||||
|
- kubelet.service
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
CA:
|
||||||
|
task:
|
||||||
|
Name: kube-proxy
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-proxy
|
||||||
|
type: client
|
||||||
|
Cert:
|
||||||
|
task:
|
||||||
|
Name: kube-proxy
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-proxy
|
||||||
|
type: client
|
||||||
|
Key:
|
||||||
|
task:
|
||||||
|
Name: kube-proxy
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-proxy
|
||||||
|
type: client
|
||||||
|
Name: kube-proxy
|
||||||
|
ServerURL: https://127.0.0.1
|
||||||
|
mode: "0400"
|
||||||
|
path: /var/lib/kube-proxy/kubeconfig
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: ""
|
||||||
|
ifNotExists: true
|
||||||
|
mode: "0400"
|
||||||
|
path: /var/log/kube-proxy.log
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
Name: kube-proxy
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-proxy
|
||||||
|
type: client
|
||||||
|
---
|
||||||
|
CA:
|
||||||
|
task:
|
||||||
|
Name: kube-proxy
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-proxy
|
||||||
|
type: client
|
||||||
|
Cert:
|
||||||
|
task:
|
||||||
|
Name: kube-proxy
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-proxy
|
||||||
|
type: client
|
||||||
|
Key:
|
||||||
|
task:
|
||||||
|
Name: kube-proxy
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-proxy
|
||||||
|
type: client
|
||||||
|
Name: kube-proxy
|
||||||
|
ServerURL: https://127.0.0.1
|
|
@ -0,0 +1,187 @@
|
||||||
|
contents: |
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
creationTimestamp: null
|
||||||
|
labels:
|
||||||
|
k8s-app: kube-scheduler
|
||||||
|
name: kube-scheduler
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- args:
|
||||||
|
- --log-file=/var/log/kube-scheduler.log
|
||||||
|
- --also-stdout
|
||||||
|
- /usr/local/bin/kube-scheduler
|
||||||
|
- --authentication-kubeconfig=/var/lib/kube-scheduler/kubeconfig
|
||||||
|
- --authorization-kubeconfig=/var/lib/kube-scheduler/kubeconfig
|
||||||
|
- --config=/var/lib/kube-scheduler/config.yaml
|
||||||
|
- --feature-gates=InTreePluginAWSUnregister=true
|
||||||
|
- --leader-elect=true
|
||||||
|
- --tls-cert-file=/srv/kubernetes/kube-scheduler/server.crt
|
||||||
|
- --tls-private-key-file=/srv/kubernetes/kube-scheduler/server.key
|
||||||
|
- --v=2
|
||||||
|
command:
|
||||||
|
- /go-runner
|
||||||
|
image: registry.k8s.io/kube-scheduler:v1.28.0
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
host: 127.0.0.1
|
||||||
|
path: /healthz
|
||||||
|
port: 10259
|
||||||
|
scheme: HTTPS
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
timeoutSeconds: 15
|
||||||
|
name: kube-scheduler
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
cpu: 100m
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /var/lib/kube-scheduler
|
||||||
|
name: varlibkubescheduler
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /srv/kubernetes/kube-scheduler
|
||||||
|
name: srvscheduler
|
||||||
|
readOnly: true
|
||||||
|
- mountPath: /var/log/kube-scheduler.log
|
||||||
|
name: logfile
|
||||||
|
hostNetwork: true
|
||||||
|
priorityClassName: system-cluster-critical
|
||||||
|
tolerations:
|
||||||
|
- key: CriticalAddonsOnly
|
||||||
|
operator: Exists
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /var/lib/kube-scheduler
|
||||||
|
name: varlibkubescheduler
|
||||||
|
- hostPath:
|
||||||
|
path: /srv/kubernetes/kube-scheduler
|
||||||
|
name: srvscheduler
|
||||||
|
- hostPath:
|
||||||
|
path: /var/log/kube-scheduler.log
|
||||||
|
name: logfile
|
||||||
|
status: {}
|
||||||
|
path: /etc/kubernetes/manifests/kube-scheduler.manifest
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
mode: "0755"
|
||||||
|
path: /srv/kubernetes/kube-scheduler
|
||||||
|
type: directory
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: kube-scheduler-server
|
||||||
|
alternateNames:
|
||||||
|
- kube-scheduler.kube-system.svc.cluster.local
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kube-scheduler
|
||||||
|
type: server
|
||||||
|
mode: "0644"
|
||||||
|
path: /srv/kubernetes/kube-scheduler/server.crt
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
Name: kube-scheduler-server
|
||||||
|
alternateNames:
|
||||||
|
- kube-scheduler.kube-system.svc.cluster.local
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kube-scheduler
|
||||||
|
type: server
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/kube-scheduler/server.key
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: |
|
||||||
|
apiVersion: kubescheduler.config.k8s.io/v1
|
||||||
|
clientConnection:
|
||||||
|
kubeconfig: /var/lib/kube-scheduler/kubeconfig
|
||||||
|
kind: KubeSchedulerConfiguration
|
||||||
|
mode: "0400"
|
||||||
|
path: /var/lib/kube-scheduler/config.yaml
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
CA:
|
||||||
|
task:
|
||||||
|
Name: kube-scheduler
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-scheduler
|
||||||
|
type: client
|
||||||
|
Cert:
|
||||||
|
task:
|
||||||
|
Name: kube-scheduler
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-scheduler
|
||||||
|
type: client
|
||||||
|
Key:
|
||||||
|
task:
|
||||||
|
Name: kube-scheduler
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-scheduler
|
||||||
|
type: client
|
||||||
|
Name: kube-scheduler
|
||||||
|
ServerURL: https://127.0.0.1
|
||||||
|
mode: "0400"
|
||||||
|
path: /var/lib/kube-scheduler/kubeconfig
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents: ""
|
||||||
|
ifNotExists: true
|
||||||
|
mode: "0400"
|
||||||
|
path: /var/log/kube-scheduler.log
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
Name: kube-scheduler
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-scheduler
|
||||||
|
type: client
|
||||||
|
---
|
||||||
|
Name: kube-scheduler-server
|
||||||
|
alternateNames:
|
||||||
|
- kube-scheduler.kube-system.svc.cluster.local
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kube-scheduler
|
||||||
|
type: server
|
||||||
|
---
|
||||||
|
CA:
|
||||||
|
task:
|
||||||
|
Name: kube-scheduler
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-scheduler
|
||||||
|
type: client
|
||||||
|
Cert:
|
||||||
|
task:
|
||||||
|
Name: kube-scheduler
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-scheduler
|
||||||
|
type: client
|
||||||
|
Key:
|
||||||
|
task:
|
||||||
|
Name: kube-scheduler
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: system:kube-scheduler
|
||||||
|
type: client
|
||||||
|
Name: kube-scheduler
|
||||||
|
ServerURL: https://127.0.0.1
|
|
@ -0,0 +1,87 @@
|
||||||
|
contents:
|
||||||
|
Asset:
|
||||||
|
AssetPath: /path/to/kubectl/asset
|
||||||
|
Key: kubectl
|
||||||
|
mode: "0755"
|
||||||
|
path: /opt/kops/bin/kubectl
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
contents:
|
||||||
|
task:
|
||||||
|
CA:
|
||||||
|
task:
|
||||||
|
Name: kubecfg
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubecfg
|
||||||
|
Organization:
|
||||||
|
- system:masters
|
||||||
|
type: client
|
||||||
|
Cert:
|
||||||
|
task:
|
||||||
|
Name: kubecfg
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubecfg
|
||||||
|
Organization:
|
||||||
|
- system:masters
|
||||||
|
type: client
|
||||||
|
Key:
|
||||||
|
task:
|
||||||
|
Name: kubecfg
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubecfg
|
||||||
|
Organization:
|
||||||
|
- system:masters
|
||||||
|
type: client
|
||||||
|
Name: kubecfg
|
||||||
|
ServerURL: https://127.0.0.1
|
||||||
|
mode: "0400"
|
||||||
|
path: /var/lib/kubectl/kubeconfig
|
||||||
|
type: file
|
||||||
|
---
|
||||||
|
Name: kubecfg
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubecfg
|
||||||
|
Organization:
|
||||||
|
- system:masters
|
||||||
|
type: client
|
||||||
|
---
|
||||||
|
CA:
|
||||||
|
task:
|
||||||
|
Name: kubecfg
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubecfg
|
||||||
|
Organization:
|
||||||
|
- system:masters
|
||||||
|
type: client
|
||||||
|
Cert:
|
||||||
|
task:
|
||||||
|
Name: kubecfg
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubecfg
|
||||||
|
Organization:
|
||||||
|
- system:masters
|
||||||
|
type: client
|
||||||
|
Key:
|
||||||
|
task:
|
||||||
|
Name: kubecfg
|
||||||
|
keypairID: "3"
|
||||||
|
signer: kubernetes-ca
|
||||||
|
subject:
|
||||||
|
CommonName: kubecfg
|
||||||
|
Organization:
|
||||||
|
- system:masters
|
||||||
|
type: client
|
||||||
|
Name: kubecfg
|
||||||
|
ServerURL: https://127.0.0.1
|
|
@ -0,0 +1,32 @@
|
||||||
|
contents: |
|
||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
MIIC2DCCAcCgAwIBAgIRALJXAkVj964tq67wMSI8oJQwDQYJKoZIhvcNAQELBQAw
|
||||||
|
FTETMBEGA1UEAxMKa3ViZXJuZXRlczAeFw0xNzEyMjcyMzUyNDBaFw0yNzEyMjcy
|
||||||
|
MzUyNDBaMBUxEzARBgNVBAMTCmt1YmVybmV0ZXMwggEiMA0GCSqGSIb3DQEBAQUA
|
||||||
|
A4IBDwAwggEKAoIBAQDgnCkSmtnmfxEgS3qNPaUCH5QOBGDH/inHbWCODLBCK9gd
|
||||||
|
XEcBl7FVv8T2kFr1DYb0HVDtMI7tixRVFDLgkwNlW34xwWdZXB7GeoFgU1xWOQSY
|
||||||
|
OACC8JgYTQ/139HBEvgq4sej67p+/s/SNcw34Kk7HIuFhlk1rRk5kMexKIlJBKP1
|
||||||
|
YYUYetsJ/QpUOkqJ5HW4GoetE76YtHnORfYvnybviSMrh2wGGaN6r/s4ChOaIbZC
|
||||||
|
An8/YiPKGIDaZGpj6GXnmXARRX/TIdgSQkLwt0aTDBnPZ4XvtpI8aaL8DYJIqAzA
|
||||||
|
NPH2b4/uNylat5jDo0b0G54agMi97+2AUrC9UUXpAgMBAAGjIzAhMA4GA1UdDwEB
|
||||||
|
/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBVGR2r
|
||||||
|
hzXzRMU5wriPQAJScszNORvoBpXfZoZ09FIupudFxBVU3d4hV9StKnQgPSGA5XQO
|
||||||
|
HE97+BxJDuA/rB5oBUsMBjc7y1cde/T6hmi3rLoEYBSnSudCOXJE4G9/0f8byAJe
|
||||||
|
rN8+No1r2VgZvZh6p74TEkXv/l3HBPWM7IdUV0HO9JDhSgOVF1fyQKJxRuLJR8jt
|
||||||
|
O6mPH2UX0vMwVa4jvwtkddqk2OAdYQvH9rbDjjbzaiW0KnmdueRo92KHAN7BsDZy
|
||||||
|
VpXHpqo1Kzg7D3fpaXCf5si7lqqrdJVXH4JC72zxsPehqgi8eIuqOBkiDWmRxAxh
|
||||||
|
8yGeRx9AbknHh4Ia
|
||||||
|
-----END CERTIFICATE-----
|
||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
MIIBZzCCARGgAwIBAgIBBDANBgkqhkiG9w0BAQsFADAaMRgwFgYDVQQDEw9zZXJ2
|
||||||
|
aWNlLWFjY291bnQwHhcNMjEwNTAyMjAzMjE3WhcNMzEwNTAyMjAzMjE3WjAaMRgw
|
||||||
|
FgYDVQQDEw9zZXJ2aWNlLWFjY291bnQwXDANBgkqhkiG9w0BAQEFAANLADBIAkEA
|
||||||
|
o4Tridlsf4Yz3UAiup/scSTiG/OqxkUW3Fz7zGKvVcLeYj9GEIKuzoB1VFk1nboD
|
||||||
|
q4cCuGLfdzaQdCQKPIsDuwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T
|
||||||
|
AQH/BAUwAwEB/zAdBgNVHQ4EFgQUhPbxEmUbwVOCa+fZgxreFhf67UEwDQYJKoZI
|
||||||
|
hvcNAQELBQADQQALMsyK2Q7C/bk27eCvXyZKUfrLvor10hEjwGhv14zsKWDeTj/J
|
||||||
|
A1LPYp7U9VtFfgFOkVbkLE9Rstc0ltNrPqxA
|
||||||
|
-----END CERTIFICATE-----
|
||||||
|
mode: "0600"
|
||||||
|
path: /srv/kubernetes/ca.crt
|
||||||
|
type: file
|
|
@ -17,6 +17,7 @@ limitations under the License.
|
||||||
package kops
|
package kops
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
"k8s.io/apimachinery/pkg/api/resource"
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
)
|
)
|
||||||
|
@ -543,6 +544,11 @@ type KubeAPIServerConfig struct {
|
||||||
DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"`
|
DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"`
|
||||||
// DefaultUnreachableTolerationSeconds indicates the tolerationSeconds of the toleration for unreachable:NoExecute that is added by default to every pod that does not already have such a toleration.
|
// DefaultUnreachableTolerationSeconds indicates the tolerationSeconds of the toleration for unreachable:NoExecute that is added by default to every pod that does not already have such a toleration.
|
||||||
DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"`
|
DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"`
|
||||||
|
|
||||||
|
// Env allows users to pass in env variables to the apiserver container.
|
||||||
|
// This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver
|
||||||
|
// This also allows the flexibility for adding any other variables for future use cases
|
||||||
|
Env []corev1.EnvVar `json:"env,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// KubeControllerManagerConfig is the configuration for the controller
|
// KubeControllerManagerConfig is the configuration for the controller
|
||||||
|
|
|
@ -17,6 +17,7 @@ limitations under the License.
|
||||||
package v1alpha2
|
package v1alpha2
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
"k8s.io/apimachinery/pkg/api/resource"
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
)
|
)
|
||||||
|
@ -550,6 +551,11 @@ type KubeAPIServerConfig struct {
|
||||||
DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"`
|
DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"`
|
||||||
// DefaultUnreachableTolerationSeconds
|
// DefaultUnreachableTolerationSeconds
|
||||||
DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"`
|
DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"`
|
||||||
|
|
||||||
|
// Env allows users to pass in env variables to the apiserver container.
|
||||||
|
// This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver
|
||||||
|
// This also allows the flexibility for adding any other variables for future use cases
|
||||||
|
Env []corev1.EnvVar `json:"env,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// KubeControllerManagerConfig is the configuration for the controller
|
// KubeControllerManagerConfig is the configuration for the controller
|
||||||
|
|
|
@ -4972,6 +4972,7 @@ func autoConvert_v1alpha2_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku
|
||||||
out.CorsAllowedOrigins = in.CorsAllowedOrigins
|
out.CorsAllowedOrigins = in.CorsAllowedOrigins
|
||||||
out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
|
out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
|
||||||
out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
|
out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
|
||||||
|
out.Env = in.Env
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5087,6 +5088,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha2_KubeAPIServerConfig(in *ko
|
||||||
out.CorsAllowedOrigins = in.CorsAllowedOrigins
|
out.CorsAllowedOrigins = in.CorsAllowedOrigins
|
||||||
out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
|
out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
|
||||||
out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
|
out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
|
||||||
|
out.Env = in.Env
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3415,6 +3415,13 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) {
|
||||||
*out = new(int64)
|
*out = new(int64)
|
||||||
**out = **in
|
**out = **in
|
||||||
}
|
}
|
||||||
|
if in.Env != nil {
|
||||||
|
in, out := &in.Env, &out.Env
|
||||||
|
*out = make([]corev1.EnvVar, len(*in))
|
||||||
|
for i := range *in {
|
||||||
|
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -36,6 +36,23 @@ func RegisterDefaults(scheme *runtime.Scheme) error {
|
||||||
|
|
||||||
func SetObjectDefaults_Cluster(in *Cluster) {
|
func SetObjectDefaults_Cluster(in *Cluster) {
|
||||||
SetDefaults_ClusterSpec(&in.Spec)
|
SetDefaults_ClusterSpec(&in.Spec)
|
||||||
|
if in.Spec.KubeAPIServer != nil {
|
||||||
|
for i := range in.Spec.KubeAPIServer.Env {
|
||||||
|
a := &in.Spec.KubeAPIServer.Env[i]
|
||||||
|
if a.ValueFrom != nil {
|
||||||
|
if a.ValueFrom.ConfigMapKeyRef != nil {
|
||||||
|
if a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name == "" {
|
||||||
|
a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if a.ValueFrom.SecretKeyRef != nil {
|
||||||
|
if a.ValueFrom.SecretKeyRef.LocalObjectReference.Name == "" {
|
||||||
|
a.ValueFrom.SecretKeyRef.LocalObjectReference.Name = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func SetObjectDefaults_ClusterList(in *ClusterList) {
|
func SetObjectDefaults_ClusterList(in *ClusterList) {
|
||||||
|
|
|
@ -17,6 +17,7 @@ limitations under the License.
|
||||||
package v1alpha3
|
package v1alpha3
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
"k8s.io/apimachinery/pkg/api/resource"
|
"k8s.io/apimachinery/pkg/api/resource"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
)
|
)
|
||||||
|
@ -541,6 +542,11 @@ type KubeAPIServerConfig struct {
|
||||||
DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"`
|
DefaultNotReadyTolerationSeconds *int64 `json:"defaultNotReadyTolerationSeconds,omitempty" flag:"default-not-ready-toleration-seconds"`
|
||||||
// DefaultUnreachableTolerationSeconds
|
// DefaultUnreachableTolerationSeconds
|
||||||
DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"`
|
DefaultUnreachableTolerationSeconds *int64 `json:"defaultUnreachableTolerationSeconds,omitempty" flag:"default-unreachable-toleration-seconds"`
|
||||||
|
|
||||||
|
// Env allows users to pass in env variables to the apiserver container.
|
||||||
|
// This can be useful to control some environment runtime settings, such as GOMEMLIMIT and GOCG to tweak the memory settings of the apiserver
|
||||||
|
// This also allows the flexibility for adding any other variables for future use cases
|
||||||
|
Env []corev1.EnvVar `json:"env,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// KubeControllerManagerConfig is the configuration for the controller
|
// KubeControllerManagerConfig is the configuration for the controller
|
||||||
|
|
|
@ -5368,6 +5368,7 @@ func autoConvert_v1alpha3_KubeAPIServerConfig_To_kops_KubeAPIServerConfig(in *Ku
|
||||||
out.CorsAllowedOrigins = in.CorsAllowedOrigins
|
out.CorsAllowedOrigins = in.CorsAllowedOrigins
|
||||||
out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
|
out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
|
||||||
out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
|
out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
|
||||||
|
out.Env = in.Env
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5483,6 +5484,7 @@ func autoConvert_kops_KubeAPIServerConfig_To_v1alpha3_KubeAPIServerConfig(in *ko
|
||||||
out.CorsAllowedOrigins = in.CorsAllowedOrigins
|
out.CorsAllowedOrigins = in.CorsAllowedOrigins
|
||||||
out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
|
out.DefaultNotReadyTolerationSeconds = in.DefaultNotReadyTolerationSeconds
|
||||||
out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
|
out.DefaultUnreachableTolerationSeconds = in.DefaultUnreachableTolerationSeconds
|
||||||
|
out.Env = in.Env
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3389,6 +3389,13 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) {
|
||||||
*out = new(int64)
|
*out = new(int64)
|
||||||
**out = **in
|
**out = **in
|
||||||
}
|
}
|
||||||
|
if in.Env != nil {
|
||||||
|
in, out := &in.Env, &out.Env
|
||||||
|
*out = make([]corev1.EnvVar, len(*in))
|
||||||
|
for i := range *in {
|
||||||
|
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -36,6 +36,23 @@ func RegisterDefaults(scheme *runtime.Scheme) error {
|
||||||
|
|
||||||
func SetObjectDefaults_Cluster(in *Cluster) {
|
func SetObjectDefaults_Cluster(in *Cluster) {
|
||||||
SetDefaults_ClusterSpec(&in.Spec)
|
SetDefaults_ClusterSpec(&in.Spec)
|
||||||
|
if in.Spec.KubeAPIServer != nil {
|
||||||
|
for i := range in.Spec.KubeAPIServer.Env {
|
||||||
|
a := &in.Spec.KubeAPIServer.Env[i]
|
||||||
|
if a.ValueFrom != nil {
|
||||||
|
if a.ValueFrom.ConfigMapKeyRef != nil {
|
||||||
|
if a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name == "" {
|
||||||
|
a.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if a.ValueFrom.SecretKeyRef != nil {
|
||||||
|
if a.ValueFrom.SecretKeyRef.LocalObjectReference.Name == "" {
|
||||||
|
a.ValueFrom.SecretKeyRef.LocalObjectReference.Name = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func SetObjectDefaults_ClusterList(in *ClusterList) {
|
func SetObjectDefaults_ClusterList(in *ClusterList) {
|
||||||
|
|
|
@ -3492,6 +3492,13 @@ func (in *KubeAPIServerConfig) DeepCopyInto(out *KubeAPIServerConfig) {
|
||||||
*out = new(int64)
|
*out = new(int64)
|
||||||
**out = **in
|
**out = **in
|
||||||
}
|
}
|
||||||
|
if in.Env != nil {
|
||||||
|
in, out := &in.Env, &out.Env
|
||||||
|
*out = make([]corev1.EnvVar, len(*in))
|
||||||
|
for i := range *in {
|
||||||
|
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue