karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Standardize the generation and management of certificates 

Signed-off-by: lonelyCZ <531187475@qq.com>
This commit is contained in:
lonelyCZ 2022-08-02 10:18:18 +08:00
parent 4224d90bfb
commit 3e89d68a23
11 changed files with 101 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
artifacts/deploy/karmada-aggregated-apiserver.yaml
View File

@ -25,8 +25,8 @@ spec:
image: swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-aggregated-apiserver:latest
imagePullPolicy: IfNotPresent
volumeMounts:
- name: k8s-certs
mountPath: /etc/kubernetes/pki
- name: karmada-certs
mountPath: /etc/karmada/pki
readOnly: true
- name: kubeconfig
subPath: kubeconfig
@ -37,11 +37,11 @@ spec:
- --authentication-kubeconfig=/etc/kubeconfig
- --authorization-kubeconfig=/etc/kubeconfig
- --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379
- --etcd-cafile=/etc/kubernetes/pki/server-ca.crt
- --etcd-certfile=/etc/kubernetes/pki/karmada.crt
- --etcd-keyfile=/etc/kubernetes/pki/karmada.key
- --tls-cert-file=/etc/kubernetes/pki/karmada.crt
- --tls-private-key-file=/etc/kubernetes/pki/karmada.key
- --etcd-cafile=/etc/karmada/pki/etcd-ca.crt
- --etcd-certfile=/etc/karmada/pki/etcd-client.crt
- --etcd-keyfile=/etc/karmada/pki/etcd-client.key
- --tls-cert-file=/etc/karmada/pki/karmada.crt
- --tls-private-key-file=/etc/karmada/pki/karmada.key
- --audit-log-path=-
- --feature-gates=APIPriorityAndFairness=false
- --audit-log-maxage=0
@ -58,7 +58,7 @@ spec:
periodSeconds: 3
timeoutSeconds: 15
volumes:
- name: k8s-certs
- name: karmada-certs
secret:
secretName: karmada-cert-secret
- name: kubeconfig

32
artifacts/deploy/karmada-apiserver.yaml
View File

@ -35,33 +35,33 @@ spec:
- kube-apiserver
- --allow-privileged=true
- --authorization-mode=Node,RBAC
- --client-ca-file=/etc/kubernetes/pki/server-ca.crt
- --client-ca-file=/etc/karmada/pki/ca.crt
- --enable-admission-plugins=NodeRestriction
- --enable-bootstrap-token-auth=true
- --etcd-cafile=/etc/kubernetes/pki/server-ca.crt
- --etcd-certfile=/etc/kubernetes/pki/karmada.crt
- --etcd-keyfile=/etc/kubernetes/pki/karmada.key
- --etcd-cafile=/etc/karmada/pki/etcd-ca.crt
- --etcd-certfile=/etc/karmada/pki/etcd-client.crt
- --etcd-keyfile=/etc/karmada/pki/etcd-client.key
- --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379
- --bind-address=0.0.0.0
- --kubelet-client-certificate=/etc/kubernetes/pki/karmada.crt
- --kubelet-client-key=/etc/kubernetes/pki/karmada.key
- --kubelet-client-certificate=/etc/karmada/pki/karmada.crt
- --kubelet-client-key=/etc/karmada/pki/karmada.key
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --disable-admission-plugins=StorageObjectInUseProtection,ServiceAccount
- --runtime-config=
- --secure-port=5443
- --service-account-issuer=https://kubernetes.default.svc.cluster.local
- --service-account-key-file=/etc/kubernetes/pki/karmada.key
- --service-account-signing-key-file=/etc/kubernetes/pki/karmada.key
- --service-account-key-file=/etc/karmada/pki/karmada.key
- --service-account-signing-key-file=/etc/karmada/pki/karmada.key
- --service-cluster-ip-range=10.96.0.0/12
- --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
- --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
- --proxy-client-cert-file=/etc/karmada/pki/front-proxy-client.crt
- --proxy-client-key-file=/etc/karmada/pki/front-proxy-client.key
- --requestheader-allowed-names=front-proxy-client
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --requestheader-client-ca-file=/etc/karmada/pki/front-proxy-ca.crt
- --requestheader-extra-headers-prefix=X-Remote-Extra-
- --requestheader-group-headers=X-Remote-Group
- --requestheader-username-headers=X-Remote-User
- --tls-cert-file=/etc/kubernetes/pki/karmada.crt
- --tls-private-key-file=/etc/kubernetes/pki/karmada.key
- --tls-cert-file=/etc/karmada/pki/apiserver.crt
- --tls-private-key-file=/etc/karmada/pki/apiserver.key
name: karmada-apiserver
image: k8s.gcr.io/kube-apiserver:v1.24.2
imagePullPolicy: IfNotPresent
@ -90,8 +90,8 @@ spec:
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
- mountPath: /etc/karmada/pki
name: karmada-certs
readOnly: true
dnsPolicy: ClusterFirstWithHostNet
enableServiceLinks: true
@ -107,7 +107,7 @@ spec:
- effect: NoExecute
operator: Exists
volumes:
- name: k8s-certs
- name: karmada-certs
secret:
secretName: karmada-cert-secret
---

20
artifacts/deploy/karmada-cert-secret.yaml
View File

@ -5,15 +5,31 @@ metadata:
namespace: karmada-system
type: Opaque
data:
server-ca.crt: |
ca.crt: |
{{ca_crt}}
ca.key: |
{{ca_key}}
karmada.crt: |
{{client_cer}}
{{client_crt}}
karmada.key: |
{{client_key}}
apiserver.crt: |
{{apiserver_crt}}
apiserver.key: |
{{apiserver_key}}
front-proxy-ca.crt: |
{{front_proxy_ca_crt}}
front-proxy-client.crt: |
{{front_proxy_client_crt}}
front-proxy-client.key: |
{{front_proxy_client_key}}
etcd-ca.crt: |
{{etcd_ca_crt}}
etcd-server.crt: |
{{etcd_server_crt}}
etcd-server.key: |
{{etcd_server_key}}
etcd-client.crt: |
{{etcd_client_crt}}
etcd-client.key: |
{{etcd_client_key}}

8
artifacts/deploy/karmada-etcd.yaml
View File

@ -55,7 +55,7 @@ spec:
volumeMounts:
- mountPath: /var/lib/etcd
name: etcd-data
- mountPath: /etc/kubernetes/pki/etcd
- mountPath: /etc/karmada/pki
name: etcd-certs
resources:
requests:
@ -75,10 +75,10 @@ spec:
- etcd0=http://etcd-0.etcd.karmada-system.svc.cluster.local:2380
- --initial-cluster-state
- new
- --cert-file=/etc/kubernetes/pki/etcd/karmada.crt
- --cert-file=/etc/karmada/pki/etcd-server.crt
- --client-cert-auth=true
- --key-file=/etc/kubernetes/pki/etcd/karmada.key
- --trusted-ca-file=/etc/kubernetes/pki/etcd/server-ca.crt
- --key-file=/etc/karmada/pki/etcd-server.key
- --trusted-ca-file=/etc/karmada/pki/etcd-ca.crt
- --data-dir=/var/lib/etcd
- --snapshot-count=10000
volumes:

16
artifacts/deploy/karmada-search.yaml
View File

@ -25,8 +25,8 @@ spec:
image: swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-search:latest
imagePullPolicy: IfNotPresent
volumeMounts:
- name: k8s-certs
mountPath: /etc/kubernetes/pki
- name: karmada-certs
mountPath: /etc/karmada/pki
readOnly: true
- name: kubeconfig
subPath: kubeconfig
@ -37,11 +37,11 @@ spec:
- --authentication-kubeconfig=/etc/kubeconfig
- --authorization-kubeconfig=/etc/kubeconfig
- --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379
- --etcd-cafile=/etc/kubernetes/pki/server-ca.crt
- --etcd-certfile=/etc/kubernetes/pki/karmada.crt
- --etcd-keyfile=/etc/kubernetes/pki/karmada.key
- --tls-cert-file=/etc/kubernetes/pki/karmada.crt
- --tls-private-key-file=/etc/kubernetes/pki/karmada.key
- --etcd-cafile=/etc/karmada/pki/etcd-ca.crt
- --etcd-certfile=/etc/karmada/pki/etcd-client.crt
- --etcd-keyfile=/etc/karmada/pki/etcd-client.key
- --tls-cert-file=/etc/karmada/pki/karmada.crt
- --tls-private-key-file=/etc/karmada/pki/karmada.key
- --audit-log-path=-
- --feature-gates=APIPriorityAndFairness=false
- --audit-log-maxage=0
@ -59,7 +59,7 @@ spec:
requests:
cpu: 100m
volumes:
- name: k8s-certs
- name: karmada-certs
secret:
secretName: karmada-cert-secret
- name: kubeconfig

12
artifacts/deploy/kube-controller-manager.yaml
View File

@ -37,16 +37,16 @@ spec:
- --authentication-kubeconfig=/etc/kubeconfig
- --authorization-kubeconfig=/etc/kubeconfig
- --bind-address=0.0.0.0
- --client-ca-file=/etc/karmada/pki/server-ca.crt
- --client-ca-file=/etc/karmada/pki/ca.crt
- --cluster-cidr=10.244.0.0/16
- --cluster-name=karmada
- --cluster-signing-cert-file=/etc/karmada/pki/server-ca.crt
- --cluster-signing-key-file=/etc/karmada/pki/server-ca.key
- --cluster-signing-cert-file=/etc/karmada/pki/ca.crt
- --cluster-signing-key-file=/etc/karmada/pki/ca.key
- --controllers=namespace,garbagecollector,serviceaccount-token,ttl-after-finished
- --kubeconfig=/etc/kubeconfig
- --leader-elect=true
- --node-cidr-mask-size=24
- --root-ca-file=/etc/karmada/pki/server-ca.crt
- --root-ca-file=/etc/karmada/pki/ca.crt
- --service-account-private-key-file=/etc/karmada/pki/karmada.key
- --service-cluster-ip-range=10.96.0.0/12
- --use-service-account-credentials=true
@ -70,14 +70,14 @@ spec:
cpu: 200m
volumeMounts:
- mountPath: /etc/karmada/pki
name: k8s-certs
name: karmada-certs
readOnly: true
- mountPath: /etc/kubeconfig
subPath: kubeconfig
name: kubeconfig
priorityClassName: system-node-critical
volumes:
- name: k8s-certs
- name: karmada-certs
secret:
secretName: karmada-cert-secret
- name: kubeconfig

2
artifacts/deploy/secret.yaml
View File

@ -18,7 +18,7 @@ stringData:
users:
- name: kind-karmada
user:
client-certificate-data: {{client_cer}}
client-certificate-data: {{client_crt}}
client-key-data: {{client_key}}
kind: Secret
metadata:

2
examples/README.md
View File

@ -60,7 +60,7 @@ webhook-configuration.sh
```bash
#!/usr/bin/env bash
export ca_string=$(cat ${HOME}/.karmada/server-ca.crt | base64 | tr "\n" " "|sed s/[[:space:]]//g)
export ca_string=$(cat ${HOME}/.karmada/ca.crt | base64 | tr "\n" " "|sed s/[[:space:]]//g)
export temp_path=$(mktemp -d)
cp -rf "examples/customresourceinterpreter/webhook-configuration.yaml" "${temp_path}/temp.yaml"

37
hack/deploy-karmada.sh
View File

@ -14,7 +14,8 @@ KARMADA_APISERVER_SECURE_PORT=${KARMADA_APISERVER_SECURE_PORT:-5443}
# The host cluster name which used to install karmada control plane components.
HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"karmada-host"}
ROOT_CA_FILE=${CERT_DIR}/server-ca.crt
ROOT_CA_FILE=${CERT_DIR}/ca.crt
ROOT_CA_KEY=${CERT_DIR}/ca.key
CFSSL_VERSION="v1.5.0"
LOAD_BALANCER=${LOAD_BALANCER:-false} # whether create a 'LoadBalancer' type service for karmada apiserver
source "${REPO_ROOT}"/hack/util.sh
@ -73,7 +74,9 @@ HOST_CLUSTER_TYPE=${3:-"local"} # the default of host cluster type is local, i.e
# generate a secret to store the certificates
function generate_cert_secret {
local karmada_ca
local karmada_ca_key
karmada_ca=$(base64 "${ROOT_CA_FILE}" | tr -d '\r\n')
karmada_ca_key=$(base64 "${ROOT_CA_KEY}" | tr -d '\r\n')
local TEMP_PATH
TEMP_PATH=$(mktemp -d)
@ -83,15 +86,24 @@ function generate_cert_secret {
cp -rf "${REPO_ROOT}"/artifacts/deploy/karmada-webhook-cert-secret.yaml "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml
sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{client_cer}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{ca_key}}/${karmada_ca_key}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{client_crt}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{apiserver_crt}}/${KARMADA_APISERVER_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{apiserver_key}}/${KARMADA_APISERVER_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{front_proxy_ca_crt}}/${FRONT_PROXY_CA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{front_proxy_client_crt}}/${FRONT_PROXY_CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{front_proxy_client_key}}/${FRONT_PROXY_CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{etcd_ca_crt}}/${ETCD_CA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{etcd_server_crt}}/${ETCD_SERVER_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{etcd_server_key}}/${ETCD_SERVER_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{etcd_client_crt}}/${ETCD_CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{etcd_client_key}}/${ETCD_CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/secret-tmp.yaml
sed -i'' -e "s/{{client_cer}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/secret-tmp.yaml
sed -i'' -e "s/{{client_crt}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/secret-tmp.yaml
sed -i'' -e "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/secret-tmp.yaml
sed -i'' -e "s/{{server_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-webhook-cert-secret-tmp.yaml
@ -121,11 +133,15 @@ interpreter_webhook_example_service_external_ip_address=${interpreter_webhook_ex
util::cmd_must_exist "openssl"
util::cmd_must_exist_cfssl ${CFSSL_VERSION}
# create CA signers
util::create_signing_certkey "" "${CERT_DIR}" server '"client auth","server auth"'
util::create_signing_certkey "" "${CERT_DIR}" front-proxy '"client auth","server auth"'
util::create_signing_certkey "" "${CERT_DIR}" ca karmada '"client auth","server auth"'
util::create_signing_certkey "" "${CERT_DIR}" front-proxy-ca front-proxy-ca '"client auth","server auth"'
util::create_signing_certkey "" "${CERT_DIR}" etcd-ca etcd-ca '"client auth","server auth"'
# signs a certificate
util::create_certkey "" "${CERT_DIR}" "server-ca" karmada system:admin kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" "${interpreter_webhook_example_service_external_ip_address}"
util::create_certkey "" "${CERT_DIR}" "front-proxy-ca" front-proxy-client front-proxy-client kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1"
util::create_certkey "" "${CERT_DIR}" "ca" karmada system:admin "system:masters" kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" "${interpreter_webhook_example_service_external_ip_address}"
util::create_certkey "" "${CERT_DIR}" "ca" apiserver karmada-apiserver "" "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1"
util::create_certkey "" "${CERT_DIR}" "front-proxy-ca" front-proxy-client front-proxy-client "" kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1"
util::create_certkey "" "${CERT_DIR}" "etcd-ca" etcd-server etcd-server "" kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1"
util::create_certkey "" "${CERT_DIR}" "etcd-ca" etcd-client etcd-client "" "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1"
# create namespace for control plane components
kubectl apply -f "${REPO_ROOT}/artifacts/deploy/namespace.yaml"
@ -137,9 +153,16 @@ kubectl apply -f "${REPO_ROOT}/artifacts/deploy/clusterrolebinding.yaml"
KARMADA_CRT=$(base64 "${CERT_DIR}/karmada.crt" | tr -d '\r\n')
KARMADA_KEY=$(base64 "${CERT_DIR}/karmada.key" | tr -d '\r\n')
KARMADA_APISERVER_CRT=$(base64 "${CERT_DIR}/apiserver.crt" | tr -d '\r\n')
KARMADA_APISERVER_KEY=$(base64 "${CERT_DIR}/apiserver.key" | tr -d '\r\n')
FRONT_PROXY_CA_CRT=$(base64 "${CERT_DIR}/front-proxy-ca.crt" | tr -d '\r\n')
FRONT_PROXY_CLIENT_CRT=$(base64 "${CERT_DIR}/front-proxy-client.crt" | tr -d '\r\n')
FRONT_PROXY_CLIENT_KEY=$(base64 "${CERT_DIR}/front-proxy-client.key" | tr -d '\r\n')
ETCD_CA_CRT=$(base64 "${CERT_DIR}/etcd-ca.crt" | tr -d '\r\n')
ETCD_SERVER_CRT=$(base64 "${CERT_DIR}/etcd-server.crt" | tr -d '\r\n')
ETCD_SERVER_KEY=$(base64 "${CERT_DIR}/etcd-server.key" | tr -d '\r\n')
ETCD_CLIENT_CRT=$(base64 "${CERT_DIR}/etcd-client.crt" | tr -d '\r\n')
ETCD_CLIENT_KEY=$(base64 "${CERT_DIR}/etcd-client.key" | tr -d '\r\n')
generate_cert_secret
# deploy karmada etcd

2
hack/pre-run-e2e.sh
View File

@ -20,7 +20,7 @@ export VERSION="latest"
export REGISTRY="swr.ap-southeast-1.myhuaweicloud.com/karmada"
CERT_DIR=${CERT_DIR:-"${HOME}/.karmada"}
ROOT_CA_FILE=${CERT_DIR}/server-ca.crt
ROOT_CA_FILE=${CERT_DIR}/ca.crt
# load interpreter webhook example image
kind load docker-image "${REGISTRY}/karmada-interpreter-webhook-example:${VERSION}" --name="${HOST_CLUSTER_NAME}"

14
hack/util.sh
View File

@ -173,13 +173,14 @@ function util::create_signing_certkey {
local sudo=$1
local dest_dir=$2
local id=$3
local purpose=$4
local cn=$4
local purpose=$5
OPENSSL_BIN=$(command -v openssl)
# Create ca
${sudo} /usr/bin/env bash -e <<EOF
rm -f "${dest_dir}/${id}-ca.crt" "${dest_dir}/${id}-ca.key"
${OPENSSL_BIN} req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "${dest_dir}/${id}-ca.key" -out "${dest_dir}/${id}-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment",${purpose}]}}}' > "${dest_dir}/${id}-ca-config.json"
rm -f "${dest_dir}/${id}.crt" "${dest_dir}/${id}.key"
${OPENSSL_BIN} req -x509 -sha256 -new -nodes -days 3650 -newkey rsa:2048 -keyout "${dest_dir}/${id}.key" -out "${dest_dir}/${id}.crt" -subj "/CN=${cn}/"
echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment",${purpose}]}}}' > "${dest_dir}/${id}-config.json"
EOF
}
@ -190,9 +191,10 @@ function util::create_certkey {
local ca=$3
local id=$4
local cn=${5:-$4}
local og=$6
local hosts=""
local SEP=""
shift 5
shift 6
while [[ -n "${1:-}" ]]; do
hosts+="${SEP}\"$1\""
SEP=","
@ -200,7 +202,7 @@ function util::create_certkey {
done
${sudo} /usr/bin/env bash -e <<EOF
cd ${dest_dir}
echo '{"CN":"${cn}","hosts":[${hosts}],"names":[{"O":"system:masters"}],"key":{"algo":"rsa","size":2048}}' | ${CFSSL_BIN} gencert -ca=${ca}.crt -ca-key=${ca}.key -config=${ca}-config.json - | ${CFSSLJSON_BIN} -bare ${id}
echo '{"CN":"${cn}","hosts":[${hosts}],"names":[{"O":"${og}"}],"key":{"algo":"rsa","size":2048}}' | ${CFSSL_BIN} gencert -ca=${ca}.crt -ca-key=${ca}.key -config=${ca}-config.json - | ${CFSSLJSON_BIN} -bare ${id}
mv "${id}-key.pem" "${id}.key"
mv "${id}.pem" "${id}.crt"
rm -f "${id}.csr"