add proxy timeout for nginx https config (#2202)

Signed-off-by: my036811 <miner.yang@broadcom.com>

.github/workflows/unittest.yaml

@@ -22,4 +22,4 @@ jobs:
 
       - name: Run unit tests
         run:
-          helm unittest -f 'test/unittest/*.yaml' .
+          helm unittest -f 'test/unittest/*/*.yaml' .

README.md

@@ -86,6 +86,7 @@ The following table lists the configurable parameters of the Harbor chart and th
 | `expose.ingress.hosts.core`                                           | The host of Harbor core service in ingress rule                                                                                                                                                                                                                                                                                                                                                                                                                  | `core.harbor.domain`                  |
 | `expose.ingress.controller`                                           | The ingress controller type. Currently supports `default`, `gce`, `alb`, `f5-bigip` and `ncp`                                                                                                                                                                                                                                                                                                                                                                    | `default`                             |
 | `expose.ingress.kubeVersionOverride`                                  | Allows the ability to override the kubernetes version used while templating the ingress                                                                                                                                                                                                                                                                                                                                                                          |                                       |
+| `expose.ingress.className`                                            | Specify the `ingressClassName` used to implement the Ingress (Kubernetes 1.18+)                                                                                                                                                                                                                                                                                                                                                                                  |                                       |
 | `expose.ingress.annotations`                                          | The annotations used commonly for ingresses                                                                                                                                                                                                                                                                                                                                                                                                                      |                                       |
 | `expose.ingress.labels`                                               | The labels specific to ingress                                                                                                                                                                                                                                                                                                                                                                                                                             | {}                                    |
 | `expose.clusterIP.name`                                               | The name of ClusterIP service                                                                                                                                                                                                                                                                                                                                                                                                                                    | `harbor`                              |

templates/_helpers.tpl

@@ -148,7 +148,21 @@ app: "{{ template "harbor.name" . }}"
 
 {{- define "harbor.redis.scheme" -}}
   {{- with .Values.redis }}
-    {{- ternary "redis+sentinel" "redis"  (and (eq .type "external" ) (not (not .external.sentinelMasterSet))) }}
+    {{- if eq .type "external" -}}
+      {{- if not (not .external.sentinelMasterSet) -}}
+        {{- ternary "rediss+sentinel" "redis+sentinel" (.external.tlsOptions.enable) }}
+      {{- else -}}
+        {{- ternary "rediss" "redis" (.external.tlsOptions.enable) }}
+      {{- end -}}
+    {{- else -}}
+      {{ print "redis" }}
+    {{- end -}}
+  {{- end }}
+{{- end -}}
+
+{{- define "harbor.redis.enableTLS" -}}
+  {{- with .Values.redis }}
+    {{- ternary "true" "false" (and ( eq .type "external") (.external.tlsOptions.enable)) }}
   {{- end }}
 {{- end -}}
 
@@ -161,7 +175,7 @@ app: "{{ template "harbor.name" . }}"
 
 {{- define "harbor.redis.masterSet" -}}
   {{- with .Values.redis }}
-    {{- ternary .external.sentinelMasterSet "" (eq "redis+sentinel" (include "harbor.redis.scheme" $)) }}
+    {{- ternary .external.sentinelMasterSet "" (contains "+sentinel" (include "harbor.redis.scheme" $)) }}
   {{- end }}
 {{- end -}}
 

templates/jobservice/jobservice-cm-env.yaml

@@ -15,6 +15,8 @@ data:
   JOBSERVICE_WEBHOOK_JOB_MAX_RETRY: "{{ .Values.jobservice.notification.webhook_job_max_retry }}"
   JOBSERVICE_WEBHOOK_JOB_HTTP_CLIENT_TIMEOUT: "{{ .Values.jobservice.notification.webhook_job_http_client_timeout }}"
 
+  LOG_LEVEL: "{{ .Values.logLevel }}"
+  
   {{- if has "jobservice" .Values.proxy.components }}
   HTTP_PROXY: "{{ .Values.proxy.httpProxy }}"
   HTTPS_PROXY: "{{ .Values.proxy.httpsProxy }}"

templates/nginx/configmap-https.yaml

@@ -138,6 +138,8 @@ data:
           proxy_set_header X-Forwarded-Proto $x_forwarded_proto;
           proxy_buffering off;
           proxy_request_buffering off;
+          proxy_send_timeout 900;
+          proxy_read_timeout 900;
         }
 
         location /service/ {

templates/registry/registry-cm.yaml

@@ -182,6 +182,7 @@ data:
       readtimeout: 10s
       writetimeout: 10s
       dialtimeout: 10s
+      enableTLS: {{ template "harbor.redis.enableTLS" . }}
       pool:
         maxidle: 100
         maxactive: 500

test/e2e/Jenkinsfile

@@ -22,8 +22,21 @@ class HarborChartFreshInstallPipelineExecutor extends FreshInstallPipelineExecut
             script.file(credentialsId: "kubeconfig", variable: "KUBE_CONFIG_FILE_PATH"),
             script.usernamePassword(credentialsId: "79e9fd98-cdf5-4f55-81fa-ecba01365534", usernameVariable: "DOCKER_HUB_USERNAME", passwordVariable: "DOCKER_HUB_PASSWORD")]) {
             script.sh """
-                # login Docker Hub to avoid the pull limit
+                # Set proxy registry or docker credential to bypass Docker Hub rate limit
-                docker login -u \${DOCKER_HUB_USERNAME} -p \${DOCKER_HUB_PASSWORD}
+                echo "PROXY_REGISTRY is \${PROXY_REGISTRY}"
+                if [ "\${PROXY_REGISTRY}" != "" ]; then
+                    # set deafult registry to a proxy registry
+                    echo '{
+                    "registry-mirrors": ["'"\${PROXY_REGISTRY}"'"]
+                    }' | sudo tee /etc/docker/daemon.json > /dev/null
+
+                    # Restart Docker to apply the changes
+                    sudo systemctl reset-failed docker.service
+                    sudo systemctl restart docker
+                else
+                    docker login -u \${DOCKER_HUB_USERNAME} -p \${DOCKER_HUB_PASSWORD}
+                fi
+
                 # build the image
                 docker build -t deployer:dev -f test/e2e/Dockerfile test/e2e
                 # clean up the namespace

test/unittest/core/core_configmap_test.yaml

@@ -35,6 +35,21 @@ tests:
           path: data._REDIS_URL_HARBOR
           value: redis://192.168.0.2:6379/test-index?idle_timeout_seconds=30
   
+  - it: RedisSentinelUrlHarborExternalTLS
+    set:
+      redis:
+        external:
+          harborDatabaseIndex: test-index
+          tlsOptions:
+            enable: true
+          sentinelMasterSet: "mymaster"
+        type: external
+    template: templates/core/core-cm.yaml
+    asserts:
+      - equal:
+          path: data._REDIS_URL_HARBOR
+          value: rediss+sentinel://192.168.0.2:6379/mymaster/test-index?idle_timeout_seconds=30
+
   - it: CacheLayerDatabaseIndex
     set:
       redis:

test/unittest/exporter/exporter_configmap_test.yaml

@@ -0,0 +1,117 @@
+suite: ExporterConfigMap
+
+tests:
+  - it: ProxyJobservice
+    set:
+      metrics:
+        enabled: true
+      proxy:
+        httpProxy: 1.1.1.1
+        httpsProxy: 2.2.2.2
+        noProxy: 127.0.0.1,localhost,.local,.internal
+        components:
+          - jobservice
+    template: templates/exporter/exporter-cm-env.yaml
+    asserts:
+      - equal:
+          path: data.HTTP_PROXY
+          value: 1.1.1.1
+      - equal:
+          path: data.HTTPS_PROXY
+          value: 2.2.2.2
+      - equal:
+          path: data.NO_PROXY
+          value: RELEASE-NAME-harbor-core,RELEASE-NAME-harbor-jobservice,RELEASE-NAME-harbor-database,RELEASE-NAME-harbor-registry,RELEASE-NAME-harbor-portal,RELEASE-NAME-harbor-trivy,RELEASE-NAME-harbor-exporter,127.0.0.1,localhost,.local,.internal
+
+  - it: ProxyNoJobservice
+    set:
+      metrics:
+        enabled: true
+      proxy:
+        httpProxy: 1.1.1.1
+        httpsProxy: 2.2.2.2
+        noProxy: 127.0.0.1,localhost,.local,.internal
+        components:
+          - testComponent
+    template: templates/exporter/exporter-cm-env.yaml
+    asserts:
+      - notExists:
+          path: data.HTTP_PROXY
+      - notExists:
+          path: data.HTTPS_PROXY
+      - notExists:
+          path: data.NO_PROXY
+
+  - it: FullSecrets
+    set:
+      metrics:
+        enabled: true
+        exporter:
+          path: /testEndpoint
+          port: 1111
+      exporter:
+        cacheDuration: 30
+        cacheCleanInterval: 1000
+      logLevel: debug
+      database:
+        maxIdleConns: 100
+        maxOpenConns: 50
+    template: templates/exporter/exporter-cm-env.yaml
+    asserts:
+      - equal:
+          path: data.LOG_LEVEL
+          value: debug
+      - equal:
+          path: data.HARBOR_EXPORTER_PORT
+          value: "1111"
+      - equal:
+          path: data.HARBOR_EXPORTER_METRICS_PATH
+          value: /testEndpoint
+      - equal:
+          path: data.HARBOR_EXPORTER_METRICS_ENABLED
+          value: "true"
+      - equal:
+          path: data.HARBOR_EXPORTER_CACHE_TIME
+          value: "30"
+      - equal:
+          path: data.HARBOR_EXPORTER_CACHE_CLEAN_INTERVAL
+          value: "1000"
+      - equal:
+          path: data.HARBOR_REDIS_URL
+          value: redis://RELEASE-NAME-harbor-redis:6379/1
+      - equal:
+          path: data.HARBOR_REDIS_NAMESPACE
+          value: harbor_job_service_namespace
+      - equal:
+          path: data.HARBOR_REDIS_TIMEOUT
+          value: "3600"
+      - equal:
+          path: data.HARBOR_SERVICE_SCHEME
+          value: http
+      - equal:
+          path: data.HARBOR_SERVICE_HOST
+          value: RELEASE-NAME-harbor-core
+      - equal:
+          path: data.HARBOR_SERVICE_PORT
+          value: "80"
+      - equal:
+          path: data.HARBOR_DATABASE_HOST
+          value: RELEASE-NAME-harbor-database
+      - equal:
+          path: data.HARBOR_DATABASE_PORT
+          value: "5432"
+      - equal:
+          path: data.HARBOR_DATABASE_USERNAME
+          value: postgres
+      - equal:
+          path: data.HARBOR_DATABASE_DBNAME
+          value: registry
+      - equal:
+          path: data.HARBOR_DATABASE_SSLMODE
+          value: disable
+      - equal:
+          path: data.HARBOR_DATABASE_MAX_IDLE_CONNS
+          value: "100"
+      - equal:
+          path: data.HARBOR_DATABASE_MAX_OPEN_CONNS
+          value: "50"

test/unittest/exporter/exporter_deployment_test.yaml

@@ -0,0 +1,313 @@
+suite: ExporterDeployment
+
+tests:
+  - it: PodLabels
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        podLabels:
+          test.label: test-label
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.template.metadata.labels["test.label"]
+          value: test-label
+
+  - it: PodAnnotations
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        podAnnotations:
+          test.annotation: test-annotation
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.template.metadata.annotations["test.annotation"]
+          value: test-annotation
+
+  - it: NoReplicas
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        replicas: 0
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 0
+
+  - it: MultipleReplicas
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        replicas: 2
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 2
+
+  - it: ServiceAccounts
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        serviceAccountName: testServiceAccount
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: testServiceAccount
+
+  - it: ImagePullSecrets
+    set:
+      metrics:
+        enabled: true
+      imagePullSecrets:
+        - name: test-secret-1
+        - name: test-secret-2
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.imagePullSecrets
+          count: 2
+      - equal:
+          path: spec.template.spec.imagePullSecrets
+          value:
+            - name: test-secret-1
+            - name: test-secret-2
+
+  - it: TopologySpreadConstraints
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        topologySpreadConstraints:
+          - maxSkew: 1
+            topologyKey: topology.kubernetes.io/zone
+            whenUnsatisfiable: ScheduleAnyway
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.topologySpreadConstraints
+          count: 1
+      - contains:
+          path: spec.template.spec.topologySpreadConstraints
+          content:
+              labelSelector:
+                matchLabels:
+                  app: harbor
+                  component: exporter
+                  release: RELEASE-NAME
+              maxSkew: 1
+              topologyKey: topology.kubernetes.io/zone
+              whenUnsatisfiable: ScheduleAnyway
+
+  - it: ContainerImage
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        image:
+          repository: test-repository/test-image
+          tag: 1.0.0
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: test-repository/test-image:1.0.0
+
+  - it: ExistingSecretAdminPassword
+    set:
+      metrics:
+        enabled: true
+      existingSecretAdminPassword: HARBOR_ADMIN_PASSWORD
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.containers[0].env
+          count: 1
+      - equal:
+          path: spec.template.spec.containers[0].env[0].name
+          value: HARBOR_ADMIN_PASSWORD
+      - equal:
+          path: spec.template.spec.containers[0].env[0].valueFrom.secretKeyRef.name
+          value: HARBOR_ADMIN_PASSWORD
+      - equal:
+          path: spec.template.spec.containers[0].env[0].valueFrom.secretKeyRef.key
+          value: HARBOR_ADMIN_PASSWORD
+
+  - it: InternalTLS
+    set:
+      metrics:
+        enabled: true
+      internalTLS:
+        enabled: true
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts[0].name
+          value: core-internal-certs
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts[0].mountPath
+          value: /etc/harbor/ssl/core
+
+  - it: DBCredentials
+    set:
+      metrics:
+        enabled: true
+      database:
+        external:
+          existingSecret: db-secret-name
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.containers[0].env
+          count: 1
+      - equal:
+          path: spec.template.spec.containers[0].env[0].name
+          value: HARBOR_DATABASE_PASSWORD
+      - equal:
+          path: spec.template.spec.containers[0].env[0].valueFrom.secretKeyRef.name
+          value: db-secret-name
+      - equal:
+          path: spec.template.spec.containers[0].env[0].valueFrom.secretKeyRef.key
+          value: password
+
+  - it: ContainerSecurityContext
+    set:
+      metrics:
+        enabled: true
+      containerSecurityContext:
+        privileged: true
+        allowPrivilegeEscalation: true
+        seccompProfile:
+          type: RuntimeDefault
+        runAsNonRoot: true
+        capabilities:
+          drop:
+            - All
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].securityContext.privileged
+          value: true
+      - equal:
+          path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation
+          value: true
+      - equal:
+          path: spec.template.spec.containers[0].securityContext.seccompProfile.type
+          value: RuntimeDefault
+      - equal:
+          path: spec.template.spec.containers[0].securityContext.runAsNonRoot
+          value: true
+      - equal:
+          path: spec.template.spec.containers[0].securityContext.capabilities.drop[0]
+          value: All
+
+  - it: Resources
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        resources:
+          requests:
+            memory: 256Mi
+            cpu: 100m
+          limits:
+            memory: 500Mi
+            cpu: 200m
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.cpu
+          value: 100m
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.memory
+          value: 256Mi
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.cpu
+          value: 200m
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.memory
+          value: 500Mi
+
+  - it: NodeSelector
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        nodeSelector:
+          node.selector/tier: test-node-selector
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.nodeSelector["node.selector/tier"]
+          value: test-node-selector
+
+  - it: Affinity
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        affinity:
+          podAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              - labelSelector:
+                  matchExpressions:
+                    - key: test-affinity
+                      operator: In
+                      values:
+                        - S1
+                topologyKey: topology.kubernetes.io/zone
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchExpressions[0].key
+          value: test-affinity
+      - equal:
+          path: spec.template.spec.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchExpressions[0].operator
+          value: In
+      - equal:
+          path: spec.template.spec.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchExpressions[0].values[0]
+          value: S1
+      - equal:
+          path: spec.template.spec.affinity.podAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey
+          value: topology.kubernetes.io/zone
+
+  - it: Tolerations
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        tolerations:
+          - effect: NoSchedule
+            key: test-label
+            value: test
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.tolerations[0].effect
+          value: NoSchedule
+      - equal:
+          path: spec.template.spec.tolerations[0].key
+          value: test-label
+      - equal:
+          path: spec.template.spec.tolerations[0].value
+          value: test
+
+  - it: PriorityClassName
+    set:
+      metrics:
+        enabled: true
+      exporter:
+        priorityClassName: test-priority
+    template: templates/exporter/exporter-dpl.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.priorityClassName
+          value: test-priority

test/unittest/exporter/exporter_secret_test.yaml

@@ -0,0 +1,36 @@
+suite: ExporterSecret
+
+tests:
+  - it: Secret
+    set:
+      metrics:
+        enabled: true
+    template: templates/exporter/exporter-secret.yaml
+    asserts:
+      - equal:
+          path: data.HARBOR_ADMIN_PASSWORD
+          value: "SGFyYm9yMTIzNDU="
+      - exists:
+          path: data.HARBOR_DATABASE_PASSWORD
+
+  - it: ExistingAdminSecret
+    set:
+      metrics:
+        enabled: true
+      existingSecretAdminPassword: test-password
+    template: templates/exporter/exporter-secret.yaml
+    asserts:
+      - notExists:
+          path: data.HARBOR_ADMIN_PASSWORD
+
+  - it: ExistingExternalDBSecret
+    set:
+      metrics:
+        enabled: true
+      database:
+        external:
+          existingSecret: test-db-secret
+    template: templates/exporter/exporter-secret.yaml
+    asserts:
+      - notExists:
+          path: data.POSTGRESQL_PASSWORD

test/unittest/exporter/exporter_svc_test.yaml

@@ -0,0 +1,14 @@
+suite: ExporterSvc
+
+tests:
+  - it: ExposeMetricsPort
+    set:
+      metrics:
+        enabled: true
+        exporter:
+          port: 1111
+    template: templates/exporter/exporter-svc.yaml
+    asserts:
+      - equal:
+          path: spec.ports[0].port
+          value: 1111

values.yaml

@@ -270,7 +270,7 @@ persistence:
 # The initial password of Harbor admin. Change it from portal after launching Harbor
 # or give an existing secret for it
 # key in secret is given via (default to HARBOR_ADMIN_PASSWORD)
-# existingSecretAdminPassword:
+existingSecretAdminPassword: ""
 existingSecretAdminPasswordKey: HARBOR_ADMIN_PASSWORD
 harborAdminPassword: "Harbor12345"
 
@@ -625,6 +625,8 @@ core:
   # If tokenKey is set, the value of tokenCert must be set as a PEM-encoded certificate signed by tokenKey, and supplied as a multiline string, indented one more than tokenCert on the following line.
   tokenCert: |
   # The XSRF key. Will be generated automatically if it isn't specified
+  # While you specified, Please make sure it is 32 characters, otherwise would have validation issue at the harbor-core runtime
+  # https://github.com/goharbor/harbor/pull/21154
   xsrfKey: ""
   # If using existingSecret, the key is defined by core.existingXsrfSecretKey
   existingXsrfSecret: ""
@@ -1006,6 +1008,14 @@ redis:
     addr: "192.168.0.2:6379"
     # The name of the set of Redis instances to monitor, it must be set to support redis+sentinel
     sentinelMasterSet: ""
+    # TLS configuration for redis connection
+    # only server-authentication is supported, mTLS for redis connection is not supported
+    # tls connection will be disable by default
+    # Once `tlsOptions.enable` set as true, tls/ssl connection will be used for redis
+    # Please set the `caBundleSecretName` in this configuration file which conatins redis server rootCA if it is self-signed.
+    # The secret must contain keys named "ca.crt" which will be injected into the trust store
+    tlsOptions:
+      enable: false
     # The "coreDatabaseIndex" must be "0" as the library Harbor
     # used doesn't support configuring it
     # harborDatabaseIndex defaults to "0", but it can be configured to "6", this config is optional