Initial check in copying KFDef specs over from kubeflow/kubeflow to kubeflow/manifests (#353)

* Initial check in copying KFDef specs over from kubeflow/kubeflow to kubeflow/manifests. * kubeflow/manifests#241 explains why we are moving the manifests * A big reason is that downloading the kubeflow/kubeflow repo just to get aditional configuration files is really expensive because that repo is bloated as a result of vendoring in different resources. * It makes sense to locate all configuration files in a single repository rather than splitting it out. * With kfctl moving to kubeflow/kfctl the KFDef specs will no longer be in the same repo as kfctl anyway. We also want to release and version the KFDef specs from the binary. * This PR is just copying over the files. Additional changes are needed to make things work with the new location. * kfctl may need to be updated to allow supplementary configuration files (e.g. GCP DM configs) to be specified in the KFDef spec rather than having the location hard coded. * For example, kubeflow/kubeflow#4118 updated the GCPPluginSpec to allow the DM configurations to be specified using a repo ref. * Once kfctl changes are in place; the KFDef specs need to be updated. * We will make those changes for GCP in a subsequent PR. * Once KFDef specs are updated we need to update E2E tests to use the new location. * File location * KFDef specs are organized in a new top level directory kfdef. * The Cloud specific configuration files were moved into subdirectories of aws & gcp respectively. * The generic/cloud agonistic configurations used by kubeflow_existing_arrikto.yaml were moved into kfdef/generic. * Update gentest targets. * Fix logic to exclude certain directories.
2019-09-12 19:26:31 -07:00 · 2019-09-12 19:26:31 -07:00 · 1bb77f3f28
parent 1c54acc214
commit 1bb77f3f28
33 changed files with 23489 additions and 2 deletions
--- a/aws/infra_configs/README.md
+++ b/aws/infra_configs/README.md
@ -0,0 +1,2 @@
+This directory contains some additional configuration files that are used by kfctl when
+deploying on AWS.
--- a/aws/infra_configs/cluster_config.yaml
+++ b/aws/infra_configs/cluster_config.yaml
@ -0,0 +1,44 @@
+# For details, Please check eksctl documentation or API specs.
+# https://github.com/weaveworks/eksctl/blob/master/pkg/apis/eksctl.io/v1alpha4/types.go
+
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+metadata:
+  # AWS_CLUSTER_NAME and AWS_REGION will override `name` and `region` here.
+  name: your_cluster_name
+  region: your_cluster_region
+  version: '1.13'
+# If your region has multiple availability zones, you can specify 3 of them.
+# cluster AZs must be set explicitly for single AZ nodegroup example to work
+#availabilityZones: ["us-west-2b", "us-west-2c", "us-west-2d"]
+
+# NodeGroup holds all configuration attributes that are specific to a nodegroup
+# You can have several node group in your cluster.
+nodeGroups:
+  - name: cpu-nodegroup
+    instanceType: m5.xlarge
+    desiredCapacity: 2
+    minSize: 0
+    maxSize: 3
+    volumeSize: 30
+
+  # Example of GPU node group
+#   - name: gpu-nodegroup
+#     instanceType: p3.2xlarge
+#     ami: auto
+#     availabilityZones: ["us-west-2b"] # GPU cluster can use single availability zone to improve network performance
+#     desiredCapacity: 0
+#     minSize: 0
+#     maxSize: 4
+#     volumeSize: 50 # Node Root Disk
+#     ssh:
+#       allow: true
+#       sshPublicKeyPath: '~/.ssh/id_rsa.pub'
+#     labels:
+#       k8s.amazonaws.com/accelerator: 'nvidia-tesla-k80'    # Customize Labels
+#     tags:
+#       k8s.io/cluster-autoscaler/enabled: 'true'
+#     iam:
+#       withAddonPolicies:
+#         autoScaler: true
+#
--- a/aws/infra_configs/cluster_features.yaml
+++ b/aws/infra_configs/cluster_features.yaml
@ -0,0 +1,17 @@
+# private_access enable private access for your Amazon EKS cluster's Kubernetes API server endpoint
+# and completely disable public access so that it's not accessible from the internet.
+# More info: https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html
+private_access: false
+endpoint_public_access: true
+endpoint_private_access: false
+# control_plane_logging provides audit and diagnostic logs directly from the EKS control plane
+# to CloudWatch Logs in your account. More info: https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html
+control_plane_logging: false
+control_plane_logging_components:
+  - api
+  - audit
+  - authenticator
+  - controllerManager
+  - scheduler
+# worker_node_group_logging provides audit and diagnostic logs from worker node groups to CloudWatch Logs in your account.
+worker_node_group_logging: false
--- a/aws/infra_configs/iam_alb_ingress_policy.json
+++ b/aws/infra_configs/iam_alb_ingress_policy.json
@ -0,0 +1,118 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+      {
+        "Effect": "Allow",
+        "Action": [
+          "acm:DescribeCertificate",
+          "acm:ListCertificates",
+          "acm:GetCertificate"
+        ],
+        "Resource": "*"
+      },
+      {
+        "Effect": "Allow",
+        "Action": [
+          "ec2:AuthorizeSecurityGroupIngress",
+          "ec2:CreateSecurityGroup",
+          "ec2:CreateTags",
+          "ec2:DeleteTags",
+          "ec2:DeleteSecurityGroup",
+          "ec2:DescribeAccountAttributes",
+          "ec2:DescribeAddresses",
+          "ec2:DescribeInstances",
+          "ec2:DescribeInstanceStatus",
+          "ec2:DescribeInternetGateways",
+          "ec2:DescribeNetworkInterfaces",
+          "ec2:DescribeSecurityGroups",
+          "ec2:DescribeSubnets",
+          "ec2:DescribeTags",
+          "ec2:DescribeVpcs",
+          "ec2:ModifyInstanceAttribute",
+          "ec2:ModifyNetworkInterfaceAttribute",
+          "ec2:RevokeSecurityGroupIngress"
+        ],
+        "Resource": "*"
+      },
+      {
+        "Effect": "Allow",
+        "Action": [
+          "elasticloadbalancing:AddListenerCertificates",
+          "elasticloadbalancing:AddTags",
+          "elasticloadbalancing:CreateListener",
+          "elasticloadbalancing:CreateLoadBalancer",
+          "elasticloadbalancing:CreateRule",
+          "elasticloadbalancing:CreateTargetGroup",
+          "elasticloadbalancing:DeleteListener",
+          "elasticloadbalancing:DeleteLoadBalancer",
+          "elasticloadbalancing:DeleteRule",
+          "elasticloadbalancing:DeleteTargetGroup",
+          "elasticloadbalancing:DeregisterTargets",
+          "elasticloadbalancing:DescribeListenerCertificates",
+          "elasticloadbalancing:DescribeListeners",
+          "elasticloadbalancing:DescribeLoadBalancers",
+          "elasticloadbalancing:DescribeLoadBalancerAttributes",
+          "elasticloadbalancing:DescribeRules",
+          "elasticloadbalancing:DescribeSSLPolicies",
+          "elasticloadbalancing:DescribeTags",
+          "elasticloadbalancing:DescribeTargetGroups",
+          "elasticloadbalancing:DescribeTargetGroupAttributes",
+          "elasticloadbalancing:DescribeTargetHealth",
+          "elasticloadbalancing:ModifyListener",
+          "elasticloadbalancing:ModifyLoadBalancerAttributes",
+          "elasticloadbalancing:ModifyRule",
+          "elasticloadbalancing:ModifyTargetGroup",
+          "elasticloadbalancing:ModifyTargetGroupAttributes",
+          "elasticloadbalancing:RegisterTargets",
+          "elasticloadbalancing:RemoveListenerCertificates",
+          "elasticloadbalancing:RemoveTags",
+          "elasticloadbalancing:SetIpAddressType",
+          "elasticloadbalancing:SetSecurityGroups",
+          "elasticloadbalancing:SetSubnets",
+          "elasticloadbalancing:SetWebACL"
+        ],
+        "Resource": "*"
+      },
+      {
+        "Effect": "Allow",
+        "Action": [
+          "iam:CreateServiceLinkedRole",
+          "iam:GetServerCertificate",
+          "iam:ListServerCertificates"
+        ],
+        "Resource": "*"
+      },
+      {
+        "Effect": "Allow",
+        "Action": [
+          "cognito-idp:DescribeUserPoolClient"
+        ],
+        "Resource": "*"
+      },
+      {
+        "Effect": "Allow",
+        "Action": [
+          "waf-regional:GetWebACLForResource",
+          "waf-regional:GetWebACL",
+          "waf-regional:AssociateWebACL",
+          "waf-regional:DisassociateWebACL"
+        ],
+        "Resource": "*"
+      },
+      {
+        "Effect": "Allow",
+        "Action": [
+          "tag:GetResources",
+          "tag:TagResources"
+        ],
+        "Resource": "*"
+      },
+      {
+        "Effect": "Allow",
+        "Action": [
+          "waf:GetWebACL"
+        ],
+        "Resource": "*"
+      }
+    ]
+}
--- a/aws/infra_configs/iam_cloudwatch_policy.json
+++ b/aws/infra_configs/iam_cloudwatch_policy.json
@ -0,0 +1,16 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "logs:DescribeLogGroups",
+                "logs:DescribeLogStreams",
+                "logs:CreateLogGroup",
+                "logs:CreateLogStream",
+                "logs:PutLogEvents"
+            ],
+            "Resource": "*",
+            "Effect": "Allow"
+        }
+    ]
+}
--- a/aws/infra_configs/iam_csi_fsx_policy.json
+++ b/aws/infra_configs/iam_csi_fsx_policy.json
@ -0,0 +1,31 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "iam:CreateServiceLinkedRole",
+        "iam:AttachRolePolicy",
+        "iam:PutRolePolicy"
+       ],
+      "Resource": "arn:aws:iam::*:role/aws-service-role/s3.data-source.lustre.fsx.amazonaws.com/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "fsx:*"
+      ],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": "s3:*",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:CreateNetworkInterface",
+      "Resource": "*"
+    }
+  ]
+}
--- a/gcp/deployment_manager_configs/README.md
+++ b/gcp/deployment_manager_configs/README.md
@ -0,0 +1,4 @@
+This directory contains some deployment manager configuration files that can be used to setup
+GCP for Kubeflow.
+
+These deployment configuration files are intended to be used with kfctl.
--- a/gcp/deployment_manager_configs/cluster-kubeflow.yaml
+++ b/gcp/deployment_manager_configs/cluster-kubeflow.yaml
@ -0,0 +1,103 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+imports:
+- path: cluster.jinja
+
+resources:
+  # Deployment manager doesn't support depends on references in template type.
+  # So the two possible work arounds are
+  # 1. Use a single template (.jinja file for all resources) or
+  # 2. Create two separate deployments and launch the boot strapper
+  # after the cluster is created.
+  #
+  # Two separate deployments doesn't make much sense; we could just use
+  # kubectl at that point. So we put all resources in a single deployment.
+- name: kubeflow
+  type: cluster.jinja
+  properties:
+    # You need to use a zone with Broadwell because that's what TFServing requires.
+    zone: SET_THE_ZONE
+    # "1.X": picks the highest valid patch+gke.N patch in the 1.X version
+    # https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.zones.clusters
+    cluster-version: "1.12"
+    # Set this to v1beta1 to use beta features such as private clusterss
+    # and the Kubernetes stackdriver agents.
+    gkeApiVersion: SET_GKE_API_VERSION
+    # Whether to enable workload identity
+    enable-workload-identity: false
+    identity-namespace: SET_IDENTITY_NAMESPACE
+    # An arbitrary string appending to name of nodepools
+    # bump this if you want to modify the node pools.
+    # This will cause existing node pools to be deleted and new ones to be created.
+    # Use prefix v so it will be treated as a string.
+    pool-version: v1
+    # CPU Pool Configs
+    # Two is small enough to fit within default quota.
+    cpu-pool-initialNodeCount: 2
+    # machine type for nodes in cpu pool. Available options: https://cloud.google.com/compute/docs/machine-types
+    cpu-pool-machine-type: n1-standard-8
+    # Autoscaling parameters
+    cpu-pool-enable-autoscaling: true
+    cpu-pool-min-nodes: 0
+    cpu-pool-max-nodes: 10
+    # GPU Pool Configs
+    gpu-pool-initialNodeCount: 0
+    # machine type for nodes in gpu pool. Available options: https://cloud.google.com/compute/docs/machine-types
+    gpu-pool-machine-type: n1-standard-8
+    # GPUs are not enabled by default. To add GPUs
+    # set gpu-pool-max-nodes to a none-zero value.
+    gpu-pool-enable-autoscaling: true
+    gpu-pool-min-nodes: 0
+    gpu-pool-max-nodes: 0
+    # Controls gpu number per node, valid input: [1, num_cpu_per_node], for n1-standard-8, num_cpu_per_node = 8
+    gpu-number-per-node: 1
+    # Check https://cloud.google.com/compute/docs/gpus/ for available GPU models and their regions
+    gpu-type: nvidia-tesla-k80
+    # Autoprovisioning parameters (only supported in gkeApiVersion v1beta1).
+    # This is configured by the gkeApiVersion setting.
+    autoprovisioning-config:
+      enabled: true
+      max-cpu: 20
+      max-memory: 200
+      max-accelerator:
+        - type: nvidia-tesla-k80
+          count: 8
+    # Whether to enable TPUs
+    enable_tpu: false
+    securityConfig:
+      # Whether to use a cluster with private IPs
+      # Use v1beta1 api
+      privatecluster: false
+      # masterIpv4CidrBlock for private clusters, if enabled
+      # Use v1beta1 api
+      masterIpv4CidrBlock: 172.16.0.16/28
+      # Protect worker node metadata from pods
+      # Use v1beta1 api
+      secureNodeMetadata: false
+      # Whether to enable Pod Security Policy Admission Controller
+      # Use v1beta1 api
+      podSecurityPolicy: false
+      masterAuthorizedNetworksConfigEnabled: false
+      masterAuthorizedNetworksConfigCidr:
+      - cidrBlock: 1.2.3.4/32
+    users:
+      # List users to grant appropriate GCP permissions to use Kubeflow.
+      # These can either be individual users (Google accounts) or Google
+      # Groups.
+      # - user:john@acme.com
+      # - group:data-scientists@acme.com
+    # This is the name of the GCP static ip address reserved for your domain.
+    # Each Kubeflow deployment in your project should use one unique ipName among all configs.
+    ipName: kubeflow-ip
--- a/gcp/deployment_manager_configs/cluster.jinja
+++ b/gcp/deployment_manager_configs/cluster.jinja
@ -0,0 +1,181 @@
+{#
+Copyright 2016 Google Inc. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+#}
+
+
+{% set NAME_PREFIX = env['deployment'] %}
+{% set CLUSTER_NAME = NAME_PREFIX %}
+{% set CPU_POOL = NAME_PREFIX + '-cpu-pool-' + properties['pool-version'] %}
+{% set GPU_POOL = NAME_PREFIX + '-gpu-pool-' + properties['pool-version'] %}
+{% set VM_OAUTH_SCOPES = ['https://www.googleapis.com/auth/logging.write',
+                          'https://www.googleapis.com/auth/monitoring',
+                          'https://www.googleapis.com/auth/devstorage.read_only'] %}
+
+{# Names for service accounts.
+   -admin is to be used for admin tasks
+   -user is to be used by users for actual jobs.
+   -vm is used for the VM service account attached to the GKE VMs.
+ #}
+{% set KF_ADMIN_NAME = NAME_PREFIX + '-admin' %}
+{% set KF_USER_NAME = NAME_PREFIX + '-user' %}
+{% set KF_VM_SA_NAME = NAME_PREFIX + '-vm' %}
+
+resources:
+- name: {{ KF_ADMIN_NAME }}
+  type: iam.v1.serviceAccount
+  properties:
+    accountId: {{ KF_ADMIN_NAME }}
+    displayName: Service Account used for Kubeflow admin actions.
+
+- name: {{ KF_USER_NAME }}
+  type: iam.v1.serviceAccount
+  properties:
+    accountId: {{ KF_USER_NAME }}
+    displayName: Service Account used for Kubeflow user actions.
+
+- name: {{ KF_VM_SA_NAME }}
+  type: iam.v1.serviceAccount
+  properties:
+    accountId: {{ KF_VM_SA_NAME }}
+    displayName: GCP Service Account to use as VM Service Account for Kubeflow Cluster VMs
+
+- name: {{ CLUSTER_NAME }}
+  {% if properties['gkeApiVersion'] == 'v1beta1' %}
+  type: gcp-types/container-v1beta1:projects.locations.clusters
+  {% else %}
+  type: container.v1.cluster
+  {% endif %}
+  properties:
+    parent: projects/{{ env['project'] }}/locations/{{ properties['zone'] }}
+    zone: {{ properties['zone'] }}
+    cluster:
+      name: {{ CLUSTER_NAME }}
+      initialClusterVersion: "{{ properties['cluster-version'] }}"
+      resourceLabels:
+        application: 'kubeflow'
+      {% if properties['gkeApiVersion'] == 'v1beta1' %}
+      # We need 1.10.2 to support Stackdriver GKE.
+      loggingService: logging.googleapis.com/kubernetes
+      monitoringService: monitoring.googleapis.com/kubernetes
+      {% if properties['enable_tpu'] %}
+      enable_tpu: {{ properties['enable_tpu'] }}
+      ipAllocationPolicy:
+        useIpAliases: {{ properties['enable_tpu'] }}
+      {% endif %}
+      podSecurityPolicyConfig:
+        enabled: {{ properties['securityConfig']['podSecurityPolicy'] }}
+      {% endif %}
+      {% if properties['enable-workload-identity'] %}
+      workloadIdentityConfig:
+        identityNamespace: {{ properties['identity-namespace'] }}
+      {% endif %}
+      {% if properties['securityConfig']['privatecluster'] %}
+      ipAllocationPolicy:
+        createSubnetwork: true
+        useIpAliases: true
+      privateClusterConfig:
+        masterIpv4CidrBlock: {{ properties['securityConfig']['masterIpv4CidrBlock'] }}
+        enablePrivateNodes: true
+      masterAuthorizedNetworksConfig:
+        enabled: {{ properties['securityConfig']['masterAuthorizedNetworksConfigEnabled'] }}
+        {% if properties['securityConfig']['masterAuthorizedNetworksConfigEnabled'] %}
+        cidrBlocks:
+          {{ properties['securityConfig']['masterAuthorizedNetworksConfigCidr'] }}
+        {% endif %}
+      {% endif %}
+      # Autoprovisioning is only supported in v1beta1.
+      {% if properties['gkeApiVersion'] == 'v1beta1' and properties['autoprovisioning-config']['enabled'] %}
+      autoscaling:
+        enableNodeAutoprovisioning: true
+        resourceLimits:
+        - resourceType: 'cpu'
+          maximum: {{ properties['autoprovisioning-config']['max-cpu'] }}
+        - resourceType: 'memory'
+          maximum: {{ properties['autoprovisioning-config']['max-memory'] }}
+          {% for accelerator in properties['autoprovisioning-config']['max-accelerator'] %}
+        - resourceType: {{ accelerator.type }}
+          maximum: {{ accelerator.count }}
+          {% endfor %}
+      {% endif %}
+      nodePools:
+      - name: {{ CPU_POOL }}
+        initialNodeCount: {{ properties['cpu-pool-initialNodeCount'] }}
+        autoscaling:
+          enabled: {{ properties['cpu-pool-enable-autoscaling'] }}
+          {% if properties['cpu-pool-enable-autoscaling'] %}
+          minNodeCount: {{ properties['cpu-pool-min-nodes'] }}
+          maxNodeCount: {{ properties['cpu-pool-max-nodes'] }}
+          {% endif %}
+        config:
+          {% if properties['securityConfig']['secureNodeMetadata'] %}
+          workloadMetadataConfig:
+            nodeMetadata: SECURE
+          {% endif %}
+          machineType: {{ properties['cpu-pool-machine-type'] }}
+          serviceAccount: {{ KF_VM_SA_NAME }}@{{ env['project'] }}.iam.gserviceaccount.com
+          oauthScopes: {{ VM_OAUTH_SCOPES }}
+          # Set min cpu platform to ensure AVX2 is supported.
+          minCpuPlatform: 'Intel Broadwell'
+  metadata:
+    dependsOn:
+    - {{ KF_VM_SA_NAME }}
+
+# We manage the node pools as separate resources.
+# We do this so that if we want to make changes we can delete the existing resource and then recreate it.
+# Updating doesn't work so well because we are limited in what changes GKE's update method supports.
+
+{% if properties['gpu-pool-max-nodes'] > 0 %}
+- name: {{ GPU_POOL }}
+  {% if properties['gkeApiVersion'] == 'v1beta1' %}
+  type: gcp-types/container-v1beta1:projects.locations.clusters.nodePools
+  {% else %}
+  type: container.v1.nodePool
+  {% endif %}
+  properties:
+    parent: projects/{{ env['project'] }}/locations/{{ properties['zone'] }}/clusters/{{ CLUSTER_NAME }}
+    project: {{ properties['securityConfig']['project'] }}
+    zone: {{ properties['zone'] }}
+    clusterId: {{ CLUSTER_NAME }}
+    nodePool:
+      name: gpu-pool
+      initialNodeCount: {{ properties['gpu-pool-initialNodeCount'] }}
+      autoscaling:
+        enabled: {{ properties['gpu-pool-enable-autoscaling'] }}
+        {% if properties['gpu-pool-enable-autoscaling'] %}
+        minNodeCount: {{ properties['gpu-pool-min-nodes'] }}
+        maxNodeCount: {{ properties['gpu-pool-max-nodes'] }}
+        {% endif %}
+      config:
+        {% if properties['securityConfig']['secureNodeMetadata'] %}
+        workloadMetadataConfig:
+          nodeMetadata: SECURE
+        {% endif %}
+        machineType: {{ properties['gpu-pool-machine-type'] }}
+        serviceAccount: {{ KF_VM_SA_NAME }}@{{ env['project'] }}.iam.gserviceaccount.com
+        oauthScopes: {{ VM_OAUTH_SCOPES }}
+        # Set min cpu platform to ensure AVX2 is supported.
+        minCpuPlatform: 'Intel Broadwell'
+        accelerators:
+          - acceleratorCount: {{ properties['gpu-number-per-node'] }}
+            acceleratorType: {{ properties['gpu-type'] }}
+
+  metadata:
+    dependsOn:
+    # We can only create 1 node pool at a time.
+    - {{ CLUSTER_NAME }}
+{% endif %}
+
+{# Project defaults to the project of the deployment. #}
+- name: {{ properties['ipName']  }}
+  type: compute.v1.globalAddress
+  properties:
+    description: "Static IP for Kubeflow ingress."
--- a/gcp/deployment_manager_configs/cluster.jinja.schema
+++ b/gcp/deployment_manager_configs/cluster.jinja.schema
@ -0,0 +1,34 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+info:
+  title: GKE cluster
+  author: Google, Inc.
+  description: |
+    Creates a GKE cluster and associated type for use in DM. The type can be
+    used in other DM configurations in the following manner:
+
+    "type: <cluster-type>:/api/v1/namespaces/{namespace}/services"
+
+required:
+- zone
+
+properties:
+  zone:
+    type: string
+    description: Zone in which the cluster should run.
+  initialNodeCount:
+    type: integer
+    description: Initial number of nodes desired in the cluster.
+    default: 4
--- a/gcp/deployment_manager_configs/gcfs.yaml
+++ b/gcp/deployment_manager_configs/gcfs.yaml
@ -0,0 +1,19 @@
+# Modify this instance to create a GCFS file store.
+# 1. Change the zone to the desired zone
+# 2. Change the instanceId to the desired id
+# 3. Change network if needed
+# 4. Change the capacity if desired.
+resources:
+- name: filestore
+  type: gcp-types/file-v1beta1:projects.locations.instances
+  properties:
+    parent: projects/isolated-project/locations/us-west1-b
+    # Any name of the instance would do
+    instanceId: YOUR_DEPLOYMENT_NAME
+    tier: STANDARD
+    description: Filestore for Kubeflow
+    networks:
+    - network: default
+    fileShares:
+    - name: kubeflow
+      capacityGb: 1024
--- a/gcp/deployment_manager_configs/iam_bindings_template.yaml
+++ b/gcp/deployment_manager_configs/iam_bindings_template.yaml
@ -0,0 +1,46 @@
+# This config is used by iam_patch.py. It is not a DM config.
+#
+# Schema for this yaml file
+# * bindings is a list of (members, roles) dict
+# * members and roles are lists
+# * each role in roles is granted to each member in members
+bindings:
+- members:
+  - set-kubeflow-admin-service-account
+  roles:
+  # Grant permissions needed to push the app to a cloud repository
+  - roles/source.admin
+  # servicemanagement.admin is needed by CloudEndpoints controller so we can create a service to get a hostname.
+  - roles/servicemanagement.admin
+  # Network admin is needed to enable IAP and configure network settings like backend timeouts and health checks
+  - roles/compute.networkAdmin
+- members:
+  - set-kubeflow-user-service-account
+  roles:
+  # Grant permissions needed to submit builds to Google Cloud Container Builder
+  - roles/cloudbuild.builds.editor
+  # roles/viewer is required for viewing the logs of a GCB build
+  - roles/viewer
+  # Grant permissions needed to push the app to a cloud repository
+  - roles/source.admin
+  - roles/storage.admin
+  - roles/bigquery.admin
+  - roles/dataflow.admin
+  - roles/ml.admin
+  - roles/dataproc.editor
+  - roles/cloudsql.admin
+- members:
+  - set-kubeflow-vm-service-account
+  roles:
+  # VM service account is used to write logs
+  - roles/logging.logWriter
+  # VM service account is used to write monitoring data
+  - roles/monitoring.metricWriter
+  # VM service account can retrieve monitoring data
+  - roles/monitoring.viewer
+  # VM service account is used to pull image from gcr
+  - roles/storage.objectViewer
+- members:
+  - set-kubeflow-iap-account
+  roles:
+  - roles/iap.httpsResourceAccessor
--- a/gcp/deployment_manager_configs/network.jinja
+++ b/gcp/deployment_manager_configs/network.jinja
@ -0,0 +1,19 @@
+# Copyright 2018 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resources:
+- type: gcp-types/compute-v1:networks
+  name: network-{{ env["deployment"] }}
+  properties:
+    autoCreateSubnetworks: true
--- a/gcp/deployment_manager_configs/network.yaml
+++ b/gcp/deployment_manager_configs/network.yaml
@ -0,0 +1,20 @@
+# Copyright 2018 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+imports:
+- path: network.jinja
+
+resources:
+- name: network
+  type: network.jinja
--- a/gcp/deployment_manager_configs/storage-kubeflow.yaml
+++ b/gcp/deployment_manager_configs/storage-kubeflow.yaml
@ -0,0 +1,35 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+imports:
+- path: storage.jinja
+
+resources:
+- name: kubeflow
+  type: storage.jinja
+  properties:
+    zone: SET_THE_ZONE
+    createPipelinePersistentStorage: SET_CREATE_PIPELINE_PERSISTENT_STORAGE
+    disks:
+     - sizeGb: 20
+       diskType: pd-standard
+       usage: metadata-store
+     - sizeGb: 200
+       diskType: pd-standard
+       usage: artifact-store
+    enable_cloudsql: false
+    database:
+      name: mlpipeline
+    dbUser:
+      user: root
--- a/gcp/deployment_manager_configs/storage.jinja
+++ b/gcp/deployment_manager_configs/storage.jinja
@ -0,0 +1,75 @@
+{#
+Copyright 2016 Google Inc. All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+#}
+
+{% macro diskName(diskObj) -%}{{ env["deployment"]}}-{{ diskObj["usage"] }}{%- endmacro %}
+
+{% set NAME_PREFIX = env['deployment'] %}
+{% set SQL_INSTANCE_NAME =  env['deployment'] + '-mysql' %}
+
+resources:
+{% if properties['createPipelinePersistentStorage'] %}
+{% for diskObj in properties["disks"] %}
+- name: {{ diskName(diskObj) }}
+  type: compute.v1.disk
+  properties:
+    zone: {{ properties["zone"] }}
+    sizeGb: {{ diskObj["sizeGb"] }}
+    type: https://www.googleapis.com/compute/v1/projects/{{ env["project"] }}/zones/{{ properties["zone"] }}/diskTypes/{{ diskObj["diskType"] }}
+{% endfor %}
+{% endif %}
+
+{% if properties['enable_cloudsql'] %}
+- name: {{ SQL_INSTANCE_NAME }}
+  type: sqladmin.v1beta4.instance
+  properties:
+    backendType: SECOND_GEN
+    instanceType: CLOUD_SQL_INSTANCE
+    databaseVersion: {{ properties['cloudsql']['databaseVersion'] }}
+    region: {{ properties['cloudsql']['region'] }}
+    settings:
+      tier: {{ properties['cloudsql']['tier'] }}
+      dataDiskSizeGb: {{ properties['cloudsql']['dataDiskSizeGb'] }}
+      dataDiskType: {{ properties['cloudsql']['dataDiskType'] }}
+      storageAutoResize: true
+      replicationType: SYNCHRONOUS
+      locationPreference:
+        zone: {{ properties['cloudsql']['zone'] }}
+      {% if properties['databaseFlags'] %}
+      databaseFlags: {{ properties['databaseFlags'] }}
+      {% endif %}
+      activationPolicy: ALWAYS
+      backupConfiguration:
+        enabled: true
+        binaryLogEnabled: true
+        startTime: {{ properties['cloudsql']['backupStartTime'] }}
+      ipConfiguration:
+        privateNetwork: projects/{{ env['project'] }}/global/networks/default
+        authorizedNetworks: {{ properties['cloudsql']['authorizedNetworks'] }}
+
+- name: {{ SQL_INSTANCE_NAME }}-db
+  type: sqladmin.v1beta4.database
+  properties:
+    name: {{ properties['database']['name'] }}
+    instance: $(ref.{{ SQL_INSTANCE_NAME }}.name)
+    charset: {{ properties['database']['charset'] }}
+
+- name: {{ SQL_INSTANCE_NAME }}-db-root
+  type: sqladmin.v1beta4.user
+  properties:
+    name: {{ properties['dbUser']['name'] }}
+    host:  "{{ properties['dbUser']['host'] }}"
+    instance: $(ref.{{ SQL_INSTANCE_NAME }}.name)
+  metadata:
+    dependsOn:
+      - {{ SQL_INSTANCE_NAME }}-db
+{% endif %}
--- a/gcp/deployment_manager_configs/storage.jinja.schema
+++ b/gcp/deployment_manager_configs/storage.jinja.schema
@ -0,0 +1,126 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+info:
+  title: Kubeflow Permanent Storage
+  author: Google, Inc.
+  description: |
+    Creates permanent storage for Kubeflow deployment
+
+required:
+- zone
+
+properties:
+  zone:
+    type: string
+  disks:
+    type: array
+    items:
+      type: object
+      required:
+      - usage
+      properties:
+        sizeGb:
+          type: integer
+          default: 200
+        diskType:
+          type: string
+          default: pd-standard
+          enum:
+          - pd-standard
+          - pd-ssd
+        usage:
+          type: string
+          description: what is the disk used for
+          enum:
+          - metadata-store
+          - artifact-store
+
+  cloudsql:
+    type: object
+    default:
+      properties:
+    properties:
+      databaseVersion:
+        type: string
+        description: MYSQL_5_7 or MYSQL_5_6
+        default: MYSQL_5_6
+      dataDiskSizeGb:
+        type: integer
+        minimum: 10
+        maximum: 1000
+        default: 10
+      dataDiskType:
+        type: string
+        decription: PD_SSD or PD_HDD
+        default: PD_SSD
+      backupStartTime:
+        type: string
+        description: HH:MM in 24 hour format
+        default: 00:00
+      tier:
+        type: string
+        description: https://cloud.google.com/sql/pricing#2nd-gen-pricing
+        default: db-n1-highmem-4
+      region:
+        type: string
+        description: i.e. us-central1
+        default: us-central1
+      zone:
+        type: string
+        description: i.e. us-central1-a
+        default: us-central1-a
+      authorizedNetworks:
+        type: array
+        description: An array of allowed CIDR blocks
+        items:
+          type: string
+
+  databaseFlags:
+    type: array
+    description: An array of https://cloud.google.com/sql/docs/mysql/flags
+    items:
+      type: object
+      required:
+        - name
+        - value
+      properties:
+        name:
+          type: string
+        value:
+          type:
+            - integer
+            - string
+
+  dbUser:
+    type: object
+    properties:
+      name:
+        type: string
+        default: root
+      host:
+        type: string
+        default: '%'
+
+  database:
+    type: object
+    required:
+      - name
+    properties:
+      name:
+        type: string
+      charset:
+        type: string
+        description: https://dev.mysql.com/doc/refman/5.7/en/charset.html
+        default: utf8
--- a/hack/gen-test-targets.sh
+++ b/hack/gen-test-targets.sh
@ -10,11 +10,25 @@ if [[ $(basename $PWD) != "manifests" ]]; then
  exit 1
 fi

+EXCLUDE_DIRS=( "kfdef" "gatekeeper" "gcp/deployment_manager_configs" "aws/infra_configs" )
 source hack/utils.sh
 rm -f $(ls tests/*_test.go | grep -v kusttestharness_test.go)
-for i in $(find * -type d -exec sh -c '(ls -p "{}"|grep />/dev/null)||echo "{}"' \; | egrep -v 'docs|gatekeeper|kfdef|tests|hack|plugins'); do
+for i in $(find * -type d -exec sh -c '(ls -p "{}"|grep />/dev/null)||echo "{}"' \; | egrep -v 'doc|tests|hack|plugins'); do
+  exclude=false
+  for item in "${EXCLUDE_DIRS[@]}"
+  do
+  	#https://stackoverflow.com/questions/2172352/in-bash-how-can-i-check-if-a-string-begins-with-some-value
+  	# Check if item is a prefix of i
+    if [[ "$i" == "$item"* ]]; then
+        exclude=true
+    fi
+  done
+
+  if $exclude; then
+    continue
+  fi
  rootdir=$(pwd)
-  absdir=$rootdir/$i
+  absdir=$rootdir/$i  
  if [[ ! $absdir  =~ overlays/test$ ]]; then
    testname=$(get-target-name $absdir)_test.go
    echo generating $testname from manifests/${absdir#*manifests/}
--- a/kfdef/README.md
+++ b/kfdef/README.md
@ -0,0 +1,3 @@
+This directory contains YAML files defining resources.
+
+These YAMl files can be used in conjuction with kfctl to deploy Kubeflow.
--- a/kfdef/generic/OWNERS
+++ b/kfdef/generic/OWNERS
@ -0,0 +1,2 @@
+approvers:
+  - yanniszark
--- a/kfdef/generic/README.md
+++ b/kfdef/generic/README.md
@ -0,0 +1,2 @@
+This directory contains some additional configuration files that are used by some KFDef resources
+when deploying with kfctl.
--- a/kfdef/generic/auth_oidc/authservice.tmpl
+++ b/kfdef/generic/auth_oidc/authservice.tmpl
@ -0,0 +1,73 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: authservice
+  namespace: istio-system
+spec:
+  type: ClusterIP
+  selector:
+    app: authservice
+  ports:
+  - port: 8080
+    name: http-authservice
+    targetPort: http-api
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: authservice
+  namespace: istio-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: authservice
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        sidecar.istio.io/inject: "false"
+      labels:
+        app: authservice
+    spec:
+      volumes:
+        - name: custom-ca
+          secret:
+            secretName: istio-ingressgateway-certs
+            items:
+              - key: tls.crt
+                path: tls.crt
+      containers:
+      - name: authservice
+        image: gcr.io/arrikto/kubeflow/oidc-authservice:v0.3
+        imagePullPolicy: Always
+        ports:
+        - name: http-api
+          containerPort: 8080
+        volumeMounts:
+          - name: custom-ca
+            mountPath: /etc/custom-ca
+            readOnly: true
+        env:
+          - name: USERID_HEADER
+            value: "kubeflow-userid"
+          - name: USERID_PREFIX
+            value: ""
+          - name: OIDC_PROVIDER_CA_FILE
+            value: "/etc/custom-ca/tls.crt"
+          - name: DISABLE_USERINFO
+            value: "true"
+          - name: PORT
+            value: "8080"
+          - name: OIDC_SCOPES
+            value: "profile email groups"
+          - name: OIDC_PROVIDER
+            value: {{.OIDCEndpoint}}
+          - name: SELF_URL
+            value: {{.KubeflowEndpoint}}
+          - name: CLIENT_ID
+            value: kubeflow-authservice-oidc
+          - name: CLIENT_SECRET
+            value: {{.AuthServiceClientSecret}} 
--- a/kfdef/generic/auth_oidc/dex.tmpl
+++ b/kfdef/generic/auth_oidc/dex.tmpl
@ -0,0 +1,121 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: dex
+  namespace: kubeflow
+spec:
+  selector:
+    app: dex
+  type: ClusterIP
+  ports:
+  - name: http
+    port: 5556
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: dex
+  name: dex
+  namespace: kubeflow
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: dex
+  template:
+    metadata:
+      labels:
+        app: dex
+    spec:
+      serviceAccountName: dex
+      containers:
+      - image: quay.io/coreos/dex:v2.10.0
+        name: dex
+        command: ["/usr/local/bin/dex", "serve", "/etc/dex/cfg/config.yaml"]
+
+        ports:
+        - name: http
+          containerPort: 5556
+
+        volumeMounts:
+        - name: config
+          mountPath: /etc/dex/cfg
+      volumes:
+      - name: config
+        configMap:
+          name: dex
+          items:
+          - key: config.yaml
+            path: config.yaml
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: dex
+  namespace: kubeflow
+data:
+  config.yaml: |
+    issuer: {{.OIDCEndpoint}}
+    storage:
+      type: kubernetes
+      config:
+        inCluster: true
+    web:
+      http: 0.0.0.0:5556
+    oauth2:
+      skipApprovalScreen: true
+    enablePasswordDB: true
+    {{if .KubeflowUser}}
+    staticPasswords:
+    - email: {{.KubeflowUser.UserEmail}}
+      # BCrypt Hash
+      hash: "{{.KubeflowUser.PasswordHash}}"
+      username: {{.KubeflowUser.Username}}
+      userID: "08a8684b-db88-4b73-90a9-3cd1661f5466"
+    {{end}}
+    staticClients:
+    - id: kubeflow-authservice-oidc
+      redirectURIs:
+      # After authenticating and giving consent, dex will redirect to
+      # this url for the specific client.
+      - {{.KubeflowEndpoint}}/login/oidc
+      name: 'Kubeflow AuthService OIDC'
+      secret: {{.AuthServiceClientSecret}} 
+    # Options for controlling the logger.
+    logger:
+      level: "debug"
+      format: "text"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: dex
+  name: dex
+  namespace: kubeflow
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: dex
+rules:
+- apiGroups: ["dex.coreos.com"] # API group created by dex
+  resources: ["*"]
+  verbs: ["*"]
+- apiGroups: ["apiextensions.k8s.io"]
+  resources: ["customresourcedefinitions"]
+  verbs: ["create"] # To manage its own resources, dex must be able to create customresourcedefinitions
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: dex
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: dex
+subjects:
+- kind: ServiceAccount
+  name: dex           # Service account assigned to the dex pod, created above
+  namespace: kubeflow
--- a/kfdef/generic/auth_oidc/envoy-filter.yaml
+++ b/kfdef/generic/auth_oidc/envoy-filter.yaml
@ -0,0 +1,34 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: EnvoyFilter
+metadata:
+  name: authn-filter
+  namespace: kubeflow
+spec:
+  workloadLabels:
+  #include namespace in the label to avoid clashes across namespaces
+    istio: ingressgateway
+  filters:
+  - filterConfig:
+      httpService:
+        serverUri:
+          uri: http://authservice.istio-system.svc.cluster.local
+          cluster: outbound|8080||authservice.istio-system.svc.cluster.local
+          failureModeAllow: false
+          timeout: 10s
+        authorizationRequest: 
+          allowedHeaders:
+            patterns:
+            - exact: "cookie"
+        authorizationResponse:
+          allowedUpstreamHeaders:
+            patterns:
+            - exact: "kubeflow-userid"
+      statusOnError:
+        code: GatewayTimeout
+    filterName: envoy.ext_authz
+    filterType: HTTP
+    insertPosition:
+      index: FIRST
+    listenerMatch:
+      portNumber: 443
+      listenerType: GATEWAY
--- a/kfdef/generic/auth_oidc/gateway.yaml
+++ b/kfdef/generic/auth_oidc/gateway.yaml
@ -0,0 +1,50 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: kubeflow-gateway 
+  namespace: kubeflow
+spec:
+  selector:
+    istio: ingressgateway
+  servers:
+  - port:
+      number: 5556
+      name: https-dex
+      protocol: HTTPS
+    hosts:
+    - "*"
+    tls:
+      mode: SIMPLE
+      serverCertificate: /etc/istio/ingressgateway-certs/tls.crt
+      privateKey: /etc/istio/ingressgateway-certs/tls.key
+  - port:
+      number: 443
+      name: https
+      protocol: HTTPS
+    hosts:
+    - "*"
+    tls:
+      mode: SIMPLE
+      serverCertificate: /etc/istio/ingressgateway-certs/tls.crt
+      privateKey: /etc/istio/ingressgateway-certs/tls.key    
+
+--- 
+
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: auth-virtual-services
+  namespace: kubeflow
+spec:
+  hosts:
+  - "*"
+  gateways:
+  - kubeflow-gateway
+  http:
+    - match:
+        - port: 5556
+      route:
+        - destination:
+            port:
+              number: 5556
+            host: dex.kubeflow.svc.cluster.local
--- a/kfdef/generic/istio/crds.yaml
+++ b/kfdef/generic/istio/crds.yaml
--- a/kfdef/generic/istio/istio-noauth.yaml
+++ b/kfdef/generic/istio/istio-noauth.yaml
--- a/kfdef/kfctl_aws.yaml
+++ b/kfdef/kfctl_aws.yaml
@ -0,0 +1,283 @@
+apiVersion: kfdef.apps.kubeflow.org/v1alpha1
+kind: KfDef
+metadata:
+  name: kubeflow-aws
+  namespace: kubeflow
+spec:
+  platform: aws
+  applications:
+    - kustomizeConfig:
+        parameters:
+          - name: namespace
+            value: istio-system
+        repoRef:
+          name: manifests
+          path: istio/istio-crds
+      name: istio-crds
+    - kustomizeConfig:
+        parameters:
+          - name: namespace
+            value: istio-system
+        repoRef:
+          name: manifests
+          path: istio/istio-install
+      name: istio-install
+    - kustomizeConfig:
+        parameters:
+          - name: clusterRbacConfig
+            value: "OFF"
+        repoRef:
+          name: manifests
+          path: istio/istio
+      name: istio
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: application/application-crds
+      name: application-crds
+    - kustomizeConfig:
+        overlays:
+          - application
+        repoRef:
+          name: manifests
+          path: application/application
+      name: application
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: metacontroller
+      name: metacontroller
+    - kustomizeConfig:
+        overlays:
+          - istio
+        repoRef:
+          name: manifests
+          path: argo
+      name: argo
+    - kustomizeConfig:
+        overlays:
+          - istio
+          - application
+        repoRef:
+          name: manifests
+          path: common/centraldashboard
+      name: centraldashboard
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: admission-webhook/webhook
+      name: webhook
+    - kustomizeConfig:
+        parameters:
+          - name: webhookNamePrefix
+            value: admission-webhook-
+        repoRef:
+          name: manifests
+          path: admission-webhook/bootstrap
+      name: bootstrap
+    - kustomizeConfig:
+        overlays:
+          - istio
+          - application
+        repoRef:
+          name: manifests
+          path: jupyter/jupyter-web-app
+      name: jupyter-web-app
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/katib-db
+      name: katib-db
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/katib-manager
+      name: katib-manager
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/katib-controller
+      name: katib-controller
+    - kustomizeConfig:
+        overlays:
+          - istio
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/katib-ui
+      name: katib-ui
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/metrics-collector
+      name: metrics-collector
+    - kustomizeConfig:
+        overlays:
+        - istio
+        repoRef:
+          name: manifests
+          path: metadata
+      name: metadata
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/suggestion
+      name: suggestion
+    - kustomizeConfig:
+        overlays:
+          - istio
+          - application
+        repoRef:
+          name: manifests
+          path: jupyter/notebook-controller
+      name: notebook-controller
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pytorch-job/pytorch-job-crds
+      name: pytorch-job-crds
+    - kustomizeConfig:
+        overlays:
+          - application
+        repoRef:
+          name: manifests
+          path: pytorch-job/pytorch-operator
+      name: pytorch-operator
+    - kustomizeConfig:
+        parameters:
+          - initRequired: true
+            name: usageId
+            value: <randomly-generated-id>
+          - initRequired: true
+            name: reportUsage
+            value: "true"
+        repoRef:
+          name: manifests
+          path: common/spartakus
+      name: spartakus
+    - kustomizeConfig:
+        overlays:
+          - istio
+        repoRef:
+          name: manifests
+          path: tensorboard
+      name: tensorboard
+    - kustomizeConfig:
+        overlays:
+          - istio
+          - application
+        repoRef:
+          name: manifests
+          path: tf-training/tf-job-operator
+      name: tf-job-operator
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pipeline/api-service
+      name: api-service
+    - kustomizeConfig:
+        parameters:
+          - name: minioPvcName
+            value: minio-pv-claim
+        repoRef:
+          name: manifests
+          path: pipeline/minio
+      name: minio
+    - kustomizeConfig:
+        parameters:
+          - name: mysqlPvcName
+            value: mysql-pv-claim
+        repoRef:
+          name: manifests
+          path: pipeline/mysql
+      name: mysql
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pipeline/persistent-agent
+      name: persistent-agent
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pipeline/pipelines-runner
+      name: pipelines-runner
+    - kustomizeConfig:
+        overlays:
+          - istio
+        repoRef:
+          name: manifests
+          path: pipeline/pipelines-ui
+      name: pipelines-ui
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pipeline/pipelines-viewer
+      name: pipelines-viewer
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pipeline/scheduledworkflow
+      name: scheduledworkflow
+    - kustomizeConfig:
+        overlays:
+          - istio
+        repoRef:
+          name: manifests
+          path: profiles
+      name: profiles
+    - kustomizeConfig:
+        overlays:
+          - application
+        repoRef:
+          name: manifests
+          path: seldon/seldon-core-operator
+    - kustomizeConfig:
+        overlays:
+          - application
+        repoRef:
+          name: manifests
+          path: mpi-job/mpi-operator
+    - kustomizeConfig:
+        parameters:
+          - initRequired: true
+            name: namespace
+            value: istio-system
+        repoRef:
+          name: manifests
+          path: aws/istio-ingress
+      name: istio-ingress
+    - kustomizeConfig:
+        parameters:
+          - initRequired: true
+            name: clusterName
+            value: kubeflow-aws
+        repoRef:
+          name: manifests
+          path: aws/aws-alb-ingress-controller
+      name: aws-alb-ingress-controller
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: aws/nvidia-device-plugin
+      name: nvidia-device-plugin
+  enableApplications: true
+  packageManager: kustomize
+  repos:
+    - name: kubeflow
+      uri: https://github.com/kubeflow/kubeflow/archive/master.tar.gz
+    - name: manifests
+      root: manifests-master
+      uri: https://github.com/kubeflow/manifests/archive/master.tar.gz
+  useBasicAuth: false
+  useIstio: true
+  version: master
+  plugins:
+    - name: aws
+      spec:
+        roles:
+          - eksctl-kubeflow-aws-nodegroup-ng-a2-NodeInstanceRole-xxxxxxx
+        region: us-west-2
+        auth:
+          basicAuth:
+            password:
+              name: password
+            username: admin
--- a/kfdef/kfctl_aws_cognito.yaml
+++ b/kfdef/kfctl_aws_cognito.yaml
@ -0,0 +1,290 @@
+apiVersion: kfdef.apps.kubeflow.org/v1alpha1
+kind: KfDef
+metadata:
+  name: kubeflow-aws
+  namespace: kubeflow
+spec:
+  platform: aws
+  applications:
+    - kustomizeConfig:
+        parameters:
+          - name: namespace
+            value: istio-system
+        repoRef:
+          name: manifests
+          path: istio/istio-crds
+      name: istio-crds
+    - kustomizeConfig:
+        parameters:
+          - name: namespace
+            value: istio-system
+        repoRef:
+          name: manifests
+          path: istio/istio-install
+      name: istio-install
+    - kustomizeConfig:
+        parameters:
+          - name: clusterRbacConfig
+            value: "OFF"
+        repoRef:
+          name: manifests
+          path: istio/istio
+      name: istio
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: application/application-crds
+      name: application-crds
+    - kustomizeConfig:
+        overlays:
+          - application
+        repoRef:
+          name: manifests
+          path: application/application
+      name: application
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: metacontroller
+      name: metacontroller
+    - kustomizeConfig:
+        overlays:
+          - istio
+        repoRef:
+          name: manifests
+          path: argo
+      name: argo
+    - kustomizeConfig:
+        overlays:
+          - istio
+          - application
+        repoRef:
+          name: manifests
+          path: common/centraldashboard
+      name: centraldashboard
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: admission-webhook/webhook
+      name: webhook
+    - kustomizeConfig:
+        parameters:
+          - name: webhookNamePrefix
+            value: admission-webhook-
+        repoRef:
+          name: manifests
+          path: admission-webhook/bootstrap
+      name: bootstrap
+    - kustomizeConfig:
+        overlays:
+          - istio
+          - application
+        repoRef:
+          name: manifests
+          path: jupyter/jupyter-web-app
+      name: jupyter-web-app
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/katib-db
+      name: katib-db
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/katib-manager
+      name: katib-manager
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/katib-controller
+      name: katib-controller
+    - kustomizeConfig:
+        overlays:
+          - istio
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/katib-ui
+      name: katib-ui
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/metrics-collector
+      name: metrics-collector
+    - kustomizeConfig:
+        overlays:
+          - istio
+        repoRef:
+          name: manifests
+          path: metadata
+      name: metadata
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: katib-v1alpha2/suggestion
+      name: suggestion
+    - kustomizeConfig:
+        overlays:
+          - istio
+          - application
+        repoRef:
+          name: manifests
+          path: jupyter/notebook-controller
+      name: notebook-controller
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pytorch-job/pytorch-job-crds
+      name: pytorch-job-crds
+    - kustomizeConfig:
+        overlays:
+          - application
+        repoRef:
+          name: manifests
+          path: pytorch-job/pytorch-operator
+      name: pytorch-operator
+    - kustomizeConfig:
+        parameters:
+          - initRequired: true
+            name: usageId
+            value: <randomly-generated-id>
+          - initRequired: true
+            name: reportUsage
+            value: "true"
+        repoRef:
+          name: manifests
+          path: common/spartakus
+      name: spartakus
+    - kustomizeConfig:
+        overlays:
+          - istio
+        repoRef:
+          name: manifests
+          path: tensorboard
+      name: tensorboard
+    - kustomizeConfig:
+        overlays:
+          - istio
+          - application
+        repoRef:
+          name: manifests
+          path: tf-training/tf-job-operator
+      name: tf-job-operator
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pipeline/api-service
+      name: api-service
+    - kustomizeConfig:
+        parameters:
+          - name: minioPvName
+            value: minio-pv
+          - name: minioPvcName
+            value: minio-pv-claim
+        repoRef:
+          name: manifests
+          path: pipeline/minio
+      name: minio
+    - kustomizeConfig:
+        parameters:
+          - name: mysqlPvName
+            value: mysql-pv
+          - name: mysqlPvcName
+            value: mysql-pv-claim
+        repoRef:
+          name: manifests
+          path: pipeline/mysql
+      name: mysql
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pipeline/persistent-agent
+      name: persistent-agent
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pipeline/pipelines-runner
+      name: pipelines-runner
+    - kustomizeConfig:
+        overlays:
+          - istio
+        repoRef:
+          name: manifests
+          path: pipeline/pipelines-ui
+      name: pipelines-ui
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pipeline/pipelines-viewer
+      name: pipelines-viewer
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: pipeline/scheduledworkflow
+      name: scheduledworkflow
+    - kustomizeConfig:
+        overlays:
+          - istio
+        repoRef:
+          name: manifests
+          path: profiles
+      name: profiles
+    - kustomizeConfig:
+        overlays:
+          - application
+        repoRef:
+          name: manifests
+          path: seldon/seldon-core-operator
+    - kustomizeConfig:
+        overlays:
+          - application
+        repoRef:
+          name: manifests
+          path: mpi-job/mpi-operator
+    - kustomizeConfig:
+        overlays:
+          - cognito
+        parameters:
+          - initRequired: true
+            name: namespace
+            value: istio-system
+        repoRef:
+          name: manifests
+          path: aws/istio-ingress
+      name: istio-ingress
+    - kustomizeConfig:
+        parameters:
+          - initRequired: true
+            name: clusterName
+            value: kubeflow-aws
+        repoRef:
+          name: manifests
+          path: aws/aws-alb-ingress-controller
+      name: aws-alb-ingress-controller
+    - kustomizeConfig:
+        repoRef:
+          name: manifests
+          path: aws/nvidia-device-plugin
+      name: nvidia-device-plugin
+  enableApplications: true
+  packageManager: kustomize
+  repos:
+    - name: kubeflow
+      uri: https://github.com/kubeflow/kubeflow/archive/master.tar.gz
+    - name: manifests
+      root: manifests-master
+      uri: https://github.com/kubeflow/manifests/archive/master.tar.gz
+  useBasicAuth: false
+  useIstio: true
+  version: master
+  plugins:
+    - name: aws
+      spec:
+        auth:
+          cognito:
+            cognitoUserPoolArn: arn:aws:cognito-idp:us-west-2:xxxxx:userpool/us-west-2_xxxxxx
+            cognitoAppClientId: xxxxxbxxxxxx
+            cognitoUserPoolDomain: your-user-pool
+            certArn: arn:aws:acm:us-west-2:xxxxx:certificate/xxxxxxxxxxxxx-xxxx
+        roles:
+          - eksctl-kubeflow-aws-nodegroup-ng-a2-NodeInstanceRole-xxxxx
+        region: us-west-2
--- a/kfdef/kfctl_existing_arrikto.yaml
+++ b/kfdef/kfctl_existing_arrikto.yaml
@ -0,0 +1,225 @@
+# This is the config to install Kubeflow on an existing K8s cluster, with support
+# for multi-user and LDAP auth using Dex.
+
+apiVersion: kfdef.apps.kubeflow.org/v1alpha1
+kind: KfDef
+metadata:
+  name: demo
+  namespace: kubeflow
+spec:
+  applications:
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: application/application-crds
+    name: application-crds
+  - kustomizeConfig:
+      overlays:
+      - application
+      repoRef:
+        name: manifests
+        path: application/application
+    name: application
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: metacontroller
+    name: metacontroller
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: argo
+    name: argo
+  - kustomizeConfig:
+      parameters:
+      - name: userid-header
+        value: kubeflow-userid
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: common/centraldashboard
+    name: centraldashboard
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: admission-webhook/webhook
+    name: webhook
+  - kustomizeConfig:
+      parameters:
+      - name: webhookNamePrefix
+        value: admission-webhook-
+      repoRef:
+        name: manifests
+        path: admission-webhook/bootstrap
+    name: bootstrap
+  - kustomizeConfig:
+      parameters:
+      - name: userid-header
+        value: kubeflow-userid
+      overlays:
+      - istio
+      - application
+      repoRef:
+        name: manifests
+        path: jupyter/jupyter-web-app
+    name: jupyter-web-app
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-db
+    name: katib-db
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-manager
+    name: katib-manager
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-controller
+    name: katib-controller
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-ui
+    name: katib-ui
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: metadata
+    name: metadata
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/metrics-collector
+    name: metrics-collector
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/suggestion
+    name: suggestion
+  - kustomizeConfig:
+      overlays:
+      - istio
+      - application
+      repoRef:
+        name: manifests
+        path: jupyter/notebook-controller
+    name: notebook-controller
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pytorch-job/pytorch-job-crds
+    name: pytorch-job-crds
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pytorch-job/pytorch-operator
+    name: pytorch-operator
+  - kustomizeConfig:
+      parameters:
+      - initRequired: true
+        name: usageId
+        value: <randomly-generated-id>
+      - initRequired: true
+        name: reportUsage
+        value: "true"
+      repoRef:
+        name: manifests
+        path: common/spartakus
+    name: spartakus
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: tensorboard
+    name: tensorboard
+  - kustomizeConfig:
+      overlays:
+      - istio
+      - application
+      repoRef:
+        name: manifests
+        path: tf-training/tf-job-operator
+    name: tf-job-operator
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/api-service
+    name: api-service
+  - kustomizeConfig:
+      parameters:
+      - name: minioPvcName
+        value: minio-pv-claim
+      repoRef:
+        name: manifests
+        path: pipeline/minio
+    name: minio
+  - kustomizeConfig:
+      parameters:
+      - name: mysqlPvcName
+        value: mysql-pv-claim
+      repoRef:
+        name: manifests
+        path: pipeline/mysql
+    name: mysql
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/persistent-agent
+    name: persistent-agent
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-runner
+    name: pipelines-runner
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-ui
+    name: pipelines-ui
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-viewer
+    name: pipelines-viewer
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/scheduledworkflow
+    name: scheduledworkflow
+  - kustomizeConfig:
+      parameters:
+      - name: userid-header
+        value: kubeflow-userid
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: profiles
+    name: profiles
+  - kustomizeConfig:
+      overlays:
+      - application
+      repoRef:
+        name: manifests
+        path: seldon/seldon-core-operator
+    name: seldon-core-operator
+  platform: existing_arrikto
+  repos:
+  - name: manifests
+    root: manifests-master
+    uri: https://github.com/kubeflow/manifests/archive/master.tar.gz
+  - name: kubeflow
+    root: kubeflow-master
+    uri: https://github.com/kubeflow/kubeflow/archive/master.tar.gz
--- a/kfdef/kfctl_gcp_basic_auth.yaml
+++ b/kfdef/kfctl_gcp_basic_auth.yaml
@ -0,0 +1,367 @@
+# Please set project and email!
+apiVersion: kfdef.apps.kubeflow.org/v1alpha1
+kind: KfDef
+metadata:
+  creationTimestamp: null
+  name: myapp2
+  namespace: kubeflow
+spec:
+  repos:
+  - name: kubeflow
+    root: kubeflow-master
+    uri: https://github.com/kubeflow/kubeflow/archive/master.tar.gz
+  - name: manifests
+    root: master/
+    uri: https://github.com/kubeflow/manifests/archive/master.tar.gz
+    # To get manifest at a PR:
+    #uri: https://github.com/kubeflow/manifests/archive/pull/235/head.tar.gz
+  appdir: /tmp/myapp2
+  applications:
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: istio-system
+      repoRef:
+        name: manifests
+        path: istio/istio-crds
+    name: istio-crds
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: istio-system
+      repoRef:
+        name: manifests
+        path: istio/istio-install
+    name: istio-install
+  - kustomizeConfig:
+      parameters:
+      - name: clusterRbacConfig
+        value: "OFF"
+      repoRef:
+        name: manifests
+        path: istio/istio
+    name: istio
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: application/application-crds
+    name: application-crds
+  - kustomizeConfig:
+      overlays:
+      - application
+      repoRef:
+        name: manifests
+        path: application/application
+    name: application
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: metacontroller
+    name: metacontroller
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: argo
+    name: argo
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: common/centraldashboard
+    name: centraldashboard
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: admission-webhook/webhook
+    name: webhook
+  - kustomizeConfig:
+      parameters:
+      - name: webhookNamePrefix
+        value: admission-webhook-
+      repoRef:
+        name: manifests
+        path: admission-webhook/bootstrap
+    name: bootstrap
+  - kustomizeConfig:
+      overlays:
+      - istio
+      - application
+      repoRef:
+        name: manifests
+        path: jupyter/jupyter-web-app
+    name: jupyter-web-app
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-db
+    name: katib-db
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-manager
+    name: katib-manager
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-controller
+    name: katib-controller
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-ui
+    name: katib-ui
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: metadata
+    name: metadata
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/metrics-collector
+    name: metrics-collector
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/suggestion
+    name: suggestion
+  - kustomizeConfig:
+      overlays:
+      - istio
+      - application
+      parameters:
+      - name: injectGcpCredentials
+        value: "true"
+      repoRef:
+        name: manifests
+        path: jupyter/notebook-controller
+    name: notebook-controller
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pytorch-job/pytorch-job-crds
+    name: pytorch-job-crds
+  - kustomizeConfig:
+      overlays:
+      - application
+      repoRef:
+        name: manifests
+        path: pytorch-job/pytorch-operator
+    name: pytorch-operator
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: knative-serving
+      repoRef:
+        name: manifests
+        path: knative/knative-serving-crds
+    name: knative-crds
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: knative-serving
+      repoRef:
+        name: manifests
+        path: knative/knative-serving-install
+    name: knative-install
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: kfserving/kfserving-crds
+    name: kfserving-crds
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: kfserving/kfserving-install
+    name: kfserving-install
+  - kustomizeConfig:
+      parameters:
+      - initRequired: true
+        name: usageId
+        value: "2700513155662330975"
+      - initRequired: true
+        name: reportUsage
+        value: "true"
+      repoRef:
+        name: manifests
+        path: common/spartakus
+    name: spartakus
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: tensorboard
+    name: tensorboard
+  - kustomizeConfig:
+      overlays:
+      - istio
+      - application
+      repoRef:
+        name: manifests
+        path: tf-training/tf-job-operator
+    name: tf-job-operator
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/api-service
+    name: api-service
+  - kustomizeConfig:
+      overlays:
+      - minioPd
+      parameters:
+      - name: minioPd
+        value: test1-storage-artifact-store
+      - name: minioPvName
+        value: minio-pv
+      - name: minioPvcName
+        value: minio-pv-claim
+      repoRef:
+        name: manifests
+        path: pipeline/minio
+    name: minio
+  - kustomizeConfig:
+      overlays:
+      - mysqlPd
+      parameters:
+      - name: mysqlPd
+        value: test1-storage-metadata-store
+      - name: mysqlPvName
+        value: mysql-pv
+      - name: mysqlPvcName
+        value: mysql-pv-claim
+      repoRef:
+        name: manifests
+        path: pipeline/mysql
+    name: mysql
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/persistent-agent
+    name: persistent-agent
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-runner
+    name: pipelines-runner
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-ui
+    name: pipelines-ui
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-viewer
+    name: pipelines-viewer
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/scheduledworkflow
+    name: scheduledworkflow
+  - kustomizeConfig:
+      overlays:
+      - gcp-credentials
+      parameters:
+      - name: secretName
+        value: admin-gcp-sa
+      - initRequired: true
+        name: ipName
+        value: ipName
+      - initRequired: true
+        name: hostname
+        # hostname will be set automatically by kfctl init & generate
+        # value: <deployName>.endpoints.<project>.cloud.goog
+      repoRef:
+        name: manifests
+        path: gcp/cloud-endpoints
+    name: cloud-endpoints
+  - kustomizeConfig:
+      overlays:
+      - istio
+      parameters:
+      - initRequired: true
+        name: admin
+        # emaill will be set automatically by kfctl init and generate
+        # value: SET_EMAIL
+      repoRef:
+        name: manifests
+        path: profiles
+    name: profiles
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: gcp/gpu-driver
+    name: gpu-driver
+  - kustomizeConfig:
+      overlays:
+      - application
+      repoRef:
+        name: manifests
+        path: seldon/seldon-core-operator
+    name: seldon-core-operator
+  - kustomizeConfig:
+      parameters:
+      - name: ambassadorServiceType
+        value: NodePort
+      - name: namespace
+        value: istio-system
+      repoRef:
+        name: manifests
+        path: common/ambassador
+    name: ambassador
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: common/basic-auth
+    name: basic-auth
+  - kustomizeConfig:
+      overlays:
+      - gcp-credentials
+      - managed-cert
+      parameters:
+      - name: namespace
+        value: istio-system
+      - initRequired: true
+        name: ipName
+        value: test1-ip
+      - initRequired: true
+        name: hostname
+        # project will be set automatically by kfctl init & generate
+        # value: test1.endpoints.SET_PROJECT.cloud.goog
+      - initRequired: true
+        name: project
+        # Project will be set automatically by kfctl init & generate
+        # value: SET_PROJECT
+      - name: ingressName
+        value: envoy-ingress
+      - name: issuer
+        value: letsencrypt-prod
+      repoRef:
+        name: manifests
+        path: gcp/basic-auth-ingress
+    name: basic-auth-ingress
+  # email should  be set the google account of the person setting up Kubeflow.
+  # If its not set kfctl generate will try to set it automatically based on the default
+  # gcloud config  
+  # email: <your_email@gmail.com>
+  enableApplications: true
+  packageManager: kustomize
+  platform: gcp
+  skipInitProject: true
+  useBasicAuth: true
+  useIstio: true
+  version: master
+  # Project should be set to the GCP project you want to use.
+  # If you run kfctl init --config=<path>/kfctl_gcp_iap.yaml 
+  # kfctl will try to automatically set it.
+  # project: <your project>
--- a/kfdef/kfctl_gcp_iap.yaml
+++ b/kfdef/kfctl_gcp_iap.yaml
@ -0,0 +1,337 @@
+# Please set project and email!
+apiVersion: kfdef.apps.kubeflow.org/v1alpha1
+kind: KfDef
+metadata:
+  creationTimestamp: null
+  name: myapp2
+  namespace: kubeflow
+spec:
+  repos:
+  - name: kubeflow
+    root: kubeflow-master
+    uri: https://github.com/kubeflow/kubeflow/archive/master.tar.gz
+  - name: manifests
+    root: master/
+    uri: https://github.com/kubeflow/manifests/archive/master.tar.gz
+    # To get manifest at a PR:
+    #uri: https://github.com/kubeflow/manifests/archive/pull/235/head.tar.gz
+  appdir: /tmp/myapp2
+  applications:
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: istio-system
+      repoRef:
+        name: manifests
+        path: istio/istio-crds
+    name: istio-crds
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: istio-system
+      repoRef:
+        name: manifests
+        path: istio/istio-install
+    name: istio-install
+  - kustomizeConfig:
+      parameters:
+      - name: clusterRbacConfig
+        value: "ON"
+      repoRef:
+        name: manifests
+        path: istio/istio
+    name: istio
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: application/application-crds
+    name: application-crds
+  - kustomizeConfig:
+      overlays:
+      - application
+      repoRef:
+        name: manifests
+        path: application/application
+    name: application
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: metacontroller
+    name: metacontroller
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: argo
+    name: argo
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: common/centraldashboard
+    name: centraldashboard
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: admission-webhook/webhook
+    name: webhook
+  - kustomizeConfig:
+      parameters:
+      - name: webhookNamePrefix
+        value: admission-webhook-
+      repoRef:
+        name: manifests
+        path: admission-webhook/bootstrap
+    name: bootstrap
+  - kustomizeConfig:
+      overlays:
+      - istio
+      - application
+      repoRef:
+        name: manifests
+        path: jupyter/jupyter-web-app
+    name: jupyter-web-app
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-db
+    name: katib-db
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-manager
+    name: katib-manager
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-controller
+    name: katib-controller
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-ui
+    name: katib-ui
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: metadata
+    name: metadata
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/metrics-collector
+    name: metrics-collector
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/suggestion
+    name: suggestion
+  - kustomizeConfig:
+      overlays:
+      - istio
+      - application
+      parameters:
+      - name: injectGcpCredentials
+        value: "true"
+      repoRef:
+        name: manifests
+        path: jupyter/notebook-controller
+    name: notebook-controller
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pytorch-job/pytorch-job-crds
+    name: pytorch-job-crds
+  - kustomizeConfig:
+      overlays:
+      - application
+      repoRef:
+        name: manifests
+        path: pytorch-job/pytorch-operator
+    name: pytorch-operator
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: knative-serving
+      repoRef:
+        name: manifests
+        path: knative/knative-serving-crds
+    name: knative-crds
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: knative-serving
+      repoRef:
+        name: manifests
+        path: knative/knative-serving-install
+    name: knative-install
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: kfserving/kfserving-crds
+    name: kfserving-crds
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: kfserving/kfserving-install
+    name: kfserving-install
+  - kustomizeConfig:
+      parameters:
+      - initRequired: true
+        name: usageId
+        value: "7439583937720421527"
+      - initRequired: true
+        name: reportUsage
+        value: "true"
+      repoRef:
+        name: manifests
+        path: common/spartakus
+    name: spartakus
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: tensorboard
+    name: tensorboard
+  - kustomizeConfig:
+      overlays:
+      - istio
+      - application
+      repoRef:
+        name: manifests
+        path: tf-training/tf-job-operator
+    name: tf-job-operator
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/api-service
+    name: api-service
+  - kustomizeConfig:
+      overlays:
+      - minioPd
+      parameters:
+      - name: minioPd
+        value: test1-storage-artifact-store
+      - name: minioPvName
+        value: minio-pv
+      - name: minioPvcName
+        value: minio-pv-claim
+      repoRef:
+        name: manifests
+        path: pipeline/minio
+    name: minio
+  - kustomizeConfig:
+      overlays:
+      - mysqlPd
+      parameters:
+      - name: mysqlPd
+        value: test1-storage-metadata-store
+      - name: mysqlPvName
+        value: mysql-pv
+      - name: mysqlPvcName
+        value: mysql-pv-claim
+      repoRef:
+        name: manifests
+        path: pipeline/mysql
+    name: mysql
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/persistent-agent
+    name: persistent-agent
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-runner
+    name: pipelines-runner
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-ui
+    name: pipelines-ui
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-viewer
+    name: pipelines-viewer
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/scheduledworkflow
+    name: scheduledworkflow
+  - kustomizeConfig:
+      overlays:
+      - gcp-credentials
+      parameters:
+      - name: secretName
+        value: admin-gcp-sa
+      repoRef:
+        name: manifests
+        path: gcp/cloud-endpoints
+    name: cloud-endpoints
+  - kustomizeConfig:
+      overlays:
+      - istio
+      parameters:
+      - initRequired: true
+        name: admin
+        value: SET_EMAIL
+      repoRef:
+        name: manifests
+        path: profiles
+    name: profiles
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: gcp/gpu-driver
+    name: gpu-driver
+  - kustomizeConfig:
+      overlays:
+      - gcp-credentials
+      - managed-cert
+      parameters:
+      - name: namespace
+        value: istio-system
+      - initRequired: true
+        name: ipName
+        value: test1-ip
+      - initRequired: true
+        name: hostname
+        # The value of hostname should be the DNS address for ingress.
+        # This will be set automatically during kfctl generate.
+        # value: test1.endpoints.SET_PROJECT.cloud.goog
+      repoRef:
+        name: manifests
+        path: gcp/iap-ingress
+    name: iap-ingress
+  - kustomizeConfig:
+      overlays:
+      - application
+      repoRef:
+        name: manifests
+        path: seldon/seldon-core-operator
+    name: seldon-core-operator
+  # email should  be set the google account of the person setting up Kubeflow.
+  # If its not set kfctl generate will try to set it automatically based on the default
+  # gcloud config  
+  # email: <your_email@gmail.com>
+  enableApplications: true
+  packageManager: kustomize
+  platform: gcp
+  skipInitProject: true
+  useBasicAuth: false
+  useIstio: true
+  version: master
+  # Project should be set to the GCP project you want to use.
+  # If you run kfctl init --config=<path>/kfctl_gcp_iap.yaml 
+  # kfctl will try to automatically set it.
+  # project: <your project>
--- a/kfdef/kfctl_k8s_istio.yaml
+++ b/kfdef/kfctl_k8s_istio.yaml
@ -0,0 +1,273 @@
+# This is the config to install Kubeflow on an existing k8s cluster.
+# If the cluster already has istio, comment out the istio install part below.
+
+apiVersion: kfdef.apps.kubeflow.org/v1alpha1
+kind: KfDef
+metadata:
+  name: kubeflow_app
+  namespace: kubeflow
+spec:
+  repos:
+  - name: manifests
+    root: manifests-master
+    uri: https://github.com/kubeflow/manifests/archive/master.tar.gz
+  - name: kubeflow
+    root: kubeflow-master
+    uri: https://github.com/kubeflow/kubeflow/archive/master.tar.gz
+  applications:
+  # Istio install. If not needed, comment out istio-crds and istio-install.
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: istio-system
+      repoRef:
+        name: manifests
+        path: istio/istio-crds
+    name: istio-crds
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: istio-system
+      repoRef:
+        name: manifests
+        path: istio/istio-install
+    name: istio-install
+  # This component is the istio resources for Kubeflow (e.g. gateway), not about installing istio.
+  - kustomizeConfig:
+      parameters:
+        - name: clusterRbacConfig
+          value: "OFF"
+      repoRef:
+        name: manifests
+        path: istio/istio
+    name: istio
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: application/application-crds
+    name: application-crds
+  - kustomizeConfig:
+      overlays:
+      - application
+      repoRef:
+        name: manifests
+        path: application/application
+    name: application
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: metacontroller
+    name: metacontroller
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: argo
+    name: argo
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: common/centraldashboard
+    name: centraldashboard
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: admission-webhook/bootstrap
+    name: bootstrap
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: admission-webhook/webhook
+    name: webhook
+  - kustomizeConfig:
+      overlays:
+      - istio
+      - application
+      repoRef:
+        name: manifests
+        path: jupyter/jupyter-web-app
+    name: jupyter-web-app
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-db
+    name: katib-db
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-manager
+    name: katib-manager
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-controller
+    name: katib-controller
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/katib-ui
+    name: katib-ui # Issue: https://github.com/kubeflow/manifests/issues/151
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: metadata
+    name: metadata
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/metrics-collector
+    name: metrics-collector
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: katib-v1alpha2/suggestion
+    name: suggestion
+  - kustomizeConfig:
+      overlays:
+      - istio
+      - application
+      repoRef:
+        name: manifests
+        path: jupyter/notebook-controller
+    name: notebook-controller
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pytorch-job/pytorch-job-crds
+    name: pytorch-job-crds
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pytorch-job/pytorch-operator
+    name: pytorch-operator
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: knative-serving
+      repoRef:
+        name: manifests
+        path: knative/knative-serving-crds
+    name: knative-crds
+  - kustomizeConfig:
+      parameters:
+      - name: namespace
+        value: knative-serving
+      repoRef:
+        name: manifests
+        path: knative/knative-serving-install
+    name: knative-install
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: kfserving/kfserving-crds
+    name: kfserving-crds
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: kfserving/kfserving-install
+    name: kfserving-install
+  - kustomizeConfig:
+      parameters:
+      - initRequired: true
+        name: usageId
+        value: <randomly-generated-id>
+      - initRequired: true
+        name: reportUsage
+        value: "true"
+      repoRef:
+        name: manifests
+        path: common/spartakus
+    name: spartakus
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: tensorboard
+    name: tensorboard
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: tf-training/tf-job-operator
+    name: tf-job-operator
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/api-service
+    name: api-service
+  - kustomizeConfig:
+      parameters:
+      - name: minioPvcName
+        value: minio-pv-claim
+      repoRef:
+        name: manifests
+        path: pipeline/minio
+    name: minio
+  - kustomizeConfig:
+      parameters:
+      - name: mysqlPvcName
+        value: mysql-pv-claim
+      repoRef:
+        name: manifests
+        path: pipeline/mysql
+    name: mysql
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/persistent-agent
+    name: persistent-agent
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-runner
+    name: pipelines-runner
+  - kustomizeConfig:
+      overlays:
+      - istio
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-ui
+    name: pipelines-ui
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/pipelines-viewer
+    name: pipelines-viewer
+  - kustomizeConfig:
+      repoRef:
+        name: manifests
+        path: pipeline/scheduledworkflow
+    name: scheduledworkflow
+  - kustomizeConfig:
+      overlays:
+      - istio
+      parameters:
+      - initRequired: true
+        name: admin
+        value: johnDoe@acme.com
+      repoRef:
+        name: manifests
+        path: profiles
+    name: profiles
+  - kustomizeConfig:
+      overlays:
+      - application
+      repoRef:
+        name: manifests
+        path: seldon/seldon-core-operator
+    name: seldon-core-operator
+  enableApplications: true
+  packageManager: kustomize
+  skipInitProject: true
+  useBasicAuth: false
+  useIstio: true
+  version: master