This commit is contained in:
Yao Xiao
GitHub
parent
bc07c3bb28
commit
5ec7ec8a00
|
|
@ -8,7 +8,6 @@ To install Kubeflow Pipelines, you have several options.
|
|||
- Via [GCP AI Platform UI](http://console.cloud.google.com/ai-platform/pipelines).
|
||||
- Via an upcoming commandline tool.
|
||||
- Via Kubectl with Kustomize, it's detailed here.
|
||||
- Community maintains a repo [here](https://github.com/e2fyi/kubeflow-aws/tree/master/pipelines) for AWS.
|
||||
|
||||
## Install via Kustomize
|
||||
|
||||
|
|
@ -51,6 +50,13 @@ Its storage is based on CloudSQL & GCS. It's better than others for production u
|
|||
|
||||
Please following [sample](sample/README.md) for a customized installation.
|
||||
|
||||
### Option-4 Install it to AWS with S3 and RDS MySQL
|
||||
Its storage is based on S3 & AWS RDS. It's more natural for AWS users to use this option.
|
||||
|
||||
Please following [AWS Instructions](env/aws/README.md) for installation.
|
||||
|
||||
Note: Community maintains a repo [e2fyi/kubeflow-aws](https://github.com/e2fyi/kubeflow-aws/tree/master/pipelines) for AWS.
|
||||
|
||||
## Uninstall
|
||||
|
||||
If the installation is based on CloudSQL/GCS, after the uninstall, the data is still there,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
approvers:
|
||||
- Jeffwan
|
||||
- PatrickXYS
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
# Sample installation
|
||||
|
||||
1. Create an EKS cluster and setup kubectl context
|
||||
|
||||
Using configuration file to simplify EKS cluster creation process:
|
||||
```
|
||||
apiVersion: eksctl.io/v1alpha5
|
||||
kind: ClusterConfig
|
||||
metadata:
|
||||
name: kfworkshop
|
||||
region: us-west-2
|
||||
version: '1.17'
|
||||
# If your region has multiple availability zones, you can specify 3 of them.
|
||||
availabilityZones: ["us-west-2b", "us-west-2c", "us-west-2d"]
|
||||
|
||||
# NodeGroup holds all configuration attributes that are specific to a nodegroup
|
||||
# You can have several node group in your cluster.
|
||||
nodeGroups:
|
||||
- name: cpu-nodegroup
|
||||
instanceType: m5.xlarge
|
||||
desiredCapacity: 2
|
||||
minSize: 0
|
||||
maxSize: 4
|
||||
volumeSize: 50
|
||||
# ssh:
|
||||
# allow: true
|
||||
# publicKeyPath: '~/.ssh/id_rsa.pub'
|
||||
|
||||
# Example of GPU node group
|
||||
- name: Tesla-V100
|
||||
instanceType: p3.8xlarge
|
||||
# Make sure the availability zone here is one of cluster availability zones.
|
||||
availabilityZones: ["us-west-2b"]
|
||||
desiredCapacity: 0
|
||||
minSize: 0
|
||||
maxSize: 4
|
||||
volumeSize: 50
|
||||
# ssh:
|
||||
# allow: true
|
||||
# publicKeyPath: '~/.ssh/id_rsa.pub'
|
||||
```
|
||||
Run this command to create EKS cluster
|
||||
```
|
||||
eksctl create cluster -f cluster.yaml
|
||||
```
|
||||
|
||||
2. Prepare S3
|
||||
|
||||
Create S3 bucket. [Console](https://console.aws.amazon.com/s3/home).
|
||||
|
||||
Run this command to create S3 bucket by changing `<YOUR_S3_BUCKET_NAME>` to your prefer s3 bucket name.
|
||||
|
||||
```
|
||||
export S3_BUCKET=<YOUR_S3_BUCKET_NAME>
|
||||
export AWS_REGION=us-west-2
|
||||
aws s3 mb s3://$S3_BUCKET --region $AWS_REGION
|
||||
```
|
||||
|
||||
3. Prepare RDS
|
||||
|
||||
Follow this [doc](https://www.kubeflow.org/docs/aws/rds/#deploy-amazon-rds-mysql-in-your-environment) to set up AWS RDS instance.
|
||||
|
||||
4. Customize your values
|
||||
- Edit [params.env](params.env), [secret.env](secret.env) and [minio-artifact-secret-patch.env](minio-artifact-secret-patch.env)
|
||||
|
||||
5. Install
|
||||
|
||||
```
|
||||
kubectl apply -k ../../cluster-scoped-resources
|
||||
|
||||
kubectl wait crd/applications.app.k8s.io --for condition=established --timeout=60s
|
||||
|
||||
kubectl apply -k ./
|
||||
# If upper one action got failed, e.x. you used wrong value, try delete, fix and apply again
|
||||
# kubectl delete -k ./
|
||||
|
||||
kubectl wait applications/mypipeline -n kubeflow --for condition=Ready --timeout=1800s
|
||||
|
||||
kubectl port-forward -n kubeflow svc/ml-pipeline-ui 8080:80
|
||||
```
|
||||
|
||||
Now you can access via `localhost:8080`
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ml-pipeline-ui
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ml-pipeline-ui
|
||||
spec:
|
||||
volumes:
|
||||
- name: config-volume
|
||||
configMap:
|
||||
name: ml-pipeline-ui-configmap
|
||||
containers:
|
||||
- name: ml-pipeline-ui
|
||||
env:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mlpipeline-minio-artifact
|
||||
key: accesskey
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mlpipeline-minio-artifact
|
||||
key: secretkey
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: ml-pipeline
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: ml-pipeline
|
||||
spec:
|
||||
containers:
|
||||
- env:
|
||||
- name: OBJECTSTORECONFIG_SECURE
|
||||
value: "true"
|
||||
- name: OBJECTSTORECONFIG_BUCKETNAME
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: pipeline-install-config
|
||||
key: bucketName
|
||||
- name: OBJECTSTORECONFIG_HOST
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: pipeline-install-config
|
||||
key: minioServiceHost
|
||||
- name: OBJECTSTORECONFIG_REGION
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: pipeline-install-config
|
||||
key: minioServiceRegion
|
||||
- name: OBJECTSTORECONFIG_PORT
|
||||
value: ""
|
||||
name: ml-pipeline-api-server
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
{
|
||||
namespace: $(kfp-namespace),
|
||||
executorImage: gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance,
|
||||
containerRuntimeExecutor: $(kfp-container-runtime-executor),
|
||||
artifactRepository:
|
||||
{
|
||||
s3: {
|
||||
bucket: $(kfp-artifact-bucket-name),
|
||||
keyPrefix: artifacts,
|
||||
endpoint: s3.amazonaws.com,
|
||||
insecure: true,
|
||||
accessKeySecret: {
|
||||
name: mlpipeline-minio-artifact,
|
||||
key: accesskey
|
||||
},
|
||||
secretKeySecret: {
|
||||
name: mlpipeline-minio-artifact,
|
||||
key: secretkey
|
||||
}
|
||||
},
|
||||
archiveLogs: true
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: kubeflow
|
||||
bases:
|
||||
- ../../env/platform-agnostic
|
||||
configMapGenerator:
|
||||
- name: pipeline-install-config
|
||||
env: params.env
|
||||
behavior: merge
|
||||
- name: workflow-controller-configmap
|
||||
behavior: replace
|
||||
files:
|
||||
- config
|
||||
- name: ml-pipeline-ui-configmap
|
||||
behavior: replace
|
||||
files:
|
||||
- viewer-pod-template.json
|
||||
secretGenerator:
|
||||
- name: mysql-secret
|
||||
env: secret.env
|
||||
behavior: merge
|
||||
- name: mlpipeline-minio-artifact
|
||||
env: minio-artifact-secret-patch.env
|
||||
behavior: merge
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
patchesStrategicMerge:
|
||||
- aws-configuration-patch.yaml
|
||||
# Identifier for application manager to apply ownerReference.
|
||||
# The ownerReference ensures the resources get garbage collected
|
||||
# when application is deleted.
|
||||
commonLabels:
|
||||
application-crd-id: kubeflow-pipelines
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
accesskey=YOUR_AWS_ACCESS_ID
|
||||
secretkey=YOUR_AWS_SECRET_KEY
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
dbHost=YOUR_RDS_ENDPOINT
|
||||
|
||||
bucketName=YOUR_S3_BUCKET_NAME
|
||||
minioServiceHost=s3.amazonaws.com
|
||||
minioServiceRegion=YOUR_AWS_REGION
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
username=YOUR_RDS_USERNAME
|
||||
password=YOUR_RDS_PASSWORD
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
{
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"env": [
|
||||
{
|
||||
"name": "AWS_ACCESS_KEY_ID",
|
||||
"valueFrom": {
|
||||
"secretKeyRef": {
|
||||
"name": "mlpipeline-minio-artifact",
|
||||
"key": "accesskey"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AWS_SECRET_ACCESS_KEY",
|
||||
"valueFrom": {
|
||||
"secretKeyRef": {
|
||||
"name": "mlpipeline-minio-artifact",
|
||||
"key": "secretkey"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "AWS_REGION",
|
||||
"valueFrom": {
|
||||
"configMapKeyRef": {
|
||||
"name": "pipeline-install-config",
|
||||
"key": "minioServiceRegion"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue