chore: update release notes (#11827)

Signed-off-by: Humair Khan <HumairAK@users.noreply.github.com>
This commit is contained in:
Humair Khan 2025-04-15 18:00:22 -04:00 committed by GitHub
parent aa95142f5a
commit 177cd3b3e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 49 additions and 995 deletions

View File

@ -1,272 +0,0 @@
# Copyright 2018 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Test before submit:
# gcloud builds submit --config=.cloudbuild.yaml --substitutions=COMMIT_SHA="$(git rev-parse HEAD)" --project=ml-pipeline-test
steps:
# # Build the Python SDK
# - name: 'python:3-alpine'
# entrypoint: '/bin/sh'
# args: ['-c', 'cd /workspace/sdk/python/; python3 setup.py sdist --format=gztar; cp dist/*.tar.gz /workspace/kfp.tar.gz']
# id: 'preparePythonSDK'
# waitFor: ["-"]
# - name: 'gcr.io/cloud-builders/gsutil'
# args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz']
# id: 'copyPythonSDK'
# waitFor: ['preparePythonSDK']
# - name: 'gcr.io/cloud-builders/gsutil'
# args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp.tar.gz']
# id: 'copyPythonSDKToLatest'
# waitFor: ['preparePythonSDK']
# Build the pipeline system images
- name: 'gcr.io/cloud-builders/docker'
entrypoint: /bin/bash
args:
- -ceux
- |
sed -i -e "s/ARG DATE/ENV DATE \"$(date -u)\"/" /workspace/frontend/Dockerfile
docker build -t gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA \
--build-arg COMMIT_HASH=$COMMIT_SHA \
--build-arg TAG_NAME="$(cat /workspace/VERSION)" \
-f /workspace/frontend/Dockerfile \
/workspace
id: 'buildFrontend'
waitFor: ['-']
- name: 'gcr.io/cloud-builders/docker'
entrypoint: /bin/bash
args:
- -ceux
- |
docker build -t gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA \
--build-arg COMMIT_SHA=$COMMIT_SHA \
--build-arg TAG_NAME="$(cat /workspace/VERSION)" \
-f /workspace/backend/Dockerfile /workspace
id: 'buildApiServer'
waitFor: ['-']
- name: 'gcr.io/cloud-builders/docker'
args: ['build', '-t', 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA', '-f',
'/workspace/backend/Dockerfile.scheduledworkflow', '/workspace']
id: 'buildScheduledWorkflow'
waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
args: ['build', '-t', 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA', '-f',
'/workspace/backend/Dockerfile.viewercontroller', '/workspace']
id: 'buildViewerCrdController'
- name: 'gcr.io/cloud-builders/docker'
args: ['build', '-t', 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA', '-f',
'/workspace/backend/Dockerfile.persistenceagent', '/workspace']
id: 'buildPersistenceAgent'
waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
args: ['build', '-t', 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA', '-f',
'/workspace/proxy/Dockerfile', '/workspace/proxy']
id: 'buildInverseProxyAgent'
waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
args: ['build', '-t', 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA',
'--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
'/workspace/backend/Dockerfile.visualization', '/workspace']
id: 'buildVisualizationServer'
waitFor: ["-"]
- id: 'buildMetadataWriter'
name: 'gcr.io/cloud-builders/docker'
args: ['build', '-t', 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA',
'--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
'/workspace/backend/metadata_writer/Dockerfile', '/workspace']
waitFor: ["-"]
- id: 'buildCacheServer'
name: 'gcr.io/cloud-builders/docker'
args: ['build', '-t', 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA',
'--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
'/workspace/backend/Dockerfile.cacheserver', '/workspace']
waitFor: ["-"]
- id: 'buildCacheDeployer'
name: 'gcr.io/cloud-builders/docker'
args: ['build', '-t', 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA',
'--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
'/workspace/backend/src/cache/deployer/Dockerfile', '/workspace']
waitFor: ["-"]
# Build marketplace deployer
- id: 'buildMarketplaceDeployer'
name: 'gcr.io/cloud-builders/docker'
args: ['build', '-t', 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA',
'--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
'/workspace/manifests/gcp_marketplace/deployer/Dockerfile', '/workspace/manifests/gcp_marketplace']
waitFor: ["-"]
# Build the Kubeflow-based pipeline component images
- name: 'gcr.io/cloud-builders/docker'
args: ['build', '-t', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA',
'/workspace/components/kubeflow/deployer']
id: 'buildDeployer'
waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
entrypoint: '/bin/bash'
args: ['-c', 'cd /workspace/components/kubeflow/launcher && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
id: 'buildTFJobLauncher'
waitFor: ["-"]
- id: 'buildCpuTrainer'
name: 'gcr.io/cloud-builders/docker'
entrypoint: '/bin/bash'
args: ['-c', 'cd /workspace/components/kubeflow/dnntrainer && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA -l ml-pipeline-kubeflow-tf-trainer -b 2.3.0']
waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
entrypoint: '/bin/bash'
args: ['-c', 'cd /workspace/components/kubeflow/dnntrainer && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA -l ml-pipeline-kubeflow-tf-trainer-gpu -b 2.3.0-gpu']
id: 'buildGpuTrainer'
waitFor: ["-"]
# Build the local pipeline component images
- name: 'gcr.io/cloud-builders/docker'
entrypoint: '/bin/bash'
args: ['-c', 'cd /workspace/components/local/confusion_matrix && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
id: 'buildConfusionMatrix'
waitFor: ["-"]
- name: 'gcr.io/cloud-builders/docker'
entrypoint: '/bin/bash'
args: ['-c', 'cd /workspace/components/local/roc && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
id: 'buildROC'
waitFor: ["-"]
# Build third_party images
- name: 'gcr.io/cloud-builders/docker'
args: ['build', '-t', 'gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA', '-f',
'/workspace/third_party/metadata_envoy/Dockerfile', '/workspace']
id: 'buildMetadataEnvoy'
# Pull third_party images
# ! Sync to the same MLMD version:
# * backend/metadata_writer/requirements.in and requirements.txt
# * @kubeflow/frontend/src/mlmd/generated
# * .cloudbuild.yaml and .release.cloudbuild.yaml
# * manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml
# * test/tag_for_hosted.sh
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0']
id: 'pullMetadataServer'
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance']
id: 'pullMinio'
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/mysql:8.0.26']
id: 'pullMysql'
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.25.0']
id: 'pullCloudsqlProxy'
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.5.14-license-compliance']
id: 'pullArgoExecutor'
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.5.14-license-compliance']
id: 'pullArgoWorkflowController'
# V2 related images
# Prerequisite: Make sure ko image is available on the same project by running the following:
# git clone https://github.com/GoogleCloudPlatform/cloud-builders-community.git
# cd cloud-builders-community/ko
# gcloud builds submit . --config=cloudbuild.yaml --project=$PROJECT_ID
# Reference: https://dev.to/amammay/effective-go-on-gcp-lean-containers-with-ko-on-cloud-build-51ek
# Temporarily disable v2 image build due to unblock kubeflow-pipeline-mkp-test
# We aren't building v2 images for MKP at this moment anyway.
#
# - name: 'gcr.io/$PROJECT_ID/ko'
# entrypoint: /bin/sh
# args:
# - -c
# - |
# cd /workspace/backend/src/v2/
# /ko publish --bare ./cmd/launcher-v2 -t $COMMIT_SHA
# env:
# - 'KO_DOCKER_REPO=gcr.io/$PROJECT_ID/kfp-launcher'
# id: 'buildLauncher'
# waitFor: ["-"]
# - name: 'gcr.io/$PROJECT_ID/ko'
# entrypoint: /bin/sh
# args:
# - -c
# - |
# cd /workspace/backend/src/v2/
# /ko publish --bare ./cmd/driver -t $COMMIT_SHA
# env:
# - 'KO_DOCKER_REPO=gcr.io/$PROJECT_ID/kfp-driver'
# id: 'buildDriver'
# waitFor: ["-"]
# Tag for Hosted - SemVersion to Major.Minor parsing
- id: "parseMajorMinorVersion"
waitFor: ["-"]
name: gcr.io/cloud-builders/docker
entrypoint: /bin/bash
args:
- -ceux
- |
# Parse major minor version and save to a file for reusing in other steps.
# e.g. 1.0.0-rc.1 and 1.0.1 are parsed as 1.0
cat /workspace/VERSION | sed -e "s#\([0-9]\+[.][0-9]\+\)[.].*#\1#" > /workspace/mm.ver
# Tag for Hosted - Tag to hosted folder with MKP friendly name
- id: 'tagForHosted'
waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildScheduledWorkflow',
'buildViewerCrdController', 'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
'buildMetadataWriter', 'buildCacheServer', 'buildCacheDeployer', 'buildMetadataEnvoy',
'buildMarketplaceDeployer', 'pullMetadataServer', 'pullMinio', 'pullMysql', 'pullCloudsqlProxy',
'pullArgoExecutor', 'pullArgoWorkflowController']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
/workspace/test/tag_for_hosted.sh $PROJECT_ID $COMMIT_SHA $(cat /workspace/VERSION) $(cat /workspace/mm.ver)
images:
# Images for the pipeline system itself
- 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA'
# Images for Marketplace
- 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA'
# Images for the Kubeflow-based pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer-gpu:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tfjob:$COMMIT_SHA'
# Images for the local components
- 'gcr.io/$PROJECT_ID/ml-pipeline-local-confusion-matrix:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-local-roc:$COMMIT_SHA'
# Images for the third_party components
- 'gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA'
timeout: '3600s'
options:
diskSizeGb: 300
machineType: 'N1_HIGHCPU_8'
tags:
- build-each-commit

View File

@ -1,652 +0,0 @@
# Copyright 2018 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
steps:
# Marketplace Major.Minor parsing
- id: "parseMajorMinorVersion"
name: gcr.io/cloud-builders/docker
entrypoint: /bin/bash
args:
- -ceux
- |
# Parse major minor version and save to a file for reusing in other steps.
# e.g. 1.0.0-rc.1 and 1.0.1 are parsed as 1.0
echo $TAG_NAME | sed -e "s#\([0-9]\+[.][0-9]\+\)[.].*#\1#" > /workspace/mm.ver
# Pull and retag images for pipeline components
- id: 'retagComponentImages'
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
waitFor: ['-']
args:
- -ceux
- |
images=(
"ml-pipeline-kubeflow-deployer"
"ml-pipeline-kubeflow-tf-trainer"
"ml-pipeline-kubeflow-tf-trainer-gpu"
"ml-pipeline-kubeflow-tfjob"
"ml-pipeline-local-confusion-matrix"
"ml-pipeline-local-roc"
)
for image in "${images[@]}"
do
from_image="gcr.io/$PROJECT_ID/$image:$COMMIT_SHA"
target_image="gcr.io/ml-pipeline/$image:$TAG_NAME"
docker pull $from_image
docker tag $from_image $target_image
docker push $target_image
done
# Pull and retag the images for the pipeline system
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA']
id: 'pullFrontend'
- id: 'tagFrontendForMarketplaceMajorMin'
waitFor: ['pullFrontend', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/ml-pipeline/frontend:$TAG_NAME
docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/ml-pipeline/frontend:$COMMIT_SHA
docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/frontend:$TAG_NAME
docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/frontend:$TAG_NAME
docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/frontend:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/frontend:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/frontend:$TAG_NAME
docker push gcr.io/ml-pipeline/frontend:$COMMIT_SHA
docker push gcr.io/ml-pipeline/google/pipelines/frontend:$TAG_NAME
docker push gcr.io/ml-pipeline/google/pipelines-test/frontend:$TAG_NAME
docker push gcr.io/ml-pipeline/google/pipelines/frontend:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/frontend:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA']
id: 'pullAPIServer'
- id: 'tagAPIServerForMarketplaceMajorMinor'
waitFor: ['pullAPIServer', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA 'gcr.io/ml-pipeline/api-server:$TAG_NAME'
docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA 'gcr.io/ml-pipeline/api-server:$COMMIT_SHA'
docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA 'gcr.io/ml-pipeline/google/pipelines/apiserver:$TAG_NAME'
docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA 'gcr.io/ml-pipeline/google/pipelines-test/apiserver:$TAG_NAME'
docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/apiserver:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/apiserver:$(cat /workspace/mm.ver)
docker push 'gcr.io/ml-pipeline/api-server:$TAG_NAME'
docker push 'gcr.io/ml-pipeline/api-server:$COMMIT_SHA'
docker push 'gcr.io/ml-pipeline/google/pipelines/apiserver:$TAG_NAME'
docker push 'gcr.io/ml-pipeline/google/pipelines-test/apiserver:$TAG_NAME'
docker push gcr.io/ml-pipeline/google/pipelines/apiserver:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/apiserver:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA']
id: 'pullScheduledworkflow'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA', 'gcr.io/ml-pipeline/scheduledworkflow:$TAG_NAME']
id: 'tagScheduledworkflowVersionNumber'
waitFor: ['pullScheduledworkflow']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA', 'gcr.io/ml-pipeline/scheduledworkflow:$COMMIT_SHA']
id: 'tagScheduledworkflowCommitSHA'
waitFor: ['pullScheduledworkflow']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines/scheduledworkflow:$TAG_NAME']
id: 'tagScheduledworkflowForMarketplace'
waitFor: ['pullScheduledworkflow']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines-test/scheduledworkflow:$TAG_NAME']
id: 'tagScheduledworkflowForMarketplaceTest'
waitFor: ['pullScheduledworkflow']
- id: 'tagScheduledworkflowForMarketplaceMajorMinor'
waitFor: ['pullScheduledworkflow', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/scheduledworkflow:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/scheduledworkflow:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/scheduledworkflow:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/scheduledworkflow:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA']
id: 'pullViewerCrdController'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA', 'gcr.io/ml-pipeline/viewer-crd-controller:$TAG_NAME']
id: 'tagViewerCrdControllerVersionNumber'
waitFor: ['pullViewerCrdController']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA', 'gcr.io/ml-pipeline/viewer-crd-controller:$COMMIT_SHA']
id: 'tagViewerCrdControllerCommitSHA'
waitFor: ['pullViewerCrdController']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines/viewercrd:$TAG_NAME']
id: 'tagViewerCrdControllerForMarketplace'
waitFor: ['pullViewerCrdController']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines-test/viewercrd:$TAG_NAME']
id: 'tagViewerCrdControllerForMarketplaceTest'
waitFor: ['pullViewerCrdController']
- id: 'tagViewerCrdControllerForMarketplaceMajorMinor'
waitFor: ['pullViewerCrdController', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/viewercrd:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/viewercrd:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/viewercrd:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/viewercrd:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA']
id: 'pullPersistenceagent'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA', 'gcr.io/ml-pipeline/persistenceagent:$TAG_NAME']
id: 'tagPersistenceagentVersionNumber'
waitFor: ['pullPersistenceagent']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA', 'gcr.io/ml-pipeline/persistenceagent:$COMMIT_SHA']
id: 'tagPersistenceagentCommitSHA'
waitFor: ['pullPersistenceagent']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines/persistenceagent:$TAG_NAME']
id: 'tagPersistenceagentForMarketplace'
waitFor: ['pullPersistenceagent']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines-test/persistenceagent:$TAG_NAME']
id: 'tagPersistenceagentForMarketplaceTest'
waitFor: ['pullPersistenceagent']
- id: 'tagPersistenceagentForMarketplaceMajorMinor'
waitFor: ['pullPersistenceagent', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/persistenceagent:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/persistenceagent:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/persistenceagent:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/persistenceagent:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA']
id: 'pullInverseProxyAgent'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA', 'gcr.io/ml-pipeline/inverse-proxy-agent:$TAG_NAME']
id: 'tagInverseProxyAgentVersionNumber'
waitFor: ['pullInverseProxyAgent']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA', 'gcr.io/ml-pipeline/inverse-proxy-agent:$COMMIT_SHA']
id: 'tagInverseProxyAgentCommitSHA'
waitFor: ['pullInverseProxyAgent']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines/proxyagent:$TAG_NAME']
id: 'tagInverseProxyAgentForMarketplace'
waitFor: ['pullInverseProxyAgent']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines-test/proxyagent:$TAG_NAME']
id: 'tagInverseProxyAgentForMarketplaceTest'
waitFor: ['pullInverseProxyAgent']
- id: 'tagInverseProxyAgentForMarketplaceMajorMinor'
waitFor: ['pullInverseProxyAgent', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/proxyagent:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/proxyagent:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/proxyagent:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/proxyagent:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA']
id: 'pullVisualizationServer'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA', 'gcr.io/ml-pipeline/visualization-server:$TAG_NAME']
id: 'tagVisualizationServerVersionNumber'
waitFor: ['pullVisualizationServer']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA', 'gcr.io/ml-pipeline/visualization-server:$COMMIT_SHA']
id: 'tagVisualizationServerCommitSHA'
waitFor: ['pullVisualizationServer']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines/visualizationserver:$TAG_NAME']
id: 'tagVisualizationServerForMarketplace'
waitFor: ['pullVisualizationServer']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines-test/visualizationserver:$TAG_NAME']
id: 'tagVisualizationServerForMarketplaceTest'
waitFor: ['pullVisualizationServer']
- id: 'tagVisualizationServerForMarketplaceMajorMinor'
waitFor: ['pullVisualizationServer', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/visualizationserver:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/visualizationserver:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/visualizationserver:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/visualizationserver:$(cat /workspace/mm.ver)
# ! Sync to the same MLMD version:
# * backend/metadata_writer/requirements.in and requirements.txt
# * @kubeflow/frontend/src/mlmd/generated
# * .cloudbuild.yaml and .release.cloudbuild.yaml
# * manifests/kustomize/base/metadata/base/metadata-grpc-deployment.yaml
# * test/tag_for_hosted.sh
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0']
id: 'pullMetadataServer'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0', 'gcr.io/ml-pipeline/google/pipelines/metadataserver:$TAG_NAME']
id: 'tagMetadataServerForMarketplace'
waitFor: ['pullMetadataServer']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0', 'gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$TAG_NAME']
id: 'tagMetadataServerForMarketplaceTest'
waitFor: ['pullMetadataServer']
- id: 'tagMetadataServerForMarketplaceMajorMinor'
waitFor: ['pullMetadataServer', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 gcr.io/ml-pipeline/google/pipelines/metadataserver:$(cat /workspace/mm.ver)
docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/metadataserver:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$(cat /workspace/mm.ver)
- id: 'pullMetadataWriter'
name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA']
waitFor: ['-']
- id: 'tagMetadataWriterVersionNumber'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA', 'gcr.io/ml-pipeline/metadata-writer:$TAG_NAME']
waitFor: ['pullMetadataWriter']
- id: 'tagMetadataWriterCommitSHA'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA', 'gcr.io/ml-pipeline/metadata-writer:$COMMIT_SHA']
waitFor: ['pullMetadataWriter']
- id: 'tagMetadataWriterForMarketplace'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines/metadatawriter:$TAG_NAME']
waitFor: ['pullMetadataWriter']
- id: 'tagMetadataWriterForMarketplaceTest'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines-test/metadatawriter:$TAG_NAME']
waitFor: ['pullMetadataWriter']
- id: 'tagMetadataWriterForMarketplaceMajorMinor'
waitFor: ['pullMetadataWriter', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/metadatawriter:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/metadatawriter:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/metadatawriter:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/metadatawriter:$(cat /workspace/mm.ver)
- id: 'pullCacheServer'
name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA']
waitFor: ['-']
- id: 'tagCacheServerVersionNumber'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA', 'gcr.io/ml-pipeline/cache-server:$TAG_NAME']
waitFor: ['pullCacheServer']
- id: 'tagCacheServerCommitSHA'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA', 'gcr.io/ml-pipeline/cache-server:$COMMIT_SHA']
waitFor: ['pullCacheServer']
- id: 'tagCacheServerForMarketplace'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines/cacheserver:$TAG_NAME']
waitFor: ['pullCacheServer']
- id: 'tagCacheServerForMarketplaceTest'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines-test/cacheserver:$TAG_NAME']
waitFor: ['pullCacheServer']
- id: 'tagCacheServerForMarketplaceMajorMinor'
waitFor: ['pullCacheServer', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/cacheserver:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/cacheserver:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/cacheserver:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/cacheserver:$(cat /workspace/mm.ver)
- id: 'pullCacheDeployer'
name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA']
waitFor: ['-']
- id: 'tagCacheDeployerVersionNumber'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA', 'gcr.io/ml-pipeline/cache-deployer:$TAG_NAME']
waitFor: ['pullCacheDeployer']
- id: 'tagCacheDeployerCommitSHA'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA', 'gcr.io/ml-pipeline/cache-deployer:$COMMIT_SHA']
waitFor: ['pullCacheDeployer']
- id: 'tagCacheDeployerForMarketplace'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines/cachedeployer:$TAG_NAME']
waitFor: ['pullCacheDeployer']
- id: 'tagCacheDeployerForMarketplaceTest'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines-test/cachedeployer:$TAG_NAME']
waitFor: ['pullCacheDeployer']
- id: 'tagCacheDeployerForMarketplaceMajorMinor'
waitFor: ['pullCacheDeployer', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/cachedeployer:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/cachedeployer:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/cachedeployer:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/cachedeployer:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA']
id: 'pullMetadataEnvoy'
- id: 'tagMetadataEnvoyVersionNumber'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA', 'gcr.io/ml-pipeline/metadata-envoy:$TAG_NAME']
waitFor: ['pullMetadataEnvoy']
- id: 'tagMetadataEnvoyCommitSHA'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA', 'gcr.io/ml-pipeline/metadata-envoy:$COMMIT_SHA']
waitFor: ['pullMetadataEnvoy']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines/metadataenvoy:$TAG_NAME']
id: 'tagMetadataEnvoyForMarketplace'
waitFor: ['pullMetadataEnvoy']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines-test/metadataenvoy:$TAG_NAME']
id: 'tagMetadataEnvoyForMarketplaceTest'
waitFor: ['pullMetadataEnvoy']
- id: 'tagMetadataEnvoyForMarketplaceMajorMinor'
waitFor: ['pullMetadataEnvoy', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/metadataenvoy:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/metadataenvoy:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/metadataenvoy:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/metadataenvoy:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance']
id: 'pullMinio'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/minio:$TAG_NAME']
id: 'tagMinioForMarketplace'
waitFor: ['pullMinio']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/minio:$TAG_NAME']
id: 'tagMinioForMarketplaceTest'
waitFor: ['pullMinio']
- id: 'tagMinioForMarketplaceMajorMinor'
waitFor: ['pullMinio', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance gcr.io/ml-pipeline/google/pipelines/minio:$(cat /workspace/mm.ver)
docker tag gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance gcr.io/ml-pipeline/google/pipelines-test/minio:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/minio:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/minio:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/mysql:8.0.26']
id: 'pullMysql'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/mysql:8.0.26', 'gcr.io/ml-pipeline/google/pipelines/mysql:$TAG_NAME']
id: 'tagMySqlForMarketplace'
waitFor: ['pullMysql']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/mysql:8.0.26', 'gcr.io/ml-pipeline/google/pipelines-test/mysql:$TAG_NAME']
id: 'tagMySqlForMarketplaceTest'
waitFor: ['pullMysql']
- id: 'tagMySqlForMarketplaceMajorMinor'
waitFor: ['pullMysql', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/ml-pipeline/mysql:8.0.26 gcr.io/ml-pipeline/google/pipelines/mysql:$(cat /workspace/mm.ver)
docker tag gcr.io/ml-pipeline/mysql:8.0.26 gcr.io/ml-pipeline/google/pipelines-test/mysql:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/mysql:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/mysql:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.25.0']
id: 'pullCloudsqlProxy'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/cloudsql-docker/gce-proxy:1.25.0', 'gcr.io/ml-pipeline/google/pipelines/cloudsqlproxy:$TAG_NAME']
id: 'tagCloudSqlProxyForMarketplace'
waitFor: ['pullCloudsqlProxy']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/cloudsql-docker/gce-proxy:1.25.0', 'gcr.io/ml-pipeline/google/pipelines-test/cloudsqlproxy:$TAG_NAME']
id: 'tagCloudSqlProxyForMarketplaceTest'
waitFor: ['pullCloudsqlProxy']
- id: 'tagCloudSqlProxyForMarketplaceMajorMinor'
waitFor: ['pullCloudsqlProxy', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/cloudsql-docker/gce-proxy:1.25.0 gcr.io/ml-pipeline/google/pipelines/cloudsqlproxy:$(cat /workspace/mm.ver)
docker tag gcr.io/cloudsql-docker/gce-proxy:1.25.0 gcr.io/ml-pipeline/google/pipelines-test/cloudsqlproxy:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/cloudsqlproxy:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/cloudsqlproxy:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.5.14-license-compliance']
id: 'pullArgoExecutor'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/argoexec:v3.5.14-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME']
id: 'tagArgoExecutorForMarketplace'
waitFor: ['pullArgoExecutor']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/argoexec:v3.5.14-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME']
id: 'tagArgoExecutorForMarketplaceTest'
waitFor: ['pullArgoExecutor']
- id: 'tagArgoExecutorForMarketplaceMajorMinor'
waitFor: ['pullArgoExecutor', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/ml-pipeline/argoexec:v3.5.14-license-compliance gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
docker tag gcr.io/ml-pipeline/argoexec:v3.5.14-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/argoexecutor:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$(cat /workspace/mm.ver)
- name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.5.14-license-compliance']
id: 'pullArgoWorkflowController'
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v3.5.14-license-compliance', 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME']
id: 'tagArgoWorkflowControllerForMarketplace'
waitFor: ['pullArgoWorkflowController']
- name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/ml-pipeline/workflow-controller:v3.5.14-license-compliance', 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME']
id: 'tagArgoWorkflowControllerForMarketplaceTest'
waitFor: ['pullArgoWorkflowController']
- id: 'tagArgoWorkflowControllerForMarketplaceMajorMinor'
waitFor: ['pullArgoWorkflowController', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/ml-pipeline/workflow-controller:v3.5.14-license-compliance gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
docker tag gcr.io/ml-pipeline/workflow-controller:v3.5.14-license-compliance gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$(cat /workspace/mm.ver)
# Marketplace specific deployer and specific primary image
- id: 'pullMarketplaceDeployer'
name: 'gcr.io/cloud-builders/docker'
args: ['pull', 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA']
waitFor: ['-']
- id: 'tagMarketplaceDeployerVersionNumber'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines/deployer:$TAG_NAME']
waitFor: ['pullMarketplaceDeployer']
- id: 'tagMarketplaceDeployerVersionNumberTest'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines-test/deployer:$TAG_NAME']
waitFor: ['pullMarketplaceDeployer']
- id: 'tagMarketplaceDeployerVersionNumberMajorMinor'
waitFor: ['pullMarketplaceDeployer', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines/deployer:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test/deployer:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines/deployer:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test/deployer:$(cat /workspace/mm.ver)
- id: 'tagMarketplacePrimaryVersionNumber'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines:$TAG_NAME']
waitFor: ['pullMarketplaceDeployer']
- id: 'tagMarketplacePrimaryVersionNumberTest'
name: 'gcr.io/cloud-builders/docker'
args: ['tag', 'gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA', 'gcr.io/ml-pipeline/google/pipelines-test:$TAG_NAME']
waitFor: ['pullMarketplaceDeployer']
- id: 'tagMarketplacePrimaryVersionNumberMajorMinor'
waitFor: ['pullMarketplaceDeployer', 'parseMajorMinorVersion']
name: 'gcr.io/cloud-builders/docker'
entrypoint: bash
args:
- -ceux
- |
docker tag gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines:$(cat /workspace/mm.ver)
docker tag gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA gcr.io/ml-pipeline/google/pipelines-test:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines:$(cat /workspace/mm.ver)
docker push gcr.io/ml-pipeline/google/pipelines-test:$(cat /workspace/mm.ver)
# # Copy the Python SDK
# - name: 'gcr.io/cloud-builders/gsutil'
# args: ['cp', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz', '/workspace/']
# id: 'copyPythonSDKLocal'
# - name: 'gcr.io/cloud-builders/gsutil'
# args: ['cp', '/workspace/kfp.tar.gz', 'gs://ml-pipeline/release/$TAG_NAME/kfp.tar.gz']
# id: 'copyPythonSDK'
# waitFor: ['copyPythonSDKLocal']
# - name: 'gcr.io/cloud-builders/gsutil'
# args: ['cp', '/workspace/kfp.tar.gz', 'gs://ml-pipeline/release/latest/kfp.tar.gz']
# id: 'copyPythonSDKToLatest'
# waitFor: ['copyPythonSDKLocal']
# # Copy the Python Component SDK
# - name: 'gcr.io/cloud-builders/gsutil'
# args: ['cp', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp-component.tar.gz', '/workspace/']
# id: 'copyPythonComponentSDKLocal'
# - name: 'gcr.io/cloud-builders/gsutil'
# args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://ml-pipeline/release/$TAG_NAME/kfp-component.tar.gz']
# id: 'copyPythonComponentSDK'
# waitFor: ['copyPythonComponentSDKLocal']
# - name: 'gcr.io/cloud-builders/gsutil'
# args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://ml-pipeline/release/latest/kfp-component.tar.gz']
# id: 'copyPythonComponentSDKToLatest'
# waitFor: ['copyPythonComponentSDKLocal']
images:
- 'gcr.io/ml-pipeline/scheduledworkflow:$TAG_NAME'
- 'gcr.io/ml-pipeline/scheduledworkflow:$COMMIT_SHA'
- 'gcr.io/ml-pipeline/persistenceagent:$TAG_NAME'
- 'gcr.io/ml-pipeline/persistenceagent:$COMMIT_SHA'
- 'gcr.io/ml-pipeline/viewer-crd-controller:$TAG_NAME'
- 'gcr.io/ml-pipeline/viewer-crd-controller:$COMMIT_SHA'
- 'gcr.io/ml-pipeline/inverse-proxy-agent:$TAG_NAME'
- 'gcr.io/ml-pipeline/inverse-proxy-agent:$COMMIT_SHA'
- 'gcr.io/ml-pipeline/visualization-server:$TAG_NAME'
- 'gcr.io/ml-pipeline/visualization-server:$COMMIT_SHA'
- 'gcr.io/ml-pipeline/metadata-envoy:$TAG_NAME'
- 'gcr.io/ml-pipeline/metadata-envoy:$COMMIT_SHA'
- 'gcr.io/ml-pipeline/metadata-writer:$TAG_NAME'
- 'gcr.io/ml-pipeline/metadata-writer:$COMMIT_SHA'
- 'gcr.io/ml-pipeline/cache-server:$TAG_NAME'
- 'gcr.io/ml-pipeline/cache-server:$COMMIT_SHA'
- 'gcr.io/ml-pipeline/cache-deployer:$TAG_NAME'
- 'gcr.io/ml-pipeline/cache-deployer:$COMMIT_SHA'
- 'gcr.io/ml-pipeline/google/pipelines/scheduledworkflow:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/viewercrd:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/persistenceagent:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/proxyagent:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/visualizationserver:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/metadataserver:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/minio:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/mysql:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/cloudsqlproxy:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/metadataenvoy:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/metadatawriter:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/deployer:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/cacheserver:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines/cachedeployer:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/scheduledworkflow:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/viewercrd:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/persistenceagent:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/proxyagent:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/visualizationserver:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/metadataserver:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/minio:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/mysql:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/cloudsqlproxy:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/argoexecutor:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/argoworkflowcontroller:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/metadataenvoy:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/metadatawriter:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/cacheserver:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/cachedeployer:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test/deployer:$TAG_NAME'
- 'gcr.io/ml-pipeline/google/pipelines-test:$TAG_NAME'
timeout: '2000s'
tags:
- release-on-tag

View File

@ -1,6 +1,5 @@
# Kubeflow Pipelines Release Process
<!-- This TOC is auto generated by "markdown all in one" VS Code plugin -->
- [Kubeflow Pipelines Release Process](#kubeflow-pipelines-release-process)
- [Schedule](#schedule)
- [Release Tags and Branches](#release-tags-and-branches)
@ -13,11 +12,16 @@
- [Cutting a release branch (Optional)](#cutting-a-release-branch-optional)
- [Before release](#before-release)
- [Releasing from release branch](#releasing-from-release-branch)
- [Release KFP Python Packages](#releasing-kfp-python-packages)
- [Create GitHub Release](#create-github-release)
- [Sync Master Branch with Release](#sync-master-branch-with-latest-release)
- [Release Process Development](#release-process-development)
## Schedule
Kubeflow Pipelines has weekly patch releases and monthly minor releases.
Kubeflow Pipelines has quarterly minor releases. Patch releases occur on a
need basis and don't currently operate on a schedule.
Patch releases only contain bug fixes, while minor releases have new features
additionally.
@ -85,9 +89,7 @@ if you only want to use or contribute to this repo.
- OS: Linux or MacOS
- Permissions needed
- Can create a branch in github.com/kubeflow/pipelines.
- (Before [#4840](https://github.com/kubeflow/pipelines/issues/4840) is resolved) one would need the admin access to kubeflow/pipelines repo.
- Can trigger cloudbuild jobs in `google.com/ml-pipeline-test` GCP project.
- Admin access to kubeflow/pipelines repo.
- Tools that should be in your `$PATH`
- docker
- python3
@ -167,11 +169,11 @@ Do the following things before a release:
console.log(Array.from(document.querySelectorAll('[id^="issue_"][id*="_link"]')).map(el => /issue_(.*)_link/.exec(el.id)[1]).join(' '))
```
1. Verify cloudbuild and postsubmit tests are passing: visit <https://github.com/kubeflow/pipelines/commits/master> for master branch.
1. Verify release branch CI is passing: visit <https://github.com/kubeflow/pipelines/commits/master> for master branch.
![How to very cloudbuild and postsubmit status](images/release-status-check.png)
![How to verify release commit status](images/release-status-check.png)
If not, contact the KFP team to determine if the failure(s) would block the release. You can also retry the failed job by opening the detail page of prow job, and click the refresh button next ot the job title.
If not, contact the KFP team to determine if the failure(s) would block the release.
### Releasing from release branch
@ -184,36 +186,7 @@ Note, when releasing from master, all the below mentions of "release branch" mea
- `1.0.1`
- `1.1.0`
- ...
Set the version by using `VERSION=<version-value>`. Contact @chensun if you are not sure what next version should be.
1. Release `driver` and `launcher` images
First build and push `driver` and `launcher` images:
```bash
pushd ./backend
export IMG_TAG_DRIVER=gcr.io/ml-pipeline/kfp-driver
make license_driver image_driver
docker push $IMG_TAG_DRIVER
export IMG_TAG_LAUNCHER=gcr.io/ml-pipeline/kfp-launcher
make license_launcher image_launcher
docker push $IMG_TAG_LAUNCHER
popd
```
If there are changes to the licenses files, make a Pull Request and merge the changes. (e.g. https://github.com/kubeflow/pipelines/pull/11177/)
Once the images are pushed, update the hard-coded hash in the code: [`DefaultLauncherImage`](https://github.com/kubeflow/pipelines/blob/4c955f4780839702dc4924f8f4e7c90aa251b826/backend/src/v2/compiler/argocompiler/container.go#L33) and [`DefaultDriverImage`](https://github.com/kubeflow/pipelines/blob/4c955f4780839702dc4924f8f4e7c90aa251b826/backend/src/v2/compiler/argocompiler/container.go#L35). Make a Pull Request and merge the change (e.g.: https://github.com/kubeflow/pipelines/pull/11178)
Pull the changes:
```bash
git checkout $BRANCH
git pull upstream $BRANCH
```
Set the version by using `VERSION=<version-value>`. Contact @chensun or @HumairAK if you are not sure what next version should be.
1. Update all version refs in release branch by
@ -224,39 +197,17 @@ Note, when releasing from master, all the below mentions of "release branch" mea
It will prompt you whether to push it to release branch. Press `y` and hit `Enter`.
Note, the script will clone kubeflow/pipelines repo into a temporary location on your computer, make those changes and attempt to push to upstream, so that it won't interfere with your current git repo.
> [!Note]
> If you see error "docker.sock: connect: permission error", you need to [allow managing docker as a non-root user](https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user).
If you see error "docker.sock: connect: permission error", you need to [allow managing docker as a non-root user](https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user).
1. Build the release images by using the [Build images from sources](https://github.com/kubeflow/pipelines/actions/workflows/image-builds.yml).
1. View related cloudbuild jobs' statuses by clicking the latest commit's status icon
in the release branch. Refer to the screenshot below to find the page.
The target tag should be `$VERSION`.
1. Wait and make sure the `build-each-commit` cloudbuild job that builds all images
in gcr.io/ml-pipeline-test succeeded. If it fails, please click "View more details
on Google Cloud Build" and then "Retry".
![Build Images From Sources](images/build-images.png))
NOTE: you can find your latest release commit in <https://github.com/kubeflow/pipelines/commits/master> and select your release branch.
![How to very cloudbuild and postsubmit status](images/release-status-check.png)
1. Select the `release-on-tag` cloudbuild job that copies built images and artifacts to
public image registry and gcs bucket. This job should have already failed because
artifacts haven't been built. Now, please click "View more details on Google Cloud Build"
and then "Retry", because after waiting for previous step, artifacts are now ready.
NOTE: **DO NOT** click the "Re-run" button from GitHub Actions status page.
It will create a build with "Branch: $BRANCH" instead of "TAG: $VERSION".
Open "View more details on Google Cloud Build", and rerun from there.
![Retry release-on-tag from the Google Cloud Build details page](images/retry-release-on-tag.png)
![Verify that you're retrying the right build](images/verify-retry-the-right-build.png)
TODO: we should have an automation KFP cluster, and the waiting and submiting
`release-on-tag` cloudbuild task should happen automatically.
NOTE: postsubmit tests will most likely fail for the release commit, this is expected, postsubmit
tests start right after the commit is in GitHub repo, but some artifacts they depend on are still
being built by the processes in these two steps.
1. Search "PyPI" in Google internal release doc for getting password of kubeflow-pipelines user.
### Releasing KFP Python Packages
1. Release `kfp-server-api` python packages to PyPI.
@ -270,15 +221,40 @@ and then "Retry", because after waiting for previous step, artifacts are now rea
```
1. Release `kfp` python packages to PyPI. (Note: Please skip this step for backend release, this step will be handled by SDK release.)
Update the SDK version in `__init__.py` and `readthedocs` `versions.json`, example PR [here](https://github.com/kubeflow/pipelines/pull/11715/files).
```bash
pip3 install twine --user
gsutil cp gs://ml-pipeline/release/$VERSION/kfp.tar.gz kfp-$VERSION.tar.gz
cd sdk/python
./build.sh kfp-$VERSION.tar.gz
python3 -m twine upload kfp-$VERSION.tar.gz
```
!!! The file name must contain the version. See <https://github.com/kubeflow/pipelines/issues/1292>
1. Release `kfp-kubernetes` python packages to PyPI. (Note: Please skip this step for backend release, this step will be handled by SDK release.)
Update the KFP Kubernetes SDK version in `__init__.py` and `readthedocs` `versions.json`, example PR [here](https://github.com/kubeflow/pipelines/pull/11380).
```bash
pip3 install twine --user
cd kubernetes_platform/python
./kubernetes_platform/python/create_release_branch.sh
```
Follow the output push instructions to push the branch to KFP, then do the following:
```bash
# set this to the appropriate version that matches what was set in __init__.py earlier
export KFP_KUBERNETES_VERSION=
./kubernetes_platform/python/release.sh
```
Note that this script will build the package, test install, and push to PyPi.
### Create GitHub Release
1. Create a GitHub release using `$VERSION` git tag and title `Version $VERSION`,
fill in the description. Detailed steps:
@ -289,7 +265,7 @@ fill in the description. Detailed steps:
1. Use this template for public releases and replace the `$VERSION` with real values.
<pre>
To deploy Kubeflow Pipelines in an existing cluster, follow the instruction in [here](https://www.kubeflow.org/docs/pipelines/standalone-deployment-gcp/) or via UI [here](https://console.cloud.google.com/ai-platform/pipelines)
To deploy Kubeflow Pipelines in an existing cluster, follow the instruction in [here](https://www.kubeflow.org/docs/components/pipelines/operator-guides/installation/)
Install python SDK (python 3.9 above) by running:
@ -304,7 +280,7 @@ fill in the description. Detailed steps:
***This is a prerelease*** checkbox in the GitHub release UI.
<pre>
To deploy Kubeflow Pipelines in an existing cluster, follow the instruction in [here](https://www.kubeflow.org/docs/pipelines/standalone-deployment-gcp/).
To deploy Kubeflow Pipelines in an existing cluster, follow the instruction in [here](https://www.kubeflow.org/docs/components/pipelines/operator-guides/installation/).
Install kfp-server-api package (python 3.9 above) by running:
@ -318,6 +294,8 @@ fill in the description. Detailed steps:
NOTE, kfp python SDK is **NOT** included and released separately.
</pre>
### Sync Master Branch with latest release
1. **(Do this step only when releasing from a NON-master release branch)**
Update master branch to the same version and include latest changelog:

BIN
images/build-images.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 177 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 347 KiB

After

Width:  |  Height:  |  Size: 197 KiB