diff --git a/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py b/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py index f7768ad33d..a14591afd6 100644 --- a/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py +++ b/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py @@ -17,7 +17,7 @@ from kfp import dsl def kubeflow_tfjob_launcher_op(container_image, command, number_of_workers: int, number_of_parameter_servers: int, tfjob_timeout_minutes: int, output_dir=None, step_name='TFJob-launcher'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--workers', number_of_workers, '--pss', number_of_parameter_servers, diff --git a/components/kubeflow/launcher/src/train.template.yaml b/components/kubeflow/launcher/src/train.template.yaml index b8c099f88c..64ba3a34ce 100644 --- a/components/kubeflow/launcher/src/train.template.yaml +++ b/components/kubeflow/launcher/src/train.template.yaml @@ -26,7 +26,7 @@ spec: spec: containers: - name: tensorflow - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:6ad2601ec7d04e842c212c50d5c78e548e12ddea + image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f command: - python - -m @@ -49,7 +49,7 @@ spec: spec: containers: - name: tensorflow - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:6ad2601ec7d04e842c212c50d5c78e548e12ddea + image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f command: - python - -m @@ -72,7 +72,7 @@ spec: spec: containers: - name: tensorflow - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:6ad2601ec7d04e842c212c50d5c78e548e12ddea + image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f command: - python - -m diff --git a/samples/kubeflow-tf/kubeflow-training-classification.py b/samples/kubeflow-tf/kubeflow-training-classification.py index 56cc0caf97..22935a13a5 100755 --- a/samples/kubeflow-tf/kubeflow-training-classification.py +++ b/samples/kubeflow-tf/kubeflow-training-classification.py @@ -21,7 +21,7 @@ import datetime def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', preprocess_mode, preprocess_module: 'GcsUri[text/code/python]', transform_output: 'GcsUri[Directory]', step_name='preprocess'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--train', train_data, '--eval', evaluation_data, @@ -38,7 +38,7 @@ def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', sc def kubeflow_tf_training_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate: float, hidden_layer_size: int, steps: int, target, preprocess_module: 'GcsUri[text/code/python]', training_output: 'GcsUri[Directory]', step_name='training', use_gpu=False): kubeflow_tf_training_op = dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--transformed-data-dir', transformed_data_dir, '--schema', schema, @@ -52,7 +52,7 @@ def kubeflow_tf_training_op(transformed_data_dir, schema: 'GcsUri[text/json]', l file_outputs = {'train': '/output.txt'} ) if use_gpu: - kubeflow_tf_training_op.image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer-gpu:6ad2601ec7d04e842c212c50d5c78e548e12ddea' + kubeflow_tf_training_op.image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer-gpu:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f' kubeflow_tf_training_op.set_gpu_limit(1) return kubeflow_tf_training_op @@ -60,7 +60,7 @@ def kubeflow_tf_training_op(transformed_data_dir, schema: 'GcsUri[text/json]', l def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', target: str, model: 'TensorFlow model', predict_mode, project: 'GcpProject', prediction_output: 'GcsUri', step_name='prediction'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--data', evaluation_data, '--schema', schema, @@ -76,7 +76,7 @@ def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json] def confusion_matrix_op(predictions, output, step_name='confusionmatrix'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--predictions', predictions, '--output', output, diff --git a/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb b/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb index 36e4d870c0..046870cc21 100644 --- a/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb +++ b/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb @@ -43,13 +43,13 @@ "EVAL_DATA = 'gs://ml-pipeline-playground/tfx/taxi-cab-classification/eval.csv'\n", "HIDDEN_LAYER_SIZE = '1500'\n", "STEPS = 3000\n", - "DATAFLOW_TFDV_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:6ad2601ec7d04e842c212c50d5c78e548e12ddea'\n", - "DATAFLOW_TFT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:6ad2601ec7d04e842c212c50d5c78e548e12ddea'\n", - "DATAFLOW_TFMA_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:6ad2601ec7d04e842c212c50d5c78e548e12ddea'\n", - "DATAFLOW_TF_PREDICT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:6ad2601ec7d04e842c212c50d5c78e548e12ddea'\n", - "KUBEFLOW_TF_TRAINER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:6ad2601ec7d04e842c212c50d5c78e548e12ddea'\n", - "KUBEFLOW_TF_TRAINER_GPU_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer-gpu:6ad2601ec7d04e842c212c50d5c78e548e12ddea'\n", - "KUBEFLOW_DEPLOYER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:6ad2601ec7d04e842c212c50d5c78e548e12ddea'\n", + "DATAFLOW_TFDV_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f'\n", + "DATAFLOW_TFT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f'\n", + "DATAFLOW_TFMA_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f'\n", + "DATAFLOW_TF_PREDICT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f'\n", + "KUBEFLOW_TF_TRAINER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f'\n", + "KUBEFLOW_TF_TRAINER_GPU_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer-gpu:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f'\n", + "KUBEFLOW_DEPLOYER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f'\n", "DEPLOYER_MODEL = 'notebook_tfx_taxi'\n", "DEPLOYER_VERSION_DEV = 'dev'\n", "DEPLOYER_VERSION_PROD = 'prod'" diff --git a/samples/resnet-cmle/resnet-train-pipeline.py b/samples/resnet-cmle/resnet-train-pipeline.py index c6bb9e4220..854ab507a8 100644 --- a/samples/resnet-cmle/resnet-train-pipeline.py +++ b/samples/resnet-cmle/resnet-train-pipeline.py @@ -22,7 +22,7 @@ def resnet_preprocess_op(project_id: 'GcpProject', output: 'GcsUri', train_csv: validation_csv: 'GcsUri[text/csv]', labels, step_name='preprocess'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/resnet-preprocess:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/resnet-preprocess:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--project_id', project_id, '--output', output, @@ -38,7 +38,7 @@ def resnet_train_op(data_dir, output: 'GcsUri', region: 'GcpRegion', depth: int, num_eval_images: int, num_label_classes: int, tf_version, step_name='train'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/resnet-train:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/resnet-train:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--data_dir', data_dir, '--output', output, @@ -60,7 +60,7 @@ def resnet_deploy_op(model_dir, model, version, project_id: 'GcpProject', region tf_version, step_name='deploy'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/resnet-deploy:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/resnet-deploy:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--model', model, '--version', version, diff --git a/samples/tfx/taxi-cab-classification-pipeline.py b/samples/tfx/taxi-cab-classification-pipeline.py index 139baf34c3..754f784161 100755 --- a/samples/tfx/taxi-cab-classification-pipeline.py +++ b/samples/tfx/taxi-cab-classification-pipeline.py @@ -21,7 +21,7 @@ import datetime def dataflow_tf_data_validation_op(inference_data: 'GcsUri', validation_data: 'GcsUri', column_names: 'GcsUri[text/json]', key_columns, project: 'GcpProject', mode, validation_output: 'GcsUri[Directory]', step_name='validation'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--csv-data-for-inference', inference_data, '--csv-data-to-validate', validation_data, @@ -40,7 +40,7 @@ def dataflow_tf_data_validation_op(inference_data: 'GcsUri', validation_data: 'G def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', preprocess_mode, preprocess_module: 'GcsUri[text/code/python]', transform_output: 'GcsUri[Directory]', step_name='preprocess'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--train', train_data, '--eval', evaluation_data, @@ -57,7 +57,7 @@ def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', sc def tf_train_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate: float, hidden_layer_size: int, steps: int, target: str, preprocess_module: 'GcsUri[text/code/python]', training_output: 'GcsUri[Directory]', step_name='training'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--transformed-data-dir', transformed_data_dir, '--schema', schema, @@ -74,7 +74,7 @@ def tf_train_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate def dataflow_tf_model_analyze_op(model: 'TensorFlow model', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', analyze_mode, analyze_slice_column, analysis_output: 'GcsUri', step_name='analysis'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--model', model, '--eval', evaluation_data, @@ -91,7 +91,7 @@ def dataflow_tf_model_analyze_op(model: 'TensorFlow model', evaluation_data: 'Gc def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', target: str, model: 'TensorFlow model', predict_mode, project: 'GcpProject', prediction_output: 'GcsUri', step_name='prediction'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--data', evaluation_data, '--schema', schema, @@ -108,7 +108,7 @@ def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json] def confusion_matrix_op(predictions: 'GcsUri', output: 'GcsUri', step_name='confusion_matrix'): return dsl.ContainerOp( name=step_name, - image='gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image='gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments=[ '--output', '%s/{{workflow.name}}/confusionmatrix' % output, '--predictions', predictions, @@ -119,7 +119,7 @@ def confusion_matrix_op(predictions: 'GcsUri', output: 'GcsUri', step_name='conf def roc_op(predictions: 'GcsUri', output: 'GcsUri', step_name='roc'): return dsl.ContainerOp( name=step_name, - image='gcr.io/ml-pipeline/ml-pipeline-local-roc:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image='gcr.io/ml-pipeline/ml-pipeline-local-roc:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments=[ '--output', '%s/{{workflow.name}}/roc' % output, '--predictions', predictions, @@ -130,7 +130,7 @@ def roc_op(predictions: 'GcsUri', output: 'GcsUri', step_name='roc'): def kubeflow_deploy_op(model: 'TensorFlow model', tf_server_name, step_name='deploy'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments = [ '--model-path', model, '--server-name', tf_server_name diff --git a/samples/xgboost-spark/xgboost-training-cm.py b/samples/xgboost-spark/xgboost-training-cm.py index 99a317b9ac..2ef8d7506c 100755 --- a/samples/xgboost-spark/xgboost-training-cm.py +++ b/samples/xgboost-spark/xgboost-training-cm.py @@ -26,7 +26,7 @@ class CreateClusterOp(dsl.ContainerOp): def __init__(self, name, project, region, staging): super(CreateClusterOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments=[ '--project', project, '--region', region, @@ -41,7 +41,7 @@ class DeleteClusterOp(dsl.ContainerOp): def __init__(self, name, project, region): super(DeleteClusterOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments=[ '--project', project, '--region', region, @@ -55,7 +55,7 @@ class AnalyzeOp(dsl.ContainerOp): def __init__(self, name, project, region, cluster_name, schema, train_data, output): super(AnalyzeOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments=[ '--project', project, '--region', region, @@ -73,7 +73,7 @@ class TransformOp(dsl.ContainerOp): target, analysis, output): super(TransformOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments=[ '--project', project, '--region', region, @@ -98,7 +98,7 @@ class TrainerOp(dsl.ContainerOp): super(TrainerOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-train:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-train:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments=[ '--project', project, '--region', region, @@ -121,7 +121,7 @@ class PredictOp(dsl.ContainerOp): def __init__(self, name, project, region, cluster_name, data, model, target, analysis, output): super(PredictOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments=[ '--project', project, '--region', region, @@ -141,7 +141,7 @@ class ConfusionMatrixOp(dsl.ContainerOp): def __init__(self, name, predictions, output): super(ConfusionMatrixOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image='gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments=[ '--output', output, '--predictions', predictions @@ -153,7 +153,7 @@ class RocOp(dsl.ContainerOp): def __init__(self, name, predictions, trueclass, output): super(RocOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-local-roc:6ad2601ec7d04e842c212c50d5c78e548e12ddea', + image='gcr.io/ml-pipeline/ml-pipeline-local-roc:5df2cdc1ed145320204e8bc73b59cdbd7b3da28f', arguments=[ '--output', output, '--predictions', predictions,