mirror of https://github.com/kubeflow/examples.git
120 lines
3.2 KiB
Python
120 lines
3.2 KiB
Python
"""Main pipeline file"""
|
|
from kubernetes import client as k8s_client
|
|
import kfp.dsl as dsl
|
|
import kfp.compiler as compiler
|
|
|
|
@dsl.pipeline(
|
|
name='Tacos vs. Burritos',
|
|
description='Simple TF CNN'
|
|
)
|
|
def tacosandburritos_train(
|
|
tenant_id,
|
|
service_principal_id,
|
|
service_principal_password,
|
|
subscription_id,
|
|
resource_group,
|
|
workspace
|
|
):
|
|
"""Pipeline steps"""
|
|
|
|
persistent_volume_path = '/mnt/azure'
|
|
data_download = 'https://aiadvocate.blob.core.windows.net/public/tacodata.zip'
|
|
epochs = 5
|
|
batch = 32
|
|
learning_rate = 0.0001
|
|
model_name = 'tacosandburritos'
|
|
profile_name = 'tacoprofile'
|
|
operations = {}
|
|
image_size = 160
|
|
training_folder = 'train'
|
|
training_dataset = 'train.txt'
|
|
model_folder = 'model'
|
|
|
|
# preprocess data
|
|
operations['preprocess'] = dsl.ContainerOp(
|
|
name='preprocess',
|
|
image='insert image name:tag',
|
|
command=['python'],
|
|
arguments=[
|
|
'/scripts/data.py',
|
|
'--base_path', persistent_volume_path,
|
|
'--data', training_folder,
|
|
'--target', training_dataset,
|
|
'--img_size', image_size,
|
|
'--zipfile', data_download
|
|
]
|
|
)
|
|
|
|
# train
|
|
operations['training'] = dsl.ContainerOp(
|
|
name='training',
|
|
image='insert image name:tag',
|
|
command=['python'],
|
|
arguments=[
|
|
'/scripts/train.py',
|
|
'--base_path', persistent_volume_path,
|
|
'--data', training_folder,
|
|
'--epochs', epochs,
|
|
'--batch', batch,
|
|
'--image_size', image_size,
|
|
'--lr', learning_rate,
|
|
'--outputs', model_folder,
|
|
'--dataset', training_dataset
|
|
]
|
|
)
|
|
operations['training'].after(operations['preprocess'])
|
|
|
|
# register model
|
|
operations['register'] = dsl.ContainerOp(
|
|
name='register',
|
|
image='insert image name:tag',
|
|
command=['python'],
|
|
arguments=[
|
|
'/scripts/register.py',
|
|
'--base_path', persistent_volume_path,
|
|
'--model', 'latest.h5',
|
|
'--model_name', model_name,
|
|
'--tenant_id', tenant_id,
|
|
'--service_principal_id', service_principal_id,
|
|
'--service_principal_password', service_principal_password,
|
|
'--subscription_id', subscription_id,
|
|
'--resource_group', resource_group,
|
|
'--workspace', workspace
|
|
]
|
|
)
|
|
operations['register'].after(operations['training'])
|
|
|
|
operations['deploy'] = dsl.ContainerOp(
|
|
name='deploy',
|
|
image='insert image name:tag',
|
|
command=['sh'],
|
|
arguments=[
|
|
'/scripts/deploy.sh',
|
|
'-n', model_name,
|
|
'-m', model_name,
|
|
'-i', '/scripts/inferenceconfig.json',
|
|
'-d', '/scripts/deploymentconfig.json',
|
|
'-t', tenant_id,
|
|
'-r', resource_group,
|
|
'-w', workspace,
|
|
'-s', service_principal_id,
|
|
'-p', service_principal_password,
|
|
'-u', subscription_id,
|
|
'-b', persistent_volume_path
|
|
]
|
|
)
|
|
operations['deploy'].after(operations['register'])
|
|
for _, op_1 in operations.items():
|
|
op_1.container.set_image_pull_policy("Always")
|
|
op_1.add_volume(
|
|
k8s_client.V1Volume(
|
|
name='azure',
|
|
persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(
|
|
claim_name='azure-managed-disk')
|
|
)
|
|
).add_volume_mount(k8s_client.V1VolumeMount(
|
|
mount_path='/mnt/azure', name='azure'))
|
|
|
|
if __name__ == '__main__':
|
|
compiler.Compiler().compile(tacosandburritos_train, __file__ + '.tar.gz')
|