updates to reflect changed node credentials, minor cleanup, update component URLs, (#675)

update serving-only pipeline as well

github_issue_summarization/pipelines/components/kubeflow-resources/tf-serving-gh/tf-serve-template.yaml

@@ -43,6 +43,10 @@ spec:
         app: MODEL_NAME
         version: v1
     spec:
+      volumes:
+      - name: gcp-credentials-user-gcp-sa
+        secret:
+          secretName: user-gcp-sa
       containers:
       - args:
         - --port=9000
@@ -52,6 +56,14 @@ spec:
         command:
         - /usr/bin/tensorflow_model_server
         image: tensorflow/serving
+        env:
+        - name: GOOGLE_APPLICATION_CREDENTIALS
+          value: /secret/gcp-credentials/user-gcp-sa.json
+        - name: CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE
+          value: /secret/gcp-credentials/user-gcp-sa.json
+        volumeMounts:
+        - mountPath: /secret/gcp-credentials
+          name: gcp-credentials-user-gcp-sa
         imagePullPolicy: IfNotPresent
         livenessProbe:
           initialDelaySeconds: 30

github_issue_summarization/pipelines/components/t2t/t2t-app/app/main.py

@@ -26,8 +26,6 @@ from flask import jsonify
 from flask import render_template
 from flask import g, request
 
-from googleapiclient import discovery  # pylint: disable=wrong-import-order
-from oauth2client.client import GoogleCredentials  # pylint: disable=wrong-import-order
 import pandas as pd
 
 import tensorflow as tf
@@ -40,8 +38,6 @@ from tensor2tensor.utils import registry
 from tensor2tensor.utils import usr_dir
 from tensor2tensor.serving import serving_utils
 
-credentials = GoogleCredentials.get_application_default()
-api = discovery.build('ml', 'v1', credentials=credentials)
 
 app = Flask(__name__)
 

github_issue_summarization/pipelines/components/t2t/webapp-launcher/t2tapp-template.yaml

@@ -62,9 +62,21 @@ spec:
         app: ghsumm
         role: frontend
     spec:
+      volumes:
+      - name: gcp-credentials-user-gcp-sa
+        secret:
+          secretName: user-gcp-sa
       containers:
       - name: MODEL_NAME-webapp
-        image: gcr.io/google-samples/ml-pipeline-t2tapp
+        image: gcr.io/google-samples/ml-pipeline-t2tapp:v3ap
+        env:
+        - name: GOOGLE_APPLICATION_CREDENTIALS
+          value: /secret/gcp-credentials/user-gcp-sa.json
+        - name: CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE
+          value: /secret/gcp-credentials/user-gcp-sa.json
+        volumeMounts:
+        - mountPath: /secret/gcp-credentials
+          name: gcp-credentials-user-gcp-sa
         # resources:
         #   limits:
         #     nvidia.com/gpu: 1

github_issue_summarization/pipelines/example_pipelines/gh_summ.py

@@ -26,11 +26,11 @@ DATASET = 'dataset'
 MODEL = 'model'
 
 copydata_op = comp.load_component_from_url(
-  'https://raw.githubusercontent.com/amygdala/kubeflow-examples/ghpl_update/github_issue_summarization/pipelines/components/t2t/datacopy_component.yaml'  # pylint: disable=line-too-long
+  'https://raw.githubusercontent.com/kubeflow/examples/master/github_issue_summarization/pipelines/components/t2t/datacopy_component.yaml'  # pylint: disable=line-too-long
   )
 
 train_op = comp.load_component_from_url(
-  'https://raw.githubusercontent.com/amygdala/kubeflow-examples/ghpl_update/github_issue_summarization/pipelines/components/t2t/train_component.yaml' # pylint: disable=line-too-long
+  'https://raw.githubusercontent.com/kubeflow/examples/master/github_issue_summarization/pipelines/components/t2t/train_component.yaml' # pylint: disable=line-too-long
   )
 
 metadata_log_op = comp.load_component_from_url(
@@ -84,7 +84,7 @@ def gh_summ(  #pylint: disable=unused-argument
 
   serve = dsl.ContainerOp(
       name='serve',
-      image='gcr.io/google-samples/ml-pipeline-kubeflow-tfserve',
+      image='gcr.io/google-samples/ml-pipeline-kubeflow-tfserve:v2',
       arguments=["--model_name", 'ghsumm-%s' % (dsl.RUN_ID_PLACEHOLDER,),
           "--model_path", train.outputs['train_output_path']
           ]
@@ -98,7 +98,7 @@ def gh_summ(  #pylint: disable=unused-argument
   with dsl.Condition(train.outputs['launch_server'] == 'true'):
     webapp = dsl.ContainerOp(
         name='webapp',
-        image='gcr.io/google-samples/ml-pipeline-webapp-launcher:v2ap',
+        image='gcr.io/google-samples/ml-pipeline-webapp-launcher:v3ap',
         arguments=["--model_name", 'ghsumm-%s' % (dsl.RUN_ID_PLACEHOLDER,),
             "--github_token", github_token]
 

github_issue_summarization/pipelines/example_pipelines/gh_summ_serve.py

@@ -14,36 +14,38 @@
 
 
 import kfp.dsl as dsl
+import kfp.gcp as gcp
+from kfp.dsl.types import String
+
 
 @dsl.pipeline(
   name='Github issue summarization',
   description='Demonstrate Tensor2Tensor-based training and TF-Serving'
 )
-def gh_summ(
-  github_token: dsl.PipelineParam = dsl.PipelineParam(
-      name='github-token', value='YOUR_GITHUB_TOKEN_HERE'),
+def gh_summ_serveonly(
+  github_token: String = 'YOUR_GITHUB_TOKEN_HERE',
   ):
 
 
   serve = dsl.ContainerOp(
       name='serve',
-      image='gcr.io/google-samples/ml-pipeline-kubeflow-tfserve',
-      arguments=["--model_name", 'ghsumm-%s' % ('{{workflow.name}}',),
+      image='gcr.io/google-samples/ml-pipeline-kubeflow-tfserve:v2',
+      arguments=["--model_name", 'ghsumm-%s' % (dsl.RUN_ID_PLACEHOLDER,),
           "--model_path",
           'gs://aju-dev-demos-codelabs/kubecon/example_t2t_model/model_output/export'
           ]
-      )
+      ).apply(gcp.use_gcp_secret('user-gcp-sa'))
 
   webapp = dsl.ContainerOp(
       name='webapp',
-      image='gcr.io/google-samples/ml-pipeline-webapp-launcher:v2ap',
-      arguments=["--model_name", 'ghsumm-%s' % ('{{workflow.name}}',),
+      image='gcr.io/google-samples/ml-pipeline-webapp-launcher:v3ap',
+      arguments=["--model_name", 'ghsumm-%s' % (dsl.RUN_ID_PLACEHOLDER,),
           "--github_token", github_token]
-
       )
+
   webapp.after(serve)
 
 
 if __name__ == '__main__':
   import kfp.compiler as compiler
-  compiler.Compiler().compile(gh_summ, __file__ + '.tar.gz')
+  compiler.Compiler().compile(gh_summ_serveonly, __file__ + '.tar.gz')