pipelines/sdk/python/kfp/compiler/main.py

# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import argparse
import kfp.dsl as dsl
import kfp.compiler
import os
import shutil
import subprocess
import sys
import tempfile
from deprecated.sphinx import deprecated

def _str2bool(v):
  if isinstance(v, bool):
    return v
  if v.lower() in ('yes', 'true', 't', 'y', '1'):
    return True
  elif v.lower() in ('no', 'false', 'f', 'n', '0'):
    return False
  else:
    raise argparse.ArgumentTypeError('Boolean value expected.')


def parse_arguments():
  """Parse command line arguments."""

  parser = argparse.ArgumentParser()
  parser.add_argument('--py',
                      type=str,
                      help='local absolute path to a py file.')
  parser.add_argument('--package',
                      type=str,
                      help='local path to a pip installable python package file.')
  parser.add_argument('--function',
                      type=str,
                      help='The name of the function to compile if there are multiple.')
  parser.add_argument('--namespace',
                      type=str,
                      help='The namespace for the pipeline function')
  parser.add_argument('--output',
                      type=str,
                      required=True,
                      help='local path to the output workflow yaml file.')
  parser.add_argument('--disable-type-check',
                      action='store_true',
                      help='disable the type check, default is enabled.')
  parser.add_argument('--disable-telemetry',
                      action='store_true',
                      help='disable adding telemetry labels, default is enabled.')

  args = parser.parse_args()
  return args


def _compile_pipeline_function(
    pipeline_funcs, function_name, output_path, type_check, allow_telemetry):
  if len(pipeline_funcs) == 0:
    raise ValueError('A function with @dsl.pipeline decorator is required in the py file.')

  if len(pipeline_funcs) > 1 and not function_name:
    func_names = [x.__name__ for x in pipeline_funcs]
    raise ValueError('There are multiple pipelines: %s. Please specify --function.' % func_names)

  if function_name:
    pipeline_func = next((x for x in pipeline_funcs if x.__name__ == function_name), None)
    if not pipeline_func:
      raise ValueError('The function "%s" does not exist. '
                       'Did you forget @dsl.pipeline decoration?' % function_name)
  else:
    pipeline_func = pipeline_funcs[0]

  kfp.compiler.Compiler().compile(
      pipeline_func, output_path, type_check, allow_telemetry=allow_telemetry)


class PipelineCollectorContext():
  def __enter__(self):
    pipeline_funcs = []
    def add_pipeline(func):
      pipeline_funcs.append(func)
      return func
    self.old_handler = dsl._pipeline._pipeline_decorator_handler
    dsl._pipeline._pipeline_decorator_handler = add_pipeline
    return pipeline_funcs
  
  def __exit__(self, *args):
    dsl._pipeline._pipeline_decorator_handler = self.old_handler


@deprecated(version='0.1.28', reason='''\
    The ability to compile pipeline from a python package is deprecated and will be removed in next release.
    Please switch to compiling pipeline files or functions.
    If you use this feature please create an issue in https://github.com/kubeflow/pipelines/issues .'''
)
def compile_package(
    package_path, namespace, function_name, output_path, type_check, allow_telemetry):
  tmpdir = tempfile.mkdtemp()
  sys.path.insert(0, tmpdir)
  try:
    subprocess.check_call(['python3', '-m', 'pip', 'install', package_path, '-t', tmpdir])
    with PipelineCollectorContext() as pipeline_funcs:
      __import__(namespace)
    _compile_pipeline_function(
        pipeline_funcs, function_name, output_path, type_check,
        allow_telemetry=allow_telemetry)
  finally:
    del sys.path[0]
    shutil.rmtree(tmpdir)


def compile_pyfile(pyfile, function_name, output_path, type_check, allow_telemetry):
  sys.path.insert(0, os.path.dirname(pyfile))
  try:
    filename = os.path.basename(pyfile)
    with PipelineCollectorContext() as pipeline_funcs:
      __import__(os.path.splitext(filename)[0])
    _compile_pipeline_function(
        pipeline_funcs, function_name, output_path, type_check,
        allow_telemetry=allow_telemetry)
  finally:
    del sys.path[0]


def main():
  args = parse_arguments()
  if ((args.py is None and args.package is None) or
      (args.py is not None and args.package is not None)):
    raise ValueError('Either --py or --package is needed but not both.')
  if args.py:
    compile_pyfile(
        args.py,
        args.function,
        args.output,
        not args.disable_type_check,
        not args.disable_telemetry
    )
  else:
    if args.namespace is None:
      raise ValueError('--namespace is required for compiling packages.')
    compile_package(
        args.package,
        args.namespace,
        args.function,
        args.output,
        not args.disable_type_check,
        not args.disable_telemetry
    )