feat(sdk): use dsl noun group for compile CLI commands (#7603)
* use kfp dsl compile for compilation commands * create and move tests
This commit is contained in:
Connor McCarthy
GitHub
parent
d1c0c75d72
commit
e12ac39ba4
|
|
@ -19,6 +19,7 @@ import click
|
|||
import kfp
|
||||
from kfp.cli import component
|
||||
from kfp.cli import diagnose_me_cli
|
||||
from kfp.cli import dsl
|
||||
from kfp.cli import experiment
|
||||
from kfp.cli import pipeline
|
||||
from kfp.cli import recurring_run
|
||||
|
|
@ -32,7 +33,7 @@ COMMANDS = {
|
|||
run.run, recurring_run.recurring_run, experiment.experiment,
|
||||
pipeline.pipeline
|
||||
},
|
||||
'no_client': {diagnose_me_cli.diagnose_me, component.component}
|
||||
'no_client': {diagnose_me_cli.diagnose_me, component.component, dsl.dsl}
|
||||
}
|
||||
|
||||
PROGRAM_NAME = 'kfp'
|
||||
|
|
|
|||
|
|
@ -14,15 +14,19 @@
|
|||
|
||||
import functools
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import unittest
|
||||
from typing import Any, Dict, List, Optional
|
||||
from unittest import mock
|
||||
|
||||
import yaml
|
||||
from absl.testing import parameterized
|
||||
from click import testing
|
||||
from kfp.cli import cli
|
||||
from kfp.cli import dsl_compile
|
||||
|
||||
|
||||
class TestCliNounAliases(unittest.TestCase):
|
||||
|
|
@ -47,6 +51,60 @@ class TestCliNounAliases(unittest.TestCase):
|
|||
result.output)
|
||||
|
||||
|
||||
def _ignore_kfp_version_helper(spec: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Ignores kfp sdk versioning in command.
|
||||
|
||||
Takes in a YAML input and ignores the kfp sdk versioning in command
|
||||
for comparison between compiled file and goldens.
|
||||
"""
|
||||
pipeline_spec = spec.get('pipelineSpec', spec)
|
||||
|
||||
if 'executors' in pipeline_spec['deploymentSpec']:
|
||||
for executor in pipeline_spec['deploymentSpec']['executors']:
|
||||
pipeline_spec['deploymentSpec']['executors'][
|
||||
executor] = yaml.safe_load(
|
||||
re.sub(
|
||||
r"'kfp==(\d+).(\d+).(\d+)(-[a-z]+.\d+)?'", 'kfp',
|
||||
yaml.dump(
|
||||
pipeline_spec['deploymentSpec']['executors']
|
||||
[executor],
|
||||
sort_keys=True)))
|
||||
return spec
|
||||
|
||||
|
||||
def load_compiled_file(filename: str) -> Dict[str, Any]:
|
||||
with open(filename, 'r') as f:
|
||||
contents = yaml.safe_load(f)
|
||||
pipeline_spec = contents[
|
||||
'pipelineSpec'] if 'pipelineSpec' in contents else contents
|
||||
# ignore the sdkVersion
|
||||
del pipeline_spec['sdkVersion']
|
||||
return _ignore_kfp_version_helper(contents)
|
||||
|
||||
|
||||
class TestAliases(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
runner = testing.CliRunner()
|
||||
cls.invoke = functools.partial(
|
||||
runner.invoke, cli=cli.cli, catch_exceptions=True, obj={})
|
||||
|
||||
def test_aliases_singular(self):
|
||||
result = self.invoke(args=['component'])
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
|
||||
def test_aliases_plural(self):
|
||||
result = self.invoke(args=['components'])
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
|
||||
def test_aliases_fails(self):
|
||||
result = self.invoke(args=['componentss'])
|
||||
self.assertEqual(result.exit_code, 2)
|
||||
self.assertEqual("Error: Unrecognized command 'componentss'\n",
|
||||
result.output)
|
||||
|
||||
|
||||
class TestCliAutocomplete(parameterized.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
|
@ -113,3 +171,90 @@ class TestCliVersion(unittest.TestCase):
|
|||
self.assertEqual(result.exit_code, 0)
|
||||
matches = re.match(r'^kfp \d\.\d\.\d.*', result.output)
|
||||
self.assertTrue(matches)
|
||||
|
||||
|
||||
COMPILER_CLI_TEST_DATA_DIR = os.path.join(
|
||||
os.path.dirname(os.path.dirname(__file__)), 'compiler_cli_tests',
|
||||
'test_data')
|
||||
|
||||
SPECIAL_TEST_PY_FILES = {'two_step_pipeline.py'}
|
||||
|
||||
TEST_PY_FILES = {
|
||||
file.split('.')[0]
|
||||
for file in os.listdir(COMPILER_CLI_TEST_DATA_DIR)
|
||||
if ".py" in file and file not in SPECIAL_TEST_PY_FILES
|
||||
}
|
||||
|
||||
|
||||
class TestDslCompile(parameterized.TestCase):
|
||||
|
||||
def invoke(self, args: List[str]) -> testing.Result:
|
||||
starting_args = ['dsl', 'compile']
|
||||
args = starting_args + args
|
||||
runner = testing.CliRunner()
|
||||
return runner.invoke(
|
||||
cli=cli.cli, args=args, catch_exceptions=False, obj={})
|
||||
|
||||
def invoke_deprecated(self, args: List[str]) -> testing.Result:
|
||||
runner = testing.CliRunner()
|
||||
return runner.invoke(
|
||||
cli=dsl_compile.dsl_compile,
|
||||
args=args,
|
||||
catch_exceptions=False,
|
||||
obj={})
|
||||
|
||||
def _test_compile_py_to_yaml(
|
||||
self,
|
||||
file_base_name: str,
|
||||
additional_arguments: Optional[List[str]] = None) -> None:
|
||||
py_file = os.path.join(COMPILER_CLI_TEST_DATA_DIR,
|
||||
f'{file_base_name}.py')
|
||||
|
||||
golden_compiled_file = os.path.join(COMPILER_CLI_TEST_DATA_DIR,
|
||||
f'{file_base_name}.yaml')
|
||||
|
||||
if additional_arguments is None:
|
||||
additional_arguments = []
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
generated_compiled_file = os.path.join(
|
||||
tmpdir, f'{file_base_name}-pipeline.yaml')
|
||||
|
||||
result = self.invoke(
|
||||
['--py', py_file, '--output', generated_compiled_file] +
|
||||
additional_arguments)
|
||||
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
|
||||
compiled = load_compiled_file(generated_compiled_file)
|
||||
|
||||
golden = load_compiled_file(golden_compiled_file)
|
||||
self.assertEqual(golden, compiled)
|
||||
|
||||
def test_two_step_pipeline(self):
|
||||
self._test_compile_py_to_yaml(
|
||||
'two_step_pipeline',
|
||||
['--pipeline-parameters', '{"text":"Hello KFP!"}'])
|
||||
|
||||
def test_two_step_pipeline_failure_parameter_parse(self):
|
||||
with self.assertRaisesRegex(json.decoder.JSONDecodeError,
|
||||
r"Unterminated string starting at:"):
|
||||
self._test_compile_py_to_yaml(
|
||||
'two_step_pipeline',
|
||||
['--pipeline-parameters', '{"text":"Hello KFP!}'])
|
||||
|
||||
@parameterized.parameters(TEST_PY_FILES)
|
||||
def test_compile_pipelines(self, file: str):
|
||||
|
||||
# To update all golden snapshots:
|
||||
# for f in test_data/*.py ; do python3 "$f" ; done
|
||||
|
||||
self._test_compile_py_to_yaml(file)
|
||||
|
||||
def test_deprecated_command_is_found(self):
|
||||
result = self.invoke_deprecated(['--help'])
|
||||
self.assertEqual(result.exit_code, 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright 2020 The Kubeflow Authors
|
||||
# Copyright 2022 The Kubeflow Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
@ -11,3 +11,12 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import click
|
||||
from kfp.cli import dsl_compile
|
||||
|
||||
|
||||
@click.group(commands={'compile': dsl_compile.dsl_compile})
|
||||
def dsl():
|
||||
"""Command group for compiling DSL to IR."""
|
||||
pass
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
# Compiler unit tests
|
||||
|
||||
To update all golden snapshots:
|
||||
|
||||
```bash
|
||||
for f in test_data/*.py ; do python3 "$f" ; done
|
||||
```
|
||||
|
|
@ -1,197 +0,0 @@
|
|||
# Copyright 2020 The Kubeflow Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def _ignore_kfp_version_helper(spec):
|
||||
"""Ignores kfp sdk versioning in command.
|
||||
|
||||
Takes in a YAML input and ignores the kfp sdk versioning in command
|
||||
for comparison between compiled file and goldens.
|
||||
"""
|
||||
pipeline_spec = spec['pipelineSpec'] if 'pipelineSpec' in spec else spec
|
||||
|
||||
if 'executors' in pipeline_spec['deploymentSpec']:
|
||||
for executor in pipeline_spec['deploymentSpec']['executors']:
|
||||
pipeline_spec['deploymentSpec']['executors'][executor] = yaml.load(
|
||||
re.sub(
|
||||
"'kfp==(\d+).(\d+).(\d+)(-[a-z]+.\d+)?'", 'kfp',
|
||||
yaml.dump(
|
||||
pipeline_spec['deploymentSpec']['executors'][executor],
|
||||
sort_keys=True)))
|
||||
return spec
|
||||
|
||||
|
||||
class CompilerCliTests(unittest.TestCase):
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.maxDiff = None
|
||||
return super().setUp()
|
||||
|
||||
def _test_compile_py_to_yaml(
|
||||
self,
|
||||
file_base_name,
|
||||
additional_arguments=None,
|
||||
):
|
||||
test_data_dir = os.path.join(os.path.dirname(__file__), 'test_data')
|
||||
py_file = os.path.join(test_data_dir, '{}.py'.format(file_base_name))
|
||||
golden_compiled_file = os.path.join(test_data_dir,
|
||||
file_base_name + '.yaml')
|
||||
|
||||
if additional_arguments is None:
|
||||
additional_arguments = []
|
||||
|
||||
def _compile(target_output_file: str):
|
||||
try:
|
||||
subprocess.check_output(
|
||||
['dsl-compile', '--py', py_file, '--output', compiled_file]
|
||||
+ additional_arguments,
|
||||
stderr=subprocess.STDOUT)
|
||||
except subprocess.CalledProcessError as exc:
|
||||
raise Exception(exc.output) from exc
|
||||
|
||||
def _load_compiled_file(filename: str):
|
||||
with open(filename, 'r') as f:
|
||||
contents = yaml.safe_load(f)
|
||||
# Correct the sdkVersion
|
||||
pipeline_spec = contents[
|
||||
'pipelineSpec'] if 'pipelineSpec' in contents else contents
|
||||
del pipeline_spec['sdkVersion']
|
||||
return _ignore_kfp_version_helper(contents)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
compiled_file = os.path.join(tmpdir,
|
||||
file_base_name + '-pipeline.yaml')
|
||||
_compile(target_output_file=compiled_file)
|
||||
|
||||
golden = _load_compiled_file(golden_compiled_file)
|
||||
compiled = _load_compiled_file(compiled_file)
|
||||
|
||||
# Devs can run the following command to update golden files:
|
||||
# UPDATE_GOLDENS=True python3 -m unittest kfp/v2/compiler_cli_tests/compiler_cli_tests.py
|
||||
# If UPDATE_GOLDENS=True, and the diff is
|
||||
# different, update the golden file and reload it.
|
||||
update_goldens = os.environ.get('UPDATE_GOLDENS', False)
|
||||
if golden != compiled and update_goldens:
|
||||
_compile(target_output_file=golden_compiled_file)
|
||||
golden = _load_compiled_file(golden_compiled_file)
|
||||
|
||||
self.assertEqual(golden, compiled)
|
||||
|
||||
def test_two_step_pipeline(self):
|
||||
self._test_compile_py_to_yaml(
|
||||
'two_step_pipeline',
|
||||
['--pipeline-parameters', '{"text":"Hello KFP!"}'])
|
||||
|
||||
def test_two_step_pipeline_failure_parameter_parse(self):
|
||||
with self.assertRaisesRegex(
|
||||
Exception, r"Failed to parse --pipeline-parameters argument:"):
|
||||
self._test_compile_py_to_yaml(
|
||||
'two_step_pipeline',
|
||||
['--pipeline-parameters', '{"text":"Hello KFP!}'])
|
||||
|
||||
def test_pipeline_with_importer(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_importer')
|
||||
|
||||
def test_pipeline_with_ontology(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_ontology')
|
||||
|
||||
def test_pipeline_with_if_placeholder(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_if_placeholder')
|
||||
|
||||
def test_pipeline_with_concat_placeholder(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_concat_placeholder')
|
||||
|
||||
def test_pipeline_with_resource_spec(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_resource_spec')
|
||||
|
||||
def test_pipeline_with_various_io_types(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_various_io_types')
|
||||
|
||||
def test_pipeline_with_reused_component(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_reused_component')
|
||||
|
||||
def test_pipeline_with_after(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_after')
|
||||
|
||||
def test_pipeline_with_condition(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_condition')
|
||||
|
||||
def test_pipeline_with_nested_conditions(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_nested_conditions')
|
||||
|
||||
def test_pipeline_with_nested_conditions_yaml(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_nested_conditions_yaml')
|
||||
|
||||
def test_pipeline_with_loops(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_loops')
|
||||
|
||||
def test_pipeline_with_nested_loops(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_nested_loops')
|
||||
|
||||
def test_pipeline_with_loops_and_conditions(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_loops_and_conditions')
|
||||
|
||||
def test_pipeline_with_params_containing_format(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_params_containing_format')
|
||||
|
||||
def test_lightweight_python_functions_v2_pipeline(self):
|
||||
self._test_compile_py_to_yaml(
|
||||
'lightweight_python_functions_v2_pipeline')
|
||||
|
||||
def test_lightweight_python_functions_v2_with_outputs(self):
|
||||
self._test_compile_py_to_yaml(
|
||||
'lightweight_python_functions_v2_with_outputs')
|
||||
|
||||
def test_xgboost_sample_pipeline(self):
|
||||
self._test_compile_py_to_yaml('xgboost_sample_pipeline')
|
||||
|
||||
def test_pipeline_with_metrics_outputs(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_metrics_outputs')
|
||||
|
||||
def test_pipeline_with_exit_handler(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_exit_handler')
|
||||
|
||||
def test_pipeline_with_env(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_env')
|
||||
|
||||
def test_v2_component_with_optional_inputs(self):
|
||||
self._test_compile_py_to_yaml('v2_component_with_optional_inputs')
|
||||
|
||||
def test_pipeline_with_gcpc_types(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_gcpc_types')
|
||||
|
||||
def test_pipeline_with_placeholders(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_placeholders')
|
||||
|
||||
def test_pipeline_with_task_final_status(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_task_final_status')
|
||||
|
||||
def test_pipeline_with_task_final_status_yaml(self):
|
||||
self._test_compile_py_to_yaml('pipeline_with_task_final_status_yaml')
|
||||
|
||||
def test_v2_component_with_pip_index_urls(self):
|
||||
self._test_compile_py_to_yaml('v2_component_with_pip_index_urls')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -89,7 +89,7 @@ setuptools.setup(
|
|||
include_package_data=True,
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'dsl-compile = kfp.cli.compile:main',
|
||||
'dsl-compile = kfp.cli.dsl_compile:main',
|
||||
'dsl-compile-deprecated = kfp.deprecated.compiler.main:main',
|
||||
'kfp=kfp.cli.__main__:main',
|
||||
]
|
||||
|
|
|
|||
Loading…
Reference in New Issue