340 lines
9.5 KiB
Python
340 lines
9.5 KiB
Python
# Copyright 2018 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
import kfp.components as comp
|
|
from kfp.components._yaml_utils import load_yaml
|
|
|
|
class LoadComponentTestCase(unittest.TestCase):
|
|
def test_load_component_from_file(self):
|
|
_this_file = Path(__file__).resolve()
|
|
_this_dir = _this_file.parent
|
|
_test_data_dir = _this_dir.joinpath('test_data')
|
|
component_path_obj = _test_data_dir.joinpath('python_add.component.yaml')
|
|
component_text = component_path_obj.read_text()
|
|
component_dict = load_yaml(component_text)
|
|
task_factory1 = comp.load_component_from_file(str(component_path_obj))
|
|
assert task_factory1.__doc__ == component_dict['description']
|
|
|
|
arg1 = 3
|
|
arg2 = 5
|
|
task1 = task_factory1(arg1, arg2)
|
|
assert task1.human_name == component_dict['name']
|
|
assert task1.image == component_dict['implementation']['dockerContainer']['image']
|
|
|
|
assert task1.arguments[0] == str(arg1)
|
|
assert task1.arguments[1] == str(arg2)
|
|
|
|
@unittest.expectedFailure #The repo is non-public and will change soon. TODO: Update the URL and enable the test once we move to a public repo
|
|
def test_load_component_from_url(self):
|
|
url = 'https://raw.githubusercontent.com/kubeflow/pipelines/638045974d688b473cda9f4516a2cf1d7d1e02dd/sdk/python/tests/components/test_data/python_add.component.yaml'
|
|
|
|
import requests
|
|
resp = requests.get(url)
|
|
component_text = resp.content
|
|
component_dict = load_yaml(component_text)
|
|
task_factory1 = comp.load_component_from_url(url)
|
|
assert task_factory1.__doc__ == component_dict['description']
|
|
|
|
arg1 = 3
|
|
arg2 = 5
|
|
task1 = task_factory1(arg1, arg2)
|
|
assert task1.human_name == component_dict['name']
|
|
assert task1.image == component_dict['implementation']['dockerContainer']['image']
|
|
|
|
assert task1.arguments[0] == str(arg1)
|
|
assert task1.arguments[1] == str(arg2)
|
|
|
|
def test_loading_minimal_component(self):
|
|
component_text = '''\
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
'''
|
|
component_dict = load_yaml(component_text)
|
|
task_factory1 = comp.load_component(text=component_text)
|
|
|
|
task1 = task_factory1()
|
|
assert task1.image == component_dict['implementation']['dockerContainer']['image']
|
|
|
|
@unittest.expectedFailure
|
|
def test_fail_on_duplicate_input_names(self):
|
|
component_text = '''\
|
|
inputs:
|
|
- {name: Data1}
|
|
- {name: Data1}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
'''
|
|
task_factory1 = comp.load_component_from_text(component_text)
|
|
|
|
@unittest.expectedFailure
|
|
def test_fail_on_duplicate_output_names(self):
|
|
component_text = '''\
|
|
outputs:
|
|
- {name: Data1}
|
|
- {name: Data1}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
'''
|
|
task_factory1 = comp.load_component_from_text(component_text)
|
|
|
|
def test_handle_underscored_input_names(self):
|
|
component_text = '''\
|
|
inputs:
|
|
- {name: Data}
|
|
- {name: _Data}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
'''
|
|
task_factory1 = comp.load_component_from_text(component_text)
|
|
|
|
def test_handle_underscored_output_names(self):
|
|
component_text = '''\
|
|
outputs:
|
|
- {name: Data}
|
|
- {name: _Data}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
'''
|
|
task_factory1 = comp.load_component_from_text(component_text)
|
|
|
|
def test_handle_input_names_with_spaces(self):
|
|
component_text = '''\
|
|
inputs:
|
|
- {name: Training data}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
'''
|
|
task_factory1 = comp.load_component_from_text(component_text)
|
|
|
|
def test_handle_output_names_with_spaces(self):
|
|
component_text = '''\
|
|
outputs:
|
|
- {name: Training data}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
'''
|
|
task_factory1 = comp.load_component_from_text(component_text)
|
|
|
|
def test_handle_file_outputs_with_spaces(self):
|
|
component_text = '''\
|
|
outputs:
|
|
- {name: Output data}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
fileOutputs:
|
|
Output data: /outputs/output-data
|
|
'''
|
|
task_factory1 = comp.load_component_from_text(component_text)
|
|
|
|
def test_handle_similar_input_names(self):
|
|
component_text = '''\
|
|
inputs:
|
|
- {name: Input 1}
|
|
- {name: Input_1}
|
|
- {name: Input-1}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
'''
|
|
task_factory1 = comp.load_component_from_text(component_text)
|
|
|
|
def test_handle_duplicate_input_output_names(self):
|
|
component_text = '''\
|
|
inputs:
|
|
- {name: Data}
|
|
outputs:
|
|
- {name: Data}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
'''
|
|
task_factory1 = comp.load_component_from_text(component_text)
|
|
|
|
@unittest.skip #TODO: FIX:
|
|
@unittest.expectedFailure
|
|
def test_fail_on_unknown_value_argument(self):
|
|
component_text = '''\
|
|
inputs:
|
|
- {name: Data}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
arguments:
|
|
- [value, Wrong]
|
|
'''
|
|
task_factory1 = comp.load_component_from_text(component_text)
|
|
|
|
@unittest.expectedFailure
|
|
def test_fail_on_unknown_file_output(self):
|
|
component_text = '''\
|
|
outputs:
|
|
- {name: Data}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
fileOutputs:
|
|
Wrong: '/outputs/output.txt'
|
|
'''
|
|
task_factory1 = comp.load_component_from_text(component_text)
|
|
|
|
@unittest.expectedFailure
|
|
def test_load_component_fail_on_no_sources(self):
|
|
comp.load_component()
|
|
|
|
@unittest.expectedFailure
|
|
def test_load_component_fail_on_multiple_sources(self):
|
|
comp.load_component(filename='', text='')
|
|
|
|
@unittest.expectedFailure
|
|
def test_load_component_fail_on_none_arguments(self):
|
|
comp.load_component(filename=None, url=None, text=None)
|
|
|
|
@unittest.expectedFailure
|
|
def test_load_component_from_file_fail_on_none_arg(self):
|
|
comp.load_component_from_file(None)
|
|
|
|
@unittest.expectedFailure
|
|
def test_load_component_from_url_fail_on_none_arg(self):
|
|
comp.load_component_from_url(None)
|
|
|
|
@unittest.expectedFailure
|
|
def test_load_component_from_text_fail_on_none_arg(self):
|
|
comp.load_component_from_text(None)
|
|
|
|
def test_input_value_resolving(self):
|
|
component_text = '''\
|
|
inputs:
|
|
- {name: Data}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
arguments:
|
|
- --data
|
|
- value: Data
|
|
'''
|
|
task_factory1 = comp.load_component(text=component_text)
|
|
task1 = task_factory1('some-data')
|
|
|
|
self.assertEqual(task1.arguments, ['--data', 'some-data'])
|
|
|
|
def test_output_resolving(self):
|
|
component_text = '''\
|
|
outputs:
|
|
- {name: Data}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
arguments:
|
|
- --output-data
|
|
- output: Data
|
|
'''
|
|
task_factory1 = comp.load_component(text=component_text)
|
|
task1 = task_factory1(data='/outputs/some-data')
|
|
|
|
self.assertEqual(task1.arguments, ['--output-data', '/outputs/some-data'])
|
|
|
|
def test_automatic_output_resolving(self):
|
|
component_text = '''\
|
|
outputs:
|
|
- {name: Data}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
arguments:
|
|
- --output-data
|
|
- {output: Data}
|
|
'''
|
|
task_factory1 = comp.load_component(text=component_text)
|
|
task1 = task_factory1()
|
|
|
|
self.assertEqual(len(task1.arguments), 2)
|
|
|
|
def test_command_concat(self):
|
|
component_text = '''\
|
|
inputs:
|
|
- {name: In1}
|
|
- {name: In2}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
arguments:
|
|
- concat: [{value: In1}, {value: In2}]
|
|
'''
|
|
task_factory1 = comp.load_component(text=component_text)
|
|
task1 = task_factory1('some', 'data')
|
|
|
|
self.assertEqual(task1.arguments, ['somedata'])
|
|
|
|
def test_command_if_then(self):
|
|
component_text = '''\
|
|
inputs:
|
|
- {name: In, required: false}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
arguments:
|
|
- if:
|
|
cond: {isPresent: In}
|
|
then: [--in, {value: In}]
|
|
#else: --no-in
|
|
'''
|
|
task_factory1 = comp.load_component(text=component_text)
|
|
|
|
task_then = task_factory1('data')
|
|
self.assertEqual(task_then.arguments, ['--in', 'data'])
|
|
|
|
#TODO: Fix optional arguments
|
|
#task_else = task_factory1() #Error: TypeError: Component() missing 1 required positional argument: 'in'
|
|
#self.assertEqual(task_else.arguments, [])
|
|
|
|
def test_command_if_then_else(self):
|
|
component_text = '''\
|
|
inputs:
|
|
- {name: In, required: false}
|
|
implementation:
|
|
dockerContainer:
|
|
image: busybox
|
|
arguments:
|
|
- if:
|
|
cond: {isPresent: In}
|
|
then: [--in, {value: In}]
|
|
else: --no-in
|
|
'''
|
|
task_factory1 = comp.load_component(text=component_text)
|
|
|
|
task_then = task_factory1('data')
|
|
self.assertEqual(task_then.arguments, ['--in', 'data'])
|
|
|
|
#TODO: Fix optional arguments
|
|
#task_else = task_factory1() #Error: TypeError: Component() missing 1 required positional argument: 'in'
|
|
#self.assertEqual(task_else.arguments, ['--no-in'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|