# Copyright 2023 The Kubeflow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import inspect import os import re import sys import textwrap from typing import List import commonmark import docstring_parser from google_cloud_pipeline_components import utils from kfp import components from kfp import dsl import yaml # setting this enables the .rst files to use the paths v1.bigquery.Component (etc.) rather than google_cloud_pipeline_components.v1.biquery.Component for shorter, readable representation in docs gcpc_root_dir = os.path.abspath( os.path.join( os.path.dirname(__file__), '..', '..', 'google_cloud_pipeline_components', ) ) # keep as append not insert, otherwise there is an issue with other package discovery sys.path.append(gcpc_root_dir) # preserve function docstrings for components by setting component decorators to passthrough decorators # also enables autodoc to document the components as functions without using the autodata directive (https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#directive-autodata) def first_order_passthrough_decorator(func): func._is_component = True return func def second_order_passthrough_decorator(*args, **kwargs): def decorator(func): func._is_component = True return func return decorator def second_order_passthrough_decorator_for_pipeline(*args, **kwargs): def decorator(func): func._is_pipeline = True return func return decorator def load_from_file(path: str): with open(path) as f: contents = f.read() component_dict = yaml.safe_load(contents) comp = components.load_component_from_text(contents) description = component_dict.get('description', '') comp.__doc__ = description return comp utils.gcpc_output_name_converter = second_order_passthrough_decorator dsl.component = second_order_passthrough_decorator dsl.container_component = first_order_passthrough_decorator dsl.pipeline = second_order_passthrough_decorator_for_pipeline components.load_component_from_file = load_from_file class OutputPath(dsl.OutputPath): def __repr__(self) -> str: type_string = getattr(self.type, '__name__', '') return f'dsl.OutputPath({type_string})' dsl.OutputPath = OutputPath class InputClass: def __getitem__(self, type_) -> str: type_string = getattr(type_, 'schema_title', getattr(type_, '__name__', '')) return f'dsl.Input[{type_string}]' Input = InputClass() dsl.Input = Input class OutputClass: def __getitem__(self, type_) -> str: type_string = getattr(type_, 'schema_title', getattr(type_, '__name__', '')) return f'dsl.Output[{type_string}]' Output = OutputClass() dsl.Output = Output # -- General configuration --------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', 'm2r2', 'sphinx_immaterial', 'autodocsumm', 'notfound.extension', ] autodoc_default_options = { 'members': True, 'member-order': 'alphabetical', 'imported-members': True, 'undoc-members': True, 'show-inheritance': False, 'inherited-members': False, 'autosummary': False, } # notfound.extension: https://sphinx-notfound-page.readthedocs.io/en/latest/configuration.html#confval-notfound_context notfound_context = { 'title': 'Page not found', 'body': textwrap.dedent(""" Page not found

404: Page not found

It's likely the object or page you're looking for doesn't exist in this version of Google Cloud Pipeline Components. Please ensure you have the correct version selected.

Back to homepage
"""), } html_theme = 'sphinx_immaterial' html_title = 'Google Cloud Pipeline Components Reference Documentation' html_static_path = ['_static'] html_css_files = ['custom.css'] html_theme_options = { 'icon': { 'repo': 'fontawesome/brands/github', }, 'repo_url': 'https://github.com/kubeflow/pipelines/tree/master/components/google-cloud', 'repo_name': 'pipelines', 'repo_type': 'github', 'edit_uri': 'https://github.com/kubeflow/pipelines/tree/master/components/google-cloud/docs/source', 'globaltoc_collapse': True, 'features': [ 'navigation.expand', # "navigation.tabs", # "toc.integrate", 'navigation.sections', # "navigation.instant", # "header.autohide", 'navigation.top', # "navigation.tracking", 'search.highlight', 'search.share', 'toc.follow', 'toc.sticky', ], 'palette': [{ 'media': '(prefers-color-scheme: light)', 'scheme': 'default', 'primary': 'googleblue', }], 'font': {'text': 'Open Sans'}, 'version_dropdown': True, 'version_json': 'https://raw.githubusercontent.com/kubeflow/pipelines/master/components/google-cloud/docs/source/versions.json', # "toc_title_is_page_title": True, } # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: source_suffix = '.rst' # The master toctree document. master_doc = 'index' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None # The default sidebars (for documents that don't match any pattern) are # defined by theme itself. Builtin themes are using these templates by # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', # 'searchbox.html']``. # # html_sidebars = {} # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. htmlhelp_basename = 'GoogleCloudPipelineComponentsDocs' def component_grouper(app, what, name, obj, section, parent): if getattr(obj, '_is_component', False): return 'Components' def pipeline_grouper(app, what, name, obj, section, parent): if getattr(obj, '_is_pipeline', False): return 'Pipelines' def autodoc_skip_member(app, what, name, obj, skip, options): skip = True if name == 'create_custom_training_job_op_from_component': return skip def make_docstring_lines_for_param( param_name: str, type_string: str, description: str, ) -> List[str]: WHITESPACE = ' ' return [ f'{WHITESPACE * 2}``{param_name}: {type_string}``', f'{WHITESPACE * 4}{description}', ] def get_return_section(component) -> List[str]: """Modifies docstring so that a return section can be treated as an args section, then parses the docstring. """ docstring = inspect.getdoc(component) type_hints = component.__annotations__ if docstring is None: return [] # Returns and Return are the only two keywords docstring_parser uses for returns # use newline to avoid replacements that aren't in the return section header return_keywords = ['Returns:\n', 'Returns\n', 'Return:\n', 'Return\n'] for keyword in return_keywords: if keyword in docstring: modified_docstring = docstring.replace(keyword.strip(), 'Args:') returns_docstring = docstring_parser.parse(modified_docstring) new_docstring_lines = [] for param in returns_docstring.params: type_string = ( repr( type_hints.get( param.arg_name, type_hints.get( param.arg_name, 'Unknown', ), ) ) .lstrip("'") .rstrip("'") ) new_docstring_lines.extend( make_docstring_lines_for_param( param_name=param.arg_name.lstrip( utils.DOCS_INTEGRATED_OUTPUT_RENAMING_PREFIX ), type_string=type_string, description=param.description, ) ) return new_docstring_lines return [] def remove__output_prefix_from_signature( app, what, name, obj, options, signature, return_annotation ): if signature is not None: signature = re.sub( rf'{utils.DOCS_INTEGRATED_OUTPUT_RENAMING_PREFIX}(\w+):', r'\1:', signature, ) return signature, return_annotation def remove_after_returns_in_place(lines: List[str]) -> bool: for i in range(len(lines)): if lines[i].startswith(':returns:'): del lines[i:] return True return False def process_named_docstring_returns(app, what, name, obj, options, lines): markdown_to_rst(lines) if getattr(obj, '_is_component', False): has_returns_section = remove_after_returns_in_place(lines) if has_returns_section: returns_section = get_return_section(obj) lines.extend([':returns:', '']) lines.extend(returns_section) def markdown_to_rst(lines: List[str]) -> List[str]: md = '\n'.join(lines) ast = commonmark.Parser().parse(md) rst = commonmark.ReStructuredTextRenderer().render(ast) lines.clear() lines += rst.splitlines() def setup(app): app.connect('autodoc-process-docstring', process_named_docstring_returns) app.connect( 'autodoc-process-signature', remove__output_prefix_from_signature, ) app.connect('autodocsumm-grouper', component_grouper) app.connect('autodocsumm-grouper', pipeline_grouper) app.connect('autodoc-skip-member', autodoc_skip_member)