# Copyright 2021 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: feature_transform_engine
description: |
  Feature Transform Engine (FTE) component to transform raw data to engineered features.

  FTE performs dataset level transformations, data splitting, data statistic generation, and
  TensorFlow-based row level transformations on the input dataset based on the provided
  transformation configuration.

  Args:
      root_dir (str):
            The Cloud Storage location to store the output.
      project (str):
          Project to run feature transform engine.
      location (str):
          Location for the created GCP services.
      dataset_level_custom_transformation_definitions (Optional[JsonArray]):
          List of dataset-level custom transformation definitions.

          Custom, bring-your-own dataset-level transform functions, where users can define and
          import their own transform function and use it with FTE's built-in transformations. Using
          custom transformations is an experimental feature and it is currently not supported during
          batch prediction.

          Example:

              .. code-block:: python

                  [
                    {
                      "transformation": "ConcatCols",
                      "module_path": "/path/to/custom_transform_fn_dlt.py",
                      "function_name": "concat_cols"
                    }
                  ]

          Using custom transform function together with FTE's built-in transformations:

              .. code-block:: python

                  [
                    {
                      "transformation": "Join",
                      "right_table_uri": "bq://test-project.dataset_test.table",
                      "join_keys": [["join_key_col", "join_key_col"]]
                    },{
                      "transformation": "ConcatCols",
                      "cols": ["feature_1", "feature_2"],
                      "output_col": "feature_1_2"
                    }
                  ]

      dataset_level_transformations (Optional[JsonArray]):
          List of dataset-level transformations.

          Example:

              .. code-block:: python

                  [
                    {
                      "transformation": "Join",
                      "right_table_uri": "bq://test-project.dataset_test.table",
                      "join_keys": [["join_key_col", "join_key_col"]]
                    },
                    ...
                  ]

          Additional information about FTE's currently supported built-in transformations:

              Join:
                  Joins features from right_table_uri. For each join key, the left table keys will
                  be included and the right table keys will be dropped.

                  Example:

                  .. code-block:: python

                      {
                        "transformation": "Join",
                        "right_table_uri": "bq://test-project.dataset_test.table",
                        "join_keys": [["join_key_col", "join_key_col"]]
                      }

                  Arguments:
                      right_table_uri (str):
                          Right table BigQuery uri to join with input_full_table_id.
                      join_keys (List[List[str]]):
                          Features to join on. For each nested list, the first element is a
                          left table column and the second is its corresponding right table column.

              TimeAggregate:
                  Creates a new feature composed of values of an existing feature from a fixed
                  time period ago or in the future. Ex: A feature for sales by store 1 year ago.

                  Example:

                  .. code-block:: python

                      {
                        "transformation": "TimeAggregate",
                        "time_difference": 40,
                        "time_difference_units": "DAY",
                        "time_series_identifier_columns": ["store_id"],
                        "time_column": "time_col",
                        "time_difference_target_column": "target_col",
                        "output_column": "output_col"
                      }

                  Arguments:
                      time_difference (int):
                          Number of time_difference_units to look back or into the
                          future on our time_difference_target_column.
                      time_difference_units (str):
                          Units of time_difference to look back or into the
                          future on our time_difference_target_column.
                          Must be one of
                              * 'DAY'
                              * 'WEEK' (Equivalent to 7 DAYs)
                              * 'MONTH'
                              * 'QUARTER'
                              * 'YEAR'
                      time_series_identifier_columns (List[str]):
                          Names of the time series identifier columns.
                      time_column (str):
                          Name of the time column.
                      time_difference_target_column (str):
                          Column we wish to get the value of time_difference time_difference_units
                          in the past or future.
                      output_column (str):
                          Name of our new time aggregate feature.
                      is_future (Optional[bool]):
                          Whether we wish to look forward in time. Defaults to False.

              PartitionByMax/PartitionByMin/PartitionByAvg/PartitionBySum:
                  Performs a partition by reduce operation (one of max, min, avg, or sum) with a
                  fixed historic time period. Ex: Getting avg sales (the reduce column) for each
                  store (partition_by_column) over the previous 5 days (time_column, time_ago_units,
                  and time_ago).

                  Example:

                  .. code-block:: python

                      {
                        "transformation": "PartitionByMax",
                        "reduce_column": "sell_price",
                        "partition_by_columns": ["store_id", "state_id"],
                        "time_column": "date",
                        "time_ago": 1,
                        "time_ago_units": "WEEK",
                        "output_column": "partition_by_reduce_max_output"
                      }

                  Arguments:
                      reduce_column (str):
                          Column to apply the reduce operation on. Reduce operations include the
                          following: Max, Min, Avg, Sum.
                      partition_by_columns (List[str]):
                          List of columns to partition by.
                      time_column (str):
                          Time column for the partition by operation's window function.
                      time_ago (int):
                          Number of time_ago_units to look back on our target_column,
                          starting from time_column (inclusive).
                      time_ago_units (str):
                          Units of time_ago to look back on our target_column. Must be
                          one of
                              * 'DAY'
                              * 'WEEK'
                      output_column (str):
                          Name of our output feature.

      forecasting_time_column (Optional[str]):
          Forecasting time column.
      forecasting_time_series_identifier_column (Optional[str]):
          Forecasting time series identifier column.
      forecasting_time_series_attribute_columns (Optional[str]):
          Forecasting time series attribute columns.
      forecasting_unavailable_at_forecast_columns (Optional[str]):
          Forecasting unavailable at forecast columns.
      forecasting_available_at_forecast_columns (Optional[str]):
          Forecasting available at forecast columns.
      forecasting_forecast_horizon (Optional[int]):
          Forecasting horizon.
      forecasting_context_window (Optional[int]):
          Forecasting context window.
      forecasting_predefined_window_column (Optional[str]):
          Forecasting predefined window column.
      forecasting_window_stride_length (Optional[int]):
          Forecasting window stride length.
      forecasting_window_max_count (Optional[int]):
          Forecasting window max count.
      forecasting_apply_windowing (Optional[bool]):
          Whether to apply window strategy.
      predefined_split_key (Optional[str]):
          Predefined split key.
      stratified_split_key (Optional[str]):
          Stratified split key.
      timestamp_split_key (Optional[str]):
          Timestamp split key.
      training_fraction (Optional[float]):
          Fraction of input data for training.
      validation_fraction (Optional[float]):
          Fraction of input data for validation.
      test_fraction (Optional[float]):
          Fraction of input data for testing.
      tf_auto_transform_features (Optional[JsonArray]):
          List of auto TF transform features.

          FTE will automatically configure a set of built-in transformations for each feature based
          on its data statistics.  If users do not want auto type resolution, but want the set of
          transformations for a given type to be automatically generated, they may specify
          pre-resolved transformations (see the tf_transformations_path argument below).
          Note that the target and weight column may not be included as an auto transformation
          unless users are running forecasting.

      tf_custom_transformation_definitions (Optional[JsonArray]):
          List of TensorFlow-based custom transformation definitions.

          Custom, bring-your-own transform functions, where users can define and import their own
          transform function and use it with FTE's built-in transformations.

          Example:

          .. code-block:: python

              [
                {
                  "transformation": "PlusOne",
                  "module_path": "gs://bucket/custom_transform_fn.py",
                  "function_name": "plus_one_transform"
                },
                {
                  "transformation": "MultiplyTwo",
                  "module_path": "gs://bucket/custom_transform_fn.py",
                  "function_name": "multiply_two_transform"
                }
              ]

          Using custom transform function together with FTE's built-in transformations:

          .. code-block:: python

              [
                {
                  "transformation": "CastToFloat",
                  "input_columns": ["feature_1"],
                  "output_columns": ["feature_1"]
                },{
                  "transformation": "PlusOne",
                  "input_columns": ["feature_1"]
                  "output_columns": ["feature_1_plused_one"]
                },{
                  "transformation": "MultiplyTwo",
                  "input_columns": ["feature_1"]
                  "output_columns": ["feature_1_multiplied_two"]
                }
              ]

      tf_transformations_path (Optional[str]):
          Path to TensorFlow-based transformation configuration.

          Path to a JSON file used to specified FTE's TF transformation configurations.  In the
          following, we provide some sample transform configurations to demonstrate FTE's
          capabilities.

          All transformations on input columns are explicitly
          specified with FTE's built-in transformations. Chaining of multiple transformations
          on a single column is also supported. For example:

          .. code-block:: python

              [
                {
                  "transformation": "ZScale",
                  "input_columns": ["feature_1"]
                }, {
                  "transformation": "ZScale",
                  "input_columns": ["feature_2"]
                }
              ]

          If users do not want auto type resolution, but want the set of
          transformations for a given type to be automatically generated, they may specify
          pre-resolved transformations for the following types in the user provided transform
          config. Pre-resolved transformations may not have an output column specified. Currently
          accepted pre-resolved transformations are as follows:
              * AutoCategorical
              * AutoText
              * AutoNumeric
              * AutoTimestamp

          Example:

                  .. code-block:: python

                    {
                      "transformation": "AutoCategorical",
                      "input_columns": ["feature_1"]
                    }

          Additional information about FTE's currently supported built-in transformations:

              Datetime:
                  Extracts datetime featues from a column containing timestamp strings.

                  Example:

                  .. code-block:: python

                    {
                      "transformation": "Datetime",
                      "input_columns": ["feature_1"],
                      "time_format": "%Y-%m-%d"
                    }

                  Arguments:
                      input_columns (List[str]):
                          A list with a single column to perform the datetime transformation on.
                      output_columns (Optional[List[str]]):
                          Names of output columns, one for each datetime_features element.
                      time_format (str):
                          Datetime format string. Time format is a combination of
                          Date + Time Delimiter (optional) + Time (optional) directives.
                          Valid date directives are as follows
                              * '%Y-%m-%d'  # 2018-11-30
                              * '%Y/%m/%d'  # 2018/11/30
                              * '%y-%m-%d'  # 18-11-30
                              * '%y/%m/%d'  # 18/11/30
                              * '%m-%d-%Y'  # 11-30-2018
                              * '%m/%d/%Y'  # 11/30/2018
                              * '%m-%d-%y'  # 11-30-18
                              * '%m/%d/%y'  # 11/30/18
                              * '%d-%m-%Y'  # 30-11-2018
                              * '%d/%m/%Y'  # 30/11/2018
                              * '%d-%B-%Y'  # 30-November-2018
                              * '%d-%m-%y'  # 30-11-18
                              * '%d/%m/%y'  # 30/11/18
                              * '%d-%B-%y'  # 30-November-18
                              * '%d%m%Y'    # 30112018
                              * '%m%d%Y'    # 11302018
                              * '%Y%m%d'    # 20181130
                          Valid time delimiters are as follows
                              * 'T'
                              * ' '
                          Valid time directives are as follows
                              * '%H:%M'          # 23:59
                              * '%H:%M:%S'       # 23:59:58
                              * '%H:%M:%S.%f'    # 23:59:58[.123456]
                              * '%H:%M:%S.%f%z'  # 23:59:58[.123456]+0000
                              * '%H:%M:%S%z',    # 23:59:58+0000
                      datetime_features (Optional[List[str]]):
                          List of datetime features to be extract. Each entry must be one of
                              * 'YEAR'
                              * 'MONTH'
                              * 'DAY'
                              * 'DAY_OF_WEEK'
                              * 'DAY_OF_YEAR'
                              * 'WEEK_OF_YEAR'
                              * 'QUARTER'
                              * 'HOUR'
                              * 'MINUTE'
                              * 'SECOND'
                          Defaults to ['YEAR', 'MONTH', 'DAY', 'DAY_OF_WEEK', 'DAY_OF_YEAR',
                          'WEEK_OF_YEAR']

              Log:
                  Performs the natural log on a numeric column.

                  Example:

                  .. code-block:: python

                    {
                      "transformation": "Log",
                      "input_columns": ["feature_1"]
                    }

                  Arguments:
                      input_columns (List[str]):
                          A list with a single column to perform the log transformation on.
                      output_columns (Optional[List[str]]):
                          A list with a single output column name, corresponding to the output of
                          our transformation.

              ZScale:
                  Performs Z-scale normalization on a numeric column.

                  Example:

                  .. code-block:: python

                    {
                      "transformation": "ZScale",
                      "input_columns": ["feature_1"]
                    }

                  Arguments:
                      input_columns (List[str]):
                          A list with a single column to perform the z-scale transformation on.
                      output_columns (Optional[List[str]]):
                          A list with a single output column name, corresponding to the output of
                          our transformation.

              Vocabulary:
                  Converts strings to integers, where each unique string gets a unique integer
                  representation.

                  Example:

                  .. code-block:: python

                    {
                      "transformation": "Vocabulary",
                      "input_columns": ["feature_1"]
                    }

                  Arguments:
                      input_columns (List[str]):
                          A list with a single column to perform the vocabulary transformation on.
                      output_columns (Optional[List[str]]):
                          A list with a single output column name, corresponding to the output of
                          our transformation.
                      top_k (Optional[int]):
                          Number of the most frequent words in the vocabulary to use for
                          generating dictionary lookup indices. If not specified, all words in the
                          vocabulary will be used. Defaults to None.
                      frequency_threshold (Optional[int]):
                          Limit the vocabulary only to words whose number of occurrences in the
                          input exceeds frequency_threshold. If not specified, all words in the
                          vocabulary will be included. If both top_k and frequency_threshold are
                          specified, a word must satisfy both conditions to be included.
                          Defaults to None.

              Categorical:
                  Transforms categorical columns to integer columns.

                  Example:

                  .. code-block:: python

                      {
                        "transformation": "Categorical",
                        "input_columns": ["feature_1"],
                        "top_k": 10
                      }

                  Arguments:
                      input_columns (List[str]):
                          A list with a single column to perform the categorical transformation on.
                      output_columns (Optional[List[str]]):
                          A list with a single output column name, corresponding to the output of
                          our transformation.
                      top_k (Optional[int]):
                          Number of the most frequent words in the vocabulary to use for
                          generating dictionary lookup indices. If not specified, all words in the
                          vocabulary will be used.
                      frequency_threshold (Optional[int]):
                          Limit the vocabulary only to words whose number of occurrences in the
                          input exceeds frequency_threshold. If not specified, all words in the
                          vocabulary will be included. If both top_k and frequency_threshold are
                          specified, a word must satisfy both conditions to be included.

              Reduce:
                  Given a column where each entry is a numeric array, reduces arrays according to
                  our reduce_mode.

                  Example:

                  .. code-block:: python

                      {
                        "transformation": "Reduce",
                        "input_columns": ["feature_1"],
                        "reduce_mode": "MEAN",
                        "output_columns": ["feature_1_mean"]
                      }

                  Arguments:
                      input_columns (List[str]):
                          A list with a single column to perform the reduce transformation on.
                      output_columns (Optional[List[str]]):
                          A list with a single output column name, corresponding to the output of
                          our transformation.
                      reduce_mode (Optional[str]):
                          One of
                            * 'MAX'
                            * 'MIN'
                            * 'MEAN'
                            * 'LAST_K'
                          Defaults to 'MEAN'.
                      last_k (Optional[int]):
                          The number of last k elements when 'LAST_K' reduce mode is used.
                          Defaults to 1.

              SplitString:
                  Given a column of strings, splits strings into token arrays.

                  Example:

                  .. code-block:: python

                      {
                        "transformation": "SplitString",
                        "input_columns": ["feature_1"],
                        "separator": "$"
                      }

                  Arguments:
                      input_columns (List[str]):
                          A list with a single column to perform the split string transformation on.
                      output_columns (Optional[List[str]]):
                          A list with a single output column name, corresponding to the output of
                          our transformation.
                      separator (Optional[str]):
                          Separator to split input string into tokens. Defaults to ' '.
                      missing_token (Optional[str]):
                          Missing token to use when no string is included.
                          Defaults to ' _MISSING_ '.

              NGram:
                  Given a column of strings, splits strings into token arrays where each token is
                  an integer.

                  Example:

                  .. code-block:: python

                      {
                        "transformation": "NGram",
                        "input_columns": ["feature_1"],
                        "min_ngram_size": 1,
                        "max_ngram_size": 2,
                        "separator": " "
                      }

                  Arguments:
                      input_columns (List[str]):
                          A list with a single column to perform the n-gram transformation on.
                      output_columns (Optional[List[str]]):
                          A list with a single output column name, corresponding to the output of
                          our transformation.
                      min_ngram_size (Optional[int]):
                          Minimum n-gram size. Must be a positive number and <= max_ngram_size.
                          Defaults to 1.
                      max_ngram_size (Optional[int]):
                          Maximum n-gram size. Must be a positive number and >= min_ngram_size.
                          Defaults to 2.
                      top_k (Optional[int]):
                          Number of the most frequent words in the vocabulary to use for
                          generating dictionary lookup indices. If not specified, all words in the
                          vocabulary will be used. Defaults to None.
                      frequency_threshold (Optional[int]):
                          Limit the dictionary's vocabulary only to words whose
                          number of occurrences in the input exceeds frequency_threshold. If not
                          specified, all words in the vocabulary will be included. If both top_k and
                          frequency_threshold are specified, a word must satisfy both conditions to
                          be included. Defaults to None.
                      separator (Optional[str]):
                          Separator to split input string into tokens. Defaults to ' '.
                      missing_token (Optional[str]):
                          Missing token to use when no string is included.
                          Defaults to ' _MISSING_ '.

              Clip:
                  Given a numeric column, clips elements such that elements < min_value are assigned
                  min_value, and elements > max_value are assigned max_value.

                  Example:

                  .. code-block:: python

                      {
                        "transformation": "Clip",
                        "input_columns": ["col1"],
                        "output_columns": ["col1_clipped"],
                        "min_value": 1.,
                        "max_value": 10.,
                      }

                  Arguments:
                      input_columns (List[str]):
                          A list with a single column to perform the n-gram transformation on.
                      output_columns (Optional[List[str]]):
                          A list with a single output column name, corresponding to the output of
                          our transformation.
                      min_value (Optional[float]):
                          Number where all values below min_value are set to min_value. If no
                          min_value is provided, min clipping will not occur. Defaults to None.
                      max_value (Optional[float]):
                        Number where all values above max_value are set to max_value If no
                        max_value is provided, max clipping will not occur. Defaults to None.

              MultiHotEncoding:
                  Performs multi-hot encoding on a categorical array column.

                  Example:

                  .. code-block:: python

                      {
                        "transformation": "MultiHotEncoding",
                        "input_columns": ["col1"],
                      }

                  The number of classes is determened by the largest number included in the input if
                  it is numeric or the total number of unique values of the input if it is type str.

                  If the input is has type str and an element contians separator tokens, the input
                  will be split at separator indices, and the each element of the split list will be
                  considered a seperate class. For example,

                  Input:

                  .. code-block:: python

                      [
                        ["foo bar"],      # Example 0
                        ["foo", "bar"],   # Example 1
                        ["foo"],          # Example 2
                        ["bar"],          # Example 3
                      ]

                  Output (with default separator=" "):

                  .. code-block:: python

                      [
                        [1, 1],          # Example 0
                        [1, 1],          # Example 1
                        [1, 0],          # Example 2
                        [0, 1],          # Example 3
                      ]

                  Arguments:
                      input_columns (List[str]):
                          A list with a single column to perform the multi-hot-encoding on.
                      output_columns (Optional[List[str]]):
                          A list with a single output column name, corresponding to the output of
                          our transformation.
                      top_k (Optional[int]):
                          Number of the most frequent words in the vocabulary to use for
                          generating dictionary lookup indices. If not specified, all words in the
                          vocabulary will be used. Defaults to None.
                      frequency_threshold (Optional[int]):
                          Limit the dictionary's vocabulary only to words whose
                          number of occurrences in the input exceeds frequency_threshold. If not
                          specified, all words in the vocabulary will be included. If both top_k and
                          frequency_threshold are specified, a word must satisfy both conditions to
                          be included. Defaults to None.
                      separator (Optional[str]):
                          Separator to split input string into tokens. Defaults to ' '.

              MaxAbsScale:
                  Performs maximum absolute scaling on a numeric column.

                  Example:

                  .. code-block:: python

                      {
                        "transformation": "MaxAbsScale",
                        "input_columns": ["col1"],
                        "output_columns": ["col1_max_abs_scaled"]
                      }

                  Arguments:
                      input_columns (List[str]):
                          A list with a single column to perform max-abs-scale on.
                      output_columns (Optional[List[str]]):
                          A list with a single output column name, corresponding to the output of
                          our transformation.

              Custom:
                  Transformations defined in tf_custom_transformation_definitions are included here
                  in the TensorFlow-based transformation configuration.  For example, given the
                  following tf_custom_transformation_definitions:

                  .. code-block:: python

                      [
                        {
                          "transformation": "PlusX",
                          "module_path": "gs://bucket/custom_transform_fn.py",
                          "function_name": "plus_one_transform"
                        }
                      ]

                  We can include the following transformation:

                  .. code-block:: python

                      {
                        "transformation": "PlusX",
                        "input_columns": ["col1"],
                        "output_columns": ["col1_max_abs_scaled"]
                        "x": 5
                      }

                  Note that input_columns must still be included in our arguments and output_columns
                  is optional. All other arguments are those defined in custom_transform_fn.py,
                  which includes `"x"` in this case. See tf_custom_transformation_definitions above.


      target_column (Optional[str]):
          Target column of input data.
      weight_column (Optional[str]):
          Weight column of input data.
      prediction_type (Optional[str]):
          Model prediction type. One of "classification", "regression", "time_series".
      run_distill (Optional[bool]):
          Whether the distillation should be applied to the training.
      run_feature_selection (Optional[bool]):
          Whether the feature selection should be applied to the dataset.
      feature_selection_algorithm (Optional[str]):
          The algorithm of feature selection. One of "AMI", "CMIM", "JMIM", "MRMR", default to be "AMI".

          The algorithms available are:
          AMI(Adjusted Mutual Information):
           Reference: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.adjusted_mutual_info_score.html
           Arrays are not yet supported in this algorithm.

          CMIM(Conditional Mutual Information Maximization):
            Reference paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, “Feature
            selection using Joint Mutual Information Maximisation,” Expert Systems with
            Applications, vol. 42, issue 22, 1 December 2015, Pages 8520-8532.

          JMIM(Joint Mutual Information Maximization):
            Reference paper: Mohamed Bennasar, Yulia Hicks, Rossitza Setchi, “Feature
            selection using Joint Mutual Information Maximisation,” Expert Systems with
            Applications, vol. 42, issue 22, 1 December 2015, Pages 8520-8532.

          MRMR(MIQ Minimum-redundancy Maximum-relevance):
            Reference paper: Hanchuan Peng, Fuhui Long, and Chris Ding.
            "Feature selection based on mutual information criteria of max-dependency,
            max-relevance, and min-redundancy."
            IEEE Transactions on pattern analysis and machine intelligence 27,
            no. 8 (2005): 1226-1238.
      materialized_examples_format (Optional[str]):
          The format to use for the materialized examples. Should be either
          'tfrecords_gzip' (default) or 'parquet'.
      max_selected_features (Optional[int]):
          Maximum number of features to select.

          If specified, the transform config will be purged by only using the selected features that
          ranked top in the feature ranking, which has the ranking value for all supported features.
          If the number of input features is smaller than max_selected_features specified, we will
          still run the feature selection process and generate the feature ranking, no features will
          be excluded.

          The value will be set to 1000 by default if run_feature_selection is enabled.
      data_source_csv_filenames (Optional[str]):
          CSV input data source to run feature transform on.
      data_source_bigquery_table_path (Optional[str]):
          BigQuery input data source to run feature transform on.
      bigquery_staging_full_dataset_id (Optional[str]):
          Dataset in "projectId.datasetId" format for storing intermediate-FTE BigQuery tables.

          If the specified dataset does not exist in BigQuery, FTE will create the dataset. If no
          bigquery_staging_full_dataset_id is specified, all intermediate tables will be stored in a
          dataset created under the provided project in the input data source's location during FTE
          execution called "vertex_feature_transform_engine_staging_{location.replace('-', '_')}".

          All tables generated by FTE will have a 30 day TTL.
      model_type (Optional[str]):
          Model type, which we wish to engineer features for. Can be one of: neural_network,
          boosted_trees.
      dataflow_machine_type (Optional[str]):
          The machine type used for dataflow jobs. If not set, default to n1-standard-16.
      dataflow_max_num_workers (Optional[int]):
          The number of workers to run the dataflow job. If not set, default to 25.
      dataflow_disk_size_gb (Optional[int]):
          The disk size, in gigabytes, to use on each Dataflow worker instance. If not set,
          default to 40.
      dataflow_subnetwork (Optional[str]):
          Dataflow's fully qualified subnetwork name, when empty the default subnetwork will be
          used. More details:
          https://cloud.google.com/dataflow/docs/guides/specifying-networks#example_network_and_subnetwork_specifications
      dataflow_use_public_ips (Optional[bool]):
          Specifies whether Dataflow workers use public IP addresses.
      dataflow_service_account (Optional[str]):
          Custom service account to run Dataflow jobs.
      encryption_spec_key_name (Optional[str]):
          Customer-managed encryption key.
      autodetect_csv_schema (Optional[bool]):
          If True, infers the column types when importing CSVs into BigQuery.

  Returns:
      dataset_stats (AutoMLTabularDatasetStats):
          The stats of the dataset.
      materialized_data (Dataset):
          The materialized dataset.
      transform_output (TransformOutput):
          The transform output artifact.
      split_example_counts (str):
          JSON string of data split example counts for train, validate, and test splits.
      bigquery_test_split_uri (str):
          BigQuery URI for the test split to pass to the batch prediction component during
          evaluation.
      bigquery_downsampled_test_split_uri (str):
          BigQuery URI for the downsampled test split to pass to the batch prediction component
          during batch explain.
      instance_schema_path (DatasetSchema):
          Schema of input data to the tf_model at serving time.
      training_schema_path (DatasetSchema):
          Schema of input data to the tf_model at training time.
      feature_ranking (TabularFeatureRanking):
          The ranking of features, all features supported in the dataset will be included.

          for "AMI" algorithm, array features won't be available in the ranking as arrays are not
          supported yet.
      gcp_resources (str):
          GCP resources created by this component.
          For more details, see https://github.com/kubeflow/pipelines/blob/master/components/google-cloud/google_cloud_pipeline_components/proto/README.md.

# LINT.IfChange
inputs:
- {name: root_dir, type: String}
- {name: project, type: String}
- {name: location, type: String}
- {name: dataset_level_custom_transformation_definitions, type: JsonArray, default: "[]"}
- {name: dataset_level_transformations, type: JsonArray, default: "[]"}
- {name: forecasting_time_column, type: String, default: ""}
- {name: forecasting_time_series_identifier_column, type: String, default: ""}
- {name: forecasting_time_series_attribute_columns, type: String, default: ""}
- {name: forecasting_unavailable_at_forecast_columns, type: String, default: ""}
- {name: forecasting_available_at_forecast_columns, type: String, default: ""}
- {name: forecasting_forecast_horizon, type: Integer, default: "-1"}
- {name: forecasting_context_window, type: Integer, default: "-1"}
- {name: forecasting_predefined_window_column, type: String, default: ""}
- {name: forecasting_window_stride_length, type: Integer, default: "-1"}
- {name: forecasting_window_max_count, type: Integer, default: "-1"}
- {name: forecasting_apply_windowing, type: Boolean, default: "true"}
- {name: predefined_split_key, type: String, default: ""}
- {name: stratified_split_key, type: String, default: ""}
- {name: timestamp_split_key, type: String, default: ""}
- {name: training_fraction, type: Float, default: "-1"}
- {name: validation_fraction, type: Float, default: "-1"}
- {name: test_fraction, type: Float, default: "-1"}
- {name: tf_auto_transform_features, type: JsonArray, default: "[]"}
- {name: tf_custom_transformation_definitions, type: JsonArray, default: "[]"}
- {name: tf_transformations_path, type: String, default: ""}
- {name: target_column, type: String, default: ""}
- {name: weight_column, type: String, default: ""}
- {name: prediction_type, type: String, default: ""}
- {name: model_type, type: String, optional: true}
- {name: run_distill, type: Boolean, default: "false"}
- {name: run_feature_selection, type: Boolean, default: "false"}
- {name: feature_selection_algorithm, type: String, default: "AMI"}
- {name: materialized_examples_format, type: String, default: "tfrecords_gzip"}
- {name: max_selected_features, type: Integer, default: 1000}
- {name: data_source_csv_filenames, type: String, default: ""}
- {name: data_source_bigquery_table_path, type: String, default: ""}
- {name: bigquery_staging_full_dataset_id, type: String, default: ""}
- {name: dataflow_machine_type, type: String, default: "n1-standard-16"}
- {name: dataflow_max_num_workers, type: Integer, default: "25"}
- {name: dataflow_disk_size_gb, type: Integer, default: "40"}
- {name: dataflow_subnetwork, type: String, default: ""}
- {name: dataflow_use_public_ips, type: Boolean, default: "true"}
- {name: dataflow_service_account, type: String, default: ""}
- {name: encryption_spec_key_name, type: String, default: ""}
- {name: autodetect_csv_schema, type: Boolean, default: "false"}

outputs:
- {name: dataset_stats, type: AutoMLTabularDatasetStats}
- {name: materialized_data, type: Dataset}
- {name: transform_output, type: TransformOutput}
- {name: split_example_counts, type: String}
- {name: instance_schema, type: DatasetSchema}
- {name: training_schema, type: DatasetSchema}
- {name: bigquery_test_split_uri, type: String}
- {name: bigquery_downsampled_test_split_uri, type: String}
- {name: feature_ranking, type: TabularFeatureRanking}
- {name: gcp_resources, type: String}
# LINT.ThenChange(//depot/google3/java/com/google/cloud/ai/platform/boq/shared/tasks/pipelinejob/tabularusagemetricshelper/FeatureTransformEngineComponentMetricsHelper.java)

implementation:
  container:
    image: us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20230322_0325
    args: [
      feature_transform_engine,
      concat: [
        "--project=",
        {inputValue: project}
      ],
      concat: [
        "--location=",
        {inputValue: location}
      ],
      concat: [
        "--dataset_level_custom_transformation_definitions=",
        {inputValue: dataset_level_custom_transformation_definitions}
      ],
      concat: [
        "--dataset_level_transformations=",
        {inputValue: dataset_level_transformations}
      ],
      concat: [
        "--forecasting_time_column=",
        {inputValue: forecasting_time_column}
      ],
      concat: [
        "--forecasting_time_series_identifier_column=",
        {inputValue: forecasting_time_series_identifier_column}
      ],
      concat: [
        "--forecasting_time_series_attribute_columns=",
        {inputValue: forecasting_time_series_attribute_columns}
      ],
      concat: [
        "--forecasting_unavailable_at_forecast_columns=",
        {inputValue: forecasting_unavailable_at_forecast_columns}
      ],
      concat: [
        "--forecasting_available_at_forecast_columns=",
        {inputValue: forecasting_available_at_forecast_columns}
      ],
      concat: [
        "--forecasting_forecast_horizon=",
        {inputValue: forecasting_forecast_horizon}
      ],
      concat: [
        "--forecasting_context_window=",
        {inputValue: forecasting_context_window}
      ],
      concat: [
        "--forecasting_predefined_window_column=",
        {inputValue: forecasting_predefined_window_column}
      ],
      concat: [
        "--forecasting_window_stride_length=",
        {inputValue: forecasting_window_stride_length}
      ],
      concat: [
        "--forecasting_window_max_count=",
        {inputValue: forecasting_window_max_count}
      ],
      concat: [
        "--forecasting_apply_windowing=",
        {inputValue: forecasting_apply_windowing}
      ],
      concat: [
        "--predefined_split_key=",
        {inputValue: predefined_split_key}
      ],
      concat: [
        "--stratified_split_key=",
        {inputValue: stratified_split_key}
      ],
      concat: [
        "--timestamp_split_key=",
        {inputValue: timestamp_split_key}
      ],
      concat: [
        "--training_fraction=",
        {inputValue: training_fraction}
      ],
      concat: [
        "--validation_fraction=",
        {inputValue: validation_fraction}
      ],
      concat: [
        "--test_fraction=",
        {inputValue: test_fraction}
      ],
      concat: [
        "--tf_auto_transform_features=",
        {inputValue: tf_auto_transform_features}
      ],
      concat: [
        "--tf_custom_transformation_definitions=",
        {inputValue: tf_custom_transformation_definitions}
      ],
      concat: [
        "--tf_transformations_path=",
        {inputValue: tf_transformations_path}
      ],
      concat: [
        "--data_source_csv_filenames=",
        {inputValue: data_source_csv_filenames}
      ],
      concat: [
        "--data_source_bigquery_table_path=",
        {inputValue: data_source_bigquery_table_path}
      ],
      concat: [
        "--bigquery_staging_full_dataset_id=",
        {inputValue: bigquery_staging_full_dataset_id}
      ],
      concat: [
        "--target_column=",
        {inputValue: target_column}
      ],
      concat: [
        "--weight_column=",
        {inputValue: weight_column}
      ],
      concat: [
        "--prediction_type=",
        {inputValue: prediction_type}
      ],
      {if: {
        cond: {isPresent: model_type},
        then:
          {concat: [
            "--model_type=",
            {inputValue: model_type}
          ]},
      }},
      concat: [
        "--run_distill=",
        {inputValue: run_distill}
      ],
      concat: [
        "--run_feature_selection=",
        {inputValue: run_feature_selection}
      ],
      concat: [
        "--materialized_examples_format=",
        {inputValue: materialized_examples_format}
      ],
      concat: [
        "--max_selected_features=",
        {inputValue: max_selected_features}
      ],
      concat: [
        "--feature_selection_staging_dir=",
        {inputValue: root_dir},
        "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/feature_selection_staging_dir",
      ],
      concat: [
        "--feature_selection_algorithm=",
        {inputValue: feature_selection_algorithm}
      ],
      concat: [
        "--feature_ranking_path=",
        {outputUri: feature_ranking}
      ],
      concat: [
        "--error_file_path=",
        {inputValue: root_dir},
        "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/error.txt",
      ],
      concat: [
        "--stats_result_path=",
        {outputUri: dataset_stats}
      ],
      concat: [
        "--transform_output_artifact_path=",
        {outputUri: transform_output}
      ],
      concat: [
        "--transform_output_path=",
        {inputValue: root_dir},
        "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/transform"
      ],
      concat: [
        "--materialized_examples_path=",
        {inputValue: root_dir},
        "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized"
      ],
      concat: [
        "--export_data_path=",
        {inputValue: root_dir},
        "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/export"
      ],
      concat: [
        "--materialized_data_path=",
        {inputValue: root_dir},
        "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/materialized_data"
      ],
      concat: [
        "--materialized_data_artifact_path=",
        {outputUri: materialized_data}
      ],
      concat: [
        "--bigquery_test_split_uri_path=",
        {outputPath: bigquery_test_split_uri}
      ],
      concat: [
        "--bigquery_downsampled_test_split_uri_path=",
        {outputPath: bigquery_downsampled_test_split_uri}
      ],
      concat: [
        "--split_example_counts_path=",
        {outputPath: split_example_counts}
      ],
      concat: [
        "--instance_schema_path=",
        {outputPath: instance_schema}
      ],
      concat: [
        "--training_schema_path=",
        {outputPath: training_schema}
      ],
      "--job_name=feature-transform-engine-{{$.pipeline_job_uuid}}-{{$.pipeline_task_uuid}}",
      concat: [
        "--dataflow_project=",
        {inputValue: project}
      ],
      concat: [
        "--dataflow_staging_dir=",
        {inputValue: root_dir},
        "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_staging",
      ],
      concat: [
        "--dataflow_tmp_dir=",
        {inputValue: root_dir},
        "/{{$.pipeline_job_uuid}}/{{$.pipeline_task_uuid}}/dataflow_tmp",
      ],
      concat: [
        "--dataflow_max_num_workers=",
        {inputValue: dataflow_max_num_workers}
      ],
      concat: [
        "--dataflow_machine_type=",
        {inputValue: dataflow_machine_type}
      ],
      "--dataflow_worker_container_image=us-docker.pkg.dev/vertex-ai/automl-tabular/dataflow-worker:20230322_0325",
      "--feature_transform_engine_docker_uri=us-docker.pkg.dev/vertex-ai/automl-tabular/feature-transform-engine:20230322_0325",
      concat: [
        "--dataflow_disk_size_gb=",
        {inputValue: dataflow_disk_size_gb}
      ],
      concat: [
        "--dataflow_subnetwork_fully_qualified=",
        {inputValue: dataflow_subnetwork}
      ],
      concat: [
        "--dataflow_use_public_ips=",
        {inputValue: dataflow_use_public_ips}
      ],
      concat: [
        "--dataflow_service_account=",
        {inputValue: dataflow_service_account}
      ],
      concat: [
        "--dataflow_kms_key=",
        {inputValue: encryption_spec_key_name}
      ],
      concat: [
        "--autodetect_csv_schema=",
        {inputValue: autodetect_csv_schema}
      ],
      concat: [
        "--gcp_resources_path=",
        {outputPath: gcp_resources}
      ]
    ]
