105 lines
2.5 KiB
Python
105 lines
2.5 KiB
Python
#!/usr/bin/env/python3
|
|
# Copyright (c) Facebook, Inc. and its affiliates.
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Utility to upload files/folders into minio"""
|
|
import os
|
|
from argparse import ArgumentParser
|
|
from pytorch_kfp_components.components.minio.component import MinIO
|
|
from pytorch_kfp_components.components.visualization.component import Visualization
|
|
|
|
# Argument parser for user defined paths
|
|
parser = ArgumentParser()
|
|
|
|
parser.add_argument(
|
|
"--bucket_name",
|
|
type=str,
|
|
help="Minio bucket name",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--folder_name",
|
|
type=str,
|
|
help="Path to destination folder",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--input_path",
|
|
type=str,
|
|
help="Input path of the file or folder to upload",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--filename",
|
|
type=str,
|
|
help="Name of the file to be uploaded",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--endpoint",
|
|
type=str,
|
|
default="minio-service.kubeflow:9000",
|
|
help="Name of the file to be uploaded",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--mlpipeline_ui_metadata",
|
|
type=str,
|
|
help="Path to write mlpipeline-ui-metadata.json",
|
|
)
|
|
|
|
args = vars(parser.parse_args())
|
|
|
|
bucket_name = args["bucket_name"]
|
|
input_path = args["input_path"]
|
|
folder_name = args["folder_name"]
|
|
filename = args["filename"]
|
|
|
|
if filename:
|
|
input_path = os.path.join(input_path, filename)
|
|
|
|
endpoint = args["endpoint"]
|
|
|
|
print("File to be uploaded: {}".format(input_path))
|
|
|
|
print("Uploading file to : {}".format(folder_name))
|
|
|
|
MinIO(
|
|
source=input_path,
|
|
bucket_name=bucket_name,
|
|
destination=folder_name,
|
|
endpoint=endpoint
|
|
)
|
|
|
|
inputs = {}
|
|
|
|
for key, value in args.items():
|
|
inputs[key] = value
|
|
|
|
outputs = {}
|
|
|
|
s3_url = f"s3://{bucket_name}/{folder_name}"
|
|
|
|
if filename:
|
|
s3_url += f"/{filename}"
|
|
|
|
outputs["minio_url"] = s3_url
|
|
|
|
visualization_arguments = {"inputs": inputs, "outputs": outputs}
|
|
|
|
markdown_dict = {"storage": "inline", "source": visualization_arguments}
|
|
|
|
visualization = Visualization(
|
|
mlpipeline_ui_metadata=args["mlpipeline_ui_metadata"],
|
|
markdown=markdown_dict,
|
|
)
|