kubeflow
/
pipelines
mirror of https://github.com/kubeflow/pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SDK/Components/PythonContainerOp - Simplified GCSHelper by extracting duplicate code (#210) 

* Simplified GCSHelper by extracting duplicate code

* Revert to using get_bucket for now
As per  @gaoning777's request
This commit is contained in:
Alexey Volkov 2018-12-02 11:59:15 -08:00 committed by Kubernetes Prow Robot
parent 9d38f717b1
commit 8a9e205d05
1 changed files with 10 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
sdk/python/kfp/compiler/_component_builder.py
View File

@ -17,11 +17,10 @@ import uuid
import os
import inspect
import re
import sys
import tempfile
import logging
from collections import OrderedDict
import sys
from google.cloud import storage
from pathlib import PurePath, Path
from .. import dsl
from ..components._components import _create_task_factory_from_component_spec
@ -31,33 +30,29 @@ class GCSHelper(object):
""" GCSHelper manages the connection with the GCS storage """
@staticmethod
def upload_gcs_file(local_path, gcs_path):
def get_blob_from_gcs_uri(gcs_path):
from google.cloud import storage
pure_path = PurePath(gcs_path)
gcs_bucket = pure_path.parts[1]
gcs_blob = '/'.join(pure_path.parts[2:])
client = storage.Client()
bucket = client.get_bucket(gcs_bucket)
blob = bucket.blob(gcs_blob)
return blob
@staticmethod
def upload_gcs_file(local_path, gcs_path):
blob = GCSHelper.get_blob_from_gcs_uri(gcs_path)
blob.upload_from_filename(local_path)
@staticmethod
def remove_gcs_blob(gcs_path):
pure_path = PurePath(gcs_path)
gcs_bucket = pure_path.parts[1]
gcs_blob = '/'.join(pure_path.parts[2:])
client = storage.Client()
bucket = client.get_bucket(gcs_bucket)
blob = bucket.blob(gcs_blob)
blob = GCSHelper.get_blob_from_gcs_uri(gcs_path)
blob.delete()
@staticmethod
def download_gcs_blob(local_path, gcs_path):
pure_path = PurePath(gcs_path)
gcs_bucket = pure_path.parts[1]
gcs_blob = '/'.join(pure_path.parts[2:])
client = storage.Client()
bucket = client.get_bucket(gcs_bucket)
blob = bucket.blob(gcs_blob)
blob = GCSHelper.get_blob_from_gcs_uri(gcs_path)
blob.download_to_filename(local_path)
class VersionedDependency(object):