fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Removed resumable downloads 

Signed-off-by: pa250194 <pa250194@ncr.com>
This commit is contained in:
pa250194 2021-09-23 12:38:38 -05:00
parent 0b971511ad
commit 057c65e939
2 changed files with 23 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
pkg/gcp/gcp.go
View File

@ -45,12 +45,6 @@ type GCPClient struct {
// client for interacting with the Google Cloud
// Storage APIs.
*gcpStorage.Client
// startRange is the starting read value for
// reading the object from bucket.
StartRange int64
// endRange is the ending read value for
// reading the object from bucket.
EndRange int64
}
// NewClient creates a new GCP storage client
@ -63,7 +57,7 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*GCPClient, er
return nil, err
}
return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil
return &GCPClient{Client: client}, nil
}
// ValidateSecret validates the credential secrets
@ -76,18 +70,11 @@ func ValidateSecret(secret map[string][]byte, name string) error {
return nil
}
// SetRange sets the startRange and endRange used to read the Object from
// the bucket. It is a helper method for resumable downloads.
func (c *GCPClient) SetRange(start, end int64) {
c.StartRange = start
c.EndRange = end
}
// BucketExists checks if the bucket with the provided name exists.
func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, error) {
_, err := c.Client.Bucket(bucketName).Attrs(ctx)
if err == gcpStorage.ErrBucketNotExist {
return false, nil
return false, err
}
if err != nil {
return false, err
@ -97,16 +84,16 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool,
// ObjectAttributes checks if the object with the provided name exists.
// If it exists the Object attributes are returned.
func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) {
attrs, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx)
func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, error) {
_, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx)
// ErrObjectNotExist is returned if the object does not exist
if err == gcpStorage.ErrObjectNotExist {
return false, nil, err
return false, err
}
if err != nil {
return false, nil, err
return false, err
}
return true, attrs, nil
return true, nil
}
// FGetObject gets the object from the bucket and downloads the object locally
@ -140,7 +127,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca
// ObjectExists verifies if object exists and you have permission to access.
// Check if the object exists and if you have permission to access it
// The Object attributes are returned if the Object exists.
exists, attrs, err := c.ObjectAttributes(ctx, bucketName, objectName)
exists, err := c.ObjectAttributes(ctx, bucketName, objectName)
if err != nil {
return err
}
@ -148,57 +135,25 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca
return ErrorObjectDoesNotExist
}
// Write to a temporary file "filename.part.gcp" before saving.
filePartPath := localPath + ".part.gcp"
// If exists, open in append mode. If not create it as a part file.
filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
objectFile, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return err
}
// If we return early with an error, be sure to close and delete
// filePart. If we have an error along the way there is a chance
// that filePart is somehow damaged, and we should discard it.
closeAndRemove := true
defer func() {
if closeAndRemove {
_ = filePart.Close()
_ = os.Remove(filePartPath)
}
}()
// Issue Stat to get the current offset.
partFileStat, err := filePart.Stat()
if err != nil {
return err
}
// Set the File size request range
// If the part file exists
if partFileStat.Size() > 0 {
c.SetRange(partFileStat.Size(), 0)
}
// Get Object from GCP Bucket
objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.StartRange, c.EndRange)
objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewReader(ctx)
if err != nil {
return err
}
defer objectReader.Close()
// Write to the part file.
if _, err := io.CopyN(filePart, objectReader, attrs.Size); err != nil {
// Write Object to file.
if _, err := io.Copy(objectFile, objectReader); err != nil {
return err
}
// Close the file before rename, this is specifically needed for Windows users.
closeAndRemove = false
if err := filePart.Close(); err != nil {
return err
}
// Safely completed. Now commit by renaming to actual filename.
if err := os.Rename(filePartPath, localPath); err != nil {
// Close the file.
if err := objectFile.Close(); err != nil {
return err
}

44
pkg/gcp/gcp_test.go
View File

@ -118,9 +118,7 @@ func TestNewClient(t *testing.T) {
func TestBucketExists(t *testing.T) {
gcpClient := &gcp.GCPClient{
Client: client,
StartRange: 0,
EndRange: -1,
Client: client,
}
exists, err := gcpClient.BucketExists(context.Background(), bucketName)
assert.NilError(t, err)
@ -130,35 +128,28 @@ func TestBucketExists(t *testing.T) {
func TestBucketNotExists(t *testing.T) {
bucket := "notexistsbucket"
gcpClient := &gcp.GCPClient{
Client: client,
StartRange: 0,
EndRange: -1,
Client: client,
}
exists, err := gcpClient.BucketExists(context.Background(), bucket)
assert.NilError(t, err)
assert.Error(t, err, "storage: bucket doesn't exist")
assert.Assert(t, !exists)
}
func TestObjectAttributes(t *testing.T) {
gcpClient := &gcp.GCPClient{
Client: client,
StartRange: 0,
EndRange: -1,
Client: client,
}
exists, objectAttrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName)
exists, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName)
if err == gcpStorage.ErrObjectNotExist {
assert.NilError(t, err)
}
assert.NilError(t, err)
assert.Assert(t, exists)
assert.Assert(t, objectAttrs != nil)
}
func TestListObjects(t *testing.T) {
gcpClient := &gcp.GCPClient{
Client: client,
StartRange: 0,
EndRange: -1,
Client: client,
}
objectInterator := gcpClient.ListObjects(context.Background(), bucketName, nil)
for {
@ -176,9 +167,7 @@ func TestFGetObject(t *testing.T) {
assert.NilError(t, err)
defer os.RemoveAll(tempDir)
gcpClient := &gcp.GCPClient{
Client: client,
StartRange: 0,
EndRange: -1,
Client: client,
}
localPath := filepath.Join(tempDir, objectName)
err = gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath)
@ -193,9 +182,7 @@ func TestFGetObjectNotExists(t *testing.T) {
assert.NilError(t, err)
defer os.RemoveAll(tempDir)
gcpClient := &gcp.GCPClient{
Client: client,
StartRange: 0,
EndRange: -1,
Client: client,
}
localPath := filepath.Join(tempDir, object)
err = gcpClient.FGetObject(context.Background(), bucketName, object, localPath)
@ -209,9 +196,7 @@ func TestFGetObjectDirectoryIsFileName(t *testing.T) {
defer os.RemoveAll(tempDir)
assert.NilError(t, err)
gcpClient := &gcp.GCPClient{
Client: client,
StartRange: 0,
EndRange: -1,
Client: client,
}
err = gcpClient.FGetObject(context.Background(), bucketName, objectName, tempDir)
if err != io.EOF {
@ -219,17 +204,6 @@ func TestFGetObjectDirectoryIsFileName(t *testing.T) {
}
}
func TestSetRange(t *testing.T) {
gcpClient := &gcp.GCPClient{
Client: client,
StartRange: 0,
EndRange: -1,
}
gcpClient.SetRange(2, 5)
assert.Equal(t, gcpClient.StartRange, int64(2))
assert.Equal(t, gcpClient.EndRange, int64(5))
}
func TestValidateSecret(t *testing.T) {
t.Parallel()
testCases := []struct {