From ecfbbd58adf94a3a326150a9b96b1e108ac0d4f6 Mon Sep 17 00:00:00 2001 From: jigargandhi Date: Tue, 14 Sep 2021 18:26:36 +0530 Subject: [PATCH 1/5] Added content type to SetRequest struct --- state/azure/blobstorage/blobstorage.go | 6 ++++++ state/requests.go | 11 ++++++----- state/responses.go | 7 ++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/state/azure/blobstorage/blobstorage.go b/state/azure/blobstorage/blobstorage.go index 6b518a86a..c81d10e57 100644 --- a/state/azure/blobstorage/blobstorage.go +++ b/state/azure/blobstorage/blobstorage.go @@ -225,6 +225,12 @@ func (r *StateStore) writeFile(req *state.SetRequest) error { blobURL := r.containerURL.NewBlockBlobURL(getFileName(req.Key)) var blobHTTPHeaders azblob.BlobHTTPHeaders + + if req.ContentType != nil && req.ContentType != "" { + blobHTTPHeaders.ContentType = req.ContentType + } + + // this is for backward compatibility where it might have come from http request if val, ok := req.Metadata[contentType]; ok && val != "" { blobHTTPHeaders.ContentType = val delete(req.Metadata, contentType) diff --git a/state/requests.go b/state/requests.go index 4a5e7202d..31f236643 100644 --- a/state/requests.go +++ b/state/requests.go @@ -43,11 +43,12 @@ type DeleteStateOption struct { // SetRequest is the object describing an upsert request type SetRequest struct { - Key string `json:"key"` - Value interface{} `json:"value"` - ETag *string `json:"etag,omitempty"` - Metadata map[string]string `json:"metadata,omitempty"` - Options SetStateOption `json:"options,omitempty"` + Key string `json:"key"` + Value interface{} `json:"value"` + ETag *string `json:"etag,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + Options SetStateOption `json:"options,omitempty"` + ContentType string `json:"contentType,omitempty"` } // GetKey gets the Key on a SetRequest diff --git a/state/responses.go b/state/responses.go index 3f6485c51..4b4da4861 100644 --- a/state/responses.go +++ b/state/responses.go @@ -7,9 +7,10 @@ package state // GetResponse is the request object for getting state type GetResponse struct { - Data []byte `json:"data"` - ETag *string `json:"etag,omitempty"` - Metadata map[string]string `json:"metadata"` + Data []byte `json:"data"` + ETag *string `json:"etag,omitempty"` + Metadata map[string]string `json:"metadata"` + ContentType string `json:"contentType,omitempty"` } // BulkGetResponse is the response object for bulk get response. From 34c20b0c0f9a4284896b1953be6ae8cccfb15e65 Mon Sep 17 00:00:00 2001 From: Jigar Gandhi Date: Thu, 21 Oct 2021 18:32:48 +0530 Subject: [PATCH 2/5] removed nil check --- state/azure/blobstorage/blobstorage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state/azure/blobstorage/blobstorage.go b/state/azure/blobstorage/blobstorage.go index 206746d17..c3bb7f6be 100644 --- a/state/azure/blobstorage/blobstorage.go +++ b/state/azure/blobstorage/blobstorage.go @@ -230,7 +230,7 @@ func (r *StateStore) writeFile(req *state.SetRequest) error { var blobHTTPHeaders azblob.BlobHTTPHeaders - if req.ContentType != nil && req.ContentType != "" { + if req.ContentType != "" { blobHTTPHeaders.ContentType = req.ContentType } From a09c7ade7b756d1f132bb7d4920ec962b7b27752 Mon Sep 17 00:00:00 2001 From: jigargandhi Date: Fri, 22 Oct 2021 18:02:01 +0530 Subject: [PATCH 3/5] Added test for blob storage --- state/azure/blobstorage/blobstorage.go | 48 ++++++++++++--------- state/azure/blobstorage/blobstorage_test.go | 13 ++++++ 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/state/azure/blobstorage/blobstorage.go b/state/azure/blobstorage/blobstorage.go index c3bb7f6be..abe7daebe 100644 --- a/state/azure/blobstorage/blobstorage.go +++ b/state/azure/blobstorage/blobstorage.go @@ -228,13 +228,38 @@ func (r *StateStore) writeFile(req *state.SetRequest) error { blobURL := r.containerURL.NewBlockBlobURL(getFileName(req.Key)) + // this is for backward compatibility where it might have come from http request + blobHTTPHeaders, err := createBlobHttpHeadersFromRequest(req) + if err != nil { + return err + } + + _, err = azblob.UploadBufferToBlockBlob(context.Background(), r.marshal(req), blobURL, azblob.UploadToBlockBlobOptions{ + Parallelism: 16, + Metadata: req.Metadata, + AccessConditions: accessConditions, + BlobHTTPHeaders: blobHTTPHeaders, + }) + if err != nil { + r.logger.Debugf("write file %s, err %s", req.Key, err) + + if req.ETag != nil { + return state.NewETagError(state.ETagMismatch, err) + } + + return err + } + + return nil +} + +func createBlobHttpHeadersFromRequest(req *state.SetRequest) (azblob.BlobHTTPHeaders, error) { var blobHTTPHeaders azblob.BlobHTTPHeaders if req.ContentType != "" { blobHTTPHeaders.ContentType = req.ContentType } - // this is for backward compatibility where it might have come from http request if val, ok := req.Metadata[contentType]; ok && val != "" { blobHTTPHeaders.ContentType = val delete(req.Metadata, contentType) @@ -242,7 +267,7 @@ func (r *StateStore) writeFile(req *state.SetRequest) error { if val, ok := req.Metadata[contentMD5]; ok && val != "" { sDec, err := b64.StdEncoding.DecodeString(val) if err != nil || len(sDec) != 16 { - return fmt.Errorf("the MD5 value specified in Content MD5 is invalid, MD5 value must be 128 bits and base64 encoded") + return azblob.BlobHTTPHeaders{}, fmt.Errorf("the MD5 value specified in Content MD5 is invalid, MD5 value must be 128 bits and base64 encoded") } blobHTTPHeaders.ContentMD5 = sDec delete(req.Metadata, contentMD5) @@ -263,24 +288,7 @@ func (r *StateStore) writeFile(req *state.SetRequest) error { blobHTTPHeaders.CacheControl = val delete(req.Metadata, cacheControl) } - - _, err := azblob.UploadBufferToBlockBlob(context.Background(), r.marshal(req), blobURL, azblob.UploadToBlockBlobOptions{ - Parallelism: 16, - Metadata: req.Metadata, - AccessConditions: accessConditions, - BlobHTTPHeaders: blobHTTPHeaders, - }) - if err != nil { - r.logger.Debugf("write file %s, err %s", req.Key, err) - - if req.ETag != nil { - return state.NewETagError(state.ETagMismatch, err) - } - - return err - } - - return nil + return blobHTTPHeaders, nil } func (r *StateStore) deleteFile(req *state.DeleteRequest) error { diff --git a/state/azure/blobstorage/blobstorage_test.go b/state/azure/blobstorage/blobstorage_test.go index 0aab9d700..4c819eff6 100644 --- a/state/azure/blobstorage/blobstorage_test.go +++ b/state/azure/blobstorage/blobstorage_test.go @@ -71,3 +71,16 @@ func TestFileName(t *testing.T) { assert.Equal(t, "key", key) }) } + +func TestBlobHttpHeaderGeneration(t *testing.T) { + + t.Run("Content type is set from request", func(t *testing.T) { + req := &state.SetRequest{ + ContentType: "application/json", + } + + blobHeaders, err := createBlobHttpHeadersFromRequest(req) + assert.Nil(t, err) + assert.Equal(t, "application/json", blobHeaders.ContentType) + }) +} From 54e2c1e01a5bb85d9170deaf977de4a265e2802d Mon Sep 17 00:00:00 2001 From: jigargandhi Date: Mon, 25 Oct 2021 15:13:40 +0530 Subject: [PATCH 4/5] Fix linting issues --- state/azure/blobstorage/blobstorage.go | 4 ++-- state/azure/blobstorage/blobstorage_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/state/azure/blobstorage/blobstorage.go b/state/azure/blobstorage/blobstorage.go index abe7daebe..984512752 100644 --- a/state/azure/blobstorage/blobstorage.go +++ b/state/azure/blobstorage/blobstorage.go @@ -229,7 +229,7 @@ func (r *StateStore) writeFile(req *state.SetRequest) error { blobURL := r.containerURL.NewBlockBlobURL(getFileName(req.Key)) // this is for backward compatibility where it might have come from http request - blobHTTPHeaders, err := createBlobHttpHeadersFromRequest(req) + blobHTTPHeaders, err := createBlobHTTPHeadersFromRequest(req) if err != nil { return err } @@ -253,7 +253,7 @@ func (r *StateStore) writeFile(req *state.SetRequest) error { return nil } -func createBlobHttpHeadersFromRequest(req *state.SetRequest) (azblob.BlobHTTPHeaders, error) { +func createBlobHTTPHeadersFromRequest(req *state.SetRequest) (azblob.BlobHTTPHeaders, error) { var blobHTTPHeaders azblob.BlobHTTPHeaders if req.ContentType != "" { diff --git a/state/azure/blobstorage/blobstorage_test.go b/state/azure/blobstorage/blobstorage_test.go index 4c819eff6..e781081ef 100644 --- a/state/azure/blobstorage/blobstorage_test.go +++ b/state/azure/blobstorage/blobstorage_test.go @@ -72,14 +72,14 @@ func TestFileName(t *testing.T) { }) } -func TestBlobHttpHeaderGeneration(t *testing.T) { +func TestBlobHTTPHeaderGeneration(t *testing.T) { t.Run("Content type is set from request", func(t *testing.T) { req := &state.SetRequest{ ContentType: "application/json", } - blobHeaders, err := createBlobHttpHeadersFromRequest(req) + blobHeaders, err := createBlobHTTPHeadersFromRequest(req) assert.Nil(t, err) assert.Equal(t, "application/json", blobHeaders.ContentType) }) From a5bb76bcfa0c6e92e2443bd17c3bd8adb732ab6a Mon Sep 17 00:00:00 2001 From: jigargandhi Date: Mon, 25 Oct 2021 15:27:32 +0530 Subject: [PATCH 5/5] fixing newline lint issue --- state/azure/blobstorage/blobstorage_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/state/azure/blobstorage/blobstorage_test.go b/state/azure/blobstorage/blobstorage_test.go index e781081ef..f8e42c748 100644 --- a/state/azure/blobstorage/blobstorage_test.go +++ b/state/azure/blobstorage/blobstorage_test.go @@ -73,7 +73,6 @@ func TestFileName(t *testing.T) { } func TestBlobHTTPHeaderGeneration(t *testing.T) { - t.Run("Content type is set from request", func(t *testing.T) { req := &state.SetRequest{ ContentType: "application/json",