Implement Azure Blob BucketProvider
This commit introduces an Azure Blob BucketProvider implementation, capable of fetching from objects from public and private "container" buckets. The supported credential types are: - ManagedIdentity with a `resourceId` Secret data field. - ManagedIdentity with a `clientId` Secret data field. - ClientSecret with `tenantId`, `clientId` and `clientSecret` Secret data fields. - SharedKey with `accountKey` Secret data field, the Account Name is extracted from the endpoint URL specified on the object. If no Secret is provided, the Bucket is assumed to be public. Co-authored-by: Zhongcheng Lao <Zhongcheng.Lao@microsoft.com> Signed-off-by: Hidde Beydals <hello@hidde.co>
This commit is contained in:
Hidde Beydals
parent
2167498736
commit
ec5bc1ac9a
5
Makefile
5
Makefile
|
|
@ -12,6 +12,9 @@ BUILD_ARGS ?=
|
|||
# Architectures to build images for
|
||||
BUILD_PLATFORMS ?= linux/amd64,linux/arm64,linux/arm/v7
|
||||
|
||||
# Go test arguments, e.g. '-tags=integration'
|
||||
GO_TEST_ARGS ?=
|
||||
|
||||
# Produce CRDs that work back to Kubernetes 1.16
|
||||
CRD_OPTIONS ?= crd:crdVersions=v1
|
||||
|
||||
|
|
@ -93,7 +96,7 @@ build: check-deps $(LIBGIT2) ## Build manager binary
|
|||
KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)"
|
||||
test: $(LIBGIT2) install-envtest test-api check-deps ## Run tests
|
||||
KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) \
|
||||
go test $(GO_STATIC_FLAGS) ./... -coverprofile cover.out
|
||||
go test $(GO_STATIC_FLAGS) $(GO_TEST_ARGS) ./... -coverprofile cover.out
|
||||
|
||||
check-deps:
|
||||
ifeq ($(shell uname -s),Darwin)
|
||||
|
|
|
|||
7
go.mod
7
go.mod
|
|
@ -6,6 +6,9 @@ replace github.com/fluxcd/source-controller/api => ./api
|
|||
|
||||
require (
|
||||
cloud.google.com/go/storage v1.16.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0
|
||||
github.com/Masterminds/semver/v3 v3.1.1
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20220113124808-70ae35bab23f
|
||||
github.com/cyphar/filepath-securejoin v0.2.3
|
||||
|
|
@ -54,7 +57,9 @@ replace helm.sh/helm/v3 v3.8.0 => github.com/hiddeco/helm/v3 v3.8.1-0.2022022311
|
|||
|
||||
require (
|
||||
cloud.google.com/go v0.99.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 // indirect
|
||||
github.com/BurntSushi/toml v0.4.1 // indirect
|
||||
github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd // indirect
|
||||
github.com/Masterminds/goutils v1.1.1 // indirect
|
||||
|
|
@ -96,6 +101,7 @@ require (
|
|||
github.com/gobwas/glob v0.2.3 // indirect
|
||||
github.com/gofrs/uuid v4.2.0+incompatible // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/google/btree v1.0.1 // indirect
|
||||
|
|
@ -146,6 +152,7 @@ require (
|
|||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.0.2 // indirect
|
||||
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
|
||||
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/prometheus/client_golang v1.12.1 // indirect
|
||||
|
|
|
|||
29
go.sum
29
go.sum
|
|
@ -53,7 +53,18 @@ cloud.google.com/go/storage v1.16.0 h1:1UwAux2OZP4310YXg5ohqBEpV16Y93uZG4+qOX7K2
|
|||
cloud.google.com/go/storage v1.16.0/go.mod h1:ieKBmUyzcftN5tbxwnXClMKH00CfcQ+xL6NN0r5QfmE=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg=
|
||||
github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c=
|
||||
github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.0/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.13.1 h1:RxemzI2cHD0A8WyMqHu/UnDjfpGES/cmjtPbQoktWqs=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.13.1/go.mod h1:+nVKciyKD2J9TyVcEQ82Bo9b+3F92PiQfHrIE/zqLqM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.1 h1:sLZ/Y+P/5RRtsXWylBjB5lkgixYfm0MQPiwrSX//JSo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.1/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
|
||||
|
|
@ -68,6 +79,8 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935
|
|||
github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
|
||||
github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
|
||||
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 h1:WVsrXCnHlDDX8ls+tootqRE87/hL9S/g4ewig9RsD/c=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
|
||||
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
|
|
@ -264,6 +277,9 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
|
|||
github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684 h1:DBZ2sN7CK6dgvHVpQsQj4sRMCbWTmd17l+5SUCjnQSY=
|
||||
github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684/go.mod h1:UfCu3YXJJCI+IdnqGgYP82dk2+Joxmv+mUTVBES6wac=
|
||||
github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
|
||||
github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko=
|
||||
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
|
||||
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
|
||||
github.com/docker/cli v20.10.9+incompatible h1:OJ7YkwQA+k2Oi51lmCojpjiygKpi76P7bg91b2eJxYU=
|
||||
github.com/docker/cli v20.10.9+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
||||
|
|
@ -437,7 +453,11 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV
|
|||
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
|
||||
github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
|
||||
github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU=
|
||||
github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
|
||||
|
|
@ -786,8 +806,10 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
|
|||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
|
||||
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
|
||||
github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
|
||||
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
||||
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
|
||||
github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
|
||||
|
|
@ -848,6 +870,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v
|
|||
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
|
||||
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
|
|
@ -1098,6 +1122,7 @@ golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPh
|
|||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
|
|
@ -1181,6 +1206,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
|
|||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
|
|
@ -1197,9 +1223,11 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
|
|||
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
|
|
@ -1336,6 +1364,7 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,312 @@
|
|||
/*
|
||||
Copyright 2022 The Flux authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package azure
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
_ "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
|
||||
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrorDirectoryExists is an error returned when the filename provided
|
||||
// is a directory.
|
||||
ErrorDirectoryExists = errors.New("filename is a directory")
|
||||
)
|
||||
|
||||
const (
|
||||
resourceIDField = "resourceId"
|
||||
clientIDField = "clientId"
|
||||
tenantIDField = "tenantId"
|
||||
clientSecretField = "clientSecret"
|
||||
accountKeyField = "accountKey"
|
||||
)
|
||||
|
||||
// BlobClient is a minimal Azure Blob client for fetching objects.
|
||||
type BlobClient struct {
|
||||
azblob.ServiceClient
|
||||
}
|
||||
|
||||
// NewClient creates a new Azure Blob storage client.
|
||||
// The credential config on the client is set based on the data from the
|
||||
// Bucket and Secret. It detects credentials in the Secret in the following
|
||||
// order:
|
||||
//
|
||||
// - azidentity.ManagedIdentityCredential for a Resource ID, when a
|
||||
// resourceIDField is found.
|
||||
// - azidentity.ManagedIdentityCredential for a User ID, when a clientIDField
|
||||
// but no tenantIDField found.
|
||||
// - azidentity.ClientSecretCredential when a tenantIDField, clientIDField and
|
||||
// clientSecretField are found.
|
||||
// - azblob.SharedKeyCredential when an accountKeyField is found. The Account
|
||||
// Name is extracted from the endpoint specified on the Bucket object.
|
||||
//
|
||||
// If no credentials are found, a simple client without credentials is
|
||||
// returned.
|
||||
func NewClient(obj *sourcev1.Bucket, secret *corev1.Secret) (c *BlobClient, err error) {
|
||||
c = &BlobClient{}
|
||||
|
||||
// Without a Secret, we can return a simple client.
|
||||
if secret == nil || len(secret.Data) == 0 {
|
||||
c.ServiceClient, err = azblob.NewServiceClientWithNoCredential(obj.Spec.Endpoint, nil)
|
||||
return
|
||||
}
|
||||
|
||||
// Attempt AAD Token Credential options first.
|
||||
var token azcore.TokenCredential
|
||||
if token, err = tokenCredentialFromSecret(secret); err != nil {
|
||||
return
|
||||
}
|
||||
if token != nil {
|
||||
c.ServiceClient, err = azblob.NewServiceClient(obj.Spec.Endpoint, token, nil)
|
||||
return
|
||||
}
|
||||
|
||||
// Fallback to Shared Key Credential.
|
||||
cred, err := sharedCredentialFromSecret(obj.Spec.Endpoint, secret)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if cred != nil {
|
||||
c.ServiceClient, err = azblob.NewServiceClientWithSharedKey(obj.Spec.Endpoint, cred, &azblob.ClientOptions{})
|
||||
return
|
||||
}
|
||||
|
||||
// Secret does not contain a valid set of credentials, fallback to simple client.
|
||||
c.ServiceClient, err = azblob.NewServiceClientWithNoCredential(obj.Spec.Endpoint, nil)
|
||||
return
|
||||
}
|
||||
|
||||
// ValidateSecret validates if the provided Secret does at least have one valid
|
||||
// set of credentials. The provided Secret may be nil.
|
||||
func ValidateSecret(secret *corev1.Secret) error {
|
||||
if secret == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var valid bool
|
||||
if _, hasTenantID := secret.Data[tenantIDField]; hasTenantID {
|
||||
if _, hasClientID := secret.Data[clientIDField]; hasClientID {
|
||||
if _, hasClientSecret := secret.Data[clientSecretField]; hasClientSecret {
|
||||
valid = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, hasResourceID := secret.Data[resourceIDField]; hasResourceID {
|
||||
valid = true
|
||||
}
|
||||
if _, hasClientID := secret.Data[clientIDField]; hasClientID {
|
||||
valid = true
|
||||
}
|
||||
if _, hasAccountKey := secret.Data[accountKeyField]; hasAccountKey {
|
||||
valid = true
|
||||
}
|
||||
|
||||
if !valid {
|
||||
return fmt.Errorf("invalid '%s' secret data: requires a '%s', '%s', or '%s' field, or a combination of '%s', '%s' and '%s'",
|
||||
secret.Name, resourceIDField, clientIDField, accountKeyField, tenantIDField, clientIDField, clientSecretField)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BucketExists returns if an object storage bucket with the provided name
|
||||
// exists, or returns a (client) error.
|
||||
func (c *BlobClient) BucketExists(ctx context.Context, bucketName string) (bool, error) {
|
||||
container := c.ServiceClient.NewContainerClient(bucketName)
|
||||
_, err := container.GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
var stgErr *azblob.StorageError
|
||||
if errors.As(err, &stgErr) {
|
||||
if stgErr.ErrorCode == azblob.StorageErrorCodeContainerNotFound {
|
||||
return false, nil
|
||||
}
|
||||
err = stgErr
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// FGetObject gets the object from the provided object storage bucket, and
|
||||
// writes it to targetPath.
|
||||
// It returns the etag of the successfully fetched file, or any error.
|
||||
func (c *BlobClient) FGetObject(ctx context.Context, bucketName, objectName, localPath string) (string, error) {
|
||||
container := c.ServiceClient.NewContainerClient(bucketName)
|
||||
blob := container.NewBlobClient(objectName)
|
||||
|
||||
// Verify if destination already exists.
|
||||
dirStatus, err := os.Stat(localPath)
|
||||
if err == nil {
|
||||
// If the destination exists and is a directory.
|
||||
if dirStatus.IsDir() {
|
||||
return "", ErrorDirectoryExists
|
||||
}
|
||||
}
|
||||
|
||||
// Proceed if file does not exist, return for all other errors.
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
// Extract top level directory.
|
||||
objectDir, _ := filepath.Split(localPath)
|
||||
if objectDir != "" {
|
||||
// Create any missing top level directories.
|
||||
if err := os.MkdirAll(objectDir, 0700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
// Download object.
|
||||
res, err := blob.Download(ctx, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Prepare target file.
|
||||
f, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Calculate hash during write.
|
||||
// NOTE: not actively used at present, as MD5 is not consistently returned
|
||||
// by API.
|
||||
hash := md5.New()
|
||||
|
||||
// Off we go.
|
||||
mw := io.MultiWriter(f, hash)
|
||||
if _, err = io.Copy(mw, res.Body(nil)); err != nil {
|
||||
if err = f.Close(); err != nil {
|
||||
ctrl.LoggerFrom(ctx).Error(err, "failed to close file after copy error")
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if err = f.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return *res.ETag, nil
|
||||
}
|
||||
|
||||
// VisitObjects iterates over the items in the provided object storage
|
||||
// bucket, calling visit for every item.
|
||||
// If the underlying client or the visit callback returns an error,
|
||||
// it returns early.
|
||||
func (c *BlobClient) VisitObjects(ctx context.Context, bucketName string, visit func(path, etag string) error) error {
|
||||
container := c.ServiceClient.NewContainerClient(bucketName)
|
||||
|
||||
items := container.ListBlobsFlat(&azblob.ContainerListBlobFlatSegmentOptions{})
|
||||
for items.NextPage(ctx) {
|
||||
resp := items.PageResponse()
|
||||
|
||||
for _, blob := range resp.ContainerListBlobFlatSegmentResult.Segment.BlobItems {
|
||||
if err := visit(*blob.Name, fmt.Sprintf("%x", *blob.Properties.Etag)); err != nil {
|
||||
err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucketName, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := items.Err(); err != nil {
|
||||
err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucketName, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close has no effect on BlobClient.
|
||||
func (c *BlobClient) Close(_ context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// ObjectIsNotFound checks if the error provided is an azblob.StorageError with
|
||||
// an azblob.StorageErrorCodeBlobNotFound error code.
|
||||
func (c *BlobClient) ObjectIsNotFound(err error) bool {
|
||||
var stgErr *azblob.StorageError
|
||||
if errors.As(err, &stgErr) {
|
||||
if stgErr.ErrorCode == azblob.StorageErrorCodeBlobNotFound {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func tokenCredentialFromSecret(secret *corev1.Secret) (azcore.TokenCredential, error) {
|
||||
var token azcore.TokenCredential
|
||||
if resourceID, ok := secret.Data[resourceIDField]; ok {
|
||||
return azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{
|
||||
ID: azidentity.ResourceID(resourceID),
|
||||
})
|
||||
}
|
||||
if clientID, hasClientID := secret.Data[clientIDField]; hasClientID {
|
||||
tenantID, hasTenantID := secret.Data[tenantIDField]
|
||||
if !hasTenantID {
|
||||
return azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{
|
||||
ID: azidentity.ClientID(clientID),
|
||||
})
|
||||
}
|
||||
if clientSecret, hasClientSecret := secret.Data[clientSecretField]; hasClientSecret {
|
||||
return azidentity.NewClientSecretCredential(string(tenantID), string(clientID), string(clientSecret), nil)
|
||||
}
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func sharedCredentialFromSecret(endpoint string, secret *corev1.Secret) (*azblob.SharedKeyCredential, error) {
|
||||
if accountKey, hasAccountKey := secret.Data[accountKeyField]; hasAccountKey {
|
||||
accountName, err := extractAccountNameFromEndpoint(endpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create shared credential from '%s' Secret data: %w", secret.Name, err)
|
||||
}
|
||||
return azblob.NewSharedKeyCredential(accountName, string(accountKey))
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// extractAccountNameFromEndpoint extracts the Azure account name from the
|
||||
// provided endpoint URL. It parses the endpoint as a URL, and returns the
|
||||
// first subdomain as the assumed account name.
|
||||
// It returns an error when it fails to parse the endpoint as a URL, or if it
|
||||
// does not have any subdomains.
|
||||
func extractAccountNameFromEndpoint(endpoint string) (string, error) {
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to extract account name from endpoint: %w", err)
|
||||
}
|
||||
hostname := u.Hostname()
|
||||
parts := strings.Split(hostname, ".")
|
||||
if len(parts) <= 2 {
|
||||
return "", fmt.Errorf("failed to extract account name from endpoint: expected '%s' to be a subdomain", hostname)
|
||||
}
|
||||
return parts[0], nil
|
||||
}
|
||||
|
|
@ -0,0 +1,319 @@
|
|||
//go:build integration
|
||||
|
||||
/*
|
||||
Copyright 2022 The Flux authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package azure
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
|
||||
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
|
||||
)
|
||||
|
||||
var (
|
||||
testTimeout = time.Second * 5
|
||||
)
|
||||
|
||||
var (
|
||||
testAccountName = os.Getenv("TEST_AZURE_ACCOUNT_NAME")
|
||||
testAccountKey = os.Getenv("TEST_AZURE_ACCOUNT_KEY")
|
||||
)
|
||||
|
||||
var (
|
||||
testContainerGenerateName = "azure-client-test-"
|
||||
testFile = "test.yaml"
|
||||
testFileData = `
|
||||
---
|
||||
test: file
|
||||
`
|
||||
testFile2 = "test2.yaml"
|
||||
testFile2Data = `
|
||||
---
|
||||
test: file2
|
||||
`
|
||||
testBucket = sourcev1.Bucket{
|
||||
Spec: sourcev1.BucketSpec{
|
||||
Endpoint: endpointURL(testAccountName),
|
||||
},
|
||||
}
|
||||
testSecret = corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
accountKeyField: []byte(testAccountKey),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
code := m.Run()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func TestBlobClient_BucketExists(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
client, err := NewClient(testBucket.DeepCopy(), testSecret.DeepCopy())
|
||||
g.Expect(err).ToNot(HaveOccurred())
|
||||
g.Expect(client).ToNot(BeNil())
|
||||
|
||||
// Generate test container name.
|
||||
testContainer := generateString(testContainerGenerateName)
|
||||
|
||||
// Create test container.
|
||||
ctx, timeout := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
g.Expect(createContainer(ctx, client, testContainer)).To(Succeed())
|
||||
t.Cleanup(func() {
|
||||
g.Expect(deleteContainer(context.Background(), client, testContainer)).To(Succeed())
|
||||
})
|
||||
|
||||
// Test if the container exists.
|
||||
ctx, timeout = context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
ok, err := client.BucketExists(ctx, testContainer)
|
||||
g.Expect(err).ToNot(HaveOccurred())
|
||||
g.Expect(ok).To(BeTrue())
|
||||
}
|
||||
|
||||
func TestBlobClient_BucketNotExists(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
client, err := NewClient(testBucket.DeepCopy(), testSecret.DeepCopy())
|
||||
g.Expect(err).ToNot(HaveOccurred())
|
||||
g.Expect(client).ToNot(BeNil())
|
||||
|
||||
// Generate test container name.
|
||||
testContainer := generateString(testContainerGenerateName)
|
||||
|
||||
// Test if the container exists.
|
||||
ctx, timeout := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
ok, err := client.BucketExists(ctx, testContainer)
|
||||
g.Expect(err).ToNot(HaveOccurred())
|
||||
g.Expect(ok).To(BeFalse())
|
||||
}
|
||||
|
||||
func TestBlobClient_FGetObject(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
tempDir := t.TempDir()
|
||||
|
||||
client, err := NewClient(testBucket.DeepCopy(), testSecret.DeepCopy())
|
||||
g.Expect(err).ToNot(HaveOccurred())
|
||||
g.Expect(client).ToNot(BeNil())
|
||||
|
||||
// Generate test container name.
|
||||
testContainer := generateString(testContainerGenerateName)
|
||||
|
||||
// Create test container.
|
||||
ctx, timeout := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
g.Expect(createContainer(ctx, client, testContainer)).To(Succeed())
|
||||
t.Cleanup(func() {
|
||||
g.Expect(deleteContainer(context.Background(), client, testContainer)).To(Succeed())
|
||||
})
|
||||
|
||||
// Create test blob.
|
||||
ctx, timeout = context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
g.Expect(createBlob(ctx, client, testContainer, testFile, testFileData))
|
||||
|
||||
localPath := filepath.Join(tempDir, testFile)
|
||||
|
||||
// Test if blob exists.
|
||||
ctx, timeout = context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
_, err = client.FGetObject(ctx, testContainer, testFile, localPath)
|
||||
|
||||
g.Expect(err).ToNot(HaveOccurred())
|
||||
g.Expect(localPath).To(BeARegularFile())
|
||||
f, _ := os.ReadFile(localPath)
|
||||
g.Expect(f).To(Equal([]byte(testFileData)))
|
||||
}
|
||||
|
||||
func TestBlobClient_FGetObject_NotFoundErr(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
client, err := NewClient(testBucket.DeepCopy(), testSecret.DeepCopy())
|
||||
g.Expect(err).ToNot(HaveOccurred())
|
||||
g.Expect(client).ToNot(BeNil())
|
||||
|
||||
// Generate test container name.
|
||||
testContainer := generateString(testContainerGenerateName)
|
||||
|
||||
// Create test container.
|
||||
ctx, timeout := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
g.Expect(createContainer(ctx, client, testContainer)).To(Succeed())
|
||||
t.Cleanup(func() {
|
||||
g.Expect(deleteContainer(context.Background(), client, testContainer)).To(Succeed())
|
||||
})
|
||||
|
||||
// Test blob does not exist.
|
||||
ctx, timeout = context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
_, err = client.FGetObject(ctx, testContainer, "doesnotexist.txt", "doesnotexist.txt")
|
||||
|
||||
g.Expect(err).To(HaveOccurred())
|
||||
g.Expect(client.ObjectIsNotFound(err)).To(BeTrue())
|
||||
}
|
||||
|
||||
func TestBlobClient_VisitObjects(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
client, err := NewClient(testBucket.DeepCopy(), testSecret.DeepCopy())
|
||||
g.Expect(err).ToNot(HaveOccurred())
|
||||
g.Expect(client).ToNot(BeNil())
|
||||
|
||||
// Generate test container name.
|
||||
testContainer := generateString(testContainerGenerateName)
|
||||
|
||||
// Create test container.
|
||||
ctx, timeout := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
g.Expect(createContainer(ctx, client, testContainer)).To(Succeed())
|
||||
t.Cleanup(func() {
|
||||
g.Expect(deleteContainer(context.Background(), client, testContainer)).To(Succeed())
|
||||
})
|
||||
|
||||
// Create test blobs.
|
||||
ctx, timeout = context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
g.Expect(createBlob(ctx, client, testContainer, testFile, testFileData))
|
||||
g.Expect(createBlob(ctx, client, testContainer, testFile2, testFile2Data))
|
||||
|
||||
visits := make(map[string]string)
|
||||
|
||||
// Visit objects.
|
||||
ctx, timeout = context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
got := client.VisitObjects(ctx, testContainer, func(path, etag string) error {
|
||||
visits[path] = etag
|
||||
return nil
|
||||
})
|
||||
|
||||
g.Expect(got).To(Succeed())
|
||||
g.Expect(visits[testFile]).ToNot(BeEmpty())
|
||||
g.Expect(visits[testFile2]).ToNot(BeEmpty())
|
||||
g.Expect(visits[testFile]).ToNot(Equal(visits[testFile2]))
|
||||
}
|
||||
|
||||
func TestBlobClient_VisitObjects_CallbackErr(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
client, err := NewClient(testBucket.DeepCopy(), testSecret.DeepCopy())
|
||||
g.Expect(err).ToNot(HaveOccurred())
|
||||
g.Expect(client).ToNot(BeNil())
|
||||
|
||||
// Generate test container name.
|
||||
testContainer := generateString(testContainerGenerateName)
|
||||
|
||||
// Create test container.
|
||||
ctx, timeout := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
g.Expect(createContainer(ctx, client, testContainer)).To(Succeed())
|
||||
t.Cleanup(func() {
|
||||
g.Expect(deleteContainer(context.Background(), client, testContainer)).To(Succeed())
|
||||
})
|
||||
|
||||
// Create test blob.
|
||||
ctx, timeout = context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
g.Expect(createBlob(ctx, client, testContainer, testFile, testFileData))
|
||||
|
||||
// Visit object.
|
||||
ctx, timeout = context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
mockErr := fmt.Errorf("mock")
|
||||
err = client.VisitObjects(ctx, testContainer, func(path, etag string) error {
|
||||
return mockErr
|
||||
})
|
||||
g.Expect(err).To(HaveOccurred())
|
||||
g.Expect(err.Error()).To(ContainSubstring("mock"))
|
||||
}
|
||||
|
||||
func createContainer(ctx context.Context, client *BlobClient, name string) error {
|
||||
if _, err := client.CreateContainer(ctx, name, nil); err != nil {
|
||||
var stgErr *azblob.StorageError
|
||||
if errors.As(err, &stgErr) {
|
||||
if stgErr.ErrorCode == azblob.StorageErrorCodeContainerAlreadyExists {
|
||||
return nil
|
||||
}
|
||||
err = stgErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func createBlob(ctx context.Context, client *BlobClient, containerName, name, data string) error {
|
||||
container := client.NewContainerClient(containerName)
|
||||
blob := container.NewAppendBlobClient(name)
|
||||
|
||||
ctx, timeout := context.WithTimeout(context.Background(), testTimeout)
|
||||
defer timeout()
|
||||
if _, err := blob.Create(ctx, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hash := md5.Sum([]byte(data))
|
||||
if _, err := blob.AppendBlock(ctx, streaming.NopCloser(strings.NewReader(data)), &azblob.AppendBlockOptions{
|
||||
TransactionalContentMD5: hash[:16],
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteContainer(ctx context.Context, client *BlobClient, name string) error {
|
||||
if _, err := client.DeleteContainer(ctx, name, nil); err != nil {
|
||||
var stgErr *azblob.StorageError
|
||||
if errors.As(err, &stgErr) {
|
||||
if code := stgErr.ErrorCode; code == azblob.StorageErrorCodeContainerNotFound ||
|
||||
code == azblob.StorageErrorCodeContainerBeingDeleted {
|
||||
return nil
|
||||
}
|
||||
err = stgErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func generateString(prefix string) string {
|
||||
randBytes := make([]byte, 16)
|
||||
rand.Read(randBytes)
|
||||
return prefix + hex.EncodeToString(randBytes)
|
||||
}
|
||||
|
|
@ -0,0 +1,318 @@
|
|||
/*
|
||||
Copyright 2022 The Flux authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package azure
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func TestValidateSecret(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
secret *corev1.Secret
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid SystemManagedIdentity Secret",
|
||||
secret: &corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
resourceIDField: []byte("/some/resource/id"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid UserManagerIdentity Secret",
|
||||
secret: &corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
clientIDField: []byte("some-client-id-"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid ServicePrincipal Secret",
|
||||
secret: &corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
tenantIDField: []byte("some-tenant-id-"),
|
||||
clientIDField: []byte("some-client-id-"),
|
||||
clientSecretField: []byte("some-client-secret-"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid SharedKey Secret",
|
||||
secret: &corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
accountKeyField: []byte("some-account-key"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid ServicePrincipal Secret with missing ClientID and ClientSecret",
|
||||
secret: &corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
tenantIDField: []byte("some-tenant-id-"),
|
||||
},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid empty secret",
|
||||
secret: &corev1.Secret{},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "valid nil secret",
|
||||
secret: nil,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
want := BeNil()
|
||||
if tt.wantErr {
|
||||
want = HaveOccurred()
|
||||
}
|
||||
g.Expect(ValidateSecret(tt.secret)).To(want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobClient_ObjectIsNotFound(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "StorageError with BlobNotFound code",
|
||||
err: &azblob.StorageError{ErrorCode: azblob.StorageErrorCodeBlobNotFound},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "StorageError with different code",
|
||||
err: &azblob.StorageError{ErrorCode: azblob.StorageErrorCodeInternalError},
|
||||
},
|
||||
{
|
||||
name: "other error",
|
||||
err: errors.New("an error"),
|
||||
},
|
||||
{
|
||||
name: "nil error",
|
||||
err: nil,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
c := &BlobClient{}
|
||||
g.Expect(c.ObjectIsNotFound(tt.err)).To(Equal(tt.want))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_extractAccountNameFromEndpoint(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "returns account name for endpoint",
|
||||
endpoint: "https://foo.blob.core.windows.net",
|
||||
want: "foo",
|
||||
},
|
||||
{
|
||||
name: "error for endpoint URL parse err",
|
||||
endpoint: "#http//foo.blob.core.windows.net",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "error for endpoint URL without subdomain",
|
||||
endpoint: "https://windows.net",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
got, err := extractAccountNameFromEndpoint(tt.endpoint)
|
||||
g.Expect(err != nil).To(Equal(tt.wantErr))
|
||||
g.Expect(got).To(Equal(tt.want))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_tokenCredentialFromSecret(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
secret *corev1.Secret
|
||||
want azcore.TokenCredential
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "with ResourceID field",
|
||||
secret: &corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
resourceIDField: []byte("resource-id"),
|
||||
},
|
||||
},
|
||||
want: &azidentity.ManagedIdentityCredential{},
|
||||
},
|
||||
{
|
||||
name: "with ClientID field",
|
||||
secret: &corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
clientIDField: []byte("client-id"),
|
||||
},
|
||||
},
|
||||
want: &azidentity.ManagedIdentityCredential{},
|
||||
},
|
||||
{
|
||||
name: "with TenantID, ClientID and ClientSecret fields",
|
||||
secret: &corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
clientIDField: []byte("client-id"),
|
||||
tenantIDField: []byte("tenant-id"),
|
||||
clientSecretField: []byte("client-secret"),
|
||||
},
|
||||
},
|
||||
want: &azidentity.ClientSecretCredential{},
|
||||
},
|
||||
{
|
||||
name: "empty secret",
|
||||
secret: &corev1.Secret{},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
got, err := tokenCredentialFromSecret(tt.secret)
|
||||
g.Expect(err != nil).To(Equal(tt.wantErr))
|
||||
if tt.want != nil {
|
||||
g.Expect(got).ToNot(BeNil())
|
||||
g.Expect(got).To(BeAssignableToTypeOf(tt.want))
|
||||
return
|
||||
}
|
||||
g.Expect(got).To(BeNil())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_sharedCredentialFromSecret(t *testing.T) {
|
||||
var testKey = []byte("dGVzdA==")
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
secret *corev1.Secret
|
||||
want *azblob.SharedKeyCredential
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "with AccountKey field",
|
||||
endpoint: "https://some.endpoint.com",
|
||||
secret: &corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
accountKeyField: testKey,
|
||||
},
|
||||
},
|
||||
want: &azblob.SharedKeyCredential{},
|
||||
},
|
||||
{
|
||||
name: "invalid endpoint",
|
||||
endpoint: "#http//some.endpoint.com",
|
||||
secret: &corev1.Secret{
|
||||
Data: map[string][]byte{
|
||||
accountKeyField: testKey,
|
||||
},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "empty secret",
|
||||
secret: &corev1.Secret{},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
got, err := sharedCredentialFromSecret(tt.endpoint, tt.secret)
|
||||
g.Expect(err != nil).To(Equal(tt.wantErr))
|
||||
if tt.want != nil {
|
||||
g.Expect(got).ToNot(BeNil())
|
||||
g.Expect(got).To(BeAssignableToTypeOf(tt.want))
|
||||
return
|
||||
}
|
||||
g.Expect(got).To(BeNil())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_extractAccountNameFromEndpoint1(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
want string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "valid URL",
|
||||
endpoint: endpointURL("foo"),
|
||||
want: "foo",
|
||||
},
|
||||
{
|
||||
name: "URL parse error",
|
||||
endpoint: " https://example.com",
|
||||
wantErr: "first path segment in URL cannot contain colon",
|
||||
},
|
||||
{
|
||||
name: "error on non subdomain",
|
||||
endpoint: "https://example.com",
|
||||
wantErr: "expected 'example.com' to be a subdomain",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
got, err := extractAccountNameFromEndpoint(tt.endpoint)
|
||||
if tt.wantErr != "" {
|
||||
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
|
||||
g.Expect(got).To(BeEmpty())
|
||||
return
|
||||
}
|
||||
g.Expect(err).To(BeNil())
|
||||
g.Expect(got).To(Equal(tt.want))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func endpointURL(accountName string) string {
|
||||
return fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
|
||||
}
|
||||
Loading…
Reference in New Issue