GCP Firestore - Certification Tests (#2754)

Signed-off-by: Roberto J Rojas <robertojrojas@gmail.com>
Signed-off-by: Bernd Verst <github@bernd.dev>
Co-authored-by: Artur Souza <artursouza.ms@outlook.com>
Co-authored-by: Bernd Verst <github@bernd.dev>

.github/infrastructure/docker-compose-gcpfirestore.yml

@@ -0,0 +1,8 @@
+version: '3'
+services:
+  pubsub:
+    image: gcr.io/google.com/cloudsdktool/cloud-sdk:425.0.0-emulators
+    ports:
+      - "8432:8432"
+    container_name: gcpfirestore
+    entrypoint: gcloud beta emulators datastore start --project conf-test-project --host-port 0.0.0.0:8432

.github/scripts/components-scripts/certification-state.gcp.firestore-setup.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+# Set variables for GitHub Actions
+echo "GCP_PROJECT_ID=$GCP_PROJECT" >> $GITHUB_ENV
+echo "GCP_FIRESTORE_ENTITY_KIND=CertificationTestEntity-$UNIQUE_ID" >> $GITHUB_ENV

.github/scripts/components-scripts/conformance-state.gcp.firestore-setup.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+
+# Set variables for GitHub Actions
+echo "GCP_PROJECT_ID=$GCP_PROJECT" >> $GITHUB_ENV
+

.github/scripts/test-info.mjs

@@ -592,6 +592,21 @@ const components = {
         requiredSecrets: ['AzureSqlServerConnectionString'],
         sourcePkg: ['state/sqlserver', 'internal/component/sql'],
     },
+    // 'state.gcp.firestore.docker': {
+    //     conformance: true,
+    //     requireDocker: true,
+    //     conformanceSetup: 'docker-compose.sh gcpfirestore',
+    // },
+    'state.gcp.firestore.cloud': {
+        conformance: true,
+        requireGCPCredentials: true,
+        conformanceSetup: 'conformance-state.gcp.firestore-setup.sh',
+    },
+    'state.gcp.firestore': {
+        certification: true,
+        requireGCPCredentials: true,
+        certificationSetup: 'certification-state.gcp.firestore-setup.sh',
+    },
     'workflows.temporal': {
         conformance: true,
         conformanceSetup: 'docker-compose.sh temporal',

state/gcp/firestore/firestore.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"reflect"
 
 	"cloud.google.com/go/datastore"
@@ -29,7 +30,10 @@ import (
 	"github.com/dapr/kit/logger"
 )
 
-const defaultEntityKind = "DaprState"
+const (
+	defaultEntityKind = "DaprState"
+	endpointKey       = "endpoint"
+)
 
 // Firestore State Store.
 type Firestore struct {
@@ -54,6 +58,7 @@ type firestoreMetadata struct {
 	ClientCertURL       string `json:"client_x509_cert_url" mapstructure:"client_x509_cert_url"`
 	EntityKind          string `json:"entity_kind" mapstructure:"entity_kind"`
 	NoIndex             bool   `json:"-"`
+	ConnectionEndpoint  string `json:"endpoint"`
 }
 
 type StateEntity struct {
@@ -78,13 +83,8 @@ func (f *Firestore) Init(ctx context.Context, metadata state.Metadata) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(meta)
-	if err != nil {
-		return err
-	}
 
-	opt := option.WithCredentialsJSON(b)
-	client, err := datastore.NewClient(ctx, meta.ProjectID, opt)
+	client, err := getGCPClient(ctx, meta, f.logger)
 	if err != nil {
 		return err
 	}
@@ -168,6 +168,13 @@ func (f *Firestore) Delete(ctx context.Context, req *state.DeleteRequest) error
 	return nil
 }
 
+func (f *Firestore) GetComponentMetadata() map[string]string {
+	metadataStruct := firestoreMetadata{}
+	metadataInfo := map[string]string{}
+	metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType)
+	return metadataInfo
+}
+
 func getFirestoreMetadata(meta state.Metadata) (*firestoreMetadata, error) {
 	m := firestoreMetadata{
 		EntityKind: defaultEntityKind,
@@ -179,8 +186,7 @@ func getFirestoreMetadata(meta state.Metadata) (*firestoreMetadata, error) {
 	}
 
 	requiredMetaProperties := []string{
-		"type", "project_id", "private_key_id", "private_key", "client_email", "client_id",
-		"auth_uri", "token_uri", "auth_provider_x509_cert_url", "client_x509_cert_url",
+		"project_id",
 	}
 
 	metadataMap := map[string]string{}
@@ -199,12 +205,45 @@ func getFirestoreMetadata(meta state.Metadata) (*firestoreMetadata, error) {
 		}
 	}
 
+	if val, found := meta.Properties[endpointKey]; found && val != "" {
+		m.ConnectionEndpoint = val
+	}
+
 	return &m, nil
 }
 
-func (f *Firestore) GetComponentMetadata() map[string]string {
-	metadataStruct := firestoreMetadata{}
-	metadataInfo := map[string]string{}
-	metadata.GetMetadataInfoFromStructType(reflect.TypeOf(metadataStruct), &metadataInfo, metadata.StateStoreType)
-	return metadataInfo
+func getGCPClient(ctx context.Context, metadata *firestoreMetadata, l logger.Logger) (*datastore.Client, error) {
+	var gcpClient *datastore.Client
+	var err error
+
+	if metadata.PrivateKeyID != "" {
+		var b []byte
+		b, err = json.Marshal(metadata)
+		if err != nil {
+			return nil, err
+		}
+
+		opt := option.WithCredentialsJSON(b)
+		gcpClient, err = datastore.NewClient(ctx, metadata.ProjectID, opt)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		l.Debugf("Using implicit credentials for GCP")
+
+		// The following allows the Google SDK to connect to
+		// the GCP Datastore Emulator.
+		// example: export DATASTORE_EMULATOR_HOST=localhost:8432
+		// see: https://cloud.google.com/pubsub/docs/emulator#env
+		if metadata.ConnectionEndpoint != "" {
+			l.Debugf("setting GCP Datastore Emulator environment variable to 'DATASTORE_EMULATOR_HOST=%s'", metadata.ConnectionEndpoint)
+			os.Setenv("DATASTORE_EMULATOR_HOST", metadata.ConnectionEndpoint)
+		}
+		gcpClient, err = datastore.NewClient(ctx, metadata.ProjectID)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return gcpClient, nil
 }

state/gcp/firestore/firestore_test.go

@@ -53,7 +53,6 @@ func TestGetFirestoreMetadata(t *testing.T) {
 	t.Run("With incorrect properties", func(t *testing.T) {
 		properties := map[string]string{
 			"type":           "service_account",
-			"project_id":     "myprojectid",
 			"private_key_id": "123",
 			"private_key":    "mykey",
 		}

tests/certification/go.mod

@@ -45,6 +45,7 @@ require (
 	cloud.google.com/go v0.110.0 // indirect
 	cloud.google.com/go/compute v1.19.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/datastore v1.11.0 // indirect
 	cloud.google.com/go/iam v1.0.0 // indirect
 	contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect
 	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
@@ -275,6 +276,7 @@ require (
 	golang.org/x/text v0.9.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/api v0.115.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

tests/certification/go.sum

@@ -32,6 +32,8 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/datastore v1.11.0 h1:iF6I/HaLs3Ado8uRKMvZRvF/ZLkWaWE9i8AiHzbC774=
+cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
 cloud.google.com/go/iam v1.0.0 h1:hlQJMovyJJwYjZcTohUH4o1L8Z8kYz+E+W/zktiLCBc=
 cloud.google.com/go/iam v1.0.0/go.mod h1:ikbQ4f1r91wTmBmmOtBCOtuEOei6taatNXytzB7Cxew=
@@ -1830,6 +1832,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
 gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
 gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=

tests/certification/state/gcp/firestore/README.md

@@ -0,0 +1,23 @@
+# GCP Firestore certification testing
+
+This project aims to test the GCP Firestore State Store component under various conditions.
+
+## Test plan
+
+### Basic Test
+1. Able to create and test connection.
+2. Able to do set, fetch and delete.
+
+### Test entity_kind
+1. Able to create and test connection.
+2. Able to specify Entity Kind to set, fetch and delete.
+
+### Test NoIndex
+1. Able to create and test connection.
+2. Able to specify Entity Kind with NoIndex to set, fetch and delete.
+
+## Run Tests
+Note: 
+**Currently, GCP Firestore in Datastore mode, does not provide a public GCP API.For setup, follow the instructions in the [GCP Firestore Documentation](https://cloud.google.com/datastore/docs/store-query-data).**
+
+To run these tests, the environment variables `GCP_PROJECT_ID` and `GCP_FIRESTORE_ENTITY_KIND`

tests/certification/state/gcp/firestore/components/basictest/gcpfirestore.yaml

@@ -0,0 +1,15 @@
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: statestore-basic
+spec:
+  type: state.gcp.firestore
+  version: v1
+  metadata:
+    - name: project_id
+      secretKeyRef:
+        name: GCP_PROJECT_ID
+        key: GCP_PROJECT_ID
+
+auth:
+  secretStore: envvar-secret-store

tests/certification/state/gcp/firestore/components/basictest/localsecrets.yaml

@@ -0,0 +1,9 @@
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: envvar-secret-store
+  namespace: default
+spec:
+  type: secretstores.local.env
+  version: v1
+  metadata:

tests/certification/state/gcp/firestore/components/entity_kind/gcpfirestore.yaml

@@ -0,0 +1,19 @@
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: statestore-basic
+spec:
+  type: state.gcp.firestore
+  version: v1
+  metadata:
+    - name: project_id
+      secretKeyRef:
+        name: GCP_PROJECT_ID
+        key: GCP_PROJECT_ID
+    - name: entity_kind
+      secretKeyRef:
+        name: GCP_FIRESTORE_ENTITY_KIND
+        key: GCP_FIRESTORE_ENTITY_KIND
+
+auth:
+  secretStore: envvar-secret-store

tests/certification/state/gcp/firestore/components/entity_kind/localsecrets.yaml

@@ -0,0 +1,9 @@
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: envvar-secret-store
+  namespace: default
+spec:
+  type: secretstores.local.env
+  version: v1
+  metadata:

tests/certification/state/gcp/firestore/components/noindex/gcpfirestore.yaml

@@ -0,0 +1,17 @@
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: statestore-basic
+spec:
+  type: state.gcp.firestore
+  version: v1
+  metadata:
+    - name: project_id
+      secretKeyRef:
+        name: GCP_PROJECT_ID
+        key: GCP_PROJECT_ID
+    - name: noindex
+      value: "true" 
+
+auth:
+  secretStore: envvar-secret-store

tests/certification/state/gcp/firestore/components/noindex/localsecrets.yaml

@@ -0,0 +1,9 @@
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: envvar-secret-store
+  namespace: default
+spec:
+  type: secretstores.local.env
+  version: v1
+  metadata:

tests/certification/state/gcp/firestore/config.yaml

@@ -0,0 +1,6 @@
+apiVersion: dapr.io/v1alpha1
+kind: Configuration
+metadata:
+  name: keyvaultconfig
+spec:
+  features:

tests/certification/state/gcp/firestore/gcp_firestore_test.go

@@ -0,0 +1,123 @@
+/*
+Copyright 2023 The Dapr Authors
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gcp_firestore_test
+
+import (
+	"fmt"
+	"testing"
+
+	firestore "github.com/dapr/components-contrib/state/gcp/firestore"
+	"github.com/dapr/components-contrib/tests/certification/embedded"
+	"github.com/dapr/components-contrib/tests/certification/flow"
+	"github.com/dapr/go-sdk/client"
+
+	secretstore_env "github.com/dapr/components-contrib/secretstores/local/env"
+	"github.com/dapr/components-contrib/tests/certification/flow/sidecar"
+	secretstores_loader "github.com/dapr/dapr/pkg/components/secretstores"
+	state_loader "github.com/dapr/dapr/pkg/components/state"
+	"github.com/dapr/dapr/pkg/runtime"
+	dapr_testing "github.com/dapr/dapr/pkg/testing"
+	"github.com/dapr/kit/logger"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	sidecarNamePrefix = "firestore-sidecar-"
+	key               = "key"
+)
+
+func TestGCPFirestoreStorage(t *testing.T) {
+	ports, err := dapr_testing.GetFreePorts(2)
+	assert.NoError(t, err)
+
+	currentGrpcPort := ports[0]
+	currentHTTPPort := ports[1]
+
+	basicTest := func(statestore string) flow.Runnable {
+		return func(ctx flow.Context) error {
+			client, err := client.NewClientWithPort(fmt.Sprint(currentGrpcPort))
+			if err != nil {
+				panic(err)
+			}
+			defer client.Close()
+
+			stateKey := key
+			stateValue := "certificationdata"
+
+			// save state, default options: strong, last-write
+			err = client.SaveState(ctx, statestore, stateKey, []byte(stateValue), nil)
+			assert.NoError(t, err)
+
+			// get state
+			item, err := client.GetState(ctx, statestore, stateKey, nil)
+			assert.NoError(t, err)
+			assert.NotNil(t, item)
+			assert.Equal(t, stateValue, string(item.Value))
+
+			// delete state
+			err = client.DeleteState(ctx, statestore, stateKey, nil)
+			assert.NoError(t, err)
+
+			return nil
+		}
+	}
+
+	flow.New(t, "Test basic operations").
+		// Run the Dapr sidecar with GCP Firestore storage.
+		Step(sidecar.Run(sidecarNamePrefix,
+			embedded.WithoutApp(),
+			embedded.WithDaprGRPCPort(currentGrpcPort),
+			embedded.WithDaprHTTPPort(currentHTTPPort),
+			embedded.WithComponentsPath("./components/basictest"),
+			componentRuntimeOptions())).
+		Step("Run basic test with master key", basicTest("statestore-basic")).
+		Run()
+
+	flow.New(t, "Test entity_kind").
+		Step(sidecar.Run(sidecarNamePrefix,
+			embedded.WithoutApp(),
+			embedded.WithDaprGRPCPort(currentGrpcPort),
+			embedded.WithDaprHTTPPort(currentHTTPPort),
+			embedded.WithComponentsPath("./components/entity_kind"),
+			componentRuntimeOptions())).
+		Step("Run basic test with master key", basicTest("statestore-basic")).
+		Run()
+
+	flow.New(t, "Test NoIndex").
+		Step(sidecar.Run(sidecarNamePrefix,
+			embedded.WithoutApp(),
+			embedded.WithDaprGRPCPort(currentGrpcPort),
+			embedded.WithDaprHTTPPort(currentHTTPPort),
+			embedded.WithComponentsPath("./components/noindex"),
+			componentRuntimeOptions())).
+		Step("Run basic test with master key", basicTest("statestore-basic")).
+		Run()
+}
+
+func componentRuntimeOptions() []runtime.Option {
+	log := logger.NewLogger("dapr.components")
+
+	stateRegistry := state_loader.NewRegistry()
+	stateRegistry.Logger = log
+	stateRegistry.RegisterComponent(firestore.NewFirestoreStateStore, "gcp.firestore")
+
+	secretstoreRegistry := secretstores_loader.NewRegistry()
+	secretstoreRegistry.Logger = log
+	secretstoreRegistry.RegisterComponent(secretstore_env.NewEnvSecretStore, "local.env")
+
+	return []runtime.Option{
+		runtime.WithStates(stateRegistry),
+		runtime.WithSecretStores(secretstoreRegistry),
+	}
+}

tests/config/state/gcp/firestore/cloud/statestore.yml

@@ -0,0 +1,14 @@
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: statestore
+spec:
+  type: state.gcp.firestore
+  version: v1
+  metadata:
+   - name: project_id
+     value: ${{GCP_PROJECT_ID}}
+   - name: entity_kind
+     value: "ConformanceTestEntity"
+   - name: noindex
+     value:  "false"

tests/config/state/gcp/firestore/docker/statestore.yml

@@ -0,0 +1,16 @@
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: statestore
+spec:
+  type: state.gcp.firestore
+  version: v1
+  metadata:
+   - name: project_id
+     value: "conf-test-project"
+   - name: endpoint
+     value: "localhost:8432"
+   - name: entity_kind
+     value: "ConformanceTestEntity"
+   - name: noindex
+     value:  "false"

tests/config/state/tests.yml

@@ -74,4 +74,10 @@ components:
     operations: [ "set", "get", "delete", "etag", "bulkset", "bulkdelete", "first-write" ]
   - component: etcd
     allOperations: false
-    operations: [ "set", "get", "delete", "bulkset", "bulkdelete", "transaction", "etag",  "first-write", "ttl" ]
+    operations: [ "set", "get", "delete", "bulkset", "bulkdelete", "transaction", "etag",  "first-write", "ttl" ]
+  - component: gcp.firestore.docker
+    allOperations: false
+    operations: [ "set", "get", "delete", "bulkset", "bulkdelete" ]
+  - component: gcp.firestore.cloud
+    allOperations: false
+    operations: [ "set", "get", "delete", "bulkset", "bulkdelete" ]

tests/conformance/common.go

@@ -91,6 +91,7 @@ import (
 	s_cloudflareworkerskv "github.com/dapr/components-contrib/state/cloudflare/workerskv"
 	s_cockroachdb "github.com/dapr/components-contrib/state/cockroachdb"
 	s_etcd "github.com/dapr/components-contrib/state/etcd"
+	s_gcpfirestore "github.com/dapr/components-contrib/state/gcp/firestore"
 	s_inmemory "github.com/dapr/components-contrib/state/in-memory"
 	s_memcached "github.com/dapr/components-contrib/state/memcached"
 	s_mongodb "github.com/dapr/components-contrib/state/mongodb"
@@ -610,6 +611,10 @@ func loadStateStore(tc TestComponent) state.Store {
 		store = s_awsdynamodb.NewDynamoDBStateStore(testLogger)
 	case "etcd":
 		store = s_etcd.NewEtcdStateStore(testLogger)
+	case "gcp.firestore.docker":
+		store = s_gcpfirestore.NewFirestoreStateStore(testLogger)
+	case "gcp.firestore.cloud":
+		store = s_gcpfirestore.NewFirestoreStateStore(testLogger)
 	default:
 		return nil
 	}