Merge branch 'master' into shubham1172/dapr-bot-enhancements

.github/workflows/conformance.yml

@@ -84,6 +84,7 @@ jobs:
         - state.postgresql
         - state.redis
         - state.sqlserver
+        - state.in-memory
         - state.cockroachdb
         - workflows.temporal
         - state.rethinkdb

state/in-memory/in_memory.go

@@ -15,22 +15,29 @@ package inmemory
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 	"sync"
 	"time"
 
+	"github.com/google/uuid"
 	jsoniter "github.com/json-iterator/go"
 
 	"github.com/dapr/kit/logger"
+	"github.com/dapr/kit/ptr"
 
 	"github.com/dapr/components-contrib/state"
+	"github.com/dapr/components-contrib/state/utils"
 )
 
 type inMemStateStoreItem struct {
-	data   []byte
-	etag   *string
-	expire int64
+	data     []byte
+	etag     *string
+	expire   *int64
+	isBinary bool
 }
 
 type inMemoryStore struct {
@@ -61,10 +68,13 @@ func (store *inMemoryStore) Close() error {
 	if store.cancel != nil {
 		store.cancel()
 	}
+
 	// release memory reference
 	store.lock.Lock()
 	defer store.lock.Unlock()
-	store.items = map[string]*inMemStateStoreItem{}
+	for k := range store.items {
+		delete(store.items, k)
+	}
 
 	return nil
 }
@@ -75,7 +85,7 @@ func (store *inMemoryStore) Features() []state.Feature {
 
 func (store *inMemoryStore) Delete(req *state.DeleteRequest) error {
 	// step1: validate parameters
-	if err := store.doDeleteValidateParameters(req); err != nil {
+	if err := state.CheckRequestOptions(req.Options); err != nil {
 		return err
 	}
 
@@ -94,13 +104,17 @@ func (store *inMemoryStore) Delete(req *state.DeleteRequest) error {
 	return nil
 }
 
-func (store *inMemoryStore) doDeleteValidateParameters(req *state.DeleteRequest) error {
-	return state.CheckRequestOptions(req.Options)
-}
-
 func (store *inMemoryStore) doValidateEtag(key string, etag *string, concurrency string) error {
-	if etag != nil && *etag != "" && concurrency == state.FirstWrite {
-		// For FirstWrite, we need to validate etag before delete
+	hasEtag := etag != nil && *etag != ""
+
+	if concurrency == state.FirstWrite && !hasEtag {
+		item := store.items[key]
+		if item != nil {
+			return state.NewETagError(state.ETagMismatch, errors.New("item already exists and no etag was passed"))
+		} else {
+			return nil
+		}
+	} else if hasEtag {
 		item := store.items[key]
 		if item == nil {
 			return state.NewETagError(state.ETagMismatch, fmt.Errorf("state not exist or expired for key=%s", key))
@@ -127,9 +141,8 @@ func (store *inMemoryStore) BulkDelete(req []state.DeleteRequest) error {
 	}
 
 	// step1: validate parameters
-
 	for i := 0; i < len(req); i++ {
-		if err := store.doDeleteValidateParameters(&req[i]); err != nil {
+		if err := state.CheckRequestOptions(&req[i].Options); err != nil {
 			return err
 		}
 	}
@@ -155,14 +168,32 @@ func (store *inMemoryStore) BulkDelete(req []state.DeleteRequest) error {
 
 func (store *inMemoryStore) Get(req *state.GetRequest) (*state.GetResponse, error) {
 	item := store.doGetWithReadLock(req.Key)
-	if item != nil && isExpired(item.expire) {
+	if item != nil && isExpired(item) {
 		item = store.doGetWithWriteLock(req.Key)
 	}
 
 	if item == nil {
 		return &state.GetResponse{Data: nil, ETag: nil}, nil
 	}
-	return &state.GetResponse{Data: unmarshal(item.data), ETag: item.etag}, nil
+
+	data := item.data
+	if item.isBinary {
+		var (
+			s   string
+			err error
+		)
+
+		if err = jsoniter.Unmarshal(data, &s); err != nil {
+			return nil, err
+		}
+
+		data, err = base64.StdEncoding.DecodeString(s)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &state.GetResponse{Data: data, ETag: item.etag}, nil
 }
 
 func (store *inMemoryStore) doGetWithReadLock(key string) *inMemStateStoreItem {
@@ -180,24 +211,36 @@ func (store *inMemoryStore) doGetWithWriteLock(key string) *inMemStateStoreItem
 	if item == nil {
 		return nil
 	}
-	if isExpired(item.expire) {
+	if isExpired(item) {
 		store.doDelete(key)
 		return nil
 	}
 	return item
 }
 
-func isExpired(expire int64) bool {
-	if expire <= 0 {
+func isExpired(item *inMemStateStoreItem) bool {
+	if item == nil || item.expire == nil {
 		return false
 	}
-	return time.Now().UnixMilli() > expire
+	return time.Now().UnixMilli() > *item.expire
 }
 
 func (store *inMemoryStore) BulkGet(req []state.GetRequest) (bool, []state.BulkGetResponse, error) {
 	return false, nil, nil
 }
 
+func (store *inMemoryStore) marshal(v any) (bt []byte, isBinary bool, err error) {
+	byteArray, isBinary := v.([]uint8)
+	if isBinary {
+		v = base64.StdEncoding.EncodeToString(byteArray)
+	}
+	bt, err = utils.Marshal(v, json.Marshal)
+	if err != nil {
+		return nil, false, err
+	}
+	return bt, isBinary, nil
+}
+
 func (store *inMemoryStore) Set(req *state.SetRequest) error {
 	// step1: validate parameters
 	ttlInSeconds, err := store.doSetValidateParameters(req)
@@ -205,31 +248,36 @@ func (store *inMemoryStore) Set(req *state.SetRequest) error {
 		return err
 	}
 
-	b, _ := marshal(req.Value)
 	// step2 and step3 should be protected by write-lock
 	store.lock.Lock()
 	defer store.lock.Unlock()
 
 	// step2: validate etag if needed
-	if err := store.doValidateEtag(req.Key, req.ETag, req.Options.Concurrency); err != nil {
+	err = store.doValidateEtag(req.Key, req.ETag, req.Options.Concurrency)
+	if err != nil {
 		return err
 	}
 
 	// step3: do really set
+	bt, isBinary, err := store.marshal(req.Value)
+	if err != nil {
+		return err
+	}
+
 	// this operation won't fail
-	store.doSet(req.Key, b, req.ETag, ttlInSeconds)
+	store.doSet(req.Key, bt, ttlInSeconds, isBinary)
 	return nil
 }
 
 func (store *inMemoryStore) doSetValidateParameters(req *state.SetRequest) (int, error) {
 	err := state.CheckRequestOptions(req.Options)
 	if err != nil {
-		return -1, err
+		return 0, err
 	}
 
 	ttlInSeconds, err := doParseTTLInSeconds(req.Metadata)
 	if err != nil {
-		return -1, err
+		return 0, err
 	}
 
 	return ttlInSeconds, nil
@@ -253,19 +301,26 @@ func doParseTTLInSeconds(metadata map[string]string) (int, error) {
 	return i, nil
 }
 
-func (store *inMemoryStore) doSet(key string, data []byte, etag *string, ttlInSeconds int) {
-	store.items[key] = &inMemStateStoreItem{
-		data:   data,
-		etag:   etag,
-		expire: time.Now().UnixMilli() + int64(ttlInSeconds)*1000,
+func (store *inMemoryStore) doSet(key string, data []byte, ttlInSeconds int, isBinary bool) {
+	etag := uuid.New().String()
+	el := &inMemStateStoreItem{
+		data:     data,
+		etag:     &etag,
+		isBinary: isBinary,
 	}
+	if ttlInSeconds > 0 {
+		el.expire = ptr.Of(time.Now().UnixMilli() + int64(ttlInSeconds)*1000)
+	}
+
+	store.items[key] = el
 }
 
 // innerSetRequest is only used to pass ttlInSeconds and data with SetRequest.
 type innerSetRequest struct {
-	req          state.SetRequest
-	ttlInSeconds int
-	data         []byte
+	req      state.SetRequest
+	ttl      int
+	data     []byte
+	isBinary bool
 }
 
 func (store *inMemoryStore) BulkSet(req []state.SetRequest) error {
@@ -281,11 +336,15 @@ func (store *inMemoryStore) BulkSet(req []state.SetRequest) error {
 			return err
 		}
 
-		b, _ := marshal(req[i].Value)
+		bt, isBinary, err := store.marshal(req[i].Value)
+		if err != nil {
+			return err
+		}
 		innerSetRequest := &innerSetRequest{
-			req:          req[i],
-			ttlInSeconds: ttlInSeconds,
-			data:         b,
+			req:      req[i],
+			ttl:      ttlInSeconds,
+			data:     bt,
+			isBinary: isBinary,
 		}
 		innerSetRequestList = append(innerSetRequestList, innerSetRequest)
 	}
@@ -305,7 +364,7 @@ func (store *inMemoryStore) BulkSet(req []state.SetRequest) error {
 	// step3: do really set
 	// these operations won't fail
 	for _, innerSetRequest := range innerSetRequestList {
-		store.doSet(innerSetRequest.req.Key, innerSetRequest.data, innerSetRequest.req.ETag, innerSetRequest.ttlInSeconds)
+		store.doSet(innerSetRequest.req.Key, innerSetRequest.data, innerSetRequest.ttl, innerSetRequest.isBinary)
 	}
 	return nil
 }
@@ -316,24 +375,28 @@ func (store *inMemoryStore) Multi(request *state.TransactionalStateRequest) erro
 	}
 
 	// step1: validate parameters
-	for _, o := range request.Operations {
+	for i, o := range request.Operations {
 		if o.Operation == state.Upsert {
 			s := o.Request.(state.SetRequest)
 			ttlInSeconds, err := store.doSetValidateParameters(&s)
 			if err != nil {
 				return err
 			}
-			b, _ := marshal(s.Value)
+			bt, isBinary, err := store.marshal(s.Value)
+			if err != nil {
+				return err
+			}
 			innerSetRequest := &innerSetRequest{
-				req:          s,
-				ttlInSeconds: ttlInSeconds,
-				data:         b,
+				req:      s,
+				ttl:      ttlInSeconds,
+				data:     bt,
+				isBinary: isBinary,
 			}
 			// replace with innerSetRequest
-			o.Request = innerSetRequest
+			request.Operations[i].Request = innerSetRequest
 		} else if o.Operation == state.Delete {
 			d := o.Request.(state.DeleteRequest)
-			err := store.doDeleteValidateParameters(&d)
+			err := state.CheckRequestOptions(&d)
 			if err != nil {
 				return err
 			}
@@ -347,7 +410,7 @@ func (store *inMemoryStore) Multi(request *state.TransactionalStateRequest) erro
 	// step2: validate etag if needed
 	for _, o := range request.Operations {
 		if o.Operation == state.Upsert {
-			s := o.Request.(innerSetRequest)
+			s := o.Request.(*innerSetRequest)
 			err := store.doValidateEtag(s.req.Key, s.req.ETag, s.req.Options.Concurrency)
 			if err != nil {
 				return err
@@ -365,8 +428,8 @@ func (store *inMemoryStore) Multi(request *state.TransactionalStateRequest) erro
 	// these operations won't fail
 	for _, o := range request.Operations {
 		if o.Operation == state.Upsert {
-			s := o.Request.(innerSetRequest)
-			store.doSet(s.req.Key, s.data, s.req.ETag, s.ttlInSeconds)
+			s := o.Request.(*innerSetRequest)
+			store.doSet(s.req.Key, s.data, s.ttl, s.isBinary)
 		} else if o.Operation == state.Delete {
 			d := o.Request.(state.DeleteRequest)
 			store.doDelete(d.Key)
@@ -375,20 +438,6 @@ func (store *inMemoryStore) Multi(request *state.TransactionalStateRequest) erro
 	return nil
 }
 
-func marshal(value interface{}) ([]byte, error) {
-	v, _ := jsoniter.MarshalToString(value)
-
-	return []byte(v), nil
-}
-
-func unmarshal(val interface{}) []byte {
-	var output string
-
-	jsoniter.UnmarshalFromString(string(val.([]byte)), &output)
-
-	return []byte(output)
-}
-
 func (store *inMemoryStore) startCleanThread() {
 	for {
 		select {
@@ -405,7 +454,7 @@ func (store *inMemoryStore) doCleanExpiredItems() {
 	defer store.lock.Unlock()
 
 	for key, item := range store.items {
-		if isExpired(item.expire) {
+		if item.expire != nil && isExpired(item) {
 			store.doDelete(key)
 		}
 	}

state/in-memory/in_memory_test.go

@@ -21,7 +21,6 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/dapr/kit/logger"
-	"github.com/dapr/kit/ptr"
 
 	"github.com/dapr/components-contrib/state"
 )
@@ -41,18 +40,19 @@ func TestReadAndWrite(t *testing.T) {
 		setReq := &state.SetRequest{
 			Key:   keyA,
 			Value: valueA,
-			ETag:  ptr.Of("the etag"),
 		}
 		err := store.Set(setReq)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		// get after set
 		getReq := &state.GetRequest{
 			Key: keyA,
 		}
 		resp, err := store.Get(getReq)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.NotNil(t, resp)
-		assert.Equal(t, valueA, string(resp.Data))
+		assert.Equal(t, `"`+valueA+`"`, string(resp.Data))
+		_ = assert.NotNil(t, resp.ETag) &&
+			assert.NotEmpty(t, *resp.ETag)
 	})
 
 	t.Run("get nothing when expired", func(t *testing.T) {
@@ -63,7 +63,7 @@ func TestReadAndWrite(t *testing.T) {
 			Metadata: map[string]string{"ttlInSeconds": "1"},
 		}
 		err := store.Set(setReq)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		// simulate expiration
 		time.Sleep(2 * time.Second)
 		// get
@@ -71,7 +71,7 @@ func TestReadAndWrite(t *testing.T) {
 			Key: keyA,
 		}
 		resp, err := store.Get(getReq)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.NotNil(t, resp)
 		assert.Nil(t, resp.Data)
 		assert.Nil(t, resp.ETag)
@@ -81,31 +81,30 @@ func TestReadAndWrite(t *testing.T) {
 		// set
 		setReq := &state.SetRequest{
 			Key:   "theSecondKey",
-			Value: "1234",
-			ETag:  ptr.Of("the etag"),
+			Value: 1234,
 		}
 		err := store.Set(setReq)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		// get
 		getReq := &state.GetRequest{
 			Key: "theSecondKey",
 		}
 		resp, err := store.Get(getReq)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.NotNil(t, resp)
-		assert.Equal(t, "1234", string(resp.Data))
+		assert.Equal(t, `1234`, string(resp.Data))
 	})
 
 	t.Run("BulkSet two keys", func(t *testing.T) {
 		err := store.BulkSet([]state.SetRequest{{
 			Key:   "theFirstKey",
-			Value: "666",
+			Value: "42",
 		}, {
 			Key:   "theSecondKey",
-			Value: "777",
+			Value: "84",
 		}})
 
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 	})
 
 	t.Run("BulkGet fails when not supported", func(t *testing.T) {
@@ -115,7 +114,7 @@ func TestReadAndWrite(t *testing.T) {
 			Key: "theSecondKey",
 		}})
 
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 		assert.Equal(t, false, supportBulk)
 	})
 
@@ -124,6 +123,6 @@ func TestReadAndWrite(t *testing.T) {
 			Key: "theFirstKey",
 		}
 		err := store.Delete(req)
-		assert.Nil(t, err)
+		assert.NoError(t, err)
 	})
 }

tests/config/state/in-memory/statestore.yaml

@@ -0,0 +1,8 @@
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: statestore
+spec:
+  type: state.in-memory
+  version: v1
+  metadata: []

tests/config/state/tests.yml

@@ -43,3 +43,6 @@ components:
   - component: rethinkdb
     allOperations: false
     operations: [ "set", "get", "delete", "bulkset", "bulkdelete"]
+  - component: in-memory
+    allOperations: false
+    operations: [ "set", "get", "delete", "bulkset", "bulkdelete", "transaction", "etag",  "first-write", "ttl" ]

tests/conformance/common.go

@@ -74,6 +74,7 @@ import (
 	s_azuretablestorage "github.com/dapr/components-contrib/state/azure/tablestorage"
 	s_cassandra "github.com/dapr/components-contrib/state/cassandra"
 	s_cockroachdb "github.com/dapr/components-contrib/state/cockroachdb"
+	s_inmemory "github.com/dapr/components-contrib/state/in-memory"
 	s_memcached "github.com/dapr/components-contrib/state/memcached"
 	s_mongodb "github.com/dapr/components-contrib/state/mongodb"
 	s_mysql "github.com/dapr/components-contrib/state/mysql"
@@ -463,6 +464,8 @@ func loadStateStore(tc TestComponent) state.Store {
 		store = s_memcached.NewMemCacheStateStore(testLogger)
 	case "rethinkdb":
 		store = s_rethinkdb.NewRethinkDBStateStore(testLogger)
+	case "in-memory":
+		store = s_inmemory.NewInMemoryStateStore(testLogger)
 	default:
 		return nil
 	}