State stores: expose TTL as a feature (#2987)

Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com>

.github/infrastructure/terraform/conformance/state/aws/dynamodb/dynamodb.tf

@@ -34,6 +34,10 @@ resource "aws_dynamodb_table" "conformance_test_basic_table" {
   billing_mode   = "PROVISIONED"
   read_capacity  = "10"
   write_capacity = "10"
+  ttl {
+    attribute_name = "expiresAt"
+    enabled        = true
+  }
   attribute {
     name = "key"
     type = "S"

internal/component/postgresql/postgresql.go

@@ -72,7 +72,12 @@ func (p *PostgreSQL) Init(ctx context.Context, metadata state.Metadata) error {
 
 // Features returns the features available in this state store.
 func (p *PostgreSQL) Features() []state.Feature {
-	return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureQueryAPI}
+	return []state.Feature{
+		state.FeatureETag,
+		state.FeatureTransactional,
+		state.FeatureQueryAPI,
+		state.FeatureTTL,
+	}
 }
 
 // Delete removes an entity from the store.

state/aws/dynamodb/dynamodb.go

@@ -94,7 +94,19 @@ func (d *StateStore) Init(_ context.Context, metadata state.Metadata) error {
 
 // Features returns the features available in this state store.
 func (d *StateStore) Features() []state.Feature {
-	return []state.Feature{state.FeatureETag, state.FeatureTransactional}
+	// TTLs are enabled only if ttlAttributeName is set
+	if d.ttlAttributeName == "" {
+		return []state.Feature{
+			state.FeatureETag,
+			state.FeatureTransactional,
+		}
+	}
+
+	return []state.Feature{
+		state.FeatureETag,
+		state.FeatureTransactional,
+		state.FeatureTTL,
+	}
 }
 
 // Get retrieves a dynamoDB item.

state/azure/cosmosdb/cosmosdb.go

@@ -204,6 +204,7 @@ func (c *StateStore) Features() []state.Feature {
 		state.FeatureETag,
 		state.FeatureTransactional,
 		state.FeatureQueryAPI,
+		state.FeatureTTL,
 	}
 }
 

state/cassandra/cassandra.go

@@ -117,7 +117,9 @@ func (c *Cassandra) Init(_ context.Context, metadata state.Metadata) error {
 
 // Features returns the features available in this state store.
 func (c *Cassandra) Features() []state.Feature {
-	return nil
+	return []state.Feature{
+		state.FeatureTTL,
+	}
 }
 
 func (c *Cassandra) tryCreateKeyspace(keyspace string, replicationFactor int) error {

state/cloudflare/workerskv/workerskv.go

@@ -90,7 +90,9 @@ func (q *CFWorkersKV) GetComponentMetadata() (metadataInfo metadata.MetadataMap)
 
 // Features returns the features supported by this state store.
 func (q CFWorkersKV) Features() []state.Feature {
-	return []state.Feature{}
+	return []state.Feature{
+		state.FeatureTTL,
+	}
 }
 
 func (q *CFWorkersKV) Delete(parentCtx context.Context, stateReq *state.DeleteRequest) error {

state/couchbase/couchbase.go

@@ -66,9 +66,11 @@ type couchbaseMetadata struct {
 // NewCouchbaseStateStore returns a new couchbase state store.
 func NewCouchbaseStateStore(logger logger.Logger) state.Store {
 	s := &Couchbase{
-		json:     jsoniter.ConfigFastest,
-		features: []state.Feature{state.FeatureETag},
-		logger:   logger,
+		json: jsoniter.ConfigFastest,
+		features: []state.Feature{
+			state.FeatureETag,
+		},
+		logger: logger,
 	}
 	s.BulkStore = state.NewDefaultBulkStore(s)
 	return s

state/etcd/etcd.go

@@ -67,9 +67,13 @@ func NewEtcdStateStoreV2(logger logger.Logger) state.Store {
 
 func newETCD(logger logger.Logger, schema schemaMarshaller) state.Store {
 	s := &Etcd{
-		schema:   schema,
-		logger:   logger,
-		features: []state.Feature{state.FeatureETag, state.FeatureTransactional},
+		schema: schema,
+		logger: logger,
+		features: []state.Feature{
+			state.FeatureETag,
+			state.FeatureTransactional,
+			state.FeatureTTL,
+		},
 	}
 	s.BulkStore = state.NewDefaultBulkStore(s)
 	return s

state/feature.go

@@ -24,9 +24,11 @@ const (
 	FeatureTransactional Feature = "TRANSACTIONAL"
 	// FeatureQueryAPI is the feature that performs query operations.
 	FeatureQueryAPI Feature = "QUERY_API"
+	// FeatureTTL is the feature that supports TTLs.
+	FeatureTTL Feature = "TTL"
 )
 
-// Feature names a feature that can be implemented by PubSub components.
+// Feature names a feature that can be implemented by state store components.
 type Feature string
 
 // IsPresent checks if a given feature is present in the list.

state/in-memory/in_memory.go

@@ -91,6 +91,7 @@ func (store *inMemoryStore) Features() []state.Feature {
 	return []state.Feature{
 		state.FeatureETag,
 		state.FeatureTransactional,
+		state.FeatureTTL,
 	}
 }
 

state/memcached/memcached.go

@@ -93,7 +93,9 @@ func (m *Memcached) Init(_ context.Context, metadata state.Metadata) error {
 
 // Features returns the features available in this state store.
 func (m *Memcached) Features() []state.Feature {
-	return nil
+	return []state.Feature{
+		state.FeatureTTL,
+	}
 }
 
 func getMemcachedMetadata(meta state.Metadata) (*memcachedMetadata, error) {

state/mongodb/metadata.yaml

@@ -17,6 +17,7 @@ capabilities:
   - transactional
   - etag
   - query
+  - ttl
 authenticationProfiles:
   - title: "Connection string"
     description: |

state/mongodb/mongodb.go

@@ -112,8 +112,13 @@ type Item struct {
 // NewMongoDB returns a new MongoDB state store.
 func NewMongoDB(logger logger.Logger) state.Store {
 	s := &MongoDB{
-		features: []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureQueryAPI},
-		logger:   logger,
+		features: []state.Feature{
+			state.FeatureETag,
+			state.FeatureTransactional,
+			state.FeatureQueryAPI,
+			state.FeatureTTL,
+		},
+		logger: logger,
 	}
 	s.BulkStore = state.NewDefaultBulkStore(s)
 	return s

state/mysql/mysql.go

@@ -225,7 +225,11 @@ func (m *MySQL) parseMetadata(md map[string]string) error {
 
 // Features returns the features available in this state store.
 func (m *MySQL) Features() []state.Feature {
-	return []state.Feature{state.FeatureETag, state.FeatureTransactional}
+	return []state.Feature{
+		state.FeatureETag,
+		state.FeatureTransactional,
+		state.FeatureTTL,
+	}
 }
 
 // Ping the database.

state/oci/objectstorage/objectstorage.go

@@ -168,10 +168,13 @@ func (r *StateStore) Ping(ctx context.Context) error {
 
 func NewOCIObjectStorageStore(logger logger.Logger) state.Store {
 	s := &StateStore{
-		json:     jsoniter.ConfigFastest,
-		features: []state.Feature{state.FeatureETag},
-		logger:   logger,
-		client:   nil,
+		json: jsoniter.ConfigFastest,
+		features: []state.Feature{
+			state.FeatureETag,
+			state.FeatureTTL,
+		},
+		logger: logger,
+		client: nil,
 	}
 	s.BulkStore = state.NewDefaultBulkStore(s)
 

state/oracledatabase/oracledatabase.go

@@ -44,7 +44,11 @@ func NewOracleDatabaseStateStore(logger logger.Logger) state.Store {
 // This unexported constructor allows injecting a dbAccess instance for unit testing.
 func newOracleDatabaseStateStore(logger logger.Logger, dba dbAccess) *OracleDatabase {
 	return &OracleDatabase{
-		features: []state.Feature{state.FeatureETag, state.FeatureTransactional},
+		features: []state.Feature{
+			state.FeatureETag,
+			state.FeatureTransactional,
+			state.FeatureTTL,
+		},
 		logger:   logger,
 		dbaccess: dba,
 	}

state/redis/redis.go

@@ -161,9 +161,9 @@ func (r *StateStore) Init(ctx context.Context, metadata state.Metadata) error {
 // Features returns the features available in this state store.
 func (r *StateStore) Features() []state.Feature {
 	if r.clientHasJSON {
-		return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureQueryAPI}
+		return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureTTL, state.FeatureQueryAPI}
 	} else {
-		return []state.Feature{state.FeatureETag, state.FeatureTransactional}
+		return []state.Feature{state.FeatureETag, state.FeatureTransactional, state.FeatureTTL}
 	}
 }
 

state/rethinkdb/rethinkdb.go

@@ -37,7 +37,7 @@ const (
 	stateArchiveTablePKName = "key"
 )
 
-// RethinkDB is a state store implementation with transactional support for RethinkDB.
+// RethinkDB is a state store implementation for RethinkDB.
 type RethinkDB struct {
 	session  *r.Session
 	config   *stateConfig

state/sqlite/sqlite.go

@@ -48,6 +48,7 @@ func newSQLiteStateStore(logger logger.Logger, dba DBAccess) *SQLiteStore {
 		features: []state.Feature{
 			state.FeatureETag,
 			state.FeatureTransactional,
+			state.FeatureTTL,
 		},
 		dbaccess: dba,
 	}

state/sqlserver/metadata.yaml

@@ -15,6 +15,7 @@ capabilities:
   - "crud"
   - "transactional"
   - "etag"
+  - "ttl"
 authenticationProfiles:
   - title: "Connection string"
     description: |

state/sqlserver/sqlserver.go

@@ -65,7 +65,11 @@ const (
 // New creates a new instance of a SQL Server transaction store.
 func New(logger logger.Logger) state.Store {
 	s := &SQLServer{
-		features:        []state.Feature{state.FeatureETag, state.FeatureTransactional},
+		features: []state.Feature{
+			state.FeatureETag,
+			state.FeatureTransactional,
+			state.FeatureTTL,
+		},
 		logger:          logger,
 		migratorFactory: newMigration,
 	}
@@ -238,12 +242,6 @@ func (s *SQLServer) executeDelete(ctx context.Context, db dbExecutor, req *state
 	return nil
 }
 
-// TvpDeleteTableStringKey defines a table type with string key.
-type TvpDeleteTableStringKey struct {
-	ID         string
-	RowVersion []byte
-}
-
 // Get returns an entity from store.
 func (s *SQLServer) Get(ctx context.Context, req *state.GetRequest) (*state.GetResponse, error) {
 	rows, err := s.db.QueryContext(ctx, s.getCommand, sql.Named(keyColumnName, req.Key))

tests/config/state/aws/dynamodb/terraform/statestore.yml

@@ -13,4 +13,6 @@ spec:
     - name: region
       value: "us-east-1"
     - name: table
-      value: ${{STATE_AWS_DYNAMODB_TABLE_1}}
+      value: ${{STATE_AWS_DYNAMODB_TABLE_1}}
+    - name: ttlAttributeName
+      value: "expiresAt"

tests/config/state/tests.yml

@@ -75,9 +75,10 @@ components:
   - component: in-memory
     operations: [ "transaction", "etag",  "first-write", "ttl" ]
   - component: aws.dynamodb.docker
+    # In the Docker variant, we do not set ttlAttributeName in the metadata, so TTLs are not enabled
     operations: [ "transaction", "etag", "first-write" ]
   - component: aws.dynamodb.terraform
-    operations: [ "transaction", "etag", "first-write" ]
+    operations: [ "transaction", "etag", "first-write", "ttl" ]
   - component: etcd.v1
     operations: [ "transaction", "etag",  "first-write", "ttl" ]
   - component: etcd.v2

tests/conformance/state/state.go

@@ -991,6 +991,10 @@ func ConformanceTests(t *testing.T, props map[string]string, statestore state.St
 
 	if config.HasOperation("ttl") {
 		t.Run("set and get with TTL", func(t *testing.T) {
+			// Check if ttl feature is listed
+			features := statestore.Features()
+			require.True(t, state.FeatureTTL.IsPresent(features))
+
 			err := statestore.Set(context.Background(), &state.SetRequest{
 				Key:   key + "-ttl",
 				Value: "⏱️",
@@ -1016,6 +1020,17 @@ func ConformanceTests(t *testing.T, props map[string]string, statestore state.St
 				return res.Data == nil
 			}, time.Second*3, 200*time.Millisecond, "expected object to have been deleted in time")
 		})
+	} else {
+		t.Run("ttl feature not present", func(t *testing.T) {
+			// We skip this check for Cloudflare Workers KV
+			// Even though the component supports TTLs, it's not tested in the conformance tests because the minimum TTL for the component is 1 minute, and the state store doesn't have strong consistency
+			if config.ComponentName == "cloudflare.workerskv" {
+				t.Skip()
+			}
+
+			features := statestore.Features()
+			require.False(t, state.FeatureTTL.IsPresent(features))
+		})
 	}
 }