Set up DynamoDB interface (#3)

This sets up a database interface for the rest of the project to use, abstracting away Dynamo.

There's an integration test that verifies the interface works against Amazon's local dynamodb, as well as a mock that's suitable for use in unit tests of other parts of the codebase. We run the same smoketest against both to verify they behave equivalently in the basic codepaths we've got in the database interface.
This commit is contained in:
Matthew McPherrin 2022-10-17 20:43:57 -04:00 committed by GitHub
parent 4d7606fb1f
commit 3f83fc810b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 423 additions and 2 deletions

1
db/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/dynamodb_local

129
db/db.go Normal file
View File

@ -0,0 +1,129 @@
package db
import (
"context"
"crypto/x509"
"fmt"
"math/big"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
)
// ddb is fulfilled by a dynamodb.Client and is used for mocking in tests.
type ddb interface {
BatchWriteItem(context.Context, *dynamodb.BatchWriteItemInput, ...func(*dynamodb.Options)) (*dynamodb.BatchWriteItemOutput, error)
PutItem(context.Context, *dynamodb.PutItemInput, ...func(*dynamodb.Options)) (*dynamodb.PutItemOutput, error)
Scan(context.Context, *dynamodb.ScanInput, ...func(*dynamodb.Options)) (*dynamodb.ScanOutput, error)
}
type Database struct {
Table string
Dynamo ddb
}
func New(cfg *aws.Config) (*Database, error) {
return &Database{
Table: "unseen-certificates",
Dynamo: dynamodb.NewFromConfig(*cfg),
}, nil
}
// CertMetadata is the entire set of attributes stored in Dynamo.
// That is the CertKey plus the revocation time today.
type CertMetadata struct {
CertKey
RevocationTime time.Time `dynamodbav:"RT,unixtime"`
}
// CertKey is the DynamoDB primary key, which is the serial number.
type CertKey struct {
SerialNumber []byte `dynamodbav:"SN"`
}
func NewCertKey(sn *big.Int) CertKey {
return CertKey{SerialNumber: sn.Bytes()}
}
// SerialString returns a consistent string representation of a SerialNumber
// It is intended for use as a map key, and is equivalent to boulder's SerialToString
func (ck CertKey) SerialString() string {
return fmt.Sprintf("%036x", ck.SerialNumber)
}
// AddCert inserts the metadata for monitoring
func (db *Database) AddCert(ctx context.Context, certificate *x509.Certificate, revocationTime time.Time) error {
item, err := attributevalue.MarshalMap(CertMetadata{
CertKey: NewCertKey(certificate.SerialNumber),
RevocationTime: revocationTime,
})
if err != nil {
return err
}
_, err = db.Dynamo.PutItem(ctx, &dynamodb.PutItemInput{
Item: item,
TableName: &db.Table,
})
if err != nil {
return err
}
return nil
}
// GetAllCerts returns all the certificates in the DynamoDB. This set is
// intended to be much smaller than the set of certificates in a CRL, so it's
// more efficient to just load the entire set instead of conditional querying.
// The map key is the serial's CertKey.SerialString.
// TODO: This could be more efficient if it was a query over issuer or shard
// TODO: However, the dataset is small enough to not matter much.
func (db *Database) GetAllCerts(ctx context.Context) (map[string]CertMetadata, error) {
resp, err := db.Dynamo.Scan(ctx, &dynamodb.ScanInput{
TableName: &db.Table,
Select: types.SelectAllAttributes,
})
if err != nil {
return nil, err
}
var certList []CertMetadata
err = attributevalue.UnmarshalListOfMaps(resp.Items, &certList)
if err != nil {
return nil, err
}
certs := make(map[string]CertMetadata, len(certList))
for _, cert := range certList {
certs[cert.CertKey.SerialString()] = cert
}
return certs, nil
}
// DeleteSerials takes a list of serials that we've seen in the CRL and thus
// no longer need to keep an eye out for.
func (db *Database) DeleteSerials(ctx context.Context, serialNumbers [][]byte) error {
var deletes []types.WriteRequest
for _, serial := range serialNumbers {
key, err := attributevalue.MarshalMap(CertKey{SerialNumber: serial})
if err != nil {
return err
}
deletes = append(deletes, types.WriteRequest{
DeleteRequest: &types.DeleteRequest{
Key: key,
},
})
}
_, err := db.Dynamo.BatchWriteItem(ctx, &dynamodb.BatchWriteItemInput{
RequestItems: map[string][]types.WriteRequest{db.Table: deletes},
})
if err != nil {
return err
}
return nil
}

71
db/db_test.go Normal file
View File

@ -0,0 +1,71 @@
package db_test
import (
"bytes"
"context"
"crypto/x509"
"math/big"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/letsencrypt/crl-monitor/db"
"github.com/letsencrypt/crl-monitor/db/mock"
)
func TestDatabaseWithMock(t *testing.T) {
smoketest(t, mock.NewMockedDB(t))
}
// smoketest goes through a set of basic actions ensuring the basics work
// It gets run with a mocked database and can also be integration tested against
// the real DynamoDB, or the downloadable version, to ensure they align.
func smoketest(t *testing.T, handle *db.Database) {
ctx := context.Background()
ts1 := time.Now()
ts2 := time.Now().Add(100 * time.Hour)
int111 := big.NewInt(111)
int4s := big.NewInt(444444)
int60s := big.NewInt(606060)
int123 := big.NewInt(123456)
// Insert 4 entries into the database with different serials and revocation times
require.NoError(t, handle.AddCert(ctx, &x509.Certificate{SerialNumber: int111}, ts1))
require.NoError(t, handle.AddCert(ctx, &x509.Certificate{SerialNumber: int4s}, ts1))
require.NoError(t, handle.AddCert(ctx, &x509.Certificate{SerialNumber: int60s}, ts2))
require.NoError(t, handle.AddCert(ctx, &x509.Certificate{SerialNumber: int123}, ts2))
// Timestamps stored in Dynamo as unix timestamps are truncated to second precision
ts1 = ts1.Truncate(time.Second)
ts2 = ts2.Truncate(time.Second)
certs, err := handle.GetAllCerts(ctx)
require.NoError(t, err)
require.Len(t, certs, 4)
require.Equal(t, certs, map[string]db.CertMetadata{
"00000000000000000000000000000000006f": {CertKey: db.CertKey{SerialNumber: int111.Bytes()}, RevocationTime: ts1},
"00000000000000000000000000000006c81c": {CertKey: db.CertKey{SerialNumber: int4s.Bytes()}, RevocationTime: ts1},
"000000000000000000000000000000093f6c": {CertKey: db.CertKey{SerialNumber: int60s.Bytes()}, RevocationTime: ts2},
"00000000000000000000000000000001e240": {CertKey: db.CertKey{SerialNumber: int123.Bytes()}, RevocationTime: ts2},
})
// Delete all the serials other than the 606060 serial
var serials [][]byte
for _, cert := range certs {
if !bytes.Equal(cert.SerialNumber, int60s.Bytes()) {
serials = append(serials, cert.SerialNumber)
}
}
require.NoError(t, handle.DeleteSerials(ctx, serials))
// The only remaining entry should be the serial 606060 one
remaining, err := handle.GetAllCerts(ctx)
require.NoError(t, err)
expected := map[string]db.CertMetadata{
"000000000000000000000000000000093f6c": {CertKey: db.CertKey{SerialNumber: int60s.Bytes()}, RevocationTime: ts2},
}
require.Equal(t, expected, remaining)
}

65
db/integration_test.go Normal file
View File

@ -0,0 +1,65 @@
//go:build integration
package db_test
import (
"context"
"fmt"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/stretchr/testify/require"
"github.com/letsencrypt/crl-monitor/db"
)
func localResolver(service, region string, opts ...interface{}) (aws.Endpoint, error) {
if service != dynamodb.ServiceID {
return aws.Endpoint{}, fmt.Errorf("unsupported service %s", service)
}
return aws.Endpoint{
PartitionID: "aws",
URL: "http://localhost:8000/",
}, nil
}
// makeTable sets up the table in the integration test DB.
// In the real Dynamo, we provision the table with Terraform
func makeTable(t *testing.T, handle *db.Database) {
_, err := handle.Dynamo.(*dynamodb.Client).CreateTable(context.Background(), &dynamodb.CreateTableInput{
AttributeDefinitions: []types.AttributeDefinition{{
AttributeName: aws.String("SN"),
AttributeType: types.ScalarAttributeTypeB,
}},
KeySchema: []types.KeySchemaElement{{
AttributeName: aws.String("SN"),
KeyType: types.KeyTypeHash,
}},
TableName: aws.String(handle.Table),
ProvisionedThroughput: &types.ProvisionedThroughput{
ReadCapacityUnits: aws.Int64(10),
WriteCapacityUnits: aws.Int64(10),
},
})
require.NoError(t, err)
}
// TestIntegrationDynamoDB runs smoketest against a local DynamoDB. The main
// goal of this test is to ensure that the in-process mock behaves similarly to
// the real dynamoDB, which is why we run the same smoketest against both.
// That means developers don't need to always be running the local DynamoDB to
// run most tests outside the db package.
func TestIntegrationDynamoDB(t *testing.T) {
cfg := aws.NewConfig()
cfg.EndpointResolverWithOptions = aws.EndpointResolverWithOptionsFunc(localResolver)
cfg.Credentials = aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) {
return aws.Credentials{AccessKeyID: "Bogus", SecretAccessKey: "Bogus"}, nil
})
handle, err := db.New(cfg)
require.NoError(t, err)
makeTable(t, handle)
smoketest(t, handle)
}

73
db/mock/mock.go Normal file
View File

@ -0,0 +1,73 @@
package mock
import (
"bytes"
"context"
"testing"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/stretchr/testify/require"
"github.com/letsencrypt/crl-monitor/db"
)
const Table = "table"
// NewMockedDB returns an in-memory Database using a mocked DynamoDB
// It is meant only for use in tests.
func NewMockedDB(t *testing.T) *db.Database {
return &db.Database{
Table: Table,
Dynamo: &dynamoMock{t: t},
}
}
type dynamoMock struct {
t *testing.T
data []map[string]types.AttributeValue
}
func has(key map[string]types.AttributeValue, item map[string]types.AttributeValue) bool {
for k, v := range key {
if !bytes.Equal(item[k].(*types.AttributeValueMemberB).Value, v.(*types.AttributeValueMemberB).Value) {
return false
}
}
return true
}
func (d *dynamoMock) BatchWriteItem(ctx context.Context, input *dynamodb.BatchWriteItemInput, opts ...func(*dynamodb.Options)) (*dynamodb.BatchWriteItemOutput, error) {
require.Empty(d.t, opts, "Options not supported")
require.NotNil(d.t, input)
for _, item := range input.RequestItems[Table] {
require.Nil(d.t, item.PutRequest, "Only delete requests supported")
require.NotNil(d.t, item.DeleteRequest)
key := item.DeleteRequest.Key
var filtered []map[string]types.AttributeValue
for _, i := range d.data {
if !has(key, i) {
filtered = append(filtered, i)
}
}
d.data = filtered
}
return &dynamodb.BatchWriteItemOutput{}, nil
}
func (d *dynamoMock) PutItem(ctx context.Context, input *dynamodb.PutItemInput, opts ...func(*dynamodb.Options)) (*dynamodb.PutItemOutput, error) {
require.Empty(d.t, opts, "Options not supported")
require.NotNil(d.t, input)
d.data = append(d.data, input.Item)
return &dynamodb.PutItemOutput{}, nil
}
func (d *dynamoMock) Scan(ctx context.Context, input *dynamodb.ScanInput, opts ...func(*dynamodb.Options)) (*dynamodb.ScanOutput, error) {
require.Empty(d.t, opts, "Options not supported")
require.NotNil(d.t, input)
return &dynamodb.ScanOutput{Items: d.data}, nil
}

22
db/run_db_integration_test.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
set -eux
SCRIPT_PATH=${0%/*}
cd "$SCRIPT_PATH"
# Fetch the local DynamoDB if there isn't one here already
if ! [ -d dynamodb_local ]; then
mkdir dynamodb_local
curl -sSL https://s3.us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.tar.gz \
| tar -xzf - -C dynamodb_local
else
echo "using existing DynamoDBLocal.jar"
fi
java -Djava.library.path=./dynamodb_local/DynamoDBLocal_lib -jar ./dynamodb_local/DynamoDBLocal.jar -inMemory &
dynamopid=$!
trap 'kill $dynamopid' EXIT
sleep 1 # Let dynamodb start
go test -tags integration .

23
go.mod
View File

@ -2,4 +2,25 @@ module github.com/letsencrypt/crl-monitor
go 1.19
require github.com/aws/aws-lambda-go v1.34.1
require (
github.com/aws/aws-lambda-go v1.34.1
github.com/aws/aws-sdk-go-v2 v1.16.16
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1
github.com/stretchr/testify v1.8.0
)
require (
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 // indirect
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17 // indirect
github.com/aws/smithy-go v1.13.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

41
go.sum
View File

@ -1,6 +1,45 @@
github.com/aws/aws-lambda-go v1.34.1 h1:M3a/uFYBjii+tDcOJ0wL/WyFi2550FHoECdPf27zvOs=
github.com/aws/aws-lambda-go v1.34.1/go.mod h1:jwFe2KmMsHmffA1X2R09hH6lFzJQxzI8qK17ewzbQMM=
github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk=
github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0 h1:bKbdstt7+PzIRSIXZ11Yo8Qh8t0AHn6jEYUfsbVcLjE=
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0/go.mod h1:+CBJZMhsb1pTUcB/NTdS505bDX10xS4xnPMqDZj2Ptw=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg=
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1 h1:1QpTkQIAaZpR387it1L+erjB5bStGFCJRvmXsodpPEU=
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1/go.mod h1:BZhn/C3z13ULTSstVi2Kymc62bgjFh/JwLO9Tm2OFYI=
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20 h1:V9q4A0qnUfDsfivspY1LQRQTOG3Y9FLHvXIaTbcU7XM=
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20/go.mod h1:7qWU48SMzlrfOlNhHpazW3psFWlOIWrq4SmOr2/ESmk=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 h1:Lh1AShsuIJTwMkoxVCAYPJgNG5H+eN6SmoUn8nOZ5wE=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw=
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17 h1:o0Ia3nb56m8+8NvhbCDiSBiZRNUwIknVWobx5vks0Vk=
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17/go.mod h1:WJD9FbkwzM2a1bZ36ntH6+5Jc+x41Q4K2AcLeHDLAS8=
github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA=
github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=