diff --git a/db/.gitignore b/db/.gitignore new file mode 100644 index 0000000..1c8bc8a --- /dev/null +++ b/db/.gitignore @@ -0,0 +1 @@ +/dynamodb_local diff --git a/db/db.go b/db/db.go new file mode 100644 index 0000000..dda9eed --- /dev/null +++ b/db/db.go @@ -0,0 +1,129 @@ +package db + +import ( + "context" + "crypto/x509" + "fmt" + "math/big" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" +) + +// ddb is fulfilled by a dynamodb.Client and is used for mocking in tests. +type ddb interface { + BatchWriteItem(context.Context, *dynamodb.BatchWriteItemInput, ...func(*dynamodb.Options)) (*dynamodb.BatchWriteItemOutput, error) + PutItem(context.Context, *dynamodb.PutItemInput, ...func(*dynamodb.Options)) (*dynamodb.PutItemOutput, error) + Scan(context.Context, *dynamodb.ScanInput, ...func(*dynamodb.Options)) (*dynamodb.ScanOutput, error) +} + +type Database struct { + Table string + Dynamo ddb +} + +func New(cfg *aws.Config) (*Database, error) { + return &Database{ + Table: "unseen-certificates", + Dynamo: dynamodb.NewFromConfig(*cfg), + }, nil +} + +// CertMetadata is the entire set of attributes stored in Dynamo. +// That is the CertKey plus the revocation time today. +type CertMetadata struct { + CertKey + RevocationTime time.Time `dynamodbav:"RT,unixtime"` +} + +// CertKey is the DynamoDB primary key, which is the serial number. +type CertKey struct { + SerialNumber []byte `dynamodbav:"SN"` +} + +func NewCertKey(sn *big.Int) CertKey { + return CertKey{SerialNumber: sn.Bytes()} +} + +// SerialString returns a consistent string representation of a SerialNumber +// It is intended for use as a map key, and is equivalent to boulder's SerialToString +func (ck CertKey) SerialString() string { + return fmt.Sprintf("%036x", ck.SerialNumber) +} + +// AddCert inserts the metadata for monitoring +func (db *Database) AddCert(ctx context.Context, certificate *x509.Certificate, revocationTime time.Time) error { + item, err := attributevalue.MarshalMap(CertMetadata{ + CertKey: NewCertKey(certificate.SerialNumber), + RevocationTime: revocationTime, + }) + if err != nil { + return err + } + + _, err = db.Dynamo.PutItem(ctx, &dynamodb.PutItemInput{ + Item: item, + TableName: &db.Table, + }) + if err != nil { + return err + } + + return nil +} + +// GetAllCerts returns all the certificates in the DynamoDB. This set is +// intended to be much smaller than the set of certificates in a CRL, so it's +// more efficient to just load the entire set instead of conditional querying. +// The map key is the serial's CertKey.SerialString. +// TODO: This could be more efficient if it was a query over issuer or shard +// TODO: However, the dataset is small enough to not matter much. +func (db *Database) GetAllCerts(ctx context.Context) (map[string]CertMetadata, error) { + resp, err := db.Dynamo.Scan(ctx, &dynamodb.ScanInput{ + TableName: &db.Table, + Select: types.SelectAllAttributes, + }) + if err != nil { + return nil, err + } + + var certList []CertMetadata + err = attributevalue.UnmarshalListOfMaps(resp.Items, &certList) + if err != nil { + return nil, err + } + + certs := make(map[string]CertMetadata, len(certList)) + for _, cert := range certList { + certs[cert.CertKey.SerialString()] = cert + } + return certs, nil +} + +// DeleteSerials takes a list of serials that we've seen in the CRL and thus +// no longer need to keep an eye out for. +func (db *Database) DeleteSerials(ctx context.Context, serialNumbers [][]byte) error { + var deletes []types.WriteRequest + for _, serial := range serialNumbers { + key, err := attributevalue.MarshalMap(CertKey{SerialNumber: serial}) + if err != nil { + return err + } + deletes = append(deletes, types.WriteRequest{ + DeleteRequest: &types.DeleteRequest{ + Key: key, + }, + }) + } + + _, err := db.Dynamo.BatchWriteItem(ctx, &dynamodb.BatchWriteItemInput{ + RequestItems: map[string][]types.WriteRequest{db.Table: deletes}, + }) + if err != nil { + return err + } + return nil +} diff --git a/db/db_test.go b/db/db_test.go new file mode 100644 index 0000000..34df34e --- /dev/null +++ b/db/db_test.go @@ -0,0 +1,71 @@ +package db_test + +import ( + "bytes" + "context" + "crypto/x509" + "math/big" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/letsencrypt/crl-monitor/db" + "github.com/letsencrypt/crl-monitor/db/mock" +) + +func TestDatabaseWithMock(t *testing.T) { + smoketest(t, mock.NewMockedDB(t)) +} + +// smoketest goes through a set of basic actions ensuring the basics work +// It gets run with a mocked database and can also be integration tested against +// the real DynamoDB, or the downloadable version, to ensure they align. +func smoketest(t *testing.T, handle *db.Database) { + ctx := context.Background() + + ts1 := time.Now() + ts2 := time.Now().Add(100 * time.Hour) + + int111 := big.NewInt(111) + int4s := big.NewInt(444444) + int60s := big.NewInt(606060) + int123 := big.NewInt(123456) + + // Insert 4 entries into the database with different serials and revocation times + require.NoError(t, handle.AddCert(ctx, &x509.Certificate{SerialNumber: int111}, ts1)) + require.NoError(t, handle.AddCert(ctx, &x509.Certificate{SerialNumber: int4s}, ts1)) + require.NoError(t, handle.AddCert(ctx, &x509.Certificate{SerialNumber: int60s}, ts2)) + require.NoError(t, handle.AddCert(ctx, &x509.Certificate{SerialNumber: int123}, ts2)) + + // Timestamps stored in Dynamo as unix timestamps are truncated to second precision + ts1 = ts1.Truncate(time.Second) + ts2 = ts2.Truncate(time.Second) + + certs, err := handle.GetAllCerts(ctx) + require.NoError(t, err) + require.Len(t, certs, 4) + require.Equal(t, certs, map[string]db.CertMetadata{ + "00000000000000000000000000000000006f": {CertKey: db.CertKey{SerialNumber: int111.Bytes()}, RevocationTime: ts1}, + "00000000000000000000000000000006c81c": {CertKey: db.CertKey{SerialNumber: int4s.Bytes()}, RevocationTime: ts1}, + "000000000000000000000000000000093f6c": {CertKey: db.CertKey{SerialNumber: int60s.Bytes()}, RevocationTime: ts2}, + "00000000000000000000000000000001e240": {CertKey: db.CertKey{SerialNumber: int123.Bytes()}, RevocationTime: ts2}, + }) + + // Delete all the serials other than the 606060 serial + var serials [][]byte + for _, cert := range certs { + if !bytes.Equal(cert.SerialNumber, int60s.Bytes()) { + serials = append(serials, cert.SerialNumber) + } + } + require.NoError(t, handle.DeleteSerials(ctx, serials)) + + // The only remaining entry should be the serial 606060 one + remaining, err := handle.GetAllCerts(ctx) + require.NoError(t, err) + expected := map[string]db.CertMetadata{ + "000000000000000000000000000000093f6c": {CertKey: db.CertKey{SerialNumber: int60s.Bytes()}, RevocationTime: ts2}, + } + require.Equal(t, expected, remaining) +} diff --git a/db/integration_test.go b/db/integration_test.go new file mode 100644 index 0000000..d82e31d --- /dev/null +++ b/db/integration_test.go @@ -0,0 +1,65 @@ +//go:build integration + +package db_test + +import ( + "context" + "fmt" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/stretchr/testify/require" + + "github.com/letsencrypt/crl-monitor/db" +) + +func localResolver(service, region string, opts ...interface{}) (aws.Endpoint, error) { + if service != dynamodb.ServiceID { + return aws.Endpoint{}, fmt.Errorf("unsupported service %s", service) + } + return aws.Endpoint{ + PartitionID: "aws", + URL: "http://localhost:8000/", + }, nil +} + +// makeTable sets up the table in the integration test DB. +// In the real Dynamo, we provision the table with Terraform +func makeTable(t *testing.T, handle *db.Database) { + _, err := handle.Dynamo.(*dynamodb.Client).CreateTable(context.Background(), &dynamodb.CreateTableInput{ + AttributeDefinitions: []types.AttributeDefinition{{ + AttributeName: aws.String("SN"), + AttributeType: types.ScalarAttributeTypeB, + }}, + KeySchema: []types.KeySchemaElement{{ + AttributeName: aws.String("SN"), + KeyType: types.KeyTypeHash, + }}, + TableName: aws.String(handle.Table), + ProvisionedThroughput: &types.ProvisionedThroughput{ + ReadCapacityUnits: aws.Int64(10), + WriteCapacityUnits: aws.Int64(10), + }, + }) + require.NoError(t, err) +} + +// TestIntegrationDynamoDB runs smoketest against a local DynamoDB. The main +// goal of this test is to ensure that the in-process mock behaves similarly to +// the real dynamoDB, which is why we run the same smoketest against both. +// That means developers don't need to always be running the local DynamoDB to +// run most tests outside the db package. +func TestIntegrationDynamoDB(t *testing.T) { + cfg := aws.NewConfig() + cfg.EndpointResolverWithOptions = aws.EndpointResolverWithOptionsFunc(localResolver) + cfg.Credentials = aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) { + return aws.Credentials{AccessKeyID: "Bogus", SecretAccessKey: "Bogus"}, nil + }) + handle, err := db.New(cfg) + require.NoError(t, err) + + makeTable(t, handle) + smoketest(t, handle) +} diff --git a/db/mock/mock.go b/db/mock/mock.go new file mode 100644 index 0000000..63c489e --- /dev/null +++ b/db/mock/mock.go @@ -0,0 +1,73 @@ +package mock + +import ( + "bytes" + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/stretchr/testify/require" + + "github.com/letsencrypt/crl-monitor/db" +) + +const Table = "table" + +// NewMockedDB returns an in-memory Database using a mocked DynamoDB +// It is meant only for use in tests. +func NewMockedDB(t *testing.T) *db.Database { + return &db.Database{ + Table: Table, + Dynamo: &dynamoMock{t: t}, + } +} + +type dynamoMock struct { + t *testing.T + + data []map[string]types.AttributeValue +} + +func has(key map[string]types.AttributeValue, item map[string]types.AttributeValue) bool { + for k, v := range key { + if !bytes.Equal(item[k].(*types.AttributeValueMemberB).Value, v.(*types.AttributeValueMemberB).Value) { + return false + } + } + return true +} + +func (d *dynamoMock) BatchWriteItem(ctx context.Context, input *dynamodb.BatchWriteItemInput, opts ...func(*dynamodb.Options)) (*dynamodb.BatchWriteItemOutput, error) { + require.Empty(d.t, opts, "Options not supported") + require.NotNil(d.t, input) + + for _, item := range input.RequestItems[Table] { + require.Nil(d.t, item.PutRequest, "Only delete requests supported") + require.NotNil(d.t, item.DeleteRequest) + + key := item.DeleteRequest.Key + + var filtered []map[string]types.AttributeValue + for _, i := range d.data { + if !has(key, i) { + filtered = append(filtered, i) + } + } + d.data = filtered + } + return &dynamodb.BatchWriteItemOutput{}, nil +} + +func (d *dynamoMock) PutItem(ctx context.Context, input *dynamodb.PutItemInput, opts ...func(*dynamodb.Options)) (*dynamodb.PutItemOutput, error) { + require.Empty(d.t, opts, "Options not supported") + require.NotNil(d.t, input) + d.data = append(d.data, input.Item) + return &dynamodb.PutItemOutput{}, nil +} + +func (d *dynamoMock) Scan(ctx context.Context, input *dynamodb.ScanInput, opts ...func(*dynamodb.Options)) (*dynamodb.ScanOutput, error) { + require.Empty(d.t, opts, "Options not supported") + require.NotNil(d.t, input) + return &dynamodb.ScanOutput{Items: d.data}, nil +} diff --git a/db/run_db_integration_test.sh b/db/run_db_integration_test.sh new file mode 100755 index 0000000..bd483e8 --- /dev/null +++ b/db/run_db_integration_test.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -eux + +SCRIPT_PATH=${0%/*} +cd "$SCRIPT_PATH" + +# Fetch the local DynamoDB if there isn't one here already +if ! [ -d dynamodb_local ]; then + mkdir dynamodb_local + curl -sSL https://s3.us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.tar.gz \ + | tar -xzf - -C dynamodb_local +else + echo "using existing DynamoDBLocal.jar" +fi + +java -Djava.library.path=./dynamodb_local/DynamoDBLocal_lib -jar ./dynamodb_local/DynamoDBLocal.jar -inMemory & +dynamopid=$! +trap 'kill $dynamopid' EXIT + +sleep 1 # Let dynamodb start + +go test -tags integration . diff --git a/go.mod b/go.mod index 86c27d9..fdc3780 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,25 @@ module github.com/letsencrypt/crl-monitor go 1.19 -require github.com/aws/aws-lambda-go v1.34.1 +require ( + github.com/aws/aws-lambda-go v1.34.1 + github.com/aws/aws-sdk-go-v2 v1.16.16 + github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1 + github.com/stretchr/testify v1.8.0 +) + +require ( + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 // indirect + github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17 // indirect + github.com/aws/smithy-go v1.13.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index 9cb9a00..aaf379a 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,45 @@ github.com/aws/aws-lambda-go v1.34.1 h1:M3a/uFYBjii+tDcOJ0wL/WyFi2550FHoECdPf27zvOs= github.com/aws/aws-lambda-go v1.34.1/go.mod h1:jwFe2KmMsHmffA1X2R09hH6lFzJQxzI8qK17ewzbQMM= +github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk= +github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0 h1:bKbdstt7+PzIRSIXZ11Yo8Qh8t0AHn6jEYUfsbVcLjE= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.0/go.mod h1:+CBJZMhsb1pTUcB/NTdS505bDX10xS4xnPMqDZj2Ptw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1 h1:1QpTkQIAaZpR387it1L+erjB5bStGFCJRvmXsodpPEU= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.1/go.mod h1:BZhn/C3z13ULTSstVi2Kymc62bgjFh/JwLO9Tm2OFYI= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20 h1:V9q4A0qnUfDsfivspY1LQRQTOG3Y9FLHvXIaTbcU7XM= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.20/go.mod h1:7qWU48SMzlrfOlNhHpazW3psFWlOIWrq4SmOr2/ESmk= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 h1:Lh1AShsuIJTwMkoxVCAYPJgNG5H+eN6SmoUn8nOZ5wE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17 h1:o0Ia3nb56m8+8NvhbCDiSBiZRNUwIknVWobx5vks0Vk= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.17/go.mod h1:WJD9FbkwzM2a1bZ36ntH6+5Jc+x41Q4K2AcLeHDLAS8= +github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= +github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=