Remove nameset-backfill tool. (#1733)
This commit is contained in:
Jacob Hoffman-Andrews
Roland Bracewell Shoemaker
parent
8c8fc01b01
commit
ed1018db6a
|
|
@ -1,182 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"flag"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/letsencrypt/boulder/cmd"
|
||||
"github.com/letsencrypt/boulder/core"
|
||||
blog "github.com/letsencrypt/boulder/log"
|
||||
"github.com/letsencrypt/boulder/rpc"
|
||||
"github.com/letsencrypt/boulder/sa"
|
||||
|
||||
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
|
||||
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
|
||||
"github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1"
|
||||
)
|
||||
|
||||
type resultHolder struct {
|
||||
Serial string
|
||||
Issued time.Time
|
||||
Expires time.Time
|
||||
DER []byte
|
||||
}
|
||||
|
||||
type backfiller struct {
|
||||
sa core.StorageAuthority
|
||||
dbMap *gorp.DbMap
|
||||
stats statsd.Statter
|
||||
log blog.Logger
|
||||
clk clock.Clock
|
||||
}
|
||||
|
||||
func new(amqpConf *cmd.AMQPConfig, syslogConf cmd.SyslogConfig, statsdURI, dbURI string) (*backfiller, error) {
|
||||
var stats statsd.Statter
|
||||
var err error
|
||||
stats, log := cmd.StatsAndLogging(cmd.StatsdConfig{Server: statsdURI, Prefix: "Boulder"}, syslogConf)
|
||||
sac, err := rpc.NewStorageAuthorityClient("nameset-backfiller", amqpConf, stats)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dbMap, err := sa.NewDbMap(dbURI)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &backfiller{sac, dbMap, stats, log, clock.Default()}, nil
|
||||
}
|
||||
|
||||
func (b *backfiller) run() error {
|
||||
added := 0
|
||||
for {
|
||||
results, err := b.findCerts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(results) == 0 {
|
||||
break
|
||||
}
|
||||
err = b.processResults(results)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
added += len(results)
|
||||
}
|
||||
b.log.Info(fmt.Sprintf("Added %d missing certificate name sets to the fqdnSets table", added))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *backfiller) findCerts() ([]resultHolder, error) {
|
||||
var allResults []resultHolder
|
||||
lastSerial := ""
|
||||
for {
|
||||
var results []resultHolder
|
||||
_, err := b.dbMap.Select(
|
||||
&results,
|
||||
`SELECT c.serial, c.issued, c.expires, c.der FROM certificates AS c
|
||||
LEFT JOIN fqdnSets AS ns ON c.serial=ns.serial
|
||||
WHERE ns.serial IS NULL
|
||||
AND c.serial > ?
|
||||
ORDER BY c.serial ASC
|
||||
LIMIT ?`,
|
||||
lastSerial,
|
||||
1000,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(results) == 0 {
|
||||
break
|
||||
}
|
||||
b.stats.Inc("db-backfill.fqdnSets.missing-found", int64(len(results)), 1.0)
|
||||
allResults = append(allResults, results...)
|
||||
lastSerial = allResults[len(allResults)-1].Serial
|
||||
}
|
||||
return allResults, nil
|
||||
}
|
||||
|
||||
func hashNames(names []string) []byte {
|
||||
names = core.UniqueLowerNames(names)
|
||||
hash := sha256.Sum256([]byte(strings.Join(names, ",")))
|
||||
return hash[:]
|
||||
}
|
||||
|
||||
func (b *backfiller) processResults(results []resultHolder) error {
|
||||
numResults := len(results)
|
||||
added := 0
|
||||
for _, r := range results {
|
||||
c, err := x509.ParseCertificate(r.DER)
|
||||
if err != nil {
|
||||
b.log.Err(fmt.Sprintf("Failed to parse certificate [serial: %s] retrieved from database: %s", r.Serial, err))
|
||||
continue
|
||||
}
|
||||
err = b.dbMap.Insert(&core.FQDNSet{
|
||||
SetHash: hashNames(c.DNSNames),
|
||||
Serial: r.Serial,
|
||||
Issued: r.Issued,
|
||||
Expires: r.Expires,
|
||||
})
|
||||
if err != nil {
|
||||
b.log.Err(fmt.Sprintf("Failed to add name set for %s to database: %s", r.Serial, err))
|
||||
continue
|
||||
}
|
||||
added++
|
||||
b.stats.Inc("db-backfill.fqdnSets.added", 1, 1.0)
|
||||
}
|
||||
if added < numResults {
|
||||
return fmt.Errorf("Didn't add all name sets, %d out of %d failed", numResults-added, numResults)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
amqpURI := flag.String("amqpURI", "", "AMQP connection URI")
|
||||
amqpURIFile := flag.String("amqpURIFile", "", "File to read AMQP connection URI from")
|
||||
amqpCert := flag.String("amqpCert", "", "AMQP client certificate to use")
|
||||
amqpKey := flag.String("amqpKey", "", "Key for AMQP client certificate")
|
||||
amqpCA := flag.String("amqpCA", "", "Root CA to trust for AMQP connections")
|
||||
|
||||
statsdURI := flag.String("statsdURI", "", "StatsD URI")
|
||||
|
||||
dbConnect := flag.String("dbConnect", "", "DB connection URI")
|
||||
dbConnectFile := flag.String("dbConnectFile", "", "File to read DB connection URI from")
|
||||
|
||||
syslogNet := flag.String("syslogNetwork", "", "Syslog network")
|
||||
syslogURI := flag.String("syslogServer", "", "Syslog URI")
|
||||
syslogLevel := flag.Int("syslogLevel", 7, "Level at which to log")
|
||||
flag.Parse()
|
||||
|
||||
dbConf := cmd.DBConfig{DBConnect: *dbConnect, DBConnectFile: *dbConnectFile}
|
||||
dbURI, err := dbConf.URL()
|
||||
|
||||
amqpConf := &cmd.AMQPConfig{
|
||||
Server: *amqpURI,
|
||||
ServerURLFile: *amqpURIFile,
|
||||
SA: &cmd.RPCServerConfig{
|
||||
Server: "SA.server",
|
||||
RPCTimeout: cmd.ConfigDuration{Duration: time.Second * 15},
|
||||
},
|
||||
}
|
||||
if *amqpCert != "" && *amqpKey != "" && *amqpCA != "" {
|
||||
amqpConf.TLS = &cmd.TLSConfig{CertFile: amqpCert, KeyFile: amqpKey, CACertFile: amqpCA}
|
||||
} else {
|
||||
amqpConf.Insecure = true
|
||||
}
|
||||
cmd.FailOnError(err, "Failed to read db URI")
|
||||
b, err := new(
|
||||
amqpConf,
|
||||
cmd.SyslogConfig{
|
||||
Network: *syslogNet,
|
||||
Server: *syslogURI,
|
||||
StdoutLevel: syslogLevel,
|
||||
},
|
||||
*statsdURI,
|
||||
dbURI,
|
||||
)
|
||||
cmd.FailOnError(err, "Failed to create backfiller")
|
||||
err = b.run()
|
||||
cmd.FailOnError(err, "Failed to backfill fqdnSets table")
|
||||
}
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
|
||||
"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
|
||||
|
||||
"github.com/letsencrypt/boulder/core"
|
||||
blog "github.com/letsencrypt/boulder/log"
|
||||
"github.com/letsencrypt/boulder/sa"
|
||||
"github.com/letsencrypt/boulder/sa/satest"
|
||||
"github.com/letsencrypt/boulder/test"
|
||||
"github.com/letsencrypt/boulder/test/vars"
|
||||
)
|
||||
|
||||
var log = blog.UseMock()
|
||||
|
||||
func TestBackfill(t *testing.T) {
|
||||
stats, _ := statsd.NewNoopClient()
|
||||
|
||||
// Create an SA
|
||||
dbMap, err := sa.NewDbMap(vars.DBConnSA)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create dbMap: %s", err)
|
||||
}
|
||||
fc := clock.NewFake()
|
||||
fc.Add(1 * time.Hour)
|
||||
sa, err := sa.NewSQLStorageAuthority(dbMap, fc, log)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create SA: %s", err)
|
||||
}
|
||||
defer test.ResetSATestDatabase(t)
|
||||
b := backfiller{sa, dbMap, stats, log, fc}
|
||||
|
||||
certDER, err := ioutil.ReadFile("test-cert.der")
|
||||
test.AssertNotError(t, err, "Couldn't read example cert DER")
|
||||
|
||||
reg := satest.CreateWorkingRegistration(t, sa)
|
||||
|
||||
err = dbMap.Insert(&core.Certificate{RegistrationID: reg.ID, DER: certDER, Serial: "serial"})
|
||||
test.AssertNotError(t, err, "Couldn't insert stub certificate")
|
||||
|
||||
results, err := b.findCerts()
|
||||
test.AssertNotError(t, err, "Failed to find missing name sets")
|
||||
test.AssertEquals(t, len(results), 1)
|
||||
test.AssertEquals(t, results[0].Serial, "serial")
|
||||
|
||||
err = b.run()
|
||||
test.AssertNotError(t, err, "Failed to find and add missing name sets")
|
||||
test.AssertEquals(t, len(log.GetAllMatching("Added 1 missing certificate name sets to the fqdnSets table")), 1)
|
||||
|
||||
results, err = b.findCerts()
|
||||
test.AssertNotError(t, err, "Failed to find missing name sets")
|
||||
test.AssertEquals(t, len(results), 0)
|
||||
}
|
||||
Binary file not shown.
Loading…
Reference in New Issue