package sa import ( "encoding/base64" "encoding/json" "errors" "fmt" "math" "net" "strconv" "strings" "time" jose "gopkg.in/square/go-jose.v2" "github.com/letsencrypt/boulder/core" corepb "github.com/letsencrypt/boulder/core/proto" "github.com/letsencrypt/boulder/db" "github.com/letsencrypt/boulder/grpc" "github.com/letsencrypt/boulder/probs" "github.com/letsencrypt/boulder/revocation" ) // errBadJSON is an error type returned when a json.Unmarshal performed by the // SA fails. It includes both the Unmarshal error and the original JSON data in // its error message to make it easier to track down the bad JSON data. type errBadJSON struct { msg string json []byte err error } // Error returns an error message that includes the json.Unmarshal error as well // as the bad JSON data. func (e errBadJSON) Error() string { return fmt.Sprintf( "%s: error unmarshaling JSON %q: %s", e.msg, string(e.json), e.err) } // badJSONError is a convenience function for constructing a errBadJSON instance // with the provided args. func badJSONError(msg string, jsonData []byte, err error) error { return errBadJSON{ msg: msg, json: jsonData, err: err, } } const regFields = "id, jwk, jwk_sha256, contact, agreement, initialIP, createdAt, LockCol, status" // selectRegistration selects all fields of one registration model func selectRegistration(s db.OneSelector, q string, args ...interface{}) (*regModel, error) { var model regModel err := s.SelectOne( &model, "SELECT "+regFields+" FROM registrations "+q, args..., ) return &model, err } const certFields = "registrationID, serial, digest, der, issued, expires" // SelectCertificate selects all fields of one certificate object func SelectCertificate(s db.OneSelector, q string, args ...interface{}) (core.Certificate, error) { var model core.Certificate err := s.SelectOne( &model, "SELECT "+certFields+" FROM certificates "+q, args..., ) return model, err } const precertFields = "registrationID, serial, der, issued, expires" // SelectPrecertificate selects all fields of one precertificate object // identified by serial. func SelectPrecertificate(s db.OneSelector, serial string) (core.Certificate, error) { var model precertificateModel err := s.SelectOne( &model, "SELECT "+precertFields+" FROM precertificates WHERE serial = ?", serial) return core.Certificate{ RegistrationID: model.RegistrationID, Serial: model.Serial, DER: model.DER, Issued: model.Issued, Expires: model.Expires, }, err } type CertWithID struct { ID int64 core.Certificate } // SelectCertificates selects all fields of multiple certificate objects func SelectCertificates(s db.Selector, q string, args map[string]interface{}) ([]CertWithID, error) { var models []CertWithID _, err := s.Select( &models, "SELECT id, "+certFields+" FROM certificates "+q, args) return models, err } func certStatusFields() []string { return []string{"serial", "status", "ocspLastUpdated", "revokedDate", "revokedReason", "lastExpirationNagSent", "ocspResponse", "notAfter", "isExpired", "issuerID"} } func certStatusFieldsSelect(restOfQuery string) string { fields := strings.Join(certStatusFields(), ",") return fmt.Sprintf("SELECT %s FROM certificateStatus %s", fields, restOfQuery) } // SelectCertificateStatus selects all fields of one certificate status model func SelectCertificateStatus(s db.OneSelector, q string, args ...interface{}) (certStatusModel, error) { var model certStatusModel err := s.SelectOne( &model, certStatusFieldsSelect(q), args..., ) return model, err } // SelectCertificateStatuses selects all fields of multiple certificate status // objects func SelectCertificateStatuses(s db.Selector, q string, args ...interface{}) ([]core.CertificateStatus, error) { var models []core.CertificateStatus _, err := s.Select( &models, certStatusFieldsSelect(q), args..., ) return models, err } var mediumBlobSize = int(math.Pow(2, 24)) type issuedNameModel struct { ID int64 `db:"id"` ReversedName string `db:"reversedName"` NotBefore time.Time `db:"notBefore"` Serial string `db:"serial"` } // regModel is the description of a core.Registration in the database before type regModel struct { ID int64 `db:"id"` Key []byte `db:"jwk"` KeySHA256 string `db:"jwk_sha256"` Contact []string `db:"contact"` Agreement string `db:"agreement"` // InitialIP is stored as sixteen binary bytes, regardless of whether it // represents a v4 or v6 IP address. InitialIP []byte `db:"initialIp"` CreatedAt time.Time `db:"createdAt"` LockCol int64 Status string `db:"status"` } type certStatusModel struct { Serial string `db:"serial"` Status core.OCSPStatus `db:"status"` OCSPLastUpdated time.Time `db:"ocspLastUpdated"` RevokedDate time.Time `db:"revokedDate"` RevokedReason revocation.Reason `db:"revokedReason"` LastExpirationNagSent time.Time `db:"lastExpirationNagSent"` OCSPResponse []byte `db:"ocspResponse"` NotAfter time.Time `db:"notAfter"` IsExpired bool `db:"isExpired"` IssuerID *int64 `db:"issuerID"` } // challModel is the description of a core.Challenge in the database // // The Validation field is a stub; the column is only there for backward compatibility. type challModel struct { ID int64 `db:"id"` AuthorizationID string `db:"authorizationID"` Type string `db:"type"` Status core.AcmeStatus `db:"status"` Error []byte `db:"error"` Token string `db:"token"` KeyAuthorization string `db:"keyAuthorization"` ValidationRecord []byte `db:"validationRecord"` // TODO(#1818): Remove, this field is unused, but is kept temporarily to avoid a database migration. Validated bool `db:"validated"` LockCol int64 } // newReg creates a reg model object from a core.Registration func registrationToModel(r *core.Registration) (*regModel, error) { key, err := json.Marshal(r.Key) if err != nil { return nil, err } sha, err := core.KeyDigestB64(r.Key) if err != nil { return nil, err } if r.InitialIP == nil { return nil, fmt.Errorf("initialIP was nil") } if r.Contact == nil { r.Contact = &[]string{} } rm := regModel{ ID: r.ID, Key: key, KeySHA256: sha, Contact: *r.Contact, Agreement: r.Agreement, InitialIP: []byte(r.InitialIP.To16()), CreatedAt: r.CreatedAt, Status: string(r.Status), } return &rm, nil } func modelToRegistration(reg *regModel) (core.Registration, error) { k := &jose.JSONWebKey{} err := json.Unmarshal(reg.Key, k) if err != nil { return core.Registration{}, badJSONError( "failed to unmarshal registration model's key", reg.Key, err) } var contact *[]string // Contact can be nil when the DB contains the literal string "null". We // prefer to represent this in memory as a pointer to an empty slice rather // than a nil pointer. if reg.Contact == nil { contact = &[]string{} } else { contact = ®.Contact } r := core.Registration{ ID: reg.ID, Key: k, Contact: contact, Agreement: reg.Agreement, InitialIP: net.IP(reg.InitialIP), CreatedAt: reg.CreatedAt, Status: core.AcmeStatus(reg.Status), } return r, nil } func modelToChallenge(cm *challModel) (core.Challenge, error) { c := core.Challenge{ Type: cm.Type, Status: cm.Status, Token: cm.Token, ProvidedKeyAuthorization: cm.KeyAuthorization, } if len(cm.Error) > 0 { var problem probs.ProblemDetails err := json.Unmarshal(cm.Error, &problem) if err != nil { return core.Challenge{}, badJSONError( "failed to unmarshal challenge model's error", cm.Error, err) } c.Error = &problem } if len(cm.ValidationRecord) > 0 { var vr []core.ValidationRecord err := json.Unmarshal(cm.ValidationRecord, &vr) if err != nil { return core.Challenge{}, badJSONError( "failed to unmarshal challenge model's validation record", cm.ValidationRecord, err) } c.ValidationRecord = vr } return c, nil } type recordedSerialModel struct { ID int64 Serial string RegistrationID int64 Created time.Time Expires time.Time } type precertificateModel struct { ID int64 Serial string RegistrationID int64 DER []byte Issued time.Time Expires time.Time } type orderModel struct { ID int64 RegistrationID int64 Expires time.Time Created time.Time Error []byte CertificateSerial string BeganProcessing bool } type requestedNameModel struct { ID int64 OrderID int64 ReversedName string } type orderToAuthzModel struct { OrderID int64 AuthzID int64 } func orderToModel(order *corepb.Order) (*orderModel, error) { om := &orderModel{ ID: *order.Id, RegistrationID: *order.RegistrationID, Expires: time.Unix(0, *order.Expires), Created: time.Unix(0, *order.Created), BeganProcessing: *order.BeganProcessing, } if order.CertificateSerial != nil { om.CertificateSerial = *order.CertificateSerial } if order.Error != nil { errJSON, err := json.Marshal(order.Error) if err != nil { return nil, err } if len(errJSON) > mediumBlobSize { return nil, fmt.Errorf("Error object is too large to store in the database") } om.Error = errJSON } return om, nil } func modelToOrder(om *orderModel) (*corepb.Order, error) { expires := om.Expires.UnixNano() created := om.Created.UnixNano() order := &corepb.Order{ Id: &om.ID, RegistrationID: &om.RegistrationID, Expires: &expires, Created: &created, CertificateSerial: &om.CertificateSerial, BeganProcessing: &om.BeganProcessing, } if len(om.Error) > 0 { var problem corepb.ProblemDetails err := json.Unmarshal(om.Error, &problem) if err != nil { return &corepb.Order{}, badJSONError( "failed to unmarshal order model's error", om.Error, err) } order.Error = &problem } return order, nil } var challTypeToUint = map[string]uint8{ "http-01": 0, "dns-01": 1, "tls-alpn-01": 2, } var uintToChallType = map[uint8]string{ 0: "http-01", 1: "dns-01", 2: "tls-alpn-01", } var identifierTypeToUint = map[string]uint8{ "dns": 0, } var uintToIdentifierType = map[uint8]string{ 0: "dns", } var statusToUint = map[string]uint8{ "pending": 0, "valid": 1, "invalid": 2, "deactivated": 3, "revoked": 4, } var uintToStatus = map[uint8]string{ 0: "pending", 1: "valid", 2: "invalid", 3: "deactivated", 4: "revoked", } func statusUint(status core.AcmeStatus) uint8 { return statusToUint[string(status)] } const authzFields = "id, identifierType, identifierValue, registrationID, status, expires, challenges, attempted, token, validationError, validationRecord" type authzModel struct { ID int64 `db:"id"` IdentifierType uint8 `db:"identifierType"` IdentifierValue string `db:"identifierValue"` RegistrationID int64 `db:"registrationID"` Status uint8 `db:"status"` Expires time.Time `db:"expires"` Challenges uint8 `db:"challenges"` Attempted *uint8 `db:"attempted"` Token []byte `db:"token"` ValidationError []byte `db:"validationError"` ValidationRecord []byte `db:"validationRecord"` } // hasMultipleNonPendingChallenges checks if a slice of challenges contains // more than one non-pending challenge func hasMultipleNonPendingChallenges(challenges []*corepb.Challenge) bool { nonPending := false for _, c := range challenges { if *c.Status == string(core.StatusValid) || *c.Status == string(core.StatusInvalid) { if !nonPending { nonPending = true } else { return true } } } return false } // authzPBToModel converts a protobuf authorization representation to the // authzModel storage representation. func authzPBToModel(authz *corepb.Authorization) (*authzModel, error) { expires := time.Unix(0, *authz.Expires).UTC() am := &authzModel{ IdentifierValue: *authz.Identifier, RegistrationID: *authz.RegistrationID, Status: statusToUint[*authz.Status], Expires: expires, } if authz.Id != nil && *authz.Id != "" { // The v1 internal authorization objects use a string for the ID, the v2 // storage format uses a integer ID. In order to maintain compatibility we // convert the integer ID to a string. id, err := strconv.Atoi(*authz.Id) if err != nil { return nil, err } am.ID = int64(id) } if hasMultipleNonPendingChallenges(authz.Challenges) { return nil, errors.New("multiple challenges are non-pending") } // In the v2 authorization style we don't store individual challenges with their own // token, validation errors/records, etc. Instead we store a single token/error/record // set, a bitmap of available challenge types, and a row indicating which challenge type // was 'attempted'. // // Since we don't currently have the singular token/error/record set abstracted out to // the core authorization type yet we need to extract these from the challenges array. // We assume that the token in each challenge is the same and that if any of the challenges // has a non-pending status that it should be considered the 'attempted' challenge and // we extract the error/record set from that particular challenge. var tokenStr string for _, chall := range authz.Challenges { // Set the challenge type bit in the bitmap am.Challenges |= 1 << challTypeToUint[*chall.Type] tokenStr = *chall.Token // If the challenge status is not core.StatusPending we assume it was the 'attempted' // challenge and extract the relevant fields we need. if *chall.Status == string(core.StatusValid) || *chall.Status == string(core.StatusInvalid) { attemptedType := challTypeToUint[*chall.Type] am.Attempted = &attemptedType // Marshal corepb.ValidationRecords to core.ValidationRecords so that we // can marshal them to JSON. records := make([]core.ValidationRecord, len(chall.Validationrecords)) for i, recordPB := range chall.Validationrecords { var err error records[i], err = grpc.PBToValidationRecord(recordPB) if err != nil { return nil, err } } var err error am.ValidationRecord, err = json.Marshal(records) if err != nil { return nil, err } // If there is a error associated with the challenge marshal it to JSON // so that we can store it in the database. if chall.Error != nil { prob, err := grpc.PBToProblemDetails(chall.Error) if err != nil { return nil, err } am.ValidationError, err = json.Marshal(prob) if err != nil { return nil, err } } } token, err := base64.RawURLEncoding.DecodeString(tokenStr) if err != nil { return nil, err } am.Token = token } return am, nil } // populateAttemptedFields takes a challenge and populates it with the validation fields status, // validation records, and error (the latter only if the validation failed) from a authzModel. func populateAttemptedFields(am authzModel, challenge *corepb.Challenge) error { if len(am.ValidationError) != 0 { // If the error is non-empty the challenge must be invalid. status := string(core.StatusInvalid) challenge.Status = &status var prob probs.ProblemDetails err := json.Unmarshal(am.ValidationError, &prob) if err != nil { return badJSONError( "failed to unmarshal authz2 model's validation error", am.ValidationError, err) } challenge.Error, err = grpc.ProblemDetailsToPB(&prob) if err != nil { return err } } else { // If the error is empty the challenge must be valid. status := string(core.StatusValid) challenge.Status = &status } var records []core.ValidationRecord err := json.Unmarshal(am.ValidationRecord, &records) if err != nil { return badJSONError( "failed to unmarshal authz2 model's validation record", am.ValidationRecord, err) } challenge.Validationrecords = make([]*corepb.ValidationRecord, len(records)) for i, r := range records { challenge.Validationrecords[i], err = grpc.ValidationRecordToPB(r) if err != nil { return err } } return nil } func modelToAuthzPB(am authzModel) (*corepb.Authorization, error) { expires := am.Expires.UTC().UnixNano() id := fmt.Sprintf("%d", am.ID) status := uintToStatus[am.Status] pb := &corepb.Authorization{ Id: &id, Status: &status, Identifier: &am.IdentifierValue, RegistrationID: &am.RegistrationID, Expires: &expires, } // Populate authorization challenge array. We do this by iterating through // the challenge type bitmap and creating a challenge of each type if its // bit is set. Each of these challenges has the token from the authorization // model and has its status set to core.StatusPending by default. If the // challenge type is equal to that in the 'attempted' row we set the status // to core.StatusValid or core.StatusInvalid depending on if there is anything // in ValidationError and populate the ValidationRecord and ValidationError // fields. for pos := uint8(0); pos < 8; pos++ { if (am.Challenges>>pos)&1 == 1 { challType := uintToChallType[pos] status := string(core.StatusPending) token := base64.RawURLEncoding.EncodeToString(am.Token) challenge := &corepb.Challenge{ Type: &challType, Status: &status, Token: &token, } // If the challenge type matches the attempted type it must be either // valid or invalid and we need to populate extra fields. // Also, once any challenge has been attempted, we consider the other // challenges "gone" per https://tools.ietf.org/html/rfc8555#section-7.1.4 if am.Attempted != nil { if uintToChallType[*am.Attempted] == challType { if err := populateAttemptedFields(am, challenge); err != nil { return nil, err } pb.Challenges = append(pb.Challenges, challenge) } } else { // When no challenge has been attempted yet, all challenges are still // present. pb.Challenges = append(pb.Challenges, challenge) } } } return pb, nil } type keyHashModel struct { ID int64 KeyHash []byte CertNotAfter time.Time CertSerial string } var stringToSourceInt = map[string]int{ "API": 1, "admin-revoker": 2, }