letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add log validator to integration tests (#4782) 

For now this mainly provides an example config and confirms that
log-validator can start up and shut down cleanly, as well as provide a
stat indicating how many log lines it has handled.

This introduces a syslog config to the boulder-tools image that will write
logs to /var/log/program.log. It also tweaks the various .json config
files so they have non-default syslogLevel, to ensure they actually
write something for log-validator to verify.
This commit is contained in:
Jacob Hoffman-Andrews 2020-04-20 13:33:42 -07:00 committed by GitHub
parent 8259925f7c
commit 87fb6028c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
41 changed files with 114 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/log-validator/main.go
View File

@ -127,11 +127,11 @@ func main() {
cmd.FailOnError(err, "failed to parse config file") cmd.FailOnError(err, "failed to parse config file")
stats, logger := cmd.StatsAndLogging(config.Syslog, config.DebugAddr) stats, logger := cmd.StatsAndLogging(config.Syslog, config.DebugAddr)
badLineCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ lineCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "bad_log_lines", Name: "log_lines",
Help: "A counter of corrupt log lines", Help: "A counter of log lines processed, with status",
}, []string{"filename"}) }, []string{"filename", "status"})
stats.MustRegister(badLineCounter) stats.MustRegister(lineCounter)
var tailers []*tail.Tail var tailers []*tail.Tail
for _, filename := range config.Files { for _, filename := range config.Files {
@ -151,8 +151,10 @@ func main() {
continue continue
} }
if err := lineValid(line.Text); err != nil { if err := lineValid(line.Text); err != nil {
badLineCounter.WithLabelValues(t.Filename).Inc() lineCounter.WithLabelValues(t.Filename, "bad").Inc()
logger.Errf("%s: %s %q", t.Filename, err, line.Text) logger.Errf("%s: %s %q", t.Filename, err, line.Text)
} else {
lineCounter.WithLabelValues(t.Filename, "ok").Inc()
} }
} }
}() }()

6
docker-compose.yml
View File

@ -2,7 +2,7 @@ version: '3'
services: services:
boulder: boulder:
# To minimize fetching this should be the same version used below # To minimize fetching this should be the same version used below
image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08 image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16
environment: environment:
FAKE_DNS: 10.77.77.77 FAKE_DNS: 10.77.77.77
PKCS11_PROXY_SOCKET: tcp://boulder-hsm:5657 PKCS11_PROXY_SOCKET: tcp://boulder-hsm:5657
@ -65,7 +65,7 @@ services:
hard: -1 hard: -1
bhsm: bhsm:
# To minimize fetching this should be the same version used above # To minimize fetching this should be the same version used above
image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08 image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16
environment: environment:
PKCS11_DAEMON_SOCKET: tcp://0.0.0.0:5657 PKCS11_DAEMON_SOCKET: tcp://0.0.0.0:5657
command: /usr/local/bin/pkcs11-daemon /usr/lib/softhsm/libsofthsm2.so command: /usr/local/bin/pkcs11-daemon /usr/lib/softhsm/libsofthsm2.so
@ -92,7 +92,7 @@ services:
logging: logging:
driver: none driver: none
netaccess: netaccess:
image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08 image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16
environment: environment:
GO111MODULE: "on" GO111MODULE: "on"
GOFLAGS: "-mod=vendor" GOFLAGS: "-mod=vendor"

5
test/boulder-tools/Dockerfile.tmpl
View File

@ -9,6 +9,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
WORKDIR $GOPATH WORKDIR $GOPATH
RUN wget -O go.tgz "https://dl.google.com/go/go${GO_VERSION_TO_INSTALL}.linux-amd64.tar.gz" && tar -C /usr/local -xzf go.tgz && rm go.tgz; RUN wget -O go.tgz "https://dl.google.com/go/go${GO_VERSION_TO_INSTALL}.linux-amd64.tar.gz" && tar -C /usr/local -xzf go.tgz && rm go.tgz;
ADD requirements.txt /tmp/requirements.txt COPY requirements.txt /tmp/requirements.txt
ADD build.sh /tmp/build.sh COPY boulder.rsyslog.conf /etc/rsyslog.d/
COPY build.sh /tmp/build.sh
RUN /tmp/build.sh RUN /tmp/build.sh

18
test/boulder-tools/boulder.rsyslog.conf Normal file
View File

@ -0,0 +1,18 @@
module( load="builtin:omfile" template="LELogFormat" )
template( name="LELogFormat" type="list" ) {
property(name="timereported" dateFormat="rfc3339")
constant(value=" ")
property(name="hostname" field.delimiter="46" field.number="1")
constant(value=" datacenter ")
property(name="syslogseverity")
constant(value=" ")
property(name="syslogtag")
property(name="msg" spifno1stsp="on" )
property(name="msg" droplastlf="on" )
constant(value="\n")
}
template( name="TmplAll" type="string" string="/var/log/%PROGRAMNAME%.log" )
action( type="omfile" dynaFile="TmplAll" )

2
test/config-next/admin-revoker.json
View File

@ -21,6 +21,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

4
test/config-next/akamai-purger.json
View File

@ -22,11 +22,11 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {
"issuerCert": "test/test-ca2.pem" "issuerCert": "test/test-ca2.pem"
} }
} }

2
test/config-next/ca-a.json
View File

@ -152,6 +152,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

2
test/config-next/ca-b.json
View File

@ -153,6 +153,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

2
test/config-next/cert-checker.json
View File

@ -18,6 +18,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

2
test/config-next/expiration-mailer.json
View File

@ -27,6 +27,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

19
test/config-next/log-validator.json Normal file
View File

@ -0,0 +1,19 @@
{
"syslog": {
"stdoutLevel": 7
},
"debugAddr": ":8016",
"files": [
"/var/log/akamai-purger.log",
"/var/log/boulder-ca.log",
"/var/log/boulder-publisher.log",
"/var/log/boulder-ra.log",
"/var/log/boulder-sa.log",
"/var/log/boulder-va.log",
"/var/log/boulder-wfe2.log",
"/var/log/boulder-wfe.log",
"/var/log/nonce-service.log",
"/var/log/ocsp-responder.log",
"/var/log/ocsp-updater.log"
]
}

3
test/config-next/nonce.json
View File

@ -3,7 +3,8 @@
"maxUsed": 131072, "maxUsed": 131072,
"noncePrefix": "taro", "noncePrefix": "taro",
"syslog": { "syslog": {
"stdoutLevel": 6 "stdoutLevel": 6,
"syslogLevel": 6
}, },
"debugAddr": ":8111", "debugAddr": ":8111",
"grpc": { "grpc": {

6
test/config-next/notify-mailer.json
View File

@ -6,5 +6,9 @@
"passwordFile": "test/secrets/smtp_password", "passwordFile": "test/secrets/smtp_password",
"dbConnectFile": "test/secrets/mailer_dburl", "dbConnectFile": "test/secrets/mailer_dburl",
"maxDBConns": 10 "maxDBConns": 10
} },
"syslog": {
"stdoutLevel": 7,
"syslogLevel": 7
}
} }

2
test/config-next/ocsp-responder.json
View File

@ -13,7 +13,7 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

2
test/config-next/ocsp-updater.json
View File

@ -33,7 +33,7 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

1
test/config-next/orphan-finder.json
View File

@ -2,6 +2,7 @@
"backdate": "1h", "backdate": "1h",
"syslog": { "syslog": {
"stdoutlevel": 7,
"stdoutlevel": 7 "stdoutlevel": 7
}, },

2
test/config-next/publisher.json
View File

@ -23,7 +23,7 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

2
test/config-next/ra.json
View File

@ -120,6 +120,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

2
test/config-next/sa.json
View File

@ -31,6 +31,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

2
test/config-next/va.json
View File

@ -50,7 +50,7 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

2
test/config-next/wfe.json
View File

@ -46,7 +46,7 @@
"syslog": { "syslog": {
"stdoutlevel": 4, "stdoutlevel": 4,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

2
test/config-next/wfe2.json
View File

@ -60,7 +60,7 @@
"syslog": { "syslog": {
"stdoutlevel": 4, "stdoutlevel": 4,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

2
test/config/admin-revoker.json
View File

@ -21,6 +21,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

4
test/config/akamai-purger.json
View File

@ -22,11 +22,11 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {
"issuerCert": "test/test-ca2.pem" "issuerCert": "test/test-ca2.pem"
} }
} }

2
test/config/ca-a.json
View File

@ -152,6 +152,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

2
test/config/ca-b.json
View File

@ -153,6 +153,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

2
test/config/cert-checker.json
View File

@ -18,6 +18,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

2
test/config/expiration-mailer.json
View File

@ -27,6 +27,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

19
test/config/log-validator.json Normal file
View File

@ -0,0 +1,19 @@
{
"syslog": {
"stdoutLevel": 7
},
"debugAddr": ":8016",
"files": [
"/var/log/akamai-purger.log",
"/var/log/boulder-ca.log",
"/var/log/boulder-publisher.log",
"/var/log/boulder-ra.log",
"/var/log/boulder-sa.log",
"/var/log/boulder-va.log",
"/var/log/boulder-wfe2.log",
"/var/log/boulder-wfe.log",
"/var/log/nonce-service.log",
"/var/log/ocsp-responder.log",
"/var/log/ocsp-updater.log"
]
}

3
test/config/nonce.json
View File

@ -3,7 +3,8 @@
"maxUsed": 131072, "maxUsed": 131072,
"noncePrefix": "taro", "noncePrefix": "taro",
"syslog": { "syslog": {
"stdoutLevel": 6 "stdoutLevel": 6,
"syslogLevel": 6
}, },
"debugAddr": ":8111", "debugAddr": ":8111",
"grpc": { "grpc": {

3
test/config/notify-mailer.json
View File

@ -8,6 +8,7 @@
"maxDBConns": 10 "maxDBConns": 10
}, },
"syslog": { "syslog": {
"stdoutLevel": 7 "stdoutLevel": 7,
"syslogLevel": 7
} }
} }

2
test/config/ocsp-responder.json
View File

@ -11,7 +11,7 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

2
test/config/ocsp-updater.json
View File

@ -32,7 +32,7 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

1
test/config/orphan-finder.json
View File

@ -2,6 +2,7 @@
"backdate": "1h", "backdate": "1h",
"syslog": { "syslog": {
"stdoutlevel": 7,
"stdoutlevel": 7 "stdoutlevel": 7
}, },

2
test/config/publisher.json
View File

@ -22,7 +22,7 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

2
test/config/ra.json
View File

@ -118,6 +118,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

2
test/config/sa.json
View File

@ -30,6 +30,6 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
} }
} }

2
test/config/va.json
View File

@ -34,7 +34,7 @@
"syslog": { "syslog": {
"stdoutlevel": 6, "stdoutlevel": 6,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

2
test/config/wfe.json
View File

@ -29,7 +29,7 @@
"syslog": { "syslog": {
"stdoutlevel": 4, "stdoutlevel": 4,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

2
test/config/wfe2.json
View File

@ -34,7 +34,7 @@
"syslog": { "syslog": {
"stdoutlevel": 4, "stdoutlevel": 4,
"sysloglevel": 4 "sysloglevel": 6
}, },
"common": { "common": {

1
test/startservers.py
View File

@ -86,6 +86,7 @@ def start(race_detection, fakeclock):
[8112, './bin/nonce-service --config %s --addr nonce2.boulder:9101 --debug-addr :8112 --prefix zinc' % os.path.join(config_dir, "nonce.json")], [8112, './bin/nonce-service --config %s --addr nonce2.boulder:9101 --debug-addr :8112 --prefix zinc' % os.path.join(config_dir, "nonce.json")],
[4431, './bin/boulder-wfe2 --config %s' % os.path.join(config_dir, "wfe2.json")], [4431, './bin/boulder-wfe2 --config %s' % os.path.join(config_dir, "wfe2.json")],
[4000, './bin/boulder-wfe --config %s' % os.path.join(config_dir, "wfe.json")], [4000, './bin/boulder-wfe --config %s' % os.path.join(config_dir, "wfe.json")],
[8016, './bin/log-validator --config %s' % os.path.join(config_dir, "log-validator.json")],
]) ])
for (port, prog) in progs: for (port, prog) in progs:
try: try: