diff --git a/cmd/log-validator/main.go b/cmd/log-validator/main.go index bf3ff54b5..252c59b5e 100644 --- a/cmd/log-validator/main.go +++ b/cmd/log-validator/main.go @@ -127,11 +127,11 @@ func main() { cmd.FailOnError(err, "failed to parse config file") stats, logger := cmd.StatsAndLogging(config.Syslog, config.DebugAddr) - badLineCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "bad_log_lines", - Help: "A counter of corrupt log lines", - }, []string{"filename"}) - stats.MustRegister(badLineCounter) + lineCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "log_lines", + Help: "A counter of log lines processed, with status", + }, []string{"filename", "status"}) + stats.MustRegister(lineCounter) var tailers []*tail.Tail for _, filename := range config.Files { @@ -151,8 +151,10 @@ func main() { continue } if err := lineValid(line.Text); err != nil { - badLineCounter.WithLabelValues(t.Filename).Inc() + lineCounter.WithLabelValues(t.Filename, "bad").Inc() logger.Errf("%s: %s %q", t.Filename, err, line.Text) + } else { + lineCounter.WithLabelValues(t.Filename, "ok").Inc() } } }() diff --git a/docker-compose.yml b/docker-compose.yml index 1fc37b67c..dc2970374 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: boulder: # To minimize fetching this should be the same version used below - image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08 + image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16 environment: FAKE_DNS: 10.77.77.77 PKCS11_PROXY_SOCKET: tcp://boulder-hsm:5657 @@ -65,7 +65,7 @@ services: hard: -1 bhsm: # To minimize fetching this should be the same version used above - image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08 + image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16 environment: PKCS11_DAEMON_SOCKET: tcp://0.0.0.0:5657 command: /usr/local/bin/pkcs11-daemon /usr/lib/softhsm/libsofthsm2.so @@ -92,7 +92,7 @@ services: logging: driver: none netaccess: - image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08 + image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16 environment: GO111MODULE: "on" GOFLAGS: "-mod=vendor" diff --git a/test/boulder-tools/Dockerfile.tmpl b/test/boulder-tools/Dockerfile.tmpl index 9a14ed3e8..8b06fb928 100644 --- a/test/boulder-tools/Dockerfile.tmpl +++ b/test/boulder-tools/Dockerfile.tmpl @@ -9,6 +9,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR $GOPATH RUN wget -O go.tgz "https://dl.google.com/go/go${GO_VERSION_TO_INSTALL}.linux-amd64.tar.gz" && tar -C /usr/local -xzf go.tgz && rm go.tgz; -ADD requirements.txt /tmp/requirements.txt -ADD build.sh /tmp/build.sh +COPY requirements.txt /tmp/requirements.txt +COPY boulder.rsyslog.conf /etc/rsyslog.d/ +COPY build.sh /tmp/build.sh RUN /tmp/build.sh diff --git a/test/boulder-tools/boulder.rsyslog.conf b/test/boulder-tools/boulder.rsyslog.conf new file mode 100644 index 000000000..a1b8d6036 --- /dev/null +++ b/test/boulder-tools/boulder.rsyslog.conf @@ -0,0 +1,18 @@ +module( load="builtin:omfile" template="LELogFormat" ) + +template( name="LELogFormat" type="list" ) { + property(name="timereported" dateFormat="rfc3339") + constant(value=" ") + property(name="hostname" field.delimiter="46" field.number="1") + constant(value=" datacenter ") + property(name="syslogseverity") + constant(value=" ") + property(name="syslogtag") + property(name="msg" spifno1stsp="on" ) + property(name="msg" droplastlf="on" ) + constant(value="\n") +} + +template( name="TmplAll" type="string" string="/var/log/%PROGRAMNAME%.log" ) + +action( type="omfile" dynaFile="TmplAll" ) diff --git a/test/config-next/admin-revoker.json b/test/config-next/admin-revoker.json index ec9eeca3a..8f672df2e 100644 --- a/test/config-next/admin-revoker.json +++ b/test/config-next/admin-revoker.json @@ -21,6 +21,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config-next/akamai-purger.json b/test/config-next/akamai-purger.json index 235023b80..a676c830e 100644 --- a/test/config-next/akamai-purger.json +++ b/test/config-next/akamai-purger.json @@ -22,11 +22,11 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { "issuerCert": "test/test-ca2.pem" } } - \ No newline at end of file + diff --git a/test/config-next/ca-a.json b/test/config-next/ca-a.json index 13bee5cd3..848591b87 100644 --- a/test/config-next/ca-a.json +++ b/test/config-next/ca-a.json @@ -152,6 +152,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config-next/ca-b.json b/test/config-next/ca-b.json index 133756dc5..5195a766c 100644 --- a/test/config-next/ca-b.json +++ b/test/config-next/ca-b.json @@ -153,6 +153,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config-next/cert-checker.json b/test/config-next/cert-checker.json index 7801d3e6a..d6eff13d6 100644 --- a/test/config-next/cert-checker.json +++ b/test/config-next/cert-checker.json @@ -18,6 +18,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config-next/expiration-mailer.json b/test/config-next/expiration-mailer.json index 444beae43..8ccba7cac 100644 --- a/test/config-next/expiration-mailer.json +++ b/test/config-next/expiration-mailer.json @@ -27,6 +27,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config-next/log-validator.json b/test/config-next/log-validator.json new file mode 100644 index 000000000..82c16cffc --- /dev/null +++ b/test/config-next/log-validator.json @@ -0,0 +1,19 @@ +{ + "syslog": { + "stdoutLevel": 7 + }, + "debugAddr": ":8016", + "files": [ + "/var/log/akamai-purger.log", + "/var/log/boulder-ca.log", + "/var/log/boulder-publisher.log", + "/var/log/boulder-ra.log", + "/var/log/boulder-sa.log", + "/var/log/boulder-va.log", + "/var/log/boulder-wfe2.log", + "/var/log/boulder-wfe.log", + "/var/log/nonce-service.log", + "/var/log/ocsp-responder.log", + "/var/log/ocsp-updater.log" + ] +} diff --git a/test/config-next/nonce.json b/test/config-next/nonce.json index 53d9773a3..5d37ee531 100644 --- a/test/config-next/nonce.json +++ b/test/config-next/nonce.json @@ -3,7 +3,8 @@ "maxUsed": 131072, "noncePrefix": "taro", "syslog": { - "stdoutLevel": 6 + "stdoutLevel": 6, + "syslogLevel": 6 }, "debugAddr": ":8111", "grpc": { diff --git a/test/config-next/notify-mailer.json b/test/config-next/notify-mailer.json index d2e28563c..a64154312 100644 --- a/test/config-next/notify-mailer.json +++ b/test/config-next/notify-mailer.json @@ -6,5 +6,9 @@ "passwordFile": "test/secrets/smtp_password", "dbConnectFile": "test/secrets/mailer_dburl", "maxDBConns": 10 - } + }, + "syslog": { + "stdoutLevel": 7, + "syslogLevel": 7 + } } diff --git a/test/config-next/ocsp-responder.json b/test/config-next/ocsp-responder.json index a99cb559f..f65fb8752 100644 --- a/test/config-next/ocsp-responder.json +++ b/test/config-next/ocsp-responder.json @@ -13,7 +13,7 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/config-next/ocsp-updater.json b/test/config-next/ocsp-updater.json index c718171ce..50473cfef 100644 --- a/test/config-next/ocsp-updater.json +++ b/test/config-next/ocsp-updater.json @@ -33,7 +33,7 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/config-next/orphan-finder.json b/test/config-next/orphan-finder.json index 84bffb3bd..0ce278e60 100644 --- a/test/config-next/orphan-finder.json +++ b/test/config-next/orphan-finder.json @@ -2,6 +2,7 @@ "backdate": "1h", "syslog": { + "stdoutlevel": 7, "stdoutlevel": 7 }, diff --git a/test/config-next/publisher.json b/test/config-next/publisher.json index 2154f90b2..bdf71ff33 100644 --- a/test/config-next/publisher.json +++ b/test/config-next/publisher.json @@ -23,7 +23,7 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/config-next/ra.json b/test/config-next/ra.json index b718e7303..e6b761e2b 100644 --- a/test/config-next/ra.json +++ b/test/config-next/ra.json @@ -120,6 +120,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config-next/sa.json b/test/config-next/sa.json index 55998f158..3e29eee5e 100644 --- a/test/config-next/sa.json +++ b/test/config-next/sa.json @@ -31,6 +31,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config-next/va.json b/test/config-next/va.json index f6e0db48f..c330d5ccd 100644 --- a/test/config-next/va.json +++ b/test/config-next/va.json @@ -50,7 +50,7 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/config-next/wfe.json b/test/config-next/wfe.json index d643adc9a..30e77689b 100644 --- a/test/config-next/wfe.json +++ b/test/config-next/wfe.json @@ -46,7 +46,7 @@ "syslog": { "stdoutlevel": 4, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/config-next/wfe2.json b/test/config-next/wfe2.json index 11f2d955d..4c94e1afd 100644 --- a/test/config-next/wfe2.json +++ b/test/config-next/wfe2.json @@ -60,7 +60,7 @@ "syslog": { "stdoutlevel": 4, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/config/admin-revoker.json b/test/config/admin-revoker.json index ec9eeca3a..8f672df2e 100644 --- a/test/config/admin-revoker.json +++ b/test/config/admin-revoker.json @@ -21,6 +21,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config/akamai-purger.json b/test/config/akamai-purger.json index 235023b80..a676c830e 100644 --- a/test/config/akamai-purger.json +++ b/test/config/akamai-purger.json @@ -22,11 +22,11 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { "issuerCert": "test/test-ca2.pem" } } - \ No newline at end of file + diff --git a/test/config/ca-a.json b/test/config/ca-a.json index 3ad5be631..54e276924 100644 --- a/test/config/ca-a.json +++ b/test/config/ca-a.json @@ -152,6 +152,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config/ca-b.json b/test/config/ca-b.json index e47dbd255..74770f914 100644 --- a/test/config/ca-b.json +++ b/test/config/ca-b.json @@ -153,6 +153,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config/cert-checker.json b/test/config/cert-checker.json index 7801d3e6a..d6eff13d6 100644 --- a/test/config/cert-checker.json +++ b/test/config/cert-checker.json @@ -18,6 +18,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config/expiration-mailer.json b/test/config/expiration-mailer.json index 444beae43..8ccba7cac 100644 --- a/test/config/expiration-mailer.json +++ b/test/config/expiration-mailer.json @@ -27,6 +27,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config/log-validator.json b/test/config/log-validator.json new file mode 100644 index 000000000..82c16cffc --- /dev/null +++ b/test/config/log-validator.json @@ -0,0 +1,19 @@ +{ + "syslog": { + "stdoutLevel": 7 + }, + "debugAddr": ":8016", + "files": [ + "/var/log/akamai-purger.log", + "/var/log/boulder-ca.log", + "/var/log/boulder-publisher.log", + "/var/log/boulder-ra.log", + "/var/log/boulder-sa.log", + "/var/log/boulder-va.log", + "/var/log/boulder-wfe2.log", + "/var/log/boulder-wfe.log", + "/var/log/nonce-service.log", + "/var/log/ocsp-responder.log", + "/var/log/ocsp-updater.log" + ] +} diff --git a/test/config/nonce.json b/test/config/nonce.json index 53d9773a3..5d37ee531 100644 --- a/test/config/nonce.json +++ b/test/config/nonce.json @@ -3,7 +3,8 @@ "maxUsed": 131072, "noncePrefix": "taro", "syslog": { - "stdoutLevel": 6 + "stdoutLevel": 6, + "syslogLevel": 6 }, "debugAddr": ":8111", "grpc": { diff --git a/test/config/notify-mailer.json b/test/config/notify-mailer.json index 25d578957..6973a9f27 100644 --- a/test/config/notify-mailer.json +++ b/test/config/notify-mailer.json @@ -8,6 +8,7 @@ "maxDBConns": 10 }, "syslog": { - "stdoutLevel": 7 + "stdoutLevel": 7, + "syslogLevel": 7 } } diff --git a/test/config/ocsp-responder.json b/test/config/ocsp-responder.json index 5b29d96d3..bb0e318ff 100644 --- a/test/config/ocsp-responder.json +++ b/test/config/ocsp-responder.json @@ -11,7 +11,7 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/config/ocsp-updater.json b/test/config/ocsp-updater.json index 8f4030303..14dcb2ac5 100644 --- a/test/config/ocsp-updater.json +++ b/test/config/ocsp-updater.json @@ -32,7 +32,7 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/config/orphan-finder.json b/test/config/orphan-finder.json index 84bffb3bd..0ce278e60 100644 --- a/test/config/orphan-finder.json +++ b/test/config/orphan-finder.json @@ -2,6 +2,7 @@ "backdate": "1h", "syslog": { + "stdoutlevel": 7, "stdoutlevel": 7 }, diff --git a/test/config/publisher.json b/test/config/publisher.json index 975f0bdfa..d3273ba05 100644 --- a/test/config/publisher.json +++ b/test/config/publisher.json @@ -22,7 +22,7 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/config/ra.json b/test/config/ra.json index 0c6c85a46..a20f862cc 100644 --- a/test/config/ra.json +++ b/test/config/ra.json @@ -118,6 +118,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config/sa.json b/test/config/sa.json index 4993b40c2..713a09cf2 100644 --- a/test/config/sa.json +++ b/test/config/sa.json @@ -30,6 +30,6 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 } } diff --git a/test/config/va.json b/test/config/va.json index ece66adda..1e63d85f0 100644 --- a/test/config/va.json +++ b/test/config/va.json @@ -34,7 +34,7 @@ "syslog": { "stdoutlevel": 6, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/config/wfe.json b/test/config/wfe.json index 3e44e0140..9d912ed4e 100644 --- a/test/config/wfe.json +++ b/test/config/wfe.json @@ -29,7 +29,7 @@ "syslog": { "stdoutlevel": 4, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/config/wfe2.json b/test/config/wfe2.json index e0af5ed13..e11c42d1e 100644 --- a/test/config/wfe2.json +++ b/test/config/wfe2.json @@ -34,7 +34,7 @@ "syslog": { "stdoutlevel": 4, - "sysloglevel": 4 + "sysloglevel": 6 }, "common": { diff --git a/test/startservers.py b/test/startservers.py index 797b8759c..7c743e0f6 100644 --- a/test/startservers.py +++ b/test/startservers.py @@ -86,6 +86,7 @@ def start(race_detection, fakeclock): [8112, './bin/nonce-service --config %s --addr nonce2.boulder:9101 --debug-addr :8112 --prefix zinc' % os.path.join(config_dir, "nonce.json")], [4431, './bin/boulder-wfe2 --config %s' % os.path.join(config_dir, "wfe2.json")], [4000, './bin/boulder-wfe --config %s' % os.path.join(config_dir, "wfe.json")], + [8016, './bin/log-validator --config %s' % os.path.join(config_dir, "log-validator.json")], ]) for (port, prog) in progs: try: