Add log validator to integration tests (#4782)

For now this mainly provides an example config and confirms that log-validator can start up and shut down cleanly, as well as provide a stat indicating how many log lines it has handled. This introduces a syslog config to the boulder-tools image that will write logs to /var/log/program.log. It also tweaks the various .json config files so they have non-default syslogLevel, to ensure they actually write something for log-validator to verify.
2020-04-20 13:33:42 -07:00 · 2020-04-20 13:33:42 -07:00 · 87fb6028c1
parent 8259925f7c
commit 87fb6028c1
41 changed files with 114 additions and 45 deletions
--- a/cmd/log-validator/main.go
+++ b/cmd/log-validator/main.go
@ -127,11 +127,11 @@ func main() {
 	cmd.FailOnError(err, "failed to parse config file")

 	stats, logger := cmd.StatsAndLogging(config.Syslog, config.DebugAddr)
-	badLineCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
-		Name: "bad_log_lines",
-		Help: "A counter of corrupt log lines",
-	}, []string{"filename"})
-	stats.MustRegister(badLineCounter)
+	lineCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "log_lines",
+		Help: "A counter of log lines processed, with status",
+	}, []string{"filename", "status"})
+	stats.MustRegister(lineCounter)

 	var tailers []*tail.Tail
 	for _, filename := range config.Files {
@ -151,8 +151,10 @@ func main() {
 					continue
 				}
 				if err := lineValid(line.Text); err != nil {
-					badLineCounter.WithLabelValues(t.Filename).Inc()
+					lineCounter.WithLabelValues(t.Filename, "bad").Inc()
 					logger.Errf("%s: %s %q", t.Filename, err, line.Text)
+				} else {
+					lineCounter.WithLabelValues(t.Filename, "ok").Inc()
 				}
 			}
 		}()
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -2,7 +2,7 @@ version: '3'
 services:
    boulder:
        # To minimize fetching this should be the same version used below
-        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08
+        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16
        environment:
            FAKE_DNS: 10.77.77.77
            PKCS11_PROXY_SOCKET: tcp://boulder-hsm:5657
@ -65,7 +65,7 @@ services:
            hard: -1
    bhsm:
        # To minimize fetching this should be the same version used above
-        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08
+        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16
        environment:
            PKCS11_DAEMON_SOCKET: tcp://0.0.0.0:5657
        command: /usr/local/bin/pkcs11-daemon /usr/lib/softhsm/libsofthsm2.so
@ -92,7 +92,7 @@ services:
        logging:
            driver: none
    netaccess:
-        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08
+        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16
        environment:
            GO111MODULE: "on"
            GOFLAGS: "-mod=vendor"
--- a/test/boulder-tools/Dockerfile.tmpl
+++ b/test/boulder-tools/Dockerfile.tmpl
@ -9,6 +9,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
 WORKDIR $GOPATH

 RUN wget -O go.tgz "https://dl.google.com/go/go${GO_VERSION_TO_INSTALL}.linux-amd64.tar.gz" && tar -C /usr/local -xzf go.tgz && rm go.tgz;
-ADD requirements.txt /tmp/requirements.txt
-ADD build.sh /tmp/build.sh
+COPY requirements.txt /tmp/requirements.txt
+COPY boulder.rsyslog.conf /etc/rsyslog.d/
+COPY build.sh /tmp/build.sh
 RUN /tmp/build.sh
--- a/test/boulder-tools/boulder.rsyslog.conf
+++ b/test/boulder-tools/boulder.rsyslog.conf
@ -0,0 +1,18 @@
+module( load="builtin:omfile" template="LELogFormat" )
+
+template( name="LELogFormat" type="list" ) {
+  property(name="timereported" dateFormat="rfc3339")
+  constant(value=" ")
+  property(name="hostname" field.delimiter="46" field.number="1")
+  constant(value=" datacenter ")
+  property(name="syslogseverity")
+  constant(value=" ")
+  property(name="syslogtag")
+  property(name="msg" spifno1stsp="on" )
+  property(name="msg" droplastlf="on" )
+  constant(value="\n")
+}
+
+template( name="TmplAll" type="string" string="/var/log/%PROGRAMNAME%.log" )
+
+action( type="omfile" dynaFile="TmplAll" )
--- a/test/config-next/admin-revoker.json
+++ b/test/config-next/admin-revoker.json
@ -21,6 +21,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config-next/akamai-purger.json
+++ b/test/config-next/akamai-purger.json
@ -22,11 +22,11 @@
  
    "syslog": {
      "stdoutlevel": 6,
-      "sysloglevel": 4
+      "sysloglevel": 6
    },
  
    "common": {
      "issuerCert": "test/test-ca2.pem"
    }
  }
-  
+  
--- a/test/config-next/ca-a.json
+++ b/test/config-next/ca-a.json
@ -152,6 +152,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config-next/ca-b.json
+++ b/test/config-next/ca-b.json
@ -153,6 +153,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config-next/cert-checker.json
+++ b/test/config-next/cert-checker.json
@ -18,6 +18,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config-next/expiration-mailer.json
+++ b/test/config-next/expiration-mailer.json
@ -27,6 +27,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config-next/log-validator.json
+++ b/test/config-next/log-validator.json
@ -0,0 +1,19 @@
+{
+  "syslog": {
+    "stdoutLevel": 7
+  },
+  "debugAddr": ":8016",
+  "files": [
+    "/var/log/akamai-purger.log",
+    "/var/log/boulder-ca.log",
+    "/var/log/boulder-publisher.log",
+    "/var/log/boulder-ra.log",
+    "/var/log/boulder-sa.log",
+    "/var/log/boulder-va.log",
+    "/var/log/boulder-wfe2.log",
+    "/var/log/boulder-wfe.log",
+    "/var/log/nonce-service.log",
+    "/var/log/ocsp-responder.log",
+    "/var/log/ocsp-updater.log"
+  ]
+}
--- a/test/config-next/nonce.json
+++ b/test/config-next/nonce.json
@ -3,7 +3,8 @@
        "maxUsed": 131072,
        "noncePrefix": "taro",
        "syslog": {
-            "stdoutLevel": 6
+            "stdoutLevel": 6,
+            "syslogLevel": 6
        },
        "debugAddr": ":8111",
        "grpc": {
--- a/test/config-next/notify-mailer.json
+++ b/test/config-next/notify-mailer.json
@ -6,5 +6,9 @@
    "passwordFile": "test/secrets/smtp_password",
    "dbConnectFile": "test/secrets/mailer_dburl",
    "maxDBConns": 10
-  }
+  },
+   "syslog": {
+    "stdoutLevel": 7,
+    "syslogLevel": 7
+   }
 }
--- a/test/config-next/ocsp-responder.json
+++ b/test/config-next/ocsp-responder.json
@ -13,7 +13,7 @@

  "syslog": {
   "stdoutlevel": 6,
-   "sysloglevel": 4
+   "sysloglevel": 6
 },

  "common": {
--- a/test/config-next/ocsp-updater.json
+++ b/test/config-next/ocsp-updater.json
@ -33,7 +33,7 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  },

  "common": {
--- a/test/config-next/orphan-finder.json
+++ b/test/config-next/orphan-finder.json
@ -2,6 +2,7 @@
  "backdate": "1h",

  "syslog": {
+    "stdoutlevel": 7,
    "stdoutlevel": 7
  },

--- a/test/config-next/publisher.json
+++ b/test/config-next/publisher.json
@ -23,7 +23,7 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  },

  "common": {
--- a/test/config-next/ra.json
+++ b/test/config-next/ra.json
@ -120,6 +120,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config-next/sa.json
+++ b/test/config-next/sa.json
@ -31,6 +31,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config-next/va.json
+++ b/test/config-next/va.json
@ -50,7 +50,7 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  },

  "common": {
--- a/test/config-next/wfe.json
+++ b/test/config-next/wfe.json
@ -46,7 +46,7 @@

  "syslog": {
    "stdoutlevel": 4,
-    "sysloglevel": 4
+    "sysloglevel": 6
  },

  "common": {
--- a/test/config-next/wfe2.json
+++ b/test/config-next/wfe2.json
@ -60,7 +60,7 @@

  "syslog": {
    "stdoutlevel": 4,
-    "sysloglevel": 4
+    "sysloglevel": 6
  },

  "common": {
--- a/test/config/admin-revoker.json
+++ b/test/config/admin-revoker.json
@ -21,6 +21,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config/akamai-purger.json
+++ b/test/config/akamai-purger.json
@ -22,11 +22,11 @@
  
    "syslog": {
      "stdoutlevel": 6,
-      "sysloglevel": 4
+      "sysloglevel": 6
    },
  
    "common": {
      "issuerCert": "test/test-ca2.pem"
    }
  }
-  
+  
--- a/test/config/ca-a.json
+++ b/test/config/ca-a.json
@ -152,6 +152,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config/ca-b.json
+++ b/test/config/ca-b.json
@ -153,6 +153,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config/cert-checker.json
+++ b/test/config/cert-checker.json
@ -18,6 +18,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config/expiration-mailer.json
+++ b/test/config/expiration-mailer.json
@ -27,6 +27,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config/log-validator.json
+++ b/test/config/log-validator.json
@ -0,0 +1,19 @@
+{
+  "syslog": {
+    "stdoutLevel": 7
+  },
+  "debugAddr": ":8016",
+  "files": [
+    "/var/log/akamai-purger.log",
+    "/var/log/boulder-ca.log",
+    "/var/log/boulder-publisher.log",
+    "/var/log/boulder-ra.log",
+    "/var/log/boulder-sa.log",
+    "/var/log/boulder-va.log",
+    "/var/log/boulder-wfe2.log",
+    "/var/log/boulder-wfe.log",
+    "/var/log/nonce-service.log",
+    "/var/log/ocsp-responder.log",
+    "/var/log/ocsp-updater.log"
+  ]
+}
--- a/test/config/nonce.json
+++ b/test/config/nonce.json
@ -3,7 +3,8 @@
        "maxUsed": 131072,
        "noncePrefix": "taro",
        "syslog": {
-            "stdoutLevel": 6
+            "stdoutLevel": 6,
+            "syslogLevel": 6
        },
        "debugAddr": ":8111",
        "grpc": {
--- a/test/config/notify-mailer.json
+++ b/test/config/notify-mailer.json
@ -8,6 +8,7 @@
    "maxDBConns": 10
  },
  "syslog": {
-    "stdoutLevel": 7
+    "stdoutLevel": 7,
+    "syslogLevel": 7
  }
 }
--- a/test/config/ocsp-responder.json
+++ b/test/config/ocsp-responder.json
@ -11,7 +11,7 @@

  "syslog": {
   "stdoutlevel": 6,
-   "sysloglevel": 4
+   "sysloglevel": 6
 },

  "common": {
--- a/test/config/ocsp-updater.json
+++ b/test/config/ocsp-updater.json
@ -32,7 +32,7 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  },

  "common": {
--- a/test/config/orphan-finder.json
+++ b/test/config/orphan-finder.json
@ -2,6 +2,7 @@
  "backdate": "1h",

  "syslog": {
+    "stdoutlevel": 7,
    "stdoutlevel": 7
  },

--- a/test/config/publisher.json
+++ b/test/config/publisher.json
@ -22,7 +22,7 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  },

  "common": {
--- a/test/config/ra.json
+++ b/test/config/ra.json
@ -118,6 +118,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config/sa.json
+++ b/test/config/sa.json
@ -30,6 +30,6 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  }
 }
--- a/test/config/va.json
+++ b/test/config/va.json
@ -34,7 +34,7 @@

  "syslog": {
    "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
  },

  "common": {
--- a/test/config/wfe.json
+++ b/test/config/wfe.json
@ -29,7 +29,7 @@

  "syslog": {
    "stdoutlevel": 4,
-    "sysloglevel": 4
+    "sysloglevel": 6
  },

  "common": {
--- a/test/config/wfe2.json
+++ b/test/config/wfe2.json
@ -34,7 +34,7 @@

  "syslog": {
    "stdoutlevel": 4,
-    "sysloglevel": 4
+    "sysloglevel": 6
  },

  "common": {
--- a/test/startservers.py
+++ b/test/startservers.py
@ -86,6 +86,7 @@ def start(race_detection, fakeclock):
        [8112, './bin/nonce-service --config %s --addr nonce2.boulder:9101 --debug-addr :8112 --prefix zinc' % os.path.join(config_dir, "nonce.json")],
        [4431, './bin/boulder-wfe2 --config %s' % os.path.join(config_dir, "wfe2.json")],
        [4000, './bin/boulder-wfe --config %s' % os.path.join(config_dir, "wfe.json")],
+        [8016, './bin/log-validator --config %s' % os.path.join(config_dir, "log-validator.json")],
    ])
    for (port, prog) in progs:
        try: