167 lines
5.0 KiB
Go
167 lines
5.0 KiB
Go
package observer
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"strconv"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"github.com/letsencrypt/boulder/cmd"
|
|
"github.com/letsencrypt/boulder/observer/probers"
|
|
)
|
|
|
|
var (
|
|
countMonitors = prometheus.NewCounterVec(
|
|
prometheus.CounterOpts{
|
|
Name: "obs_monitors",
|
|
Help: "details of each configured monitor",
|
|
},
|
|
[]string{"kind", "valid"},
|
|
)
|
|
histObservations *prometheus.HistogramVec
|
|
)
|
|
|
|
// ObsConf is exported to receive YAML configuration.
|
|
type ObsConf struct {
|
|
DebugAddr string `yaml:"debugaddr" validate:"omitempty,hostname_port"`
|
|
Buckets []float64 `yaml:"buckets" validate:"min=1,dive"`
|
|
Syslog cmd.SyslogConfig `yaml:"syslog"`
|
|
OpenTelemetry cmd.OpenTelemetryConfig
|
|
MonConfs []*MonConf `yaml:"monitors" validate:"min=1,dive"`
|
|
}
|
|
|
|
// validateSyslog ensures the `Syslog` field received by `ObsConf`
|
|
// contains valid log levels.
|
|
func (c *ObsConf) validateSyslog() error {
|
|
syslog, stdout := c.Syslog.SyslogLevel, c.Syslog.StdoutLevel
|
|
if stdout < 0 || stdout > 7 || syslog < 0 || syslog > 7 {
|
|
return fmt.Errorf(
|
|
"invalid 'syslog', '%+v', valid log levels are 0-7", c.Syslog)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// validateDebugAddr ensures the `debugAddr` received by `ObsConf` is
|
|
// properly formatted and a valid port.
|
|
func (c *ObsConf) validateDebugAddr() error {
|
|
_, p, err := net.SplitHostPort(c.DebugAddr)
|
|
if err != nil {
|
|
return fmt.Errorf(
|
|
"invalid 'debugaddr', %q, not expected format", c.DebugAddr)
|
|
}
|
|
port, _ := strconv.Atoi(p)
|
|
if port <= 0 || port > 65535 {
|
|
return fmt.Errorf(
|
|
"invalid 'debugaddr','%d' is not a valid port", port)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *ObsConf) makeMonitors(metrics prometheus.Registerer) ([]*monitor, []error, error) {
|
|
var errs []error
|
|
var monitors []*monitor
|
|
proberSpecificMetrics := make(map[string]map[string]prometheus.Collector)
|
|
for e, m := range c.MonConfs {
|
|
entry := strconv.Itoa(e + 1)
|
|
proberConf, err := probers.GetConfigurer(m.Kind)
|
|
if err != nil {
|
|
// append error to errs
|
|
errs = append(errs, fmt.Errorf("'monitors' entry #%s couldn't be validated: %w", entry, err))
|
|
// increment metrics
|
|
countMonitors.WithLabelValues(m.Kind, "false").Inc()
|
|
// bail out before constructing the monitor. with no configurer, it will fail
|
|
continue
|
|
}
|
|
kind := proberConf.Kind()
|
|
|
|
// set up custom metrics internal to each prober kind
|
|
_, exist := proberSpecificMetrics[kind]
|
|
if !exist {
|
|
// we haven't seen this prober kind before, so we need to request
|
|
// any custom metrics it may have and register them with the
|
|
// prometheus registry
|
|
proberSpecificMetrics[kind] = make(map[string]prometheus.Collector)
|
|
for name, collector := range proberConf.Instrument() {
|
|
// register the collector with the prometheus registry
|
|
metrics.MustRegister(collector)
|
|
// store the registered collector so we can pass it to every
|
|
// monitor that will construct this kind of prober
|
|
proberSpecificMetrics[kind][name] = collector
|
|
}
|
|
}
|
|
|
|
monitor, err := m.makeMonitor(proberSpecificMetrics[kind])
|
|
if err != nil {
|
|
// append validation error to errs
|
|
errs = append(errs, fmt.Errorf("'monitors' entry #%s couldn't be validated: %w", entry, err))
|
|
|
|
// increment metrics
|
|
countMonitors.WithLabelValues(kind, "false").Inc()
|
|
} else {
|
|
// append monitor to monitors
|
|
monitors = append(monitors, monitor)
|
|
|
|
// increment metrics
|
|
countMonitors.WithLabelValues(kind, "true").Inc()
|
|
}
|
|
}
|
|
if len(c.MonConfs) == len(errs) {
|
|
return nil, errs, errors.New("no valid monitors, cannot continue")
|
|
}
|
|
return monitors, errs, nil
|
|
}
|
|
|
|
// MakeObserver constructs an `Observer` object from the contents of the
|
|
// bound `ObsConf`. If the `ObsConf` cannot be validated, an error
|
|
// appropriate for end-user consumption is returned instead.
|
|
func (c *ObsConf) MakeObserver() (*Observer, error) {
|
|
err := c.validateSyslog()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = c.validateDebugAddr()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(c.MonConfs) == 0 {
|
|
return nil, errors.New("no monitors provided")
|
|
}
|
|
|
|
if len(c.Buckets) == 0 {
|
|
return nil, errors.New("no histogram buckets provided")
|
|
}
|
|
|
|
// Start monitoring and logging.
|
|
metrics, logger, shutdown := cmd.StatsAndLogging(c.Syslog, c.OpenTelemetry, c.DebugAddr)
|
|
histObservations = prometheus.NewHistogramVec(
|
|
prometheus.HistogramOpts{
|
|
Name: "obs_observations",
|
|
Help: "details of each probe attempt",
|
|
Buckets: c.Buckets,
|
|
}, []string{"name", "kind", "success"})
|
|
metrics.MustRegister(countMonitors)
|
|
metrics.MustRegister(histObservations)
|
|
defer cmd.AuditPanic()
|
|
logger.Info(cmd.VersionString())
|
|
logger.Infof("Initializing boulder-observer daemon")
|
|
logger.Debugf("Using config: %+v", c)
|
|
|
|
monitors, errs, err := c.makeMonitors(metrics)
|
|
if len(errs) != 0 {
|
|
logger.Errf("%d of %d monitors failed validation", len(errs), len(c.MonConfs))
|
|
for _, err := range errs {
|
|
logger.Errf("%s", err)
|
|
}
|
|
} else {
|
|
logger.Info("all monitors passed validation")
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Observer{logger, monitors, shutdown}, nil
|
|
}
|