open-telemetry
/
opentelemetry-collector
mirror of https://github.com/open-telemetry/opentelemetry-collector.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor config: pipelines now under service (#376) 

* Refactor config: pipelines now under service

This change moves 'pipelines' sections from the top level to under the 'service' section.

* Fix typo in old comment
This commit is contained in:
Paulo Janotti 2019-10-02 17:42:18 -07:00 committed by GitHub
parent 402b80cabd
commit 0e505d53e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
84 changed files with 555 additions and 486 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
README.md
View File

@ -139,7 +139,8 @@ exporters:
...
processors:
...
pipelines:
service:
pipelines:
...
```
@ -218,12 +219,12 @@ a receiver/exporter is reference by all the pipelines.
The following is an example pipeline configuration. For more information, refer
to [pipeline documentation](docs/pipelines.md)
```yaml
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
```
### <a name="config-diagnostics"></a>Diagnostics

38
config/config.go
View File

@ -137,7 +137,6 @@ func Load(
Receivers map[string]interface{} `mapstructure:"receivers"`
Processors map[string]interface{} `mapstructure:"processors"`
Exporters map[string]interface{} `mapstructure:"exporters"`
Pipelines map[string]interface{} `mapstructure:"pipelines"`
}
if err := v.UnmarshalExact(&topLevelSections); err != nil {
@ -147,7 +146,7 @@ func Load(
}
}
// Start with extensions and service.
// Start with the service extensions.
extensions, err := loadExtensions(v, factories.Extensions)
if err != nil {
@ -155,13 +154,7 @@ func Load(
}
config.Extensions = extensions
service, err := loadService(v)
if err != nil {
return nil, err
}
config.Service = service
// Load data components (receivers, exporters, processores, and pipelines).
// Load data components (receivers, exporters, and processors).
receivers, err := loadReceivers(v, factories.Receivers)
if err != nil {
@ -181,11 +174,12 @@ func Load(
}
config.Processors = processors
pipelines, err := loadPipelines(v)
// Load the service and its data pipelines.
service, err := loadService(v)
if err != nil {
return nil, err
}
config.Pipelines = pipelines
config.Service = service
// Config is loaded. Now validate it.
@ -296,13 +290,27 @@ func loadExtensions(v *viper.Viper, factories map[string]extension.Factory) (con
func loadService(v *viper.Viper) (configmodels.Service, error) {
var service configmodels.Service
if err := v.UnmarshalKey(serviceKeyName, &service, errorOnUnused); err != nil {
serviceSub := getConfigSection(v, serviceKeyName)
// Process the pipelines first so in case of error on them it can be properly
// reported.
pipelines, err := loadPipelines(serviceSub)
if err != nil {
return service, err
}
// Do an exact match to find any unused section on config.
if err := serviceSub.UnmarshalExact(&service); err != nil {
return service, &configError{
code: errUnmarshalErrorOnService,
msg: fmt.Sprintf("error reading settings for %q: %v", serviceKeyName, err),
}
}
// Unmarshal cannot properly build Pipelines field, set it to the value
// previously loaded.
service.Pipelines = pipelines
return service, nil
}
@ -640,12 +648,12 @@ func validateServiceExtensions(
func validatePipelines(cfg *configmodels.Config, logger *zap.Logger) error {
// Must have at least one pipeline.
if len(cfg.Pipelines) < 1 {
if len(cfg.Service.Pipelines) < 1 {
return &configError{code: errMissingPipelines, msg: "must have at least one pipeline"}
}
// Validate pipelines.
for _, pipeline := range cfg.Pipelines {
for _, pipeline := range cfg.Service.Pipelines {
if err := validatePipeline(cfg, pipeline, logger); err != nil {
return err
}
@ -849,7 +857,7 @@ func validateProcessors(cfg *configmodels.Config) {
// getConfigSection returns a sub-config from the viper config that has the corresponding given key.
// It also expands all the string values.
func getConfigSection(v *viper.Viper, key string) *viper.Viper {
// Unmarsh only the subconfig for this processor.
// Unmarshal only the subconfig for this processor.
sv := v.Sub(key)
if sv == nil {
// When the config for this key is empty Sub returns nil. In order to avoid nil checks

8
config/config_test.go
View File

@ -111,7 +111,7 @@ func TestDecodeConfig(t *testing.T) {
"Did not load processor config correctly")
// Verify Pipelines
assert.Equal(t, 1, len(config.Pipelines), "Incorrect pipelines count")
assert.Equal(t, 1, len(config.Service.Pipelines), "Incorrect pipelines count")
assert.Equal(t,
&configmodels.Pipeline{
@ -121,7 +121,7 @@ func TestDecodeConfig(t *testing.T) {
Processors: []string{"exampleprocessor"},
Exporters: []string{"exampleexporter"},
},
config.Pipelines["traces"],
config.Service.Pipelines["traces"],
"Did not load pipeline config correctly")
}
@ -275,7 +275,7 @@ func TestSimpleConfig(t *testing.T) {
"TEST[%s] Did not load processor config correctly", test.name)
// Verify Pipelines
assert.Equalf(t, 1, len(config.Pipelines), "TEST[%s]", test.name)
assert.Equalf(t, 1, len(config.Service.Pipelines), "TEST[%s]", test.name)
assert.Equalf(t,
&configmodels.Pipeline{
@ -285,7 +285,7 @@ func TestSimpleConfig(t *testing.T) {
Processors: []string{"exampleprocessor"},
Exporters: []string{"exampleexporter"},
},
config.Pipelines["traces"],
config.Service.Pipelines["traces"],
"TEST[%s] Did not load pipeline config correctly", test.name)
}
}

4
config/configmodels/configmodels.go
View File

@ -36,7 +36,6 @@ type Config struct {
Receivers Receivers
Exporters Exporters
Processors Processors
Pipelines Pipelines
Extensions Extensions
Service Service
}
@ -144,6 +143,9 @@ type Extensions map[string]Extension
type Service struct {
// Extensions is the ordered list of extensions configured for the service.
Extensions []string `mapstructure:"extensions"`
// Pipelines is the set of data pipelines configured for the service.
Pipelines Pipelines `mapstructure:"pipelines"`
}
// Below are common setting structs for Receivers, Exporters and Processors.

11
config/testdata/duplicate-exporter.yaml vendored
View File

@ -5,8 +5,9 @@ exporters:
exampleexporter/ exp :
processors:
exampleprocessor:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]

19
config/testdata/duplicate-pipeline.yaml vendored
View File

@ -4,12 +4,13 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces/default:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
traces/ default:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
traces/default:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
traces/ default:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]

11
config/testdata/duplicate-processor.yaml vendored
View File

@ -5,8 +5,9 @@ exporters:
processors:
exampleprocessor/ abc:
exampleprocessor/abc:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]

11
config/testdata/duplicate-receiver.yaml vendored
View File

@ -5,8 +5,9 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]

11
config/testdata/invalid-bool-value.yaml vendored
View File

@ -5,8 +5,9 @@ exporters:
processors:
exampleprocessor:
disabled: [2,3]
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]

11
config/testdata/invalid-disabled-bool-value.yaml vendored
View File

@ -5,8 +5,9 @@ exporters:
disabled: "string for bool"
processors:
exampleprocessor:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]

11
config/testdata/invalid-disabled-bool-value2.yaml vendored
View File

@ -5,8 +5,9 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]

16
config/testdata/invalid-exporter-section.yaml vendored
View File

@ -10,11 +10,11 @@ extensions:
service:
extensions:
- exampleextension
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter

15
config/testdata/invalid-extension-name.yaml vendored
View File

@ -1,16 +1,15 @@
extensions:
exampleextension:
service:
extensions: [exampleextension, nosuchextension, and, another, three]
receivers:
examplereceiver:
processors:
exampleprocessor:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
service:
extensions: [exampleextension, nosuchextension, and, another, three]
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]

16
config/testdata/invalid-extension-section.yaml vendored
View File

@ -11,11 +11,11 @@ extensions:
service:
extensions:
- examapleextension
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter

18
config/testdata/invalid-pipeline-section.yaml vendored
View File

@ -9,12 +9,12 @@ extensions:
service:
extensions:
- exampleextension
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter
unknown_section: 1
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter
unknown_section: 1

11
config/testdata/invalid-pipeline-type-and-name.yaml vendored
View File

@ -5,8 +5,9 @@ processors:
exporters:
exampleexporter:
pipelines:
/metrics:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
service:
pipelines:
/metrics:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]

11
config/testdata/invalid-pipeline-type.yaml vendored
View File

@ -5,8 +5,9 @@ processors:
exporters:
exampleexporter:
pipelines:
wrongdatatype:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
service:
pipelines:
wrongdatatype:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]

16
config/testdata/invalid-processor-section.yaml vendored
View File

@ -11,11 +11,11 @@ extensions:
service:
extensions:
- examapleextension
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter

11
config/testdata/invalid-receiver-name.yaml vendored
View File

@ -4,8 +4,9 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces:
receivers: [1,2,3]
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
traces:
receivers: [1,2,3]
exporters: [exampleexporter]
processors: [exampleprocessor]

7
config/testdata/invalid-receiver-reference.yaml vendored
View File

@ -4,6 +4,7 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces:
receivers: [invalidreceivername]
service:
pipelines:
traces:
receivers: [invalidreceivername]

16
config/testdata/invalid-receiver-section.yaml vendored
View File

@ -11,11 +11,11 @@ extensions:
service:
extensions:
- examapleextension
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter

14
config/testdata/invalid-sequence-value.yaml vendored
View File

@ -4,9 +4,11 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces:
receivers:
examplereceiver:
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
traces:
receivers:
examplereceiver:
some: config
exporters: [exampleexporter]
processors: [exampleprocessor]

19
config/testdata/invalid-service-extensions-value.yaml vendored
View File

@ -1,18 +1,17 @@
extensions:
exampleextension:
service:
extensions:
exampleextension:
disabled: true
receivers:
examplereceiver:
processors:
exampleprocessor:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
service:
extensions:
exampleextension:
disabled: true
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]

16
config/testdata/invalid-service-section.yaml vendored
View File

@ -11,11 +11,11 @@ service:
- examapleextension
unknown_section:
a_num: 2
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter

16
config/testdata/invalid-top-level-section.yaml vendored
View File

@ -9,13 +9,13 @@ extensions:
service:
extenstions:
- examapleextension
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter
pipelines:
traces:
receivers:
- examplereceiver
processors:
- exampleprocessor
exporters:
- exampleexporter
unknown_section:
a_num: 2

11
config/testdata/metric-pipeline-cannot-have-processors.yaml vendored
View File

@ -4,8 +4,9 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
metrics:
receivers: [multireceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
metrics:
receivers: [multireceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]

3
config/testdata/missing-all-sections.yaml vendored
View File

@ -1,4 +1,5 @@
receivers:
exporters:
processors:
pipeline:
service:
pipeline:

11
config/testdata/missing-enabled-exporters.yaml vendored
View File

@ -8,8 +8,9 @@ exporters:
processors:
exampleprocessor:
pipelines:
traces:
receivers: [examplereceiver/one]
processors: [exampleprocessor]
exporters: [exampleexporter/disabled]
service:
pipelines:
traces:
receivers: [examplereceiver/one]
processors: [exampleprocessor]
exporters: [exampleexporter/disabled]

11
config/testdata/missing-enabled-receivers.yaml vendored
View File

@ -16,8 +16,9 @@ exporters:
disabled: true
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver, examplereceiver/disabled]
processors: [exampleprocessor, exampleprocessor/disabled]
exporters: [exampleexporter/disabled, exampleexporter]
service:
pipelines:
traces:
receivers: [examplereceiver, examplereceiver/disabled]
processors: [exampleprocessor, exampleprocessor/disabled]
exporters: [exampleexporter/disabled, exampleexporter]

7
config/testdata/missing-exporter-name-after-slash.yaml vendored
View File

@ -4,6 +4,7 @@ exporters:
exampleexporter/:
processors:
exampleprocessor:
pipelines:
traces:
receivers: [somereceiver]
service:
pipelines:
traces:
receivers: [somereceiver]

7
config/testdata/missing-exporters.yaml vendored
View File

@ -3,6 +3,7 @@ receivers:
exporters:
processors:
exampleprocessor:
pipelines:
traces:
receivers: [invalidreceivername]
service:
pipelines:
traces:
receivers: [invalidreceivername]

3
config/testdata/missing-pipelines.yaml vendored
View File

@ -4,4 +4,5 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
service:
pipelines:

7
config/testdata/missing-processor-type.yaml vendored
View File

@ -4,6 +4,7 @@ exporters:
exampleexporter:
processors:
/exampleprocessor:
pipelines:
traces:
receivers: [somereceiver]
service:
pipelines:
traces:
receivers: [somereceiver]

7
config/testdata/missing-processors.yaml vendored
View File

@ -3,6 +3,7 @@ receivers:
exporters:
exampleexporter:
processors:
pipelines:
traces:
receivers: [invalidreceivername]
service:
pipelines:
traces:
receivers: [invalidreceivername]

7
config/testdata/missing-receiver-type.yaml vendored
View File

@ -4,6 +4,7 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces:
receivers: [somereceiver]
service:
pipelines:
traces:
receivers: [somereceiver]

5
config/testdata/missing-receivers.yaml vendored
View File

@ -3,5 +3,6 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces:
service:
pipelines:
traces:

17
config/testdata/multiproto-config.yaml vendored
View File

@ -17,11 +17,12 @@ exporters:
exampleexporter:
extra: "locahost:1010"
pipelines:
traces:
receivers: [multireceiver/myreceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
metrics:
receivers: [multireceiver/myreceiver]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [multireceiver/myreceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
metrics:
receivers: [multireceiver/myreceiver]
exporters: [exampleexporter]

9
config/testdata/pipeline-exporter-not-exists.yaml vendored
View File

@ -4,7 +4,8 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
metrics:
receivers: [multireceiver]
exporters: [nosuchexporter]
service:
pipelines:
metrics:
receivers: [multireceiver]
exporters: [nosuchexporter]

7
config/testdata/pipeline-must-have-exporter.yaml vendored
View File

@ -4,6 +4,7 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces:
receivers: [multireceiver]
service:
pipelines:
traces:
receivers: [multireceiver]

7
config/testdata/pipeline-must-have-exporter2.yaml vendored
View File

@ -4,6 +4,7 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
metrics:
receivers: [multireceiver]
service:
pipelines:
metrics:
receivers: [multireceiver]

9
config/testdata/pipeline-must-have-processors.yaml vendored
View File

@ -4,7 +4,8 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces:
receivers: [multireceiver]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [multireceiver]
exporters: [exampleexporter]

7
config/testdata/pipeline-must-have-receiver.yaml vendored
View File

@ -4,6 +4,7 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
metrics:
exporters: [exampleexporter]
service:
pipelines:
metrics:
exporters: [exampleexporter]

17
config/testdata/pipeline-processor-not-exists.yaml vendored
View File

@ -4,11 +4,12 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces:
receivers:
- multireceiver
processors:
- nosuchprocessor
exporters:
- exampleexporter
service:
pipelines:
traces:
receivers:
- multireceiver
processors:
- nosuchprocessor
exporters:
- exampleexporter

12
config/testdata/simple-config-with-all-env.yaml vendored
View File

@ -31,12 +31,6 @@ exporters:
- "${EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_1}"
- "${EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_2}"
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
extensions:
exampleextension:
extra: "${EXTENSIONS_EXAMPLEEXTENSION_EXTRA}"
@ -49,3 +43,9 @@ extensions:
service:
extensions: [exampleextension]
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]

12
config/testdata/simple-config-with-no-env.yaml vendored
View File

@ -31,12 +31,6 @@ exporters:
- "some exporter list value_1"
- "some exporter list value_2"
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
extensions:
exampleextension:
extra: "some extension string"
@ -49,3 +43,9 @@ extensions:
service:
extensions: [exampleextension]
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]

12
config/testdata/simple-config-with-partial-env.yaml vendored
View File

@ -31,12 +31,6 @@ exporters:
- "${EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_1}"
- "${EXPORTERS_EXAMPLEEXPORTER_EXTRA_LIST_VALUE_2}"
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
extensions:
exampleextension:
extra: "${EXTENSIONS_EXAMPLEEXTENSION_EXTRA}"
@ -49,3 +43,9 @@ extensions:
service:
extensions: [exampleextension]
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]

11
config/testdata/unknown-exporter-type.yaml vendored
View File

@ -4,8 +4,9 @@ exporters:
nosuchexporter:
processors:
exampleprocessor:
pipelines:
traces:
receivers: [multireceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
traces:
receivers: [multireceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]

11
config/testdata/unknown-processor-type.yaml vendored
View File

@ -4,8 +4,9 @@ exporters:
exampleexporter:
processors:
nosuchprocessor:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]
service:
pipelines:
traces:
receivers: [examplereceiver]
exporters: [exampleexporter]
processors: [exampleprocessor]

17
config/testdata/unknown-receiver-type.yaml vendored
View File

@ -4,11 +4,12 @@ exporters:
exampleexporter:
processors:
exampleprocessor:
pipelines:
traces:
receivers:
- multireceiver
exporters:
- exampleexporter
processors:
- exampleprocessor
service:
pipelines:
traces:
receivers:
- multireceiver
exporters:
- exampleexporter
processors:
- exampleprocessor

12
config/testdata/valid-config.yaml vendored
View File

@ -18,12 +18,6 @@ exporters:
disabled: true
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver, examplereceiver/disabled]
processors: [exampleprocessor, exampleprocessor/disabled]
exporters: [exampleexporter/disabled, exampleexporter]
extensions:
exampleextension/0:
exampleextension/disabled:
@ -33,3 +27,9 @@ extensions:
service:
extensions: [exampleextension/0, exampleextension/disabled, exampleextension/1]
pipelines:
traces:
receivers: [examplereceiver, examplereceiver/disabled]
processors: [exampleprocessor, exampleprocessor/disabled]
exporters: [exampleexporter/disabled, exampleexporter]

68
docs/design.md
View File

@ -26,11 +26,12 @@ The pipeline is constructed during Collector startup based on pipeline definitio
A pipeline configuration typically looks like this:
```yaml
pipelines: # section that can contain multiple subsections, one per pipeline
traces: # type of the pipeline
receivers: [opencensus, jaeger, zipkin]
processors: [tags, tail_sampling, batch, queued_retry]
exporters: [opencensus, jaeger, stackdriver, zipkin]
service:
pipelines: # section that can contain multiple subsections, one per pipeline
traces: # type of the pipeline
receivers: [opencensus, jaeger, zipkin]
processors: [tags, tail_sampling, batch, queued_retry]
exporters: [opencensus, jaeger, stackdriver, zipkin]
```
The above example defines a pipeline for “traces” type of telemetry data, with 3 receivers, 4 processors and 4 exporters.
@ -46,15 +47,16 @@ receivers:
opencensus:
endpoint: "localhost:55678”
pipelines:
traces: # a pipeline of “traces” type
receivers: [opencensus]
processors: [tags, tail_sampling, batch, queued_retry]
exporters: [jaeger]
traces/2: # another pipeline of “traces” type
receivers: [opencensus]
processors: [batch]
exporters: [opencensus]
service:
pipelines:
traces: # a pipeline of “traces” type
receivers: [opencensus]
processors: [tags, tail_sampling, batch, queued_retry]
exporters: [jaeger]
traces/2: # another pipeline of “traces” type
receivers: [opencensus]
processors: [batch]
exporters: [opencensus]
```
In the above example “opencensus” receiver will send the same data to pipeline “traces” and to pipeline “traces/2”. (Note: the configuration uses composite key names in the form of `type[/name]` as defined in this [this document](https://docs.google.com/document/d/1GWOzV0H0RTN1adiwo7fTmkjfCATDDFGuOB4jp3ldCc8/edit#)).
@ -88,15 +90,16 @@ exporters:
grpc:
endpoint: "localhost:14250”
pipelines:
traces: # a pipeline of “traces” type
receivers: [zipkin]
processors: [tags, tail_sampling, batch, queued_retry]
exporters: [jaeger]
traces/2: # another pipeline of “traces” type
receivers: [opencensus]
processors: [batch]
exporters: [jaeger]
service:
pipelines:
traces: # a pipeline of “traces” type
receivers: [zipkin]
processors: [tags, tail_sampling, batch, queued_retry]
exporters: [jaeger]
traces/2: # another pipeline of “traces” type
receivers: [opencensus]
processors: [batch]
exporters: [jaeger]
```
In the above example “jaeger” exporter will get data from pipeline “traces” and from pipeline “traces/2”. When the Collector loads this config the result will look like this (part of processors and receivers are omitted from the diagram for brevity):
@ -120,15 +123,16 @@ processors:
per-exporter: true
enabled: true
pipelines:
traces: # a pipeline of “traces” type
receivers: [zipkin]
processors: [queued_retry]
exporters: [jaeger]
traces/2: # another pipeline of “traces” type
receivers: [opencensus]
processors: [queued_retry]
exporters: [opencensus]
service:
pipelines:
traces: # a pipeline of “traces” type
receivers: [zipkin]
processors: [queued_retry]
exporters: [jaeger]
traces/2: # another pipeline of “traces” type
receivers: [opencensus]
processors: [queued_retry]
exporters: [opencensus]
```
When the Collector loads this config the result will look like this:

19
examples/demo/otel-agent-config.yaml
View File

@ -1,7 +1,6 @@
receivers:
opencensus:
endpoint: localhost:55678
reconnection_delay: 2s
jaeger:
protocols:
thrift-http:
@ -18,15 +17,6 @@ processors:
batch:
queued_retry:
pipelines:
traces:
receivers: [opencensus, jaeger]
exporters: [opencensus, logging]
processors: [batch, queued_retry]
metrics:
receivers: [opencensus]
exporters: [logging,opencensus]
extensions:
pprof:
endpoint: localhost:1777
@ -35,3 +25,12 @@ extensions:
service:
extensions: [pprof, zpages]
pipelines:
traces:
receivers: [opencensus, jaeger]
exporters: [opencensus, logging]
processors: [batch, queued_retry]
metrics:
receivers: [opencensus]
exporters: [logging,opencensus]

17
examples/demo/otel-collector-config.yaml
View File

@ -26,15 +26,6 @@ processors:
batch:
queued_retry:
pipelines:
traces:
receivers: [opencensus]
exporters: [logging, zipkin, jaeger_grpc]
processors: [batch, queued_retry]
metrics:
receivers: [opencensus]
exporters: [logging,prometheus]
extensions:
health_check:
pprof:
@ -44,4 +35,12 @@ extensions:
service:
extensions: [pprof, zpages, health_check]
pipelines:
traces:
receivers: [opencensus]
exporters: [logging, zipkin, jaeger_grpc]
processors: [batch, queued_retry]
metrics:
receivers: [opencensus]
exporters: [logging,prometheus]

11
exporter/jaeger/jaegergrpcexporter/testdata/config.yaml vendored
View File

@ -10,8 +10,9 @@ exporters:
jaeger_grpc/2:
endpoint: "a.new.target:1234"
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [jaeger_grpc, jaeger_grpc/2]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [jaeger_grpc, jaeger_grpc/2]

11
exporter/jaeger/jaegerthrifthttpexporter/testdata/config.yaml vendored
View File

@ -14,8 +14,9 @@ exporters:
added-entry: "added value"
dot.test: test
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [jaeger_thrift_http, jaeger_thrift_http/2]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [jaeger_thrift_http, jaeger_thrift_http/2]

17
exporter/loggingexporter/testdata/config.yaml vendored
View File

@ -9,11 +9,12 @@ exporters:
logging/2:
loglevel: debug
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [logging]
metrics:
receivers: [examplereceiver]
exporters: [logging,logging/2]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [logging]
metrics:
receivers: [examplereceiver]
exporters: [logging,logging/2]

11
exporter/opencensusexporter/testdata/config.yaml vendored
View File

@ -22,8 +22,9 @@ exporters:
timeout: 30
permit_without_stream: true
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [opencensus]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [opencensus]

11
exporter/prometheusexporter/testdata/config.yaml vendored
View File

@ -13,8 +13,9 @@ exporters:
label1: value1
"another label": spaced value
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [prometheus]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [prometheus]

11
exporter/zipkinexporter/testdata/config.yaml vendored
View File

@ -10,8 +10,9 @@ exporters:
zipkin/2:
url: "https://somedest:1234/api/v2/spans"
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [zipkin, zipkin/2]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [zipkin, zipkin/2]

10
extension/healthcheckextension/testdata/config.yaml vendored
View File

@ -5,6 +5,11 @@ extensions:
service:
extensions: [health_check/1]
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
# Data pipeline is required to load the config.
receivers:
@ -13,8 +18,3 @@ processors:
exampleprocessor:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]

10
extension/pprofextension/testdata/config.yaml vendored
View File

@ -7,6 +7,11 @@ extensions:
service:
extensions: [pprof/1]
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
# Data pipeline is required to load the config.
receivers:
@ -15,8 +20,3 @@ processors:
exampleprocessor:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]

10
extension/zpagesextension/testdata/config.yaml vendored
View File

@ -5,6 +5,11 @@ extensions:
service:
extensions: [zpages/1]
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]
# Data pipeline is required to load the config.
receivers:
@ -13,8 +18,3 @@ processors:
exampleprocessor:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver]
processors: [exampleprocessor]
exporters: [exampleexporter]

11
processor/attributesprocessor/testdata/config.yaml vendored
View File

@ -198,10 +198,11 @@ receivers:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver]
processors: [attributes/insert]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [attributes/insert]
exporters: [exampleexporter]

11
processor/nodebatcherprocessor/testdata/config.yaml vendored
View File

@ -13,8 +13,9 @@ processors:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver]
processors: [batch/2]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [batch/2]
exporters: [exampleexporter]

11
processor/probabilisticsamplerprocessor/testdata/config.yaml vendored
View File

@ -9,8 +9,9 @@ processors:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver]
processors: [probabilistic_sampler]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [probabilistic_sampler]
exporters: [exampleexporter]

11
processor/probabilisticsamplerprocessor/testdata/empty.yaml vendored
View File

@ -8,8 +8,9 @@ processors:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver]
processors: [probabilistic_sampler]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [probabilistic_sampler]
exporters: [exampleexporter]

11
processor/queuedprocessor/testdata/config.yaml vendored
View File

@ -12,8 +12,9 @@ processors:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver]
processors: [queued_retry/2]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [queued_retry/2]
exporters: [exampleexporter]

11
processor/spanprocessor/testdata/config.yaml vendored
View File

@ -46,8 +46,9 @@ processors:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [examplereceiver]
processors: [span/custom]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [span/custom]
exporters: [exampleexporter]

11
processor/tailsamplingprocessor/testdata/tail_sampling_config.yaml vendored
View File

@ -32,8 +32,9 @@ processors:
}
]
pipelines:
traces:
receivers: [examplereceiver]
processors: [tail_sampling]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [examplereceiver]
processors: [tail_sampling]
exporters: [exampleexporter]

11
receiver/jaegerreceiver/testdata/config.yaml vendored
View File

@ -53,9 +53,10 @@ processors:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [jaeger]
processors: [exampleprocessor]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [jaeger]
processors: [exampleprocessor]
exporters: [exampleexporter]

17
receiver/opencensusreceiver/testdata/config.yaml vendored
View File

@ -60,11 +60,12 @@ processors:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [opencensus/customname]
processors: [exampleprocessor]
exporters: [exampleexporter]
metrics:
receivers: [opencensus]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [opencensus/customname]
processors: [exampleprocessor]
exporters: [exampleexporter]
metrics:
receivers: [opencensus]
exporters: [exampleexporter]

11
receiver/prometheusreceiver/testdata/config.yaml vendored
View File

@ -19,8 +19,9 @@ processors:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [prometheus]
processors: [exampleprocessor]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [prometheus]
processors: [exampleprocessor]
exporters: [exampleexporter]

11
receiver/prometheusreceiver/testdata/invalid-config-section.yaml vendored
View File

@ -12,8 +12,9 @@ processors:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [prometheus]
processors: [exampleprocessor]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [prometheus]
processors: [exampleprocessor]
exporters: [exampleexporter]

11
receiver/vmmetricsreceiver/testdata/config.yaml vendored
View File

@ -12,8 +12,9 @@ processors:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [vmmetrics]
processors: [exampleprocessor]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [vmmetrics]
processors: [exampleprocessor]
exporters: [exampleexporter]

11
receiver/zipkinreceiver/testdata/config.yaml vendored
View File

@ -9,9 +9,10 @@ processors:
exporters:
exampleexporter:
pipelines:
traces:
receivers: [zipkin]
processors: [exampleprocessor]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [zipkin]
processors: [exampleprocessor]
exporters: [exampleexporter]

2
service/builder/exporters_builder.go
View File

@ -121,7 +121,7 @@ func (eb *ExportersBuilder) calcExportersRequiredDataTypes() exportersRequiredDa
result := make(exportersRequiredDataTypes)
// Iterate over pipelines.
for _, pipeline := range eb.config.Pipelines {
for _, pipeline := range eb.config.Service.Pipelines {
// Iterate over all exporters for this pipeline.
for _, expName := range pipeline.Exporters {
// Find the exporter config by name.

14
service/builder/exporters_builder_test.go
View File

@ -46,11 +46,13 @@ func TestExportersBuilder_Build(t *testing.T) {
},
},
Pipelines: map[string]*configmodels.Pipeline{
"trace": {
Name: "trace",
InputType: configmodels.TracesDataType,
Exporters: []string{"opencensus"},
Service: configmodels.Service{
Pipelines: map[string]*configmodels.Pipeline{
"trace": {
Name: "trace",
InputType: configmodels.TracesDataType,
Exporters: []string{"opencensus"},
},
},
},
}
@ -80,7 +82,7 @@ func TestExportersBuilder_Build(t *testing.T) {
// Remove the pipeline so that the exporter is not attached to any pipeline.
// This should result in creating an exporter that has none of consumption
// functions set.
delete(cfg.Pipelines, "trace")
delete(cfg.Service.Pipelines, "trace")
exporters, err = NewExportersBuilder(zap.NewNop(), cfg, factories.Exporters).Build()
assert.NotNil(t, exporters)
assert.Nil(t, err)

2
service/builder/pipelines_builder.go
View File

@ -62,7 +62,7 @@ func NewPipelinesBuilder(
func (pb *PipelinesBuilder) Build() (BuiltPipelines, error) {
pipelineProcessors := make(BuiltPipelines)
for _, pipeline := range pb.config.Pipelines {
for _, pipeline := range pb.config.Service.Pipelines {
firstProcessor, err := pb.buildPipeline(pipeline)
if err != nil {
return nil, err

4
service/builder/pipelines_builder_test.go
View File

@ -94,7 +94,7 @@ func testPipeline(t *testing.T, pipelineName string, exporterNames []string) {
assert.NoError(t, err)
require.NotNil(t, pipelineProcessors)
processor := pipelineProcessors[cfg.Pipelines[pipelineName]]
processor := pipelineProcessors[cfg.Service.Pipelines[pipelineName]]
// Ensure pipeline has its fields correctly populated.
require.NotNil(t, processor)
@ -164,7 +164,7 @@ func TestPipelinesBuilder_Error(t *testing.T) {
// Corrupt the pipeline, change data type to metrics. We have to forcedly do it here
// since there is no way to have such config loaded by LoadConfigFile, it would not
// pass validation. We are doing this to test failure mode of PipelinesBuilder.
pipeline := cfg.Pipelines["traces"]
pipeline := cfg.Service.Pipelines["traces"]
pipeline.InputType = configmodels.MetricsDataType
exporters, err := NewExportersBuilder(zap.NewNop(), cfg, factories.Exporters).Build()

2
service/builder/receivers_builder.go
View File

@ -154,7 +154,7 @@ func (rb *ReceiversBuilder) findPipelinesToAttach(config configmodels.Receiver)
pipelinesToAttach[configmodels.MetricsDataType] = make([]*builtPipeline, 0)
// Iterate over all pipelines.
for _, pipelineCfg := range rb.config.Pipelines {
for _, pipelineCfg := range rb.config.Service.Pipelines {
// Get the first processor of the pipeline.
pipelineProcessor := rb.builtPipelines[pipelineCfg]
if pipelineProcessor == nil {

37
service/builder/testdata/pipelines_builder.yaml vendored
View File

@ -15,25 +15,26 @@ exporters:
exampleexporter:
exampleexporter/2:
pipelines:
traces:
receivers: [examplereceiver, examplereceiver/multi]
processors: [attributes]
exporters: [exampleexporter]
service:
pipelines:
traces:
receivers: [examplereceiver, examplereceiver/multi]
processors: [attributes]
exporters: [exampleexporter]
traces/2:
receivers: [examplereceiver/2, examplereceiver/multi]
processors: [attributes]
exporters: [exampleexporter, exampleexporter/2]
traces/2:
receivers: [examplereceiver/2, examplereceiver/multi]
processors: [attributes]
exporters: [exampleexporter, exampleexporter/2]
metrics:
receivers: [examplereceiver]
exporters: [exampleexporter]
metrics:
receivers: [examplereceiver]
exporters: [exampleexporter]
metrics/2:
receivers: [examplereceiver/3]
exporters: [exampleexporter]
metrics/2:
receivers: [examplereceiver/3]
exporters: [exampleexporter]
metrics/3:
receivers: [examplereceiver/3]
exporters: [exampleexporter/2]
metrics/3:
receivers: [examplereceiver/3]
exporters: [exampleexporter/2]

12
service/testdata/otelcol-config.yaml vendored
View File

@ -14,12 +14,6 @@ processors:
queued_retry:
batch:
pipelines:
traces:
receivers: [jaeger]
processors: [attributes, batch, queued_retry]
exporters: [opencensus]
extensions:
health_check:
pprof:
@ -27,3 +21,9 @@ extensions:
service:
extensions: [pprof, zpages, health_check]
pipelines:
traces:
receivers: [jaeger]
processors: [attributes, batch, queued_retry]
exporters: [opencensus]

11
testbed/tests/testdata/add-attributes-config.yaml vendored
View File

@ -28,8 +28,9 @@ processors:
value: test-datacenter
action: insert
pipelines:
traces:
receivers: [jaeger]
processors: [attributes,queued_retry]
exporters: [opencensus]
service:
pipelines:
traces:
receivers: [jaeger]
processors: [attributes,queued_retry]
exporters: [opencensus]

11
testbed/tests/testdata/agent-config.yaml vendored
View File

@ -11,8 +11,9 @@ exporters:
processors:
queued_retry:
pipelines:
traces:
receivers: [jaeger]
processors: [queued_retry]
exporters: [opencensus]
service:
pipelines:
traces:
receivers: [jaeger]
processors: [queued_retry]
exporters: [opencensus]