Add deprecation warning

Signed-off-by: Yuri Shkuro <yurishkuro@users.noreply.github.com>

.github/workflows/build.yaml

@@ -1,10 +1,36 @@
-name: Test, format and lint
+name: Build, test, format and lint
 
 on:
   push:
   pull_request:
 
 jobs:
+  build-binaries:
+    runs-on: ubuntu-latest
+    name: Build binary for ${{ matrix.platform.name }}
+    strategy:
+      matrix:
+        platform:
+        - name: linux on amd64
+          task: build-linux-amd64
+        - name: linux on arm64
+          task: build-linux-arm64
+        - name: osx on amd64
+          task: build-darwin-amd64
+        - name: osx on arm64
+          task: build-darwin-arm64
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          submodules: true
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ^1.19
+
+      - name: Build binaries
+        run: make ${{ matrix.platform.task }}
+
   format-lint:
     runs-on: ubuntu-latest
     name: Format and lint
@@ -15,7 +41,7 @@ jobs:
 
       - uses: actions/setup-go@v2
         with:
-          go-version: ^1.16
+          go-version: ^1.19
 
       - name: Install tools
         run: make install-tools
@@ -36,7 +62,7 @@ jobs:
 
       - uses: actions/setup-go@v2
         with:
-          go-version: ^1.16
+          go-version: ^1.19
 
       - name: Run e2e test
         run: make e2e-tests
@@ -51,7 +77,25 @@ jobs:
 
       - uses: actions/setup-go@v2
         with:
-          go-version: ^1.16
+          go-version: ^1.19
 
       - name: Run unit test
         run: make test
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    name: Integration tests
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          submodules: true
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ^1.19
+
+      - name: Setup database
+        run: docker run --rm -d -p9000:9000 --name test-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server:22
+
+      - name: Run integration tests
+        run: make integration-test

.github/workflows/release.yaml

@@ -15,10 +15,10 @@ jobs:
 
       - uses: actions/setup-go@v2
         with:
-          go-version: ^1.16
+          go-version: ^1.19
 
       - name: Create release distribution
-        run: make build tar
+        run: make build-all-platforms tar-all-platforms
 
       - name: Log in to the Container registry
         uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9

.golangci.yml

@@ -125,6 +125,10 @@ issues:
     - text: "G402:"
       linters:
         - gosec
+    - path: grpc_test.go
+      linters:
+        # See https://github.com/golangci/golangci-lint/issues/2286
+        - typecheck
 
   # The list of ids of default excludes to include or disable. By default it's empty.
   # See the list of default excludes here https://golangci-lint.run/usage/configuration.

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.13
+FROM docker.io/library/alpine:3.16
 
 ADD jaeger-clickhouse-linux-amd64 /go/bin/jaeger-clickhouse
 

Makefile

@@ -3,15 +3,34 @@ GOARCH ?= $(shell go env GOARCH)
 GOBUILD=CGO_ENABLED=0 installsuffix=cgo go build -trimpath
 
 TOOLS_MOD_DIR = ./internal/tools
-JAEGER_VERSION ?= 1.24.0
+JAEGER_VERSION ?= 1.32.0
 
-DOCKER_REPO ?= ghcr.io/pavolloffay/jaeger-clickhouse
+DOCKER_REPO ?= ghcr.io/jaegertracing/jaeger-clickhouse
 DOCKER_TAG ?= latest
 
 .PHONY: build
 build:
 	${GOBUILD} -o jaeger-clickhouse-$(GOOS)-$(GOARCH) ./cmd/jaeger-clickhouse/main.go
 
+.PHONY: build-linux-amd64
+build-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) build
+
+.PHONY: build-linux-arm64
+build-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) build
+
+.PHONY: build-darwin-amd64
+build-darwin-amd64:
+	GOOS=darwin GOARCH=amd64 $(MAKE) build
+
+.PHONY: build-darwin-arm64
+build-darwin-arm64:
+	GOOS=darwin GOARCH=arm64 $(MAKE) build
+
+.PHONY: build-all-platforms
+build-all-platforms: build-linux-amd64 build-linux-arm64 build-darwin-amd64 build-darwin-arm64
+
 .PHONY: e2e-tests
 e2e-tests:
 	GOOS=linux GOARCH=amd64 $(MAKE) build
@@ -28,20 +47,46 @@ run-hotrod:
 .PHONY: fmt
 fmt:
 	go fmt ./...
-	goimports -w  -local github.com/pavolloffay/jaeger-clickhouse ./
+	goimports -w -local github.com/jaegertracing/jaeger-clickhouse ./
 
 .PHONY: lint
 lint:
-	golangci-lint run --allow-parallel-runners ./...
+	golangci-lint -v run --allow-parallel-runners ./...
 
 .PHONY: test
 test:
 	go test ./...
 
+.PHONY: integration-test
+integration-test: build
+	STORAGE=grpc-plugin \
+	PLUGIN_BINARY_PATH=$(PWD)/jaeger-clickhouse-linux-amd64 \
+	PLUGIN_CONFIG_PATH=$(PWD)/integration/config-local.yaml \
+	go test ./integration
+
 .PHONY: tar
 tar:
 	tar -czvf jaeger-clickhouse-$(GOOS)-$(GOARCH).tar.gz  jaeger-clickhouse-$(GOOS)-$(GOARCH) config.yaml
 
+.PHONY: tar-linux-amd64
+tar-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) tar
+
+.PHONY: tar-linux-arm64
+tar-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) tar
+
+.PHONY: tar-darwin-amd64
+tar-darwin-amd64:
+	GOOS=darwin GOARCH=amd64 $(MAKE) tar
+
+.PHONY: tar-darwin-arm64
+tar-darwin-arm64:
+	GOOS=darwin GOARCH=arm64 $(MAKE) tar
+
+.PHONY: tar-all-platforms
+tar-all-platforms: tar-linux-amd64 tar-linux-arm64 tar-darwin-amd64 tar-darwin-arm64
+
 .PHONY: docker
 docker: build
 	docker build -t ${DOCKER_REPO}:${DOCKER_TAG} -f Dockerfile .

README.md

@@ -1,13 +1,39 @@
-# Jaeger ClickHouse
+# Jaeger ClickHouse (experimental)
 
-This is implementation of Jaeger ClickHouse [storage plugin](https://github.com/jaegertracing/jaeger/tree/master/plugin/storage/grpc).
-See as well [jaegertracing/jaeger/issues/1438](https://github.com/jaegertracing/jaeger/issues/1438) for historical discussion regarding Clickhouse plugin.
+⚠️ This module only implements grpc-plugin API that has been deprecated in Jaeger (https://github.com/jaegertracing/jaeger/issues/4647).
+
+This is a [Jaeger gRPC storage plugin](https://github.com/jaegertracing/jaeger/tree/master/plugin/storage/grpc) implementation for storing traces in ClickHouse.
+
+## Project status
+
+This is a community-driven project, and we would love to hear your issues and feature requests.
+Pull requests are also greatly appreciated.
+
+## Why use ClickHouse for Jaeger?
+
+[ClickHouse](https://clickhouse.com) is an analytical column-oriented database management system.
+It is designed to analyze streams of events which are kind of resemblant to spans.
+It's open-source, optimized for performance, and actively developed.
+
+## How it works
+
+Jaeger spans are stored in 2 tables. The first contains the whole span encoded either in JSON or Protobuf.
+The second stores key information about spans for searching. This table is indexed by span duration and tags.
+Also, info about operations is stored in the materialized view. There are not indexes for archived spans.
+Storing data in replicated local tables with distributed global tables is natively supported. Spans are bufferized.
+Span buffers are flushed to DB either by timer or after reaching max batch size. Timer interval and batch size can be
+set in [config file](./config.yaml).
+
+Database schema generated by JetBrains DataGrip
+![Picture of tables](./pictures/tables.png)
+
+# How to start using Jaeger over ClickHouse
 
 ## Documentation
 
 Refer to the [config.yaml](./config.yaml) for all supported configuration options.
 
-* [Kubernetes deployment documentation](./guide-kubernetes.md)
+* [Kubernetes deployment](./guide-kubernetes.md)
 * [Sharding and replication](./guide-sharding-and-replication.md)
 * [Multi-tenancy](./guide-multitenancy.md)
 
@@ -16,7 +42,7 @@ Refer to the [config.yaml](./config.yaml) for all supported configuration option
 ### Docker database example
 
 ```bash
-docker run --rm -it -p9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server:21
+docker run --rm -it -p9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server:22
 GOOS=linux make build run
 make run-hotrod
 ```
@@ -25,7 +51,7 @@ Open [localhost:16686](http://localhost:16686) and [localhost:8080](http://local
 
 ### Custom database
 
-You need to specify connection options in config.yaml file, then you can run 
+You need to specify connection options in `config.yaml`, then you can run
 
 ```bash
 make build
@@ -34,4 +60,6 @@ SPAN_STORAGE_TYPE=grpc-plugin {Jaeger binary adress} --query.ui-config=jaeger-ui
 
 ## Credits
 
-This project is based on https://github.com/bobrik/jaeger/tree/ivan/clickhouse/plugin/storage/clickhouse.
+This project is originally based on [this clickhouse plugin implementation](https://github.com/bobrik/jaeger/tree/ivan/clickhouse/plugin/storage/clickhouse).
+
+See also [jaegertracing/jaeger/issues/1438](https://github.com/jaegertracing/jaeger/issues/1438) for historical discussion regarding the implementation of a ClickHouse plugin.

blog/post1.md

@@ -0,0 +1,96 @@
+# Jaeger ClickHouse
+This is an implementation of Jaeger's [storage plugin](https://github.com/jaegertracing/jaeger/tree/master/plugin/storage/grpc) for ClickHouse.
+See as well [jaegertracing/jaeger/issues/1438](https://github.com/jaegertracing/jaeger/issues/1438) for historical discussion regarding Clickhouse plugin.
+
+## Project status
+
+Jaeger ClickHouse is a community-driven project, we would love to hear your feature requests.
+Pull requests also will be greatly appreciated.
+
+## Why use ClickHouse for Jaeger?
+
+[ClickHouse](https://github.com/clickhouse/clickhouse) is an analytical column-oriented database management system. It is designed to analyze streams of clicks which are kind of resemblant to spans. It's open-source, optimized for performance, and actively developed.
+
+## How does it work?
+
+Jaeger spans are stored in 2 tables. First one contains whole span encoded either in JSON or Protobuf.
+Second stores key information about spans for searching. This table is indexed by span duration and tags.
+Also, info about operations is stored in the materialized view. There are no indexes for archived spans.
+Storing data in replicated local tables with distributed global tables is natively supported. Spans are buffered.
+Span buffers are flushed to DB either by timer or after reaching max batch size.
+Timer interval and batch size can be set in [config file](../config.yaml).
+
+![Picture of tables](post1_pics/tables.png)
+
+## Benchmarks
+
+10^8 traces were flushed using [jaeger-tracegen](https://www.jaegertracing.io/docs/1.25/tools/) to Clickhouse and ElasticSearch servers.
+Clickhouse server consisted of 3 shards, 2 hosts in each, and 3 Zookeeper hosts. Elasticsearch server consisted of 6 hosts,
+with 5 shards for primary index and 1 replica. All hosts were equal(8 vCPU, 32 GiB RAM, 20 GiB SSD).
+
+### General stats
+
+Cpu usage, [% of 1 host CPU]
+![CPU usage](post1_pics/cpu-usage.png)
+
+Memory usage, [bytes]
+![Memory usage](post1_pics/memory-usage.png)
+
+IO write, [operations]
+![IO write](post1_pics/io-write.png)
+
+Disk usage, [bytes]
+![Disk usage](post1_pics/disk-usage.png)
+
+### Recorded
+
+#### ClickHouse
+
+```sql
+SELECT count()
+FROM jaeger_index
+WHERE service = 'tracegen'
+
+┌──count()─┐
+│ 57026426 │
+└──────────┘
+```
+
+#### Elasticsearch
+
+![Elasticsearch span count](post1_pics/elastic-count.png?raw=true)
+
+# How to start using Jaeger over ClickHouse
+
+## Documentation
+
+Refer to the [config.yaml](../config.yaml) for all supported configuration options.
+
+* [Kubernetes deployment](../guide-kubernetes.md)
+* [Sharding and replication](../guide-sharding-and-replication.md)
+* [Multi-tenancy](../guide-multitenancy.md)
+
+## Build & Run
+
+### Docker database example
+
+```bash
+docker run --rm -it -p9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server:22
+GOOS=linux make build run
+make run-hotrod
+```
+
+Open [localhost:16686](http://localhost:16686) and [localhost:8080](http://localhost:8080).
+
+### Custom database
+
+You need to specify connection options in config.yaml file, then you can run
+
+```bash
+make build
+SPAN_STORAGE_TYPE=grpc-plugin {Jaeger binary adress} --query.ui-config=jaeger-ui.json --grpc-storage-plugin.binary=./{name of built binary} --grpc-storage-plugin.configuration-file=config.yaml --grpc-storage-plugin.log-level=debug
+```
+
+## Credits
+
+This project is based on https://github.com/bobrik/jaeger/tree/ivan/clickhouse/plugin/storage/clickhouse.

cmd/jaeger-clickhouse/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"flag"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -10,13 +9,13 @@ import (
 	// Package contains time zone info for connecting to ClickHouse servers with non-UTC time zone
 	_ "time/tzdata"
 
-	"github.com/hashicorp/go-hclog"
+	hclog "github.com/hashicorp/go-hclog"
 	"github.com/jaegertracing/jaeger/plugin/storage/grpc"
 	"github.com/jaegertracing/jaeger/plugin/storage/grpc/shared"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"gopkg.in/yaml.v3"
+	yaml "gopkg.in/yaml.v3"
 
-	"github.com/pavolloffay/jaeger-clickhouse/storage"
+	"github.com/jaegertracing/jaeger-clickhouse/storage"
 )
 
 func main() {
@@ -32,7 +31,7 @@ func main() {
 		JSONFormat: true,
 	})
 
-	cfgFile, err := ioutil.ReadFile(filepath.Clean(configPath))
+	cfgFile, err := os.ReadFile(filepath.Clean(configPath))
 	if err != nil {
 		logger.Error("Could not read config file", "config", configPath, "error", err)
 		os.Exit(1)
@@ -59,6 +58,7 @@ func main() {
 	}
 	pluginServices.Store = store
 	pluginServices.ArchiveStore = store
+	pluginServices.StreamingSpanWriter = store
 
 	grpc.Serve(&pluginServices)
 	if err = store.Close(); err != nil {

config.yaml

@@ -1,6 +1,19 @@
-address: tcp://some-clickhouse-server:9000
-# When empty the embedded scripts from sqlscripts directory are used
+address: some-clickhouse-server:9000
+# Directory with .sql files to run at plugin startup, mainly for integration tests.
+# Depending on the value of "init_tables", this can be run as a
+# replacement or supplement to creating default tables for span storage.
+# If init_tables is also enabled, the scripts in this directory will be run first.
 init_sql_scripts_dir:
+# Whether to automatically attempt to create tables in ClickHouse.
+# By default, this is enabled if init_sql_scripts_dir is empty,
+# or disabled if init_sql_scripts_dir is provided.
+init_tables:
+# Maximal amount of spans that can be pending writes at a time.
+# New spans exceeding this limit will be discarded,
+# keeping memory in check if there are issues writing to ClickHouse.
+# Check the "jaeger_clickhouse_discarded_spans" metric to keep track of discards.
+# If 0, no limit is set. Default 10_000_000.
+max_span_count:
 # Batch write size. Default 10_000.
 batch_write_size:
 # Batch flush interval. Default 5s.
@@ -9,13 +22,16 @@ batch_flush_interval:
 encoding:
 # Path to CA TLS certificate.
 ca_file:
-# Username for connection. Default is "default".
+# Username for connection to ClickHouse. Default is "default".
 username:
-# Password for connection.
+# Password for connection to ClickHouse.
 password:
-# Database name. The database has to be created manually before Jaeger starts. Default is "default".
+# ClickHouse database name. The database must be created manually before Jaeger starts. Default is "default".
 database:
-# Endpoint for scraping prometheus metrics. Default localhost:9090.
+# If non-empty, enables a tenant column in tables, and uses the provided tenant name for this instance.
+# Default is empty. See guide-multitenancy.md for more information.
+tenant:
+# Endpoint for serving prometheus metrics. Default localhost:9090.
 metrics_endpoint: localhost:9090
 # Whether to use sql scripts supporting replication and sharding.
 # Replication can be used only on database with Atomic engine.
@@ -27,3 +43,7 @@ spans_table:
 spans_index_table:
 # Operations table. Default "jaeger_operations_local" or "jaeger_operations" when replication is enabled.
 operations_table:
+# TTL for data in tables in days. If 0, no TTL is set. Default 0.
+ttl:
+# The maximum number of spans to fetch per trace. If 0, no limit is set. Default 0.
+max_num_spans:

e2etests/clickhouse-replicated.xml

@@ -0,0 +1,47 @@
+<!-- Minimal configuration to enable cluster mode in a single clickhouse process -->
+<yandex>
+    <macros>
+        <installation>cluster</installation>
+        <all-sharded-shard>0</all-sharded-shard>
+        <cluster>cluster</cluster>
+        <shard>0</shard>
+        <replica>cluster-0-0</replica>
+    </macros>
+
+    <remote_servers>
+        <cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster>
+    </remote_servers>
+
+    <keeper_server>
+        <tcp_port>2181</tcp_port>
+        <server_id>0</server_id>
+        <log_storage_path>/var/log/clickhouse-server/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <raft_configuration>
+            <server>
+                <id>0</id>
+                <hostname>localhost</hostname>
+                <port>9444</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+
+    <zookeeper>
+        <!-- Clickhouse Keeper -->
+        <node>
+            <host>localhost</host>
+            <port>2181</port>
+        </node>
+    </zookeeper>
+    <distributed_ddl>
+        <path>/clickhouse/cluster/task_queue/ddl</path>
+    </distributed_ddl>
+</yandex>

e2etests/config-local-multi1.yaml

@@ -0,0 +1,4 @@
+address: chi:9000
+tenant: multi1
+# For test purposes flush on every write
+batch_write_size: 1

e2etests/config-local-multi2.yaml

@@ -0,0 +1,4 @@
+address: chi:9000
+tenant: multi2
+# For test purposes flush on every write
+batch_write_size: 1

e2etests/config-local-single.yaml

@@ -1,3 +1,3 @@
-address: tcp://chi:9000
+address: chi:9000
 # For test purposes flush on every write
 batch_write_size: 1

e2etests/config-replication-multi1.yaml

@@ -0,0 +1,5 @@
+address: chi:9000
+replication: true
+tenant: multi1
+# For test purposes flush on every write
+batch_write_size: 1

e2etests/config-replication-multi2.yaml

@@ -0,0 +1,5 @@
+address: chi:9000
+replication: true
+tenant: multi2
+# For test purposes flush on every write
+batch_write_size: 1

e2etests/config-replication-single.yaml

@@ -0,0 +1,4 @@
+address: chi:9000
+replication: true
+# For test purposes flush on every write
+batch_write_size: 1

e2etests/e2e_test.go

@@ -2,7 +2,6 @@ package e2etests
 
 import (
 	"context"
-	"database/sql"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -11,17 +10,17 @@ import (
 	"testing"
 	"time"
 
-	_ "github.com/ClickHouse/clickhouse-go" // import driver
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/ecodia/golang-awaitility/awaitility"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/testcontainers/testcontainers-go"
+	testcontainers "github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
 )
 
 const (
-	clickHouseImage = "yandex/clickhouse-server:21"
-	jaegerImage     = "jaegertracing/all-in-one:1.24.0"
+	clickHouseImage = "clickhouse/clickhouse-server:22"
+	jaegerImage     = "jaegertracing/all-in-one:1.32.0"
 
 	networkName     = "chi-jaeger-test"
 	clickhousePort  = "9000/tcp"
@@ -29,11 +28,45 @@ const (
 	jaegerAdminPort = "14269/tcp"
 )
 
+type testCase struct {
+	configs []string
+	chiconf *string
+}
+
 func TestE2E(t *testing.T) {
 	if os.Getenv("E2E_TEST") == "" {
 		t.Skip("Set E2E_TEST=true to run the test")
 	}
 
+	// Minimal additional configuration (config.d) to enable cluster mode
+	chireplconf := "clickhouse-replicated.xml"
+
+	tests := map[string]testCase{
+		"local-single": {
+			configs: []string{"config-local-single.yaml"},
+			chiconf: nil,
+		},
+		"local-multi": {
+			configs: []string{"config-local-multi1.yaml", "config-local-multi2.yaml"},
+			chiconf: nil,
+		},
+		"replication-single": {
+			configs: []string{"config-replication-single.yaml"},
+			chiconf: &chireplconf,
+		},
+		"replication-multi": {
+			configs: []string{"config-replication-multi1.yaml", "config-replication-multi2.yaml"},
+			chiconf: &chireplconf,
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			testE2E(t, test)
+		})
+	}
+}
+
+func testE2E(t *testing.T, test testCase) {
 	ctx := context.Background()
 	workingDir, err := os.Getwd()
 	require.NoError(t, err)
@@ -44,12 +77,21 @@ func TestE2E(t *testing.T) {
 	require.NoError(t, err)
 	defer network.Remove(ctx)
 
+	var bindMounts map[string]string
+	if test.chiconf != nil {
+		bindMounts = map[string]string{
+			fmt.Sprintf("%s/%s", workingDir, *test.chiconf): "/etc/clickhouse-server/config.d/testconf.xml",
+		}
+	} else {
+		bindMounts = map[string]string{}
+	}
 	chReq := testcontainers.ContainerRequest{
 		Image:        clickHouseImage,
 		ExposedPorts: []string{clickhousePort},
 		WaitingFor:   &clickhouseWaitStrategy{test: t, pollInterval: time.Millisecond * 200, startupTimeout: time.Minute},
 		Networks:     []string{networkName},
 		Hostname:     "chi",
+		BindMounts:   bindMounts,
 	}
 	chContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
 		ContainerRequest: chReq,
@@ -58,53 +100,61 @@ func TestE2E(t *testing.T) {
 	require.NoError(t, err)
 	defer chContainer.Terminate(ctx)
 
-	jaegerReq := testcontainers.ContainerRequest{
-		Image:        jaegerImage,
-		ExposedPorts: []string{jaegerQueryPort, jaegerAdminPort},
-		WaitingFor:   wait.ForHTTP("/").WithPort(jaegerAdminPort).WithStartupTimeout(time.Second * 10),
-		Env: map[string]string{
-			"SPAN_STORAGE_TYPE": "grpc-plugin",
-		},
-		Cmd: []string{
-			"--grpc-storage-plugin.binary=/project-dir/jaeger-clickhouse-linux-amd64",
-			"--grpc-storage-plugin.configuration-file=/project-dir/e2etests/config.yaml",
-			"--grpc-storage-plugin.log-level=debug",
-		},
-		BindMounts: map[string]string{
-			workingDir + "/..": "/project-dir",
-		},
-		Networks: []string{networkName},
+	jaegerContainers := make([]testcontainers.Container, 0)
+	for _, pluginConfig := range test.configs {
+		jaegerReq := testcontainers.ContainerRequest{
+			Image:        jaegerImage,
+			ExposedPorts: []string{jaegerQueryPort, jaegerAdminPort},
+			WaitingFor:   wait.ForHTTP("/").WithPort(jaegerAdminPort).WithStartupTimeout(time.Second * 10),
+			Env: map[string]string{
+				"SPAN_STORAGE_TYPE": "grpc-plugin",
+			},
+			Cmd: []string{
+				"--grpc-storage-plugin.binary=/project-dir/jaeger-clickhouse-linux-amd64",
+				fmt.Sprintf("--grpc-storage-plugin.configuration-file=/project-dir/e2etests/%s", pluginConfig),
+				"--grpc-storage-plugin.log-level=debug",
+			},
+			BindMounts: map[string]string{
+				workingDir + "/..": "/project-dir",
+			},
+			Networks: []string{networkName},
+		}
+		// Call Start() manually here so that if it fails then we can still access the logs.
+		jaegerContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+			ContainerRequest: jaegerReq,
+		})
+		require.NoError(t, err)
+		defer func() {
+			logs, errLogs := jaegerContainer.Logs(ctx)
+			require.NoError(t, errLogs)
+			all, errLogs := ioutil.ReadAll(logs)
+			require.NoError(t, errLogs)
+			fmt.Printf("Jaeger logs:\n---->\n%s<----\n\n", string(all))
+			jaegerContainer.Terminate(ctx)
+		}()
+		err = jaegerContainer.Start(ctx)
+		require.NoError(t, err)
+
+		jaegerContainers = append(jaegerContainers, jaegerContainer)
 	}
-	jaegerContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
-		ContainerRequest: jaegerReq,
-		Started:          true,
-	})
-	require.NoError(t, err)
-	defer func() {
-		logs, errLogs := jaegerContainer.Logs(ctx)
-		require.NoError(t, errLogs)
-		all, errLogs := ioutil.ReadAll(logs)
-		require.NoError(t, errLogs)
-		fmt.Printf("Jaeger logs:\n---->\n%s<----\n\n", string(all))
-		jaegerContainer.Terminate(ctx)
-	}()
 
-	chContainer.MappedPort(ctx, clickhousePort)
-	jaegerQueryPort, err := jaegerContainer.MappedPort(ctx, jaegerQueryPort)
-	require.NoError(t, err)
+	for _, jaegerContainer := range jaegerContainers {
+		jaegerQueryPort, err := jaegerContainer.MappedPort(ctx, jaegerQueryPort)
+		require.NoError(t, err)
 
-	err = awaitility.Await(100*time.Millisecond, time.Second*3, func() bool {
-		// Jaeger traces itself so this request generates some spans
-		response, errHTTP := http.Get(fmt.Sprintf("http://localhost:%d/api/services", jaegerQueryPort.Int()))
-		require.NoError(t, errHTTP)
-		body, errHTTP := ioutil.ReadAll(response.Body)
-		require.NoError(t, errHTTP)
-		var r result
-		errHTTP = json.Unmarshal(body, &r)
-		require.NoError(t, errHTTP)
-		return len(r.Data) == 1 && r.Data[0] == "jaeger-query"
-	})
-	assert.NoError(t, err)
+		err = awaitility.Await(100*time.Millisecond, time.Second*3, func() bool {
+			// Jaeger traces itself so this request generates some spans
+			response, errHTTP := http.Get(fmt.Sprintf("http://localhost:%d/api/services", jaegerQueryPort.Int()))
+			require.NoError(t, errHTTP)
+			body, errHTTP := ioutil.ReadAll(response.Body)
+			require.NoError(t, errHTTP)
+			var r result
+			errHTTP = json.Unmarshal(body, &r)
+			require.NoError(t, errHTTP)
+			return len(r.Data) == 1 && r.Data[0] == "jaeger-query"
+		})
+		assert.NoError(t, err)
+	}
 }
 
 type result struct {
@@ -125,7 +175,18 @@ func (c *clickhouseWaitStrategy) WaitUntilReady(ctx context.Context, target wait
 
 	port, err := target.MappedPort(ctx, clickhousePort)
 	require.NoError(c.test, err)
-	db, err := sql.Open("clickhouse", fmt.Sprintf("tcp://localhost:%d?database=default", port.Int()))
+
+	db := clickhouse.OpenDB(&clickhouse.Options{
+		Addr: []string{
+			fmt.Sprintf("localhost:%d", port.Int()),
+		},
+		Auth: clickhouse.Auth{
+			Database: "default",
+		},
+		Compression: &clickhouse.Compression{
+			Method: clickhouse.CompressionLZ4,
+		},
+	})
 	require.NoError(c.test, err)
 
 	for {

embed.go

@@ -2,8 +2,5 @@ package jaegerclickhouse
 
 import "embed"
 
-//go:embed sqlscripts/local/*
-var EmbeddedFilesNoReplication embed.FS
-
-//go:embed sqlscripts/replication/*
-var EmbeddedFilesReplication embed.FS
+//go:embed sqlscripts/*
+var SQLScripts embed.FS

go.mod

@@ -1,19 +1,102 @@
-module github.com/pavolloffay/jaeger-clickhouse
+module github.com/jaegertracing/jaeger-clickhouse
 
-go 1.16
+go 1.19
 
 require (
-	github.com/ClickHouse/clickhouse-go v1.4.5
+	github.com/ClickHouse/clickhouse-go/v2 v2.3.0
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/ecodia/golang-awaitility v0.0.0-20180710094957-fb55e59708c7
 	github.com/gogo/protobuf v1.3.2
-	github.com/google/go-cmp v0.5.6 // indirect
-	github.com/hashicorp/go-hclog v0.16.1
-	github.com/jaegertracing/jaeger v1.24.0
+	github.com/hashicorp/go-hclog v1.3.1
+	github.com/jaegertracing/jaeger v1.38.2-0.20221007043206-b4c88ddf6cdd
 	github.com/opentracing/opentracing-go v1.2.0
-	github.com/prometheus/client_golang v1.11.0
-	github.com/stretchr/testify v1.7.0
+	github.com/prometheus/client_golang v1.13.0
+	github.com/stretchr/testify v1.8.0
 	github.com/testcontainers/testcontainers-go v0.11.1
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
-	gotest.tools v2.2.0+incompatible // indirect
+	go.uber.org/zap v1.23.0
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/ClickHouse/ch-go v0.47.3 // indirect
+	github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3 // indirect
+	github.com/Microsoft/hcsshim v0.8.16 // indirect
+	github.com/andybalholm/brotli v1.0.4 // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 // indirect
+	github.com/containerd/containerd v1.5.0-beta.4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v20.10.7+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/fatih/color v1.13.0 // indirect
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.6.1 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
+	github.com/hashicorp/go-plugin v1.4.5 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/klauspost/compress v1.15.10 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/moby/sys/mountinfo v0.4.1 // indirect
+	github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
+	github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect
+	github.com/oklog/run v1.1.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opencontainers/runc v1.0.0-rc93 // indirect
+	github.com/paulmach/orb v0.7.1 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.1 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/spf13/afero v1.8.2 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/cobra v1.5.0 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.13.0 // indirect
+	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
+	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
+	go.opencensus.io v0.23.0 // indirect
+	go.opentelemetry.io/otel v1.10.0 // indirect
+	go.opentelemetry.io/otel/trace v1.10.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	golang.org/x/net v0.0.0-20221002022538-bcab6841153b // indirect
+	golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
+	google.golang.org/grpc v1.50.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

guide-kubernetes.md

@@ -4,7 +4,7 @@ This is a guide to deploy Jaeger with Clickhouse storage on Kubernetes.
 
 ## Prerequisites
 
-1. Deploy [Jaeger operator](https://github.com/jaegertracing/jaeger-operator). Note that `gprc-plugin` storage type is supported since version 1.25.0.
+1. Deploy [Jaeger operator](https://github.com/jaegertracing/jaeger-operator). Note that `grpc-plugin` storage type is supported since version 1.25.0.
 2. Deploy [Clickhouse operator](https://github.com/Altinity/clickhouse-operator)
 3. Deploy [Zookeeper](https://github.com/Altinity/clickhouse-operator/blob/master/docs/replication_setup.md) (if replication is used)
 
@@ -41,7 +41,7 @@ metadata:
     jaeger-clickhouse: demo
 data:
   config.yaml: |
-    address: tcp://clickhouse-jaeger:9000
+    address: clickhouse-jaeger:9000
     username: clickhouse_operator
     password: clickhouse_operator_password
     spans_table:
@@ -64,7 +64,7 @@ spec:
   storage:
     type: grpc-plugin
     grpcPlugin:
-      image: ghcr.io/pavolloffay/jaeger-clickhouse:0.5.1
+      image: ghcr.io/jaegertracing/jaeger-clickhouse:0.7.0
     options:
       grpc-storage-plugin:
         binary: /plugin/jaeger-clickhouse

guide-multitenancy.md

@@ -1,16 +1,78 @@
 # Multi-tenant deployment
 
-Multi-tenant deployment requires using a database per tenant. Each tenant will talk
-to a separate Jaeger query and collector (or all-in-one).
+It may be desirable to share a common ClickHouse instance across multiple Jaeger instances.
+There are two ways of doing this, depending on whether spanning the tenants across separate databases is preferable.
 
-Create a database:
+## Shared database/tables
 
-```sql
-CREATE DATABASE tenant_1 ENGINE=Atomic;
-```
+If you wish to reuse the same ClickHouse database/tables across all tenants, you can specify a different `tenant: "<name>"` in each jaeger-clickhouse instance config.
+
+When a non-empty `tenant` is specified, all tables will be created with a `tenant` column, and all reads/writes for a given Jaeger instance will be applied against the configured tenant name for that instance.
+
+1. Create a shared database:
+    ```sql
+    CREATE DATABASE shared ENGINE=Atomic
+    ```
+2. Configure the per-tenant jaeger-clickhouse clients to specify tenant names:
+    ```yaml
+    database: shared
+    tenant: tenant_1
+    ```
+    ```yaml
+    database: shared
+    tenant: tenant_2
+    ```
+
+Multitenant mode must be enabled when the deployment is first created and cannot be toggled later, except perhaps by manually adding/removing the `tenant` column from all tables.
+Multitenant/singletenant instances must not be mixed within the same database - the two modes are mutually exclusive of each other.
+
+## Separate databases
+
+If you wish to keep instances fully separate, you can configure one ClickHouse database per tenant.
+This may be useful when different per-database configuration across tenants is desirable.
+
+1. Create a database for each tenant:
+    ```sql
+    CREATE DATABASE tenant_1 ENGINE=Atomic;
+    CREATE DATABASE tenant_2 ENGINE=Atomic;
+    ```
+2. Configure the per-tenant jaeger-clickhouse plugins matching databases:
+    ```yaml
+    database: tenant_1
+    ```
+    ```yaml
+    database: tenant_2
+    ```
+
+## Mixing methods in the same ClickHouse instance
+
+Each of the methods applies on a per-database basis. The methods require different schemas and must not be mixed in a single database, but it is possible to have different databases using different methods in the same ClickHouse instance.
+
+For example, there could be a `shared` database where multiple tenants are sharing the same tables:
+
+    ```sql
+    CREATE DATABASE shared ENGINE=Atomic
+    ```
+    ```yaml
+    database: shared
+    tenant: tenant_1
+    ```
+    ```yaml
+    database: shared
+    tenant: tenant_2
+    ```
+
+Then there could be separate `isolated_x` databases for tenants that should be provided with their own dedicated tables, enabling e.g. better ACL isolation:
+
+    ```sql
+    CREATE DATABASE isolated_1 ENGINE=Atomic
+    CREATE DATABASE isolated_2 ENGINE=Atomic
+    ```
+    ```yaml
+    database: isolated_1
+    ```
+    ```yaml
+    database: isolated_2
+    ```
 
-Then configure the plugin to use tenant's database:
 
-```yaml
-database: tenant_1
-```

guide-sharding-and-replication.md

@@ -64,7 +64,7 @@ kubectl exec -it statefulset.apps/chi-jaeger-cluster1-0-0 -- clickhouse-client
 The plugin has to be configured to write and read that from the global tables:
 
 ```yaml
-address: tcp://clickhouse-jaeger:9000
+address: clickhouse-jaeger:9000
 # database: jaeger
 spans_table: jaeger_spans
 spans_index_table: jaeger_index

integration/config-local.yaml

@@ -0,0 +1,3 @@
+address: localhost:9000
+init_sql_scripts_dir: init_sql_scripts
+init_tables: true

integration/grpc_test.go

@@ -0,0 +1,107 @@
+// Copyright (c) 2019 The Jaeger Authors.
+// Copyright (c) 2018 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+	"os"
+	"testing"
+
+	"github.com/jaegertracing/jaeger/pkg/config"
+	"github.com/jaegertracing/jaeger/pkg/metrics"
+	"github.com/jaegertracing/jaeger/pkg/testutils"
+	"github.com/jaegertracing/jaeger/plugin/storage/grpc"
+	"github.com/jaegertracing/jaeger/plugin/storage/integration"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+)
+
+const defaultPluginBinaryPath = "../../../examples/memstore-plugin/memstore-plugin"
+
+type GRPCStorageIntegrationTestSuite struct {
+	integration.StorageIntegration
+	logger           *zap.Logger
+	pluginBinaryPath string
+	pluginConfigPath string
+}
+
+func (s *GRPCStorageIntegrationTestSuite) initialize() error {
+	s.logger, _ = testutils.NewLogger()
+
+	f := grpc.NewFactory()
+	v, command := config.Viperize(f.AddFlags)
+	flags := []string{
+		"--grpc-storage-plugin.binary",
+		s.pluginBinaryPath,
+		"--grpc-storage-plugin.log-level",
+		"debug",
+	}
+	if s.pluginConfigPath != "" {
+		flags = append(flags,
+			"--grpc-storage-plugin.configuration-file",
+			s.pluginConfigPath,
+		)
+	}
+	err := command.ParseFlags(flags)
+	if err != nil {
+		return err
+	}
+	f.InitFromViper(v, zap.NewNop())
+	if err = f.Initialize(metrics.NullFactory, s.logger); err != nil {
+		return err
+	}
+
+	if s.SpanWriter, err = f.CreateSpanWriter(); err != nil {
+		return err
+	}
+	if s.SpanReader, err = f.CreateSpanReader(); err != nil {
+		return err
+	}
+
+	// TODO DependencyWriter is not implemented in grpc store
+
+	s.Refresh = s.refresh
+	s.CleanUp = s.cleanUp
+	return nil
+}
+
+func (s *GRPCStorageIntegrationTestSuite) refresh() error {
+	return nil
+}
+
+func (s *GRPCStorageIntegrationTestSuite) cleanUp() error {
+	return s.initialize()
+}
+
+func TestGRPCStorage(t *testing.T) {
+	if os.Getenv("STORAGE") != "grpc-plugin" {
+		t.Skip("Integration test against grpc skipped; set STORAGE env var to grpc-plugin to run this")
+	}
+	binaryPath := os.Getenv("PLUGIN_BINARY_PATH")
+	if binaryPath == "" {
+		t.Logf("PLUGIN_BINARY_PATH env var not set, using %s", defaultPluginBinaryPath)
+		binaryPath = defaultPluginBinaryPath
+	}
+	configPath := os.Getenv("PLUGIN_CONFIG_PATH")
+	if configPath == "" {
+		t.Log("PLUGIN_CONFIG_PATH env var not set")
+	}
+	s := &GRPCStorageIntegrationTestSuite{
+		pluginBinaryPath: binaryPath,
+		pluginConfigPath: configPath,
+	}
+	require.NoError(t, s.initialize())
+	s.IntegrationTestAll(t)
+}

integration/init_sql_scripts/0001-drop-database.sql

@@ -0,0 +1 @@
+DROP DATABASE IF EXISTS default;

integration/init_sql_scripts/0002-create-database.sql

@@ -0,0 +1 @@
+CREATE DATABASE IF NOT EXISTS default;

internal/tools/go.mod

@@ -1,19 +1,145 @@
-module github.com/pavolloffay/jaeger-clickhouse/internal/tools
+module github.com/jaegertracing/jaeger-clickhouse/internal/tools
 
-go 1.16
+go 1.19
 
 require (
-	github.com/go-lintpack/lintpack v0.5.2 // indirect
-	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
-	github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 // indirect
-	github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
-	github.com/golangci/golangci-lint v1.41.1 // indirect
-	github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc // indirect
-	github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 // indirect
-	github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 // indirect
-	github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada // indirect
-	github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa // indirect
-	github.com/ugorji/go v1.1.4 // indirect
+	github.com/golangci/golangci-lint v1.41.1
 	golang.org/x/tools v0.1.5
-	sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 // indirect
+)
+
+require (
+	4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a // indirect
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/OpenPeeDeeP/depguard v1.0.1 // indirect
+	github.com/alexkohler/prealloc v1.0.0 // indirect
+	github.com/ashanbrown/forbidigo v1.2.0 // indirect
+	github.com/ashanbrown/makezero v0.0.0-20210520155254-b6261585ddde // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bkielbasa/cyclop v1.2.0 // indirect
+	github.com/bombsimon/wsl/v3 v3.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/charithe/durationcheck v0.0.8 // indirect
+	github.com/chavacava/garif v0.0.0-20210405164556-e8a0a408d6af // indirect
+	github.com/daixiang0/gci v0.2.8 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/denis-tingajkin/go-header v0.4.2 // indirect
+	github.com/esimonov/ifshort v1.0.2 // indirect
+	github.com/ettle/strcase v0.1.1 // indirect
+	github.com/fatih/color v1.12.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/fzipp/gocyclo v0.3.1 // indirect
+	github.com/go-critic/go-critic v0.5.6 // indirect
+	github.com/go-toolsmith/astcast v1.0.0 // indirect
+	github.com/go-toolsmith/astcopy v1.0.0 // indirect
+	github.com/go-toolsmith/astequal v1.0.0 // indirect
+	github.com/go-toolsmith/astfmt v1.0.0 // indirect
+	github.com/go-toolsmith/astp v1.0.0 // indirect
+	github.com/go-toolsmith/strparse v1.0.0 // indirect
+	github.com/go-toolsmith/typep v1.0.2 // indirect
+	github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gofrs/flock v0.8.0 // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
+	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
+	github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 // indirect
+	github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a // indirect
+	github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 // indirect
+	github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca // indirect
+	github.com/golangci/misspell v0.3.5 // indirect
+	github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5 // indirect
+	github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
+	github.com/google/go-cmp v0.5.4 // indirect
+	github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254 // indirect
+	github.com/gostaticanalysis/analysisutil v0.4.1 // indirect
+	github.com/gostaticanalysis/comment v1.4.1 // indirect
+	github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5 // indirect
+	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jgautheron/goconst v1.5.1 // indirect
+	github.com/jingyugao/rowserrcheck v1.1.0 // indirect
+	github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
+	github.com/julz/importas v0.0.0-20210419104244-841f0c0fe66d // indirect
+	github.com/kisielk/errcheck v1.6.0 // indirect
+	github.com/kisielk/gotool v1.0.0 // indirect
+	github.com/kulti/thelper v0.4.0 // indirect
+	github.com/kunwardeep/paralleltest v1.0.2 // indirect
+	github.com/kyoh86/exportloopref v0.1.8 // indirect
+	github.com/ldez/gomoddirectives v0.2.1 // indirect
+	github.com/ldez/tagliatelle v0.2.0 // indirect
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/maratori/testpackage v1.0.1 // indirect
+	github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mbilski/exhaustivestruct v1.2.0 // indirect
+	github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 // indirect
+	github.com/mgechev/revive v1.0.7 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/moricho/tparallel v0.2.1 // indirect
+	github.com/nakabonne/nestif v0.3.0 // indirect
+	github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
+	github.com/nishanths/exhaustive v0.1.0 // indirect
+	github.com/nishanths/predeclared v0.2.1 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pelletier/go-toml v1.2.0 // indirect
+	github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/polyfloyd/go-errorlint v0.0.0-20210510181950-ab96adb96fea // indirect
+	github.com/prometheus/client_golang v1.7.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.10.0 // indirect
+	github.com/prometheus/procfs v0.1.3 // indirect
+	github.com/quasilyte/go-ruleguard v0.3.4 // indirect
+	github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect
+	github.com/ryancurrah/gomodguard v1.2.2 // indirect
+	github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect
+	github.com/sanposhiho/wastedassign/v2 v2.0.6 // indirect
+	github.com/securego/gosec/v2 v2.8.0 // indirect
+	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/sonatard/noctx v0.0.1 // indirect
+	github.com/sourcegraph/go-diff v0.6.1 // indirect
+	github.com/spf13/afero v1.1.2 // indirect
+	github.com/spf13/cast v1.3.0 // indirect
+	github.com/spf13/cobra v1.1.3 // indirect
+	github.com/spf13/jwalterweatherman v1.0.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.7.1 // indirect
+	github.com/ssgreg/nlreturn/v2 v2.1.0 // indirect
+	github.com/stretchr/objx v0.1.1 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b // indirect
+	github.com/tetafro/godot v1.4.7 // indirect
+	github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 // indirect
+	github.com/tomarrell/wrapcheck/v2 v2.1.0 // indirect
+	github.com/tommy-muehle/go-mnd/v2 v2.4.0 // indirect
+	github.com/ultraware/funlen v0.0.3 // indirect
+	github.com/ultraware/whitespace v0.0.4 // indirect
+	github.com/uudashr/gocognit v1.0.1 // indirect
+	github.com/yeya24/promlinter v0.1.0 // indirect
+	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
+	golang.org/x/text v0.3.5 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/ini.v1 v1.51.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	honnef.co/go/tools v0.2.0 // indirect
+	mvdan.cc/gofumpt v0.1.1 // indirect
+	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
+	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
+	mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 // indirect
 )

internal/tools/go.sum

@@ -50,7 +50,6 @@ github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuN
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
 github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
-github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -83,7 +82,6 @@ github.com/bkielbasa/cyclop v1.2.0/go.mod h1:qOI0yy6A7dYC4Zgsa72Ppm9kONl0RoIlPbz
 github.com/bombsimon/wsl/v3 v3.3.0 h1:Mka/+kRLoQJq7g2rggtgQsjuI/K5Efd87WX96EWFxjM=
 github.com/bombsimon/wsl/v3 v3.3.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -155,10 +153,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
@@ -169,17 +165,14 @@ github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR
 github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
 github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
 github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
-github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
 github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
 github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
-github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
 github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
 github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
 github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
-github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
 github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
 github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
-github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
+github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
 github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
 github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
 github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
@@ -229,23 +222,18 @@ github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5
 github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
 github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
 github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
-github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
 github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
 github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
-github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
-github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
 github.com/golangci/golangci-lint v1.41.1 h1:KH28pTSqRu6DTXIAANl1sPXNCmqg4VEH21z6G9Wj4SM=
 github.com/golangci/golangci-lint v1.41.1/go.mod h1:LPtcY3aAAU8wydHrKpnanx9Og8K/cblZSyGmI5CJZUk=
-github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
 github.com/golangci/misspell v0.3.5 h1:pLzmVdl3VxTOncgzHcvLOKirdvcx/TydsClUQXTehjo=
 github.com/golangci/misspell v0.3.5/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
-github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
 github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5 h1:c9Mqqrm/Clj5biNaG7rABrmwUq88nHh0uABo2b/WYmc=
 github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5/go.mod h1:LK+zW4MpyytAWQRz0M4xnzEk50lSvqDQKfx304apFkY=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
@@ -280,6 +268,7 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU=
 github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254 h1:Nb2aRlC404yz7gQIfRZxX9/MLvQiqXyiBTJtgAy6yrI=
@@ -353,7 +342,6 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
-github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.2.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -361,6 +349,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -382,6 +371,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kulti/thelper v0.4.0 h1:2Nx7XbdbE/BYZeoip2mURKUdtHQRuy6Ug+wR7K9ywNM=
 github.com/kulti/thelper v0.4.0/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5exl2U=
@@ -466,6 +456,7 @@ github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaP
 github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
 github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 h1:4kuARK6Y6FxaNu/BnU2OAaLF86eTVhP2hjTB6iMvItA=
 github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nishanths/exhaustive v0.1.0 h1:kVlMw8h2LHPMGUVqUj6230oQjjTMFjwcZrnkhXzFfl8=
 github.com/nishanths/exhaustive v0.1.0/go.mod h1:S1j9110vxV1ECdCudXRkeMnFQ/DQk9ajLT0Uf2MYZQQ=
@@ -473,6 +464,7 @@ github.com/nishanths/predeclared v0.0.0-20190419143655-18a43bb90ffc/go.mod h1:62
 github.com/nishanths/predeclared v0.2.1 h1:1TXtjmy4f3YCFjTxRd8zcFHOmoUir+gp0ESzjFzG2sw=
 github.com/nishanths/predeclared v0.2.1/go.mod h1:HvkGJcA3naj4lOwnFXFDkFxVtSqQMB9sbB1usJ+xjQE=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
@@ -483,9 +475,11 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54=
 github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug=
 github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -558,9 +552,7 @@ github.com/securego/gosec/v2 v2.8.0/go.mod h1:hJZ6NT5TqoY+jmOsaxAV4cXoEdrMRLVaNP
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
 github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
-github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/gopsutil/v3 v3.21.5/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
-github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -570,7 +562,9 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/sonatard/noctx v0.0.1 h1:VC1Qhl6Oxx9vvWo3UDgrGXYCeKCe3Wbw7qAWL6FrmTY=
@@ -626,11 +620,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1
 github.com/tomarrell/wrapcheck/v2 v2.1.0 h1:LTzwrYlgBUwi9JldazhbJN84fN9nS2UNGrZIo2syqxE=
 github.com/tomarrell/wrapcheck/v2 v2.1.0/go.mod h1:crK5eI4RGSUrb9duDTQ5GqcukbKZvi85vX6nbhsBAeI=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
-github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa h1:RC4maTWLKKwb7p1cnoygsbKIgNlJqSYBeAFON3Ar8As=
-github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
 github.com/tommy-muehle/go-mnd/v2 v2.4.0 h1:1t0f8Uiaq+fqKteUR4N9Umr6E99R+lDnLnq7PwX2PPE=
 github.com/tommy-muehle/go-mnd/v2 v2.4.0/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw=
-github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA=
 github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
@@ -762,6 +753,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -850,10 +842,8 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190307163923-6a08e3108db3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -1023,6 +1013,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
@@ -1032,6 +1023,7 @@ gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
@@ -1069,4 +1061,3 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=

sqlscripts/distributed-table.tmpl.sql

@@ -0,0 +1,3 @@
+CREATE TABLE IF NOT EXISTS {{.Table}}
+    ON CLUSTER '{cluster}' AS {{.Database}}.{{.Table}}_local
+    ENGINE = Distributed('{cluster}', {{.Database}}, {{.Table}}_local, {{.Hash}})

sqlscripts/jaeger-index.tmpl.sql

@@ -0,0 +1,28 @@
+CREATE TABLE IF NOT EXISTS {{.SpansIndexTable}}
+{{if .Replication}}ON CLUSTER '{cluster}'{{end}}
+(
+    {{if .Multitenant -}}
+    tenant     LowCardinality(String) CODEC (ZSTD(1)),
+    {{- end -}}
+    timestamp  DateTime CODEC (Delta, ZSTD(1)),
+    traceID    String CODEC (ZSTD(1)),
+    service    LowCardinality(String) CODEC (ZSTD(1)),
+    operation  LowCardinality(String) CODEC (ZSTD(1)),
+    durationUs UInt64 CODEC (ZSTD(1)),
+    tags Nested
+    (
+        key LowCardinality(String),
+        value String
+    ) CODEC (ZSTD(1)),
+    INDEX idx_tag_keys tags.key TYPE bloom_filter(0.01) GRANULARITY 64,
+    INDEX idx_duration durationUs TYPE minmax GRANULARITY 1
+) ENGINE {{if .Replication}}ReplicatedMergeTree{{else}}MergeTree(){{end}}
+    {{.TTLTimestamp}}
+    PARTITION BY (
+        {{if .Multitenant -}}
+        tenant,
+        {{- end -}}
+        toDate(timestamp)
+    )
+    ORDER BY (service, -toUnixTimestamp(timestamp))
+    SETTINGS index_granularity = 1024

sqlscripts/jaeger-operations.tmpl.sql

@@ -0,0 +1,43 @@
+CREATE MATERIALIZED VIEW IF NOT EXISTS {{.OperationsTable}}
+{{if .Replication}}ON CLUSTER '{cluster}'{{end}}
+    ENGINE {{if .Replication}}ReplicatedSummingMergeTree{{else}}SummingMergeTree{{end}}
+    {{.TTLDate}}
+    PARTITION BY (
+        {{if .Multitenant -}}
+        tenant,
+        {{- end -}}
+        toYYYYMM(date)
+    )
+    ORDER BY (
+        {{if .Multitenant -}}
+        tenant,
+        {{- end -}}
+        date,
+        service,
+        operation
+    )
+    SETTINGS index_granularity = 32
+    POPULATE
+AS SELECT
+    {{if .Multitenant -}}
+    tenant,
+    {{- end -}}
+    toDate(timestamp) AS date,
+    service,
+    operation,
+    count() AS count,
+    if(
+        has(tags.key, 'span.kind'),
+        tags.value[indexOf(tags.key, 'span.kind')],
+        ''
+    ) AS spankind
+FROM {{.Database}}.{{.SpansIndexTable}}
+GROUP BY
+    {{if .Multitenant -}}
+    tenant,
+    {{- end -}}
+    date,
+    service,
+    operation,
+    tags.key,
+    tags.value

sqlscripts/jaeger-spans-archive.tmpl.sql

@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS {{.SpansArchiveTable}}
+{{if .Replication}}ON CLUSTER '{cluster}'{{end}}
+(
+    {{if .Multitenant -}}
+    tenant    LowCardinality(String) CODEC (ZSTD(1)),
+    {{- end -}}
+    timestamp DateTime CODEC (Delta, ZSTD(1)),
+    traceID   String CODEC (ZSTD(1)),
+    model     String CODEC (ZSTD(3))
+) ENGINE {{if .Replication}}ReplicatedMergeTree{{else}}MergeTree(){{end}}
+    {{.TTLTimestamp}}
+    PARTITION BY (
+        {{if .Multitenant -}}
+        tenant,
+        {{- end -}}
+        toYYYYMM(timestamp)
+    )
+    ORDER BY traceID
+    SETTINGS index_granularity = 1024

sqlscripts/jaeger-spans.tmpl.sql

@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS {{.SpansTable}}
+{{if .Replication}}ON CLUSTER '{cluster}'{{end}}
+(
+    {{if .Multitenant -}}
+    tenant    LowCardinality(String) CODEC (ZSTD(1)),
+    {{- end -}}
+    timestamp DateTime CODEC (Delta, ZSTD(1)),
+    traceID   String CODEC (ZSTD(1)),
+    model     String CODEC (ZSTD(3))
+) ENGINE {{if .Replication}}ReplicatedMergeTree{{else}}MergeTree(){{end}}
+    {{.TTLTimestamp}}
+    PARTITION BY (
+        {{if .Multitenant -}}
+        tenant,
+        {{- end -}}
+        toDate(timestamp)
+    )
+    ORDER BY traceID
+    SETTINGS index_granularity = 1024

sqlscripts/local/0001-jaeger-index.sql

@@ -1,13 +0,0 @@
-CREATE TABLE IF NOT EXISTS %s (
-     timestamp DateTime CODEC(Delta, ZSTD(1)),
-     traceID String CODEC(ZSTD(1)),
-     service LowCardinality(String) CODEC(ZSTD(1)),
-     operation LowCardinality(String) CODEC(ZSTD(1)),
-     durationUs UInt64 CODEC(ZSTD(1)),
-     tags Array(String) CODEC(ZSTD(1)),
-     INDEX idx_tags tags TYPE bloom_filter(0.01) GRANULARITY 64,
-     INDEX idx_duration durationUs TYPE minmax GRANULARITY 1
-) ENGINE MergeTree()
-PARTITION BY toDate(timestamp)
-ORDER BY (service, -toUnixTimestamp(timestamp))
-SETTINGS index_granularity=1024

sqlscripts/local/0002-jaeger-spans.sql

@@ -1,8 +0,0 @@
-CREATE TABLE IF NOT EXISTS %s (
-    timestamp DateTime CODEC(Delta, ZSTD(1)),
-    traceID String CODEC(ZSTD(1)),
-    model String CODEC(ZSTD(3))
-) ENGINE MergeTree()
-PARTITION BY toDate(timestamp)
-ORDER BY traceID
-SETTINGS index_granularity=1024

sqlscripts/local/0003-jaeger-operations.sql

@@ -1,12 +0,0 @@
-CREATE MATERIALIZED VIEW IF NOT EXISTS %s
-ENGINE SummingMergeTree
-PARTITION BY toYYYYMM(date) ORDER BY (date, service, operation)
-SETTINGS index_granularity=32
-POPULATE
-AS SELECT
-    toDate(timestamp) AS date,
-    service,
-    operation,
-    count() as count
-FROM %s -- Here goes local jaeger index table's name
-GROUP BY date, service, operation

sqlscripts/local/0004-jaeger-spans-archive.sql

@@ -1,8 +0,0 @@
-CREATE TABLE IF NOT EXISTS %s (
-    timestamp DateTime CODEC(Delta, ZSTD(1)),
-    traceID String CODEC(ZSTD(1)),
-    model String CODEC(ZSTD(3))
-) ENGINE MergeTree()
-PARTITION BY toYYYYMM(timestamp)
-ORDER BY traceID
-SETTINGS index_granularity=1024

sqlscripts/replication/0001-jaeger-index-local.sql

@@ -1,14 +0,0 @@
-CREATE TABLE IF NOT EXISTS %s ON CLUSTER '{cluster}'
-(
-    timestamp  DateTime CODEC (Delta, ZSTD(1)),
-    traceID    String CODEC (ZSTD(1)),
-    service    LowCardinality(String) CODEC (ZSTD(1)),
-    operation  LowCardinality(String) CODEC (ZSTD(1)),
-    durationUs UInt64 CODEC (ZSTD(1)),
-    tags       Array(String) CODEC (ZSTD(1)),
-    INDEX idx_tags tags TYPE bloom_filter(0.01) GRANULARITY 64,
-    INDEX idx_duration durationUs TYPE minmax GRANULARITY 1
-) ENGINE ReplicatedMergeTree
-      PARTITION BY toDate(timestamp)
-      ORDER BY (service, -toUnixTimestamp(timestamp))
-      SETTINGS index_granularity = 1024;

sqlscripts/replication/0002-jaeger-spans-local.sql

@@ -1,9 +0,0 @@
-CREATE TABLE IF NOT EXISTS %s ON CLUSTER '{cluster}'
-(
-    timestamp DateTime CODEC (Delta, ZSTD(1)),
-    traceID   String CODEC (ZSTD(1)),
-    model     String CODEC (ZSTD(3))
-) ENGINE ReplicatedMergeTree
-      PARTITION BY toDate(timestamp)
-      ORDER BY traceID
-      SETTINGS index_granularity = 1024;

sqlscripts/replication/0003-jaeger-operations-local.sql

@@ -1,11 +0,0 @@
-CREATE MATERIALIZED VIEW IF NOT EXISTS %s ON CLUSTER '{cluster}'
-        ENGINE ReplicatedMergeTree
-            PARTITION BY toYYYYMM(date) ORDER BY (date, service, operation)
-            SETTINGS index_granularity=32
-        POPULATE
-AS SELECT toDate(timestamp) AS date,
-          service,
-          operation,
-          count()           as count
-   FROM %s -- here goes local index table
-   GROUP BY date, service, operation;

sqlscripts/replication/0004-jaeger-spans-archive-local.sql

@@ -1,9 +0,0 @@
-CREATE TABLE IF NOT EXISTS %s ON CLUSTER '{cluster}'
-(
-    timestamp DateTime CODEC (Delta, ZSTD(1)),
-    traceID   String CODEC (ZSTD(1)),
-    model     String CODEC (ZSTD(3))
-) ENGINE ReplicatedMergeTree
-      PARTITION BY toYYYYMM(timestamp)
-      ORDER BY traceID
-      SETTINGS index_granularity = 1024

sqlscripts/replication/0005-distributed-city-hash.sql

@@ -1,3 +0,0 @@
-CREATE TABLE IF NOT EXISTS %s -- global table name
-    ON CLUSTER '{cluster}' AS %s -- local table name
-    ENGINE = Distributed('{cluster}', %s, %s, cityHash64(traceID)); -- local table name

sqlscripts/replication/0006-distributed-rand.sql

@@ -1,3 +0,0 @@
-CREATE TABLE IF NOT EXISTS %s -- operations table
-    ON CLUSTER '{cluster}' AS %s -- local operations table
-    ENGINE = Distributed('{cluster}', %s, %s, rand()); -- local operations table

storage/clickhousespanstore/heap.go

@@ -0,0 +1,85 @@
+package clickhousespanstore
+
+import (
+	"container/heap"
+	"fmt"
+	"time"
+)
+
+var (
+	_                 heap.Interface = workerHeap{}
+	errWorkerNotFound                = fmt.Errorf("worker not found in heap")
+)
+
+type heapItem struct {
+	pushTime time.Time
+	worker   *WriteWorker
+}
+
+// workerHeap is a heap for WriteWorkers where worker's push time is the key.
+type workerHeap struct {
+	elems   *[]*heapItem
+	indexes map[*WriteWorker]int
+}
+
+func newWorkerHeap(cap int) workerHeap {
+	elems := make([]*heapItem, 0, cap)
+	return workerHeap{
+		elems:   &elems,
+		indexes: make(map[*WriteWorker]int),
+	}
+}
+
+func (workerHeap workerHeap) AddWorker(worker *WriteWorker) {
+	heap.Push(workerHeap, heapItem{
+		worker:   worker,
+		pushTime: time.Now(),
+	})
+}
+
+func (workerHeap *workerHeap) RemoveWorker(worker *WriteWorker) error {
+	idx, ok := workerHeap.indexes[worker]
+	if !ok {
+		return errWorkerNotFound
+	}
+	heap.Remove(workerHeap, idx)
+	return nil
+}
+
+func (workerHeap *workerHeap) CloseWorkers() {
+	for _, item := range *workerHeap.elems {
+		item.worker.Close()
+	}
+}
+
+func (workerHeap workerHeap) Len() int {
+	return len(*workerHeap.elems)
+}
+
+func (workerHeap workerHeap) Less(i, j int) bool {
+	return (*workerHeap.elems)[i].pushTime.Before((*workerHeap.elems)[j].pushTime)
+}
+
+func (workerHeap workerHeap) Swap(i, j int) {
+	(*workerHeap.elems)[i], (*workerHeap.elems)[j] = (*workerHeap.elems)[j], (*workerHeap.elems)[i]
+	workerHeap.indexes[(*workerHeap.elems)[i].worker] = i
+	workerHeap.indexes[(*workerHeap.elems)[j].worker] = j
+}
+
+func (workerHeap workerHeap) Push(x interface{}) {
+	switch t := x.(type) {
+	case heapItem:
+		*workerHeap.elems = append(*workerHeap.elems, &t)
+		workerHeap.indexes[t.worker] = len(*workerHeap.elems) - 1
+	default:
+		panic("Unknown type")
+	}
+}
+
+func (workerHeap workerHeap) Pop() interface{} {
+	lastInd := len(*workerHeap.elems) - 1
+	last := (*workerHeap.elems)[lastInd]
+	delete(workerHeap.indexes, last.worker)
+	*workerHeap.elems = (*workerHeap.elems)[:lastInd]
+	return last.worker
+}

storage/clickhousespanstore/mocks/converter.go

@@ -28,6 +28,10 @@ func (conv ConverterMock) ConvertValue(v interface{}) (driver.Value, error) {
 		return driver.Value(t), nil
 	case int64:
 		return driver.Value(t), nil
+	case uint64:
+		return driver.Value(t), nil
+	case int:
+		return driver.Value(t), nil
 	case []string:
 		return driver.Value(fmt.Sprint(t)), nil
 	default:

storage/clickhousespanstore/mocks/converter_test.go

@@ -27,6 +27,7 @@ func TestConverterMock_ConvertValue(t *testing.T) {
 			expectedResult: driver.Value(int64(12340123456789)),
 		},
 		"int64 value":         {valueToConvert: int64(1823), expectedResult: driver.Value(int64(1823))},
+		"int value":           {valueToConvert: 1823, expectedResult: driver.Value(1823)},
 		"model.SpanID value":  {valueToConvert: model.SpanID(318148), expectedResult: driver.Value(model.SpanID(318148))},
 		"model.TraceID value": {valueToConvert: model.TraceID{Low: 0xabd5, High: 0xa31}, expectedResult: driver.Value("0000000000000a31000000000000abd5")},
 		"uint8 slice value":   {valueToConvert: []uint8("asdkja"), expectedResult: driver.Value([]uint8{0x61, 0x73, 0x64, 0x6b, 0x6a, 0x61})},

storage/clickhousespanstore/mocks/db.go

@@ -0,0 +1,14 @@
+package mocks
+
+import (
+	"database/sql"
+
+	sqlmock "github.com/DATA-DOG/go-sqlmock"
+)
+
+func GetDbMock() (*sql.DB, sqlmock.Sqlmock, error) {
+	return sqlmock.New(
+		sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual),
+		sqlmock.ValueConverterOption(ConverterMock{}),
+	)
+}

storage/clickhousespanstore/mocks/spylogger.go

@@ -5,9 +5,8 @@ import (
 	"log"
 	"testing"
 
+	hclog "github.com/hashicorp/go-hclog"
 	"github.com/stretchr/testify/assert"
-
-	"github.com/hashicorp/go-hclog"
 )
 
 const levelCount = 5

storage/clickhousespanstore/mocks/spylogger_test.go

@@ -5,9 +5,8 @@ import (
 	"strconv"
 	"testing"
 
+	hclog "github.com/hashicorp/go-hclog"
 	"github.com/stretchr/testify/assert"
-
-	"github.com/hashicorp/go-hclog"
 )
 
 const (

storage/clickhousespanstore/params.go

@@ -0,0 +1,19 @@
+package clickhousespanstore
+
+import (
+	"database/sql"
+	"time"
+
+	hclog "github.com/hashicorp/go-hclog"
+)
+
+// WorkerParams contains parameters that are shared between WriteWorkers
+type WorkerParams struct {
+	logger     hclog.Logger
+	db         *sql.DB
+	indexTable TableName
+	spansTable TableName
+	tenant     string
+	encoding   Encoding
+	delay      time.Duration
+}

storage/clickhousespanstore/pool.go

@@ -0,0 +1,131 @@
+package clickhousespanstore
+
+import (
+	"math"
+	"sync"
+
+	"github.com/jaegertracing/jaeger/model"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	numDiscardedSpans = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "jaeger_clickhouse_discarded_spans",
+		Help: "Count of spans that have been discarded due to pending writes exceeding max_span_count",
+	})
+	numPendingSpans = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "jaeger_clickhouse_pending_spans",
+		Help: "Number of spans that are currently pending, counts against max_span_count",
+	})
+)
+
+// WriteWorkerPool is a worker pool for writing batches of spans.
+// Given a new batch, WriteWorkerPool creates a new WriteWorker.
+// If the number of currently processed spans if more than maxSpanCount, then the oldest worker is removed.
+type WriteWorkerPool struct {
+	params *WorkerParams
+
+	finish  chan bool
+	done    sync.WaitGroup
+	batches chan []*model.Span
+
+	maxSpanCount int
+	mutex        sync.Mutex
+	workers      workerHeap
+	workerDone   chan *WriteWorker
+}
+
+var registerPoolMetrics sync.Once
+
+func NewWorkerPool(params *WorkerParams, maxSpanCount int) WriteWorkerPool {
+	registerPoolMetrics.Do(func() {
+		prometheus.MustRegister(numDiscardedSpans, numPendingSpans)
+	})
+
+	return WriteWorkerPool{
+		params:  params,
+		finish:  make(chan bool),
+		done:    sync.WaitGroup{},
+		batches: make(chan []*model.Span),
+
+		mutex:      sync.Mutex{},
+		workers:    newWorkerHeap(100),
+		workerDone: make(chan *WriteWorker),
+
+		maxSpanCount: maxSpanCount,
+	}
+}
+
+func (pool *WriteWorkerPool) Work() {
+	finish := false
+	nextWorkerID := int32(1)
+	pendingSpanCount := 0
+	for {
+		// Initialize to zero, or update value from previous loop
+		numPendingSpans.Set(float64(pendingSpanCount))
+
+		pool.done.Add(1)
+		select {
+		case batch := <-pool.batches:
+			batchSize := len(batch)
+			if pool.checkLimit(pendingSpanCount, batchSize) {
+				// Limit disabled or batch fits within limit, write the batch.
+				worker := WriteWorker{
+					workerID: nextWorkerID,
+
+					params: pool.params,
+					batch:  batch,
+
+					finish:     make(chan bool),
+					workerDone: pool.workerDone,
+					done:       sync.WaitGroup{},
+				}
+				if nextWorkerID == math.MaxInt32 {
+					nextWorkerID = 1
+				} else {
+					nextWorkerID++
+				}
+				pool.workers.AddWorker(&worker)
+				pendingSpanCount += batchSize
+				go worker.Work()
+			} else {
+				// Limit exceeded, complain
+				numDiscardedSpans.Add(float64(batchSize))
+				pool.params.logger.Error("Discarding batch of spans due to exceeding pending span count", "batch_size", batchSize, "pending_span_count", pendingSpanCount, "max_span_count", pool.maxSpanCount)
+			}
+		case worker := <-pool.workerDone:
+			// The worker has finished, subtract its work from the count and clean it from the heap.
+			pendingSpanCount -= len(worker.batch)
+			if err := pool.workers.RemoveWorker(worker); err != nil {
+				pool.params.logger.Error("could not remove worker", "worker", worker, "error", err)
+			}
+		case <-pool.finish:
+			pool.workers.CloseWorkers()
+			finish = true
+		}
+		pool.done.Done()
+
+		if finish {
+			break
+		}
+	}
+}
+
+func (pool *WriteWorkerPool) WriteBatch(batch []*model.Span) {
+	pool.batches <- batch
+}
+
+func (pool *WriteWorkerPool) Close() {
+	pool.finish <- true
+	pool.done.Wait()
+}
+
+// checkLimit returns whether batchSize fits within the maxSpanCount
+func (pool *WriteWorkerPool) checkLimit(pendingSpanCount int, batchSize int) bool {
+	if pool.maxSpanCount <= 0 {
+		return true
+	}
+
+	// Check limit, add batchSize if within limit
+	return pendingSpanCount+batchSize <= pool.maxSpanCount
+}

storage/clickhousespanstore/reader.go

@@ -10,10 +10,9 @@ import (
 	"time"
 
 	"github.com/gogo/protobuf/proto"
-	"github.com/opentracing/opentracing-go"
-
 	"github.com/jaegertracing/jaeger/model"
 	"github.com/jaegertracing/jaeger/storage/spanstore"
+	opentracing "github.com/opentracing/opentracing-go"
 )
 
 const (
@@ -34,17 +33,21 @@ type TraceReader struct {
 	operationsTable TableName
 	indexTable      TableName
 	spansTable      TableName
+	tenant          string
+	maxNumSpans     uint
 }
 
 var _ spanstore.Reader = (*TraceReader)(nil)
 
 // NewTraceReader returns a TraceReader for the database
-func NewTraceReader(db *sql.DB, operationsTable, indexTable, spansTable TableName) *TraceReader {
+func NewTraceReader(db *sql.DB, operationsTable, indexTable, spansTable TableName, tenant string, maxNumSpans uint) *TraceReader {
 	return &TraceReader{
 		db:              db,
 		operationsTable: operationsTable,
 		indexTable:      indexTable,
 		spansTable:      spansTable,
+		tenant:          tenant,
+		maxNumSpans:     maxNumSpans,
 	}
 }
 
@@ -58,20 +61,29 @@ func (r *TraceReader) getTraces(ctx context.Context, traceIDs []model.TraceID) (
 	span, _ := opentracing.StartSpanFromContext(ctx, "getTraces")
 	defer span.Finish()
 
-	values := make([]interface{}, len(traceIDs))
+	args := make([]interface{}, len(traceIDs))
 	for i, traceID := range traceIDs {
-		values[i] = traceID.String()
+		args[i] = traceID.String()
 	}
 
 	// It's more efficient to do PREWHERE on traceID to the only read needed models:
 	// * https://clickhouse.tech/docs/en/sql-reference/statements/select/prewhere/
 	//nolint:gosec  , G201: SQL string formatting
-	query := fmt.Sprintf("SELECT model FROM %s PREWHERE traceID IN (%s)", r.spansTable, "?"+strings.Repeat(",?", len(values)-1))
+	query := fmt.Sprintf("SELECT model FROM %s PREWHERE traceID IN (%s)", r.spansTable, "?"+strings.Repeat(",?", len(traceIDs)-1))
+
+	if r.tenant != "" {
+		query += " AND tenant = ?"
+		args = append(args, r.tenant)
+	}
+
+	if r.maxNumSpans > 0 {
+		query += fmt.Sprintf(" ORDER BY timestamp LIMIT %d BY traceID", r.maxNumSpans)
+	}
 
 	span.SetTag("db.statement", query)
-	span.SetTag("db.args", values)
+	span.SetTag("db.args", args)
 
-	rows, err := r.db.QueryContext(ctx, query, values...)
+	rows, err := r.db.QueryContext(ctx, query, args...)
 	if err != nil {
 		return nil, err
 	}
@@ -171,11 +183,19 @@ func (r *TraceReader) GetServices(ctx context.Context) ([]string, error) {
 		return nil, errNoOperationsTable
 	}
 
-	query := fmt.Sprintf("SELECT service FROM %s GROUP BY service", r.operationsTable)
+	query := fmt.Sprintf("SELECT service FROM %s", r.operationsTable)
+	args := make([]interface{}, 0)
 
+	if r.tenant != "" {
+		query += " WHERE tenant = ?"
+		args = append(args, r.tenant)
+	}
+
+	query += " GROUP BY service"
 	span.SetTag("db.statement", query)
+	span.SetTag("db.args", args)
 
-	return r.getStrings(ctx, query)
+	return r.getStrings(ctx, query, args...)
 }
 
 // GetOperations fetches operations in the service and empty slice if service does not exists
@@ -190,20 +210,44 @@ func (r *TraceReader) GetOperations(
 		return nil, errNoOperationsTable
 	}
 
-	query := fmt.Sprintf("SELECT operation FROM %s WHERE service = ? GROUP BY operation", r.operationsTable)
-	args := []interface{}{params.ServiceName}
+	//nolint:gosec  , G201: SQL string formatting
+	query := fmt.Sprintf("SELECT operation, spankind FROM %s WHERE", r.operationsTable)
+	args := make([]interface{}, 0)
+
+	if r.tenant != "" {
+		query += " tenant = ? AND"
+		args = append(args, r.tenant)
+	}
+
+	query += " service = ? GROUP BY operation, spankind ORDER BY operation"
+	args = append(args, params.ServiceName)
 
 	span.SetTag("db.statement", query)
 	span.SetTag("db.args", args)
 
-	names, err := r.getStrings(ctx, query, args...)
+	rows, err := r.db.QueryContext(ctx, query, args...)
 	if err != nil {
 		return nil, err
 	}
 
-	operations := make([]spanstore.Operation, len(names))
-	for i, name := range names {
-		operations[i].Name = name
+	defer rows.Close()
+
+	operations := make([]spanstore.Operation, 0)
+
+	for rows.Next() {
+		var name, spanKind string
+		if err := rows.Scan(&name, &spanKind); err != nil {
+			return nil, err
+		}
+		operation := spanstore.Operation{Name: name}
+		if spanKind != "" {
+			operation.SpanKind = spanKind
+		}
+		operations = append(operations, operation)
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, err
 	}
 
 	return operations, nil
@@ -303,16 +347,18 @@ func (r *TraceReader) findTraceIDsInRange(ctx context.Context, params *spanstore
 	query := fmt.Sprintf("SELECT DISTINCT traceID FROM %s WHERE service = ?", r.indexTable)
 	args := []interface{}{params.ServiceName}
 
+	if r.tenant != "" {
+		query += " AND tenant = ?"
+		args = append(args, r.tenant)
+	}
+
 	if params.OperationName != "" {
 		query += " AND operation = ?"
 		args = append(args, params.OperationName)
 	}
 
-	query += " AND timestamp >= ?"
-	args = append(args, start)
-
-	query += " AND timestamp <= ?"
-	args = append(args, end)
+	query += " AND timestamp >= ? AND timestamp <= ?"
+	args = append(args, start, end)
 
 	if params.DurationMin != 0 {
 		query += " AND durationUs >= ?"
@@ -325,8 +371,8 @@ func (r *TraceReader) findTraceIDsInRange(ctx context.Context, params *spanstore
 	}
 
 	for key, value := range params.Tags {
-		query += " AND has(tags, ?)"
-		args = append(args, fmt.Sprintf("%s=%s", key, value))
+		query += " AND has(tags.key, ?) AND has(splitByChar(',', tags.value[indexOf(tags.key, ?)]), ?)"
+		args = append(args, key, key, value)
 	}
 
 	if len(skip) > 0 {

storage/clickhousespanstore/tablename.go

@@ -1,16 +1,7 @@
 package clickhousespanstore
 
-import (
-	"fmt"
-	"strings"
-)
-
 type TableName string
 
-func (tableName TableName) ToGlobal() TableName {
-	return TableName(strings.ReplaceAll(string(tableName), "_local", ""))
-}
-
-func (tableName TableName) AddDbName(databaseName string) TableName {
-	return TableName(fmt.Sprintf("%s.%s", databaseName, tableName))
+func (tableName TableName) ToLocal() TableName {
+	return tableName + "_local"
 }

storage/clickhousespanstore/tablename_test.go

@@ -0,0 +1,13 @@
+package clickhousespanstore
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTableName_ToLocal(t *testing.T) {
+	tableName := TableName("some_table")
+	assert.Equal(t, tableName+"_local", tableName.ToLocal())
+
+}

storage/clickhousespanstore/worker.go

@@ -0,0 +1,274 @@
+package clickhousespanstore
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/jaegertracing/jaeger/model"
+)
+
+var delays = []int{2, 3, 5, 8}
+
+// WriteWorker writes spans to CLickHouse.
+// Given a batch of spans, WriteWorker attempts to write them to database.
+// Interval in seconds between attempts changes due to delays slice, then it remains the same as the last value in delays.
+type WriteWorker struct {
+	// workerID is an arbitrary identifier for keeping track of this worker in logs
+	workerID   int32
+	params     *WorkerParams
+	batch      []*model.Span
+	finish     chan bool
+	workerDone chan *WriteWorker
+	done       sync.WaitGroup
+}
+
+func (worker *WriteWorker) Work() {
+	worker.done.Add(1)
+
+	defer worker.done.Done()
+
+	// TODO: look for specific error(connection refused | database error)
+	if err := worker.writeBatch(worker.batch); err != nil {
+		worker.params.logger.Error("Could not write a batch of spans", "error", err, "worker_id", worker.workerID)
+	} else {
+		worker.close()
+		return
+	}
+	attempt := 0
+	for {
+		currentDelay := worker.getCurrentDelay(&attempt, worker.params.delay)
+		timer := time.After(currentDelay)
+		select {
+		case <-worker.finish:
+			worker.close()
+			return
+		case <-timer:
+			if err := worker.writeBatch(worker.batch); err != nil {
+				worker.params.logger.Error("Could not write a batch of spans", "error", err, "worker_id", worker.workerID)
+			} else {
+				worker.close()
+				return
+			}
+		}
+	}
+}
+
+func (worker *WriteWorker) Close() {
+	worker.finish <- true
+	worker.done.Wait()
+}
+
+func (worker *WriteWorker) getCurrentDelay(attempt *int, delay time.Duration) time.Duration {
+	if *attempt < len(delays) {
+		*attempt++
+	}
+	return time.Duration(int64(delays[*attempt-1]) * delay.Nanoseconds())
+}
+
+func (worker *WriteWorker) close() {
+	worker.workerDone <- worker
+}
+
+func (worker *WriteWorker) writeBatch(batch []*model.Span) error {
+	worker.params.logger.Debug("Writing spans", "size", len(batch))
+	if err := worker.writeModelBatch(batch); err != nil {
+		return err
+	}
+
+	if worker.params.indexTable != "" {
+		if err := worker.writeIndexBatch(batch); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (worker *WriteWorker) writeModelBatch(batch []*model.Span) error {
+	tx, err := worker.params.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	committed := false
+
+	defer func() {
+		if !committed {
+			// Clickhouse does not support real rollback
+			_ = tx.Rollback()
+		}
+	}()
+
+	var query string
+	if worker.params.tenant == "" {
+		query = fmt.Sprintf("INSERT INTO %s (timestamp, traceID, model) VALUES (?, ?, ?)", worker.params.spansTable)
+	} else {
+		query = fmt.Sprintf("INSERT INTO %s (tenant, timestamp, traceID, model) VALUES (?, ?, ?, ?)", worker.params.spansTable)
+	}
+
+	statement, err := tx.Prepare(query)
+	if err != nil {
+		return err
+	}
+
+	defer statement.Close()
+
+	for _, span := range batch {
+		var serialized []byte
+
+		if worker.params.encoding == EncodingJSON {
+			serialized, err = json.Marshal(span)
+		} else {
+			serialized, err = proto.Marshal(span)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if worker.params.tenant == "" {
+			_, err = statement.Exec(span.StartTime, span.TraceID.String(), serialized)
+		} else {
+			_, err = statement.Exec(worker.params.tenant, span.StartTime, span.TraceID.String(), serialized)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	committed = true
+
+	return tx.Commit()
+}
+
+func (worker *WriteWorker) writeIndexBatch(batch []*model.Span) error {
+	tx, err := worker.params.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	committed := false
+
+	defer func() {
+		if !committed {
+			// Clickhouse does not support real rollback
+			_ = tx.Rollback()
+		}
+	}()
+
+	var query string
+	if worker.params.tenant == "" {
+		query = fmt.Sprintf(
+			"INSERT INTO %s (timestamp, traceID, service, operation, durationUs, tags.key, tags.value) VALUES (?, ?, ?, ?, ?, ?, ?)",
+			worker.params.indexTable,
+		)
+	} else {
+		query = fmt.Sprintf(
+			"INSERT INTO %s (tenant, timestamp, traceID, service, operation, durationUs, tags.key, tags.value) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+			worker.params.indexTable,
+		)
+	}
+
+	statement, err := tx.Prepare(query)
+	if err != nil {
+		return err
+	}
+
+	defer statement.Close()
+
+	for _, span := range batch {
+		keys, values := uniqueTagsForSpan(span)
+		if worker.params.tenant == "" {
+			_, err = statement.Exec(
+				span.StartTime,
+				span.TraceID.String(),
+				span.Process.ServiceName,
+				span.OperationName,
+				uint64(span.Duration.Microseconds()),
+				keys,
+				values,
+			)
+		} else {
+			_, err = statement.Exec(
+				worker.params.tenant,
+				span.StartTime,
+				span.TraceID.String(),
+				span.Process.ServiceName,
+				span.OperationName,
+				uint64(span.Duration.Microseconds()),
+				keys,
+				values,
+			)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	committed = true
+
+	return tx.Commit()
+}
+
+func uniqueTagsForSpan(span *model.Span) (keys, values []string) {
+	uniqueTags := make(map[string][]string, len(span.Tags)+len(span.Process.Tags))
+
+	for i := range span.Tags {
+		key := tagKey(&span.GetTags()[i])
+		uniqueTags[key] = append(uniqueTags[key], tagValue(&span.GetTags()[i]))
+	}
+
+	for i := range span.Process.Tags {
+		key := tagKey(&span.GetProcess().GetTags()[i])
+		uniqueTags[key] = append(uniqueTags[key], tagValue(&span.GetProcess().GetTags()[i]))
+	}
+
+	for _, event := range span.Logs {
+		for i := range event.Fields {
+			key := tagKey(&event.GetFields()[i])
+			uniqueTags[key] = append(uniqueTags[key], tagValue(&event.GetFields()[i]))
+		}
+	}
+
+	keys = make([]string, 0, len(uniqueTags))
+	for k := range uniqueTags {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	values = make([]string, 0, len(uniqueTags))
+	for _, key := range keys {
+		values = append(values, strings.Join(unique(uniqueTags[key]), ","))
+	}
+
+	return keys, values
+}
+
+func tagKey(kv *model.KeyValue) string {
+	return kv.Key
+}
+
+func tagValue(kv *model.KeyValue) string {
+	return kv.AsString()
+}
+
+func unique(slice []string) []string {
+	if len(slice) == 1 {
+		return slice
+	}
+
+	keys := make(map[string]bool)
+	list := []string{}
+	for _, entry := range slice {
+		if _, value := keys[entry]; !value {
+			keys[entry] = true
+			list = append(list, entry)
+		}
+	}
+	return list
+}

storage/clickhousespanstore/worker_test.go

@@ -0,0 +1,552 @@
+package clickhousespanstore
+
+import (
+	"database/sql"
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"strconv"
+	"testing"
+	"time"
+
+	sqlmock "github.com/DATA-DOG/go-sqlmock"
+	"github.com/gogo/protobuf/proto"
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/jaegertracing/jaeger/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore/mocks"
+)
+
+const (
+	testTagCount      = 10
+	testLogCount      = 5
+	testLogFieldCount = 5
+	testIndexTable    = "test_index_table"
+	testSpansTable    = "test_spans_table"
+	testTenant        = "test_tenant"
+)
+
+type expectation struct {
+	preparation string
+	execArgs    [][]driver.Value
+}
+
+var (
+	errorMock = fmt.Errorf("error mock")
+	process   = model.NewProcess("test_service", []model.KeyValue{model.String("test_process_key", "test_process_value")})
+	testSpan  = model.Span{
+		TraceID:       model.NewTraceID(1, 2),
+		SpanID:        model.NewSpanID(3),
+		OperationName: "GET /unit_test",
+		StartTime:     testStartTime,
+		Process:       process,
+		Tags:          []model.KeyValue{model.String("test_string_key", "test_string_value"), model.Int64("test_int64_key", 4)},
+		Logs:          []model.Log{{Timestamp: testStartTime, Fields: []model.KeyValue{model.String("test_log_key", "test_log_value")}}},
+		Duration:      time.Minute,
+	}
+	testSpans             = []*model.Span{&testSpan}
+	keys, values          = uniqueTagsForSpan(&testSpan)
+	indexWriteExpectation = expectation{
+		preparation: fmt.Sprintf("INSERT INTO %s (timestamp, traceID, service, operation, durationUs, tags.key, tags.value) VALUES (?, ?, ?, ?, ?, ?, ?)", testIndexTable),
+		execArgs: [][]driver.Value{{
+			testSpan.StartTime,
+			testSpan.TraceID.String(),
+			testSpan.Process.GetServiceName(),
+			testSpan.OperationName,
+			uint64(testSpan.Duration.Microseconds()),
+			keys,
+			values,
+		}}}
+	indexWriteExpectationTenant = expectation{
+		preparation: fmt.Sprintf("INSERT INTO %s (tenant, timestamp, traceID, service, operation, durationUs, tags.key, tags.value) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", testIndexTable),
+		execArgs: [][]driver.Value{{
+			testTenant,
+			testSpan.StartTime,
+			testSpan.TraceID.String(),
+			testSpan.Process.GetServiceName(),
+			testSpan.OperationName,
+			uint64(testSpan.Duration.Microseconds()),
+			keys,
+			values,
+		}}}
+	writeBatchLogs = []mocks.LogMock{{Msg: "Writing spans", Args: []interface{}{"size", len(testSpans)}}}
+)
+
+func TestSpanWriter_TagKeyValue(t *testing.T) {
+	tests := map[string]struct {
+		kv       model.KeyValue
+		expected string
+	}{
+		"string value":       {kv: model.String("tag_key", "tag_string_value"), expected: "tag_string_value"},
+		"true value":         {kv: model.Bool("tag_key", true), expected: "true"},
+		"false value":        {kv: model.Bool("tag_key", false), expected: "false"},
+		"positive int value": {kv: model.Int64("tag_key", 1203912), expected: "1203912"},
+		"negative int value": {kv: model.Int64("tag_key", -1203912), expected: "-1203912"},
+		"float value":        {kv: model.Float64("tag_key", 0.005009), expected: "0.005009"},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			assert.Equal(t, test.expected, tagValue(&test.kv), "Incorrect tag value string")
+		})
+	}
+}
+
+func TestSpanWriter_UniqueTagsForSpan(t *testing.T) {
+	tests := map[string]struct {
+		tags           []model.KeyValue
+		processTags    []model.KeyValue
+		logs           []model.Log
+		expectedKeys   []string
+		expectedValues []string
+	}{
+		"default": {
+			tags:           []model.KeyValue{model.String("key2", "value")},
+			processTags:    []model.KeyValue{model.Int64("key3", 412)},
+			logs:           []model.Log{{Fields: []model.KeyValue{model.Float64("key1", .5)}}},
+			expectedKeys:   []string{"key1", "key2", "key3"},
+			expectedValues: []string{"0.5", "value", "412"},
+		},
+		"repeating tags": {
+			tags:           []model.KeyValue{model.String("key2", "value"), model.String("key2", "value")},
+			processTags:    []model.KeyValue{model.Int64("key3", 412)},
+			logs:           []model.Log{{Fields: []model.KeyValue{model.Float64("key1", .5)}}},
+			expectedKeys:   []string{"key1", "key2", "key3"},
+			expectedValues: []string{"0.5", "value", "412"},
+		},
+		"repeating keys": {
+			tags:           []model.KeyValue{model.String("key2", "value_a"), model.String("key2", "value_b")},
+			processTags:    []model.KeyValue{model.Int64("key3", 412)},
+			logs:           []model.Log{{Fields: []model.KeyValue{model.Float64("key1", .5)}}},
+			expectedKeys:   []string{"key1", "key2", "key3"},
+			expectedValues: []string{"0.5", "value_a,value_b", "412"},
+		},
+		"repeating values": {
+			tags:           []model.KeyValue{model.String("key2", "value"), model.Int64("key4", 412)},
+			processTags:    []model.KeyValue{model.Int64("key3", 412)},
+			logs:           []model.Log{{Fields: []model.KeyValue{model.Float64("key1", .5)}}},
+			expectedKeys:   []string{"key1", "key2", "key3", "key4"},
+			expectedValues: []string{"0.5", "value", "412", "412"},
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			process := model.Process{Tags: test.processTags}
+			span := model.Span{Tags: test.tags, Process: &process, Logs: test.logs}
+			actualKeys, actualValues := uniqueTagsForSpan(&span)
+			assert.Equal(t, test.expectedKeys, actualKeys)
+			assert.Equal(t, test.expectedValues, actualValues)
+		})
+	}
+}
+
+func TestSpanWriter_General(t *testing.T) {
+	spanJSON, err := json.Marshal(&testSpan)
+	require.NoError(t, err)
+	modelWriteExpectationJSON := getModelWriteExpectation(spanJSON, "")
+	modelWriteExpectationJSONTenant := getModelWriteExpectation(spanJSON, testTenant)
+	spanProto, err := proto.Marshal(&testSpan)
+	require.NoError(t, err)
+	modelWriteExpectationProto := getModelWriteExpectation(spanProto, "")
+	modelWriteExpectationProtoTenant := getModelWriteExpectation(spanProto, testTenant)
+	tests := map[string]struct {
+		encoding     Encoding
+		indexTable   TableName
+		tenant       string
+		spans        []*model.Span
+		expectations []expectation
+		action       func(writeWorker *WriteWorker, spans []*model.Span) error
+		expectedLogs []mocks.LogMock
+	}{
+		"write index batch": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			spans:        testSpans,
+			expectations: []expectation{indexWriteExpectation},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeIndexBatch(spans) },
+		},
+		"write index tenant batch": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			spans:        testSpans,
+			expectations: []expectation{indexWriteExpectationTenant},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeIndexBatch(spans) },
+		},
+		"write model batch JSON": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationJSON},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeModelBatch(spans) },
+		},
+		"write model tenant batch JSON": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationJSONTenant},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeModelBatch(spans) },
+		},
+		"write model batch Proto": {
+			encoding:     EncodingProto,
+			indexTable:   testIndexTable,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationProto},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeModelBatch(spans) },
+		},
+		"write model tenant batch Proto": {
+			encoding:     EncodingProto,
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationProtoTenant},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeModelBatch(spans) },
+		},
+		"write batch no index JSON": {
+			encoding:     EncodingJSON,
+			indexTable:   "",
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationJSON},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch no index Proto": {
+			encoding:     EncodingProto,
+			indexTable:   "",
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationProto},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch JSON": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationJSON, indexWriteExpectation},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch tenant JSON": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationJSONTenant, indexWriteExpectationTenant},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch Proto": {
+			encoding:     EncodingProto,
+			indexTable:   testIndexTable,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationProto, indexWriteExpectation},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch tenant Proto": {
+			encoding:     EncodingProto,
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationProtoTenant, indexWriteExpectationTenant},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			db, mock, err := mocks.GetDbMock()
+			require.NoError(t, err, "an error was not expected when opening a stub database connection")
+			defer db.Close()
+
+			spyLogger := mocks.NewSpyLogger()
+			worker := getWriteWorker(spyLogger, db, test.encoding, test.indexTable, test.tenant)
+
+			for _, expectation := range test.expectations {
+				mock.ExpectBegin()
+				prep := mock.ExpectPrepare(expectation.preparation)
+				for _, args := range expectation.execArgs {
+					prep.ExpectExec().WithArgs(args...).WillReturnResult(sqlmock.NewResult(1, 1))
+				}
+				mock.ExpectCommit()
+			}
+
+			assert.NoError(t, test.action(&worker, test.spans))
+			assert.NoError(t, mock.ExpectationsWereMet())
+			spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, test.expectedLogs)
+		})
+	}
+}
+
+func TestSpanWriter_BeginError(t *testing.T) {
+	tests := map[string]struct {
+		action       func(writeWorker *WriteWorker) error
+		expectedLogs []mocks.LogMock
+	}{
+		"write model batch": {action: func(writeWorker *WriteWorker) error { return writeWorker.writeModelBatch(testSpans) }},
+		"write index batch": {action: func(writeWorker *WriteWorker) error { return writeWorker.writeIndexBatch(testSpans) }},
+		"write batch": {
+			action:       func(writeWorker *WriteWorker) error { return writeWorker.writeBatch(testSpans) },
+			expectedLogs: writeBatchLogs,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			db, mock, err := mocks.GetDbMock()
+			require.NoError(t, err, "an error was not expected when opening a stub database connection")
+			defer db.Close()
+
+			spyLogger := mocks.NewSpyLogger()
+			writeWorker := getWriteWorker(spyLogger, db, EncodingJSON, testIndexTable, "")
+
+			mock.ExpectBegin().WillReturnError(errorMock)
+
+			assert.ErrorIs(t, test.action(&writeWorker), errorMock)
+			assert.NoError(t, mock.ExpectationsWereMet())
+			spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, test.expectedLogs)
+		})
+	}
+}
+
+func TestSpanWriter_PrepareError(t *testing.T) {
+	spanJSON, err := json.Marshal(&testSpan)
+	require.NoError(t, err)
+	modelWriteExpectation := getModelWriteExpectation(spanJSON, "")
+	modelWriteExpectationTenant := getModelWriteExpectation(spanJSON, testTenant)
+
+	tests := map[string]struct {
+		action       func(writeWorker *WriteWorker) error
+		tenant       string
+		expectation  expectation
+		expectedLogs []mocks.LogMock
+	}{
+		"write model batch": {
+			action:      func(writeWorker *WriteWorker) error { return writeWorker.writeModelBatch(testSpans) },
+			expectation: modelWriteExpectation,
+		},
+		"write model tenant batch": {
+			action:      func(writeWorker *WriteWorker) error { return writeWorker.writeModelBatch(testSpans) },
+			tenant:      testTenant,
+			expectation: modelWriteExpectationTenant,
+		},
+		"write index batch": {
+			action:      func(writeWorker *WriteWorker) error { return writeWorker.writeIndexBatch(testSpans) },
+			expectation: indexWriteExpectation,
+		},
+		"write index tenant batch": {
+			action:      func(writeWorker *WriteWorker) error { return writeWorker.writeIndexBatch(testSpans) },
+			tenant:      testTenant,
+			expectation: indexWriteExpectationTenant,
+		},
+		"write batch": {
+			action:       func(writeWorker *WriteWorker) error { return writeWorker.writeBatch(testSpans) },
+			expectation:  modelWriteExpectation,
+			expectedLogs: writeBatchLogs,
+		},
+		"write tenant batch": {
+			action:       func(writeWorker *WriteWorker) error { return writeWorker.writeBatch(testSpans) },
+			tenant:       testTenant,
+			expectation:  modelWriteExpectationTenant,
+			expectedLogs: writeBatchLogs,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			db, mock, err := mocks.GetDbMock()
+			require.NoError(t, err, "an error was not expected when opening a stub database connection")
+			defer db.Close()
+
+			spyLogger := mocks.NewSpyLogger()
+			spanWriter := getWriteWorker(spyLogger, db, EncodingJSON, testIndexTable, test.tenant)
+
+			mock.ExpectBegin()
+			mock.ExpectPrepare(test.expectation.preparation).WillReturnError(errorMock)
+			mock.ExpectRollback()
+
+			assert.ErrorIs(t, test.action(&spanWriter), errorMock)
+			assert.NoError(t, mock.ExpectationsWereMet())
+			spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, test.expectedLogs)
+		})
+	}
+}
+
+func TestSpanWriter_ExecError(t *testing.T) {
+	spanJSON, err := json.Marshal(&testSpan)
+	require.NoError(t, err)
+	modelWriteExpectation := getModelWriteExpectation(spanJSON, "")
+	modelWriteExpectationTenant := getModelWriteExpectation(spanJSON, testTenant)
+	tests := map[string]struct {
+		indexTable   TableName
+		tenant       string
+		expectations []expectation
+		action       func(writer *WriteWorker) error
+		expectedLogs []mocks.LogMock
+	}{
+		"write model batch": {
+			indexTable:   testIndexTable,
+			expectations: []expectation{modelWriteExpectation},
+			action:       func(writer *WriteWorker) error { return writer.writeModelBatch(testSpans) },
+		},
+		"write model tenant batch": {
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			expectations: []expectation{modelWriteExpectationTenant},
+			action:       func(writer *WriteWorker) error { return writer.writeModelBatch(testSpans) },
+		},
+		"write index batch": {
+			indexTable:   testIndexTable,
+			expectations: []expectation{indexWriteExpectation},
+			action:       func(writer *WriteWorker) error { return writer.writeIndexBatch(testSpans) },
+		},
+		"write index tenant batch": {
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			expectations: []expectation{indexWriteExpectationTenant},
+			action:       func(writer *WriteWorker) error { return writer.writeIndexBatch(testSpans) },
+		},
+		"write batch no index": {
+			indexTable:   "",
+			expectations: []expectation{modelWriteExpectation},
+			action:       func(writer *WriteWorker) error { return writer.writeBatch(testSpans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch": {
+			indexTable:   testIndexTable,
+			expectations: []expectation{modelWriteExpectation, indexWriteExpectation},
+			action:       func(writer *WriteWorker) error { return writer.writeBatch(testSpans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write tenant batch": {
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			expectations: []expectation{modelWriteExpectationTenant, indexWriteExpectationTenant},
+			action:       func(writer *WriteWorker) error { return writer.writeBatch(testSpans) },
+			expectedLogs: writeBatchLogs,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			db, mock, err := mocks.GetDbMock()
+			require.NoError(t, err, "an error was not expected when opening a stub database connection")
+			defer db.Close()
+
+			spyLogger := mocks.NewSpyLogger()
+			writeWorker := getWriteWorker(spyLogger, db, EncodingJSON, testIndexTable, test.tenant)
+
+			for i, expectation := range test.expectations {
+				mock.ExpectBegin()
+				prep := mock.ExpectPrepare(expectation.preparation)
+				if i < len(test.expectations)-1 {
+					for _, args := range expectation.execArgs {
+						prep.ExpectExec().WithArgs(args...).WillReturnResult(sqlmock.NewResult(1, 1))
+					}
+					mock.ExpectCommit()
+				} else {
+					prep.ExpectExec().WithArgs(expectation.execArgs[0]...).WillReturnError(errorMock)
+					mock.ExpectRollback()
+				}
+			}
+
+			assert.ErrorIs(t, test.action(&writeWorker), errorMock)
+			assert.NoError(t, mock.ExpectationsWereMet())
+			spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, test.expectedLogs)
+		})
+	}
+}
+
+func getWriteWorker(spyLogger mocks.SpyLogger, db *sql.DB, encoding Encoding, indexTable TableName, tenant string) WriteWorker {
+	return WriteWorker{
+		params: &WorkerParams{
+			logger:     spyLogger,
+			db:         db,
+			spansTable: testSpansTable,
+			indexTable: indexTable,
+			tenant:     tenant,
+			encoding:   encoding,
+		},
+		workerDone: make(chan *WriteWorker),
+	}
+}
+
+func generateRandomSpans(count int) []*model.Span {
+	spans := make([]*model.Span, count)
+	for i := 0; i < count; i++ {
+		span := generateRandomSpan()
+		spans[i] = &span
+	}
+	return spans
+}
+
+func generateRandomSpan() model.Span {
+	processTags := generateRandomKeyValues(testTagCount)
+	process := model.Process{
+		ServiceName: "service" + strconv.FormatUint(rand.Uint64(), 10),
+		Tags:        processTags,
+	}
+	span := model.Span{
+		TraceID:       model.NewTraceID(rand.Uint64(), rand.Uint64()),
+		SpanID:        model.NewSpanID(rand.Uint64()),
+		OperationName: "operation" + strconv.FormatUint(rand.Uint64(), 10),
+		StartTime:     getRandomTime(),
+		Process:       &process,
+		Tags:          generateRandomKeyValues(testTagCount),
+		Logs:          generateRandomLogs(),
+		Duration:      time.Unix(rand.Int63n(1<<32), 0).Sub(time.Unix(0, 0)),
+	}
+	return span
+}
+
+func generateRandomLogs() []model.Log {
+	logs := make([]model.Log, 0, testLogCount)
+	for i := 0; i < testLogCount; i++ {
+		timestamp := getRandomTime()
+		logs = append(logs, model.Log{Timestamp: timestamp, Fields: generateRandomKeyValues(testLogFieldCount)})
+	}
+	return logs
+}
+
+func getRandomTime() time.Time {
+	return time.Unix(rand.Int63n(time.Now().Unix()), 0)
+}
+
+func generateRandomKeyValues(count int) []model.KeyValue {
+	tags := make([]model.KeyValue, 0, count)
+	for i := 0; i < count; i++ {
+		key := "key" + strconv.FormatUint(rand.Uint64(), 16)
+		value := "key" + strconv.FormatUint(rand.Uint64(), 16)
+		kv := model.KeyValue{Key: key, VType: model.ValueType_STRING, VStr: value}
+		tags = append(tags, kv)
+	}
+
+	return tags
+}
+
+func getModelWriteExpectation(spanJSON []byte, tenant string) expectation {
+	if tenant == "" {
+		return expectation{
+			preparation: fmt.Sprintf("INSERT INTO %s (timestamp, traceID, model) VALUES (?, ?, ?)", testSpansTable),
+			execArgs: [][]driver.Value{{
+				testSpan.StartTime,
+				testSpan.TraceID.String(),
+				spanJSON,
+			}},
+		}
+	} else {
+		return expectation{
+			preparation: fmt.Sprintf("INSERT INTO %s (tenant, timestamp, traceID, model) VALUES (?, ?, ?, ?)", testSpansTable),
+			execArgs: [][]driver.Value{{
+				tenant,
+				testSpan.StartTime,
+				testSpan.TraceID.String(),
+				spanJSON,
+			}},
+		}
+	}
+}

storage/clickhousespanstore/writer.go

@@ -3,14 +3,10 @@ package clickhousespanstore
 import (
 	"context"
 	"database/sql"
-	"encoding/json"
-	"fmt"
-	"sort"
 	"sync"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
-	"github.com/hashicorp/go-hclog"
+	hclog "github.com/hashicorp/go-hclog"
 	"github.com/jaegertracing/jaeger/model"
 	"github.com/jaegertracing/jaeger/storage/spanstore"
 	"github.com/prometheus/client_golang/prometheus"
@@ -38,52 +34,63 @@ var (
 
 // SpanWriter for writing spans to ClickHouse
 type SpanWriter struct {
-	logger     hclog.Logger
-	db         *sql.DB
-	indexTable TableName
-	spansTable TableName
-	encoding   Encoding
-	delay      time.Duration
-	size       int64
-	spans      chan *model.Span
-	finish     chan bool
-	done       sync.WaitGroup
+	workerParams WorkerParams
+
+	size   int64
+	spans  chan *model.Span
+	finish chan bool
+	done   sync.WaitGroup
 }
 
-var registerMetrics sync.Once
+var registerWriterMetrics sync.Once
 var _ spanstore.Writer = (*SpanWriter)(nil)
 
 // NewSpanWriter returns a SpanWriter for the database
-func NewSpanWriter(logger hclog.Logger, db *sql.DB, indexTable, spansTable TableName, encoding Encoding, delay time.Duration, size int64) *SpanWriter {
+func NewSpanWriter(
+	logger hclog.Logger,
+	db *sql.DB,
+	indexTable,
+	spansTable TableName,
+	tenant string,
+	encoding Encoding,
+	delay time.Duration,
+	size int64,
+	maxSpanCount int,
+) *SpanWriter {
 	writer := &SpanWriter{
-		logger:     logger,
-		db:         db,
-		indexTable: indexTable,
-		spansTable: spansTable,
-		encoding:   encoding,
-		delay:      delay,
-		size:       size,
-		spans:      make(chan *model.Span, size),
-		finish:     make(chan bool),
+		workerParams: WorkerParams{
+			logger:     logger,
+			db:         db,
+			indexTable: indexTable,
+			spansTable: spansTable,
+			tenant:     tenant,
+			encoding:   encoding,
+			delay:      delay,
+		},
+		size:   size,
+		spans:  make(chan *model.Span, size),
+		finish: make(chan bool),
 	}
 
 	writer.registerMetrics()
-	go writer.backgroundWriter()
+	go writer.backgroundWriter(maxSpanCount)
 
 	return writer
 }
 
 func (w *SpanWriter) registerMetrics() {
-	registerMetrics.Do(func() {
+	registerWriterMetrics.Do(func() {
 		prometheus.MustRegister(numWritesWithBatchSize)
 		prometheus.MustRegister(numWritesWithFlushInterval)
 	})
 }
 
-func (w *SpanWriter) backgroundWriter() {
+func (w *SpanWriter) backgroundWriter(maxSpanCount int) {
+	pool := NewWorkerPool(&w.workerParams, maxSpanCount)
+	go pool.Work()
 	batch := make([]*model.Span, 0, w.size)
 
-	timer := time.After(w.delay)
+	timer := time.After(w.workerParams.delay)
 	last := time.Now()
 
 	for {
@@ -97,31 +104,32 @@ func (w *SpanWriter) backgroundWriter() {
 			batch = append(batch, span)
 			flush = len(batch) == cap(batch)
 			if flush {
-				w.logger.Debug("Flush due to batch size", "size", len(batch))
+				w.workerParams.logger.Debug("Flush due to batch size", "size", len(batch))
 				numWritesWithBatchSize.Inc()
 			}
 		case <-timer:
-			timer = time.After(w.delay)
-			flush = time.Since(last) > w.delay && len(batch) > 0
+			timer = time.After(w.workerParams.delay)
+			flush = time.Since(last) > w.workerParams.delay && len(batch) > 0
 			if flush {
-				w.logger.Debug("Flush due to timer")
+				w.workerParams.logger.Debug("Flush due to timer")
 				numWritesWithFlushInterval.Inc()
 			}
 		case <-w.finish:
 			finish = true
 			flush = len(batch) > 0
-			w.logger.Debug("Finish channel")
+			w.workerParams.logger.Debug("Finish channel")
 		}
 
 		if flush {
-			if err := w.writeBatch(batch); err != nil {
-				w.logger.Error("Could not write a batch of spans", "error", err)
-			}
+			pool.WriteBatch(batch)
 
 			batch = make([]*model.Span, 0, w.size)
 			last = time.Now()
 		}
 
+		if finish {
+			pool.Close()
+		}
 		w.done.Done()
 
 		if finish {
@@ -130,108 +138,6 @@ func (w *SpanWriter) backgroundWriter() {
 	}
 }
 
-func (w *SpanWriter) writeBatch(batch []*model.Span) error {
-	w.logger.Debug("Writing spans", "size", len(batch))
-	if err := w.writeModelBatch(batch); err != nil {
-		return err
-	}
-
-	if w.indexTable != "" {
-		if err := w.writeIndexBatch(batch); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (w *SpanWriter) writeModelBatch(batch []*model.Span) error {
-	tx, err := w.db.Begin()
-	if err != nil {
-		return err
-	}
-
-	committed := false
-
-	defer func() {
-		if !committed {
-			// Clickhouse does not support real rollback
-			_ = tx.Rollback()
-		}
-	}()
-
-	statement, err := tx.Prepare(fmt.Sprintf("INSERT INTO %s (timestamp, traceID, model) VALUES (?, ?, ?)", w.spansTable))
-	if err != nil {
-		return err
-	}
-
-	defer statement.Close()
-
-	for _, span := range batch {
-		var serialized []byte
-
-		if w.encoding == EncodingJSON {
-			serialized, err = json.Marshal(span)
-		} else {
-			serialized, err = proto.Marshal(span)
-		}
-
-		if err != nil {
-			return err
-		}
-
-		_, err = statement.Exec(span.StartTime, span.TraceID.String(), serialized)
-		if err != nil {
-			return err
-		}
-	}
-
-	committed = true
-
-	return tx.Commit()
-}
-
-func (w *SpanWriter) writeIndexBatch(batch []*model.Span) error {
-	tx, err := w.db.Begin()
-	if err != nil {
-		return err
-	}
-
-	committed := false
-
-	defer func() {
-		if !committed {
-			// Clickhouse does not support real rollback
-			_ = tx.Rollback()
-		}
-	}()
-
-	statement, err := tx.Prepare(fmt.Sprintf("INSERT INTO %s (timestamp, traceID, service, operation, durationUs, tags) VALUES (?, ?, ?, ?, ?, ?)", w.indexTable))
-	if err != nil {
-		return err
-	}
-
-	defer statement.Close()
-
-	for _, span := range batch {
-		_, err = statement.Exec(
-			span.StartTime,
-			span.TraceID.String(),
-			span.Process.ServiceName,
-			span.OperationName,
-			span.Duration.Microseconds(),
-			uniqueTagsForSpan(span),
-		)
-		if err != nil {
-			return err
-		}
-	}
-
-	committed = true
-
-	return tx.Commit()
-}
-
 // WriteSpan writes the encoded span
 func (w *SpanWriter) WriteSpan(_ context.Context, span *model.Span) error {
 	w.spans <- span
@@ -244,35 +150,3 @@ func (w *SpanWriter) Close() error {
 	w.done.Wait()
 	return nil
 }
-
-func uniqueTagsForSpan(span *model.Span) []string {
-	uniqueTags := make(map[string]struct{}, len(span.Tags)+len(span.Process.Tags))
-
-	for i := range span.Tags {
-		uniqueTags[tagString(&span.GetTags()[i])] = struct{}{}
-	}
-
-	for i := range span.Process.Tags {
-		uniqueTags[tagString(&span.GetProcess().GetTags()[i])] = struct{}{}
-	}
-
-	for _, event := range span.Logs {
-		for i := range event.Fields {
-			uniqueTags[tagString(&event.GetFields()[i])] = struct{}{}
-		}
-	}
-
-	tags := make([]string, 0, len(uniqueTags))
-
-	for kv := range uniqueTags {
-		tags = append(tags, kv)
-	}
-
-	sort.Strings(tags)
-
-	return tags
-}
-
-func tagString(kv *model.KeyValue) string {
-	return fmt.Sprintf("%s=%s", kv.Key, kv.AsString())
-}

storage/clickhousespanstore/writer_test.go

@@ -1,528 +0,0 @@
-package clickhousespanstore
-
-import (
-	"database/sql"
-	"encoding/json"
-	"fmt"
-	"math/rand"
-	"sort"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/hashicorp/go-hclog"
-
-	"github.com/DATA-DOG/go-sqlmock"
-	"github.com/gogo/protobuf/proto"
-	"github.com/jaegertracing/jaeger/model"
-
-	"github.com/pavolloffay/jaeger-clickhouse/storage/clickhousespanstore/mocks"
-)
-
-const (
-	testSpanCount     = 100
-	testTagCount      = 10
-	testLogCount      = 5
-	testLogFieldCount = 5
-	testIndexTable    = "test_index_table"
-	testSpansTable    = "test_spans_table"
-)
-
-var errorMock = fmt.Errorf("error mock")
-
-func TestSpanWriter_TagString(t *testing.T) {
-	tests := map[string]struct {
-		kv       model.KeyValue
-		expected string
-	}{
-		"string value":       {kv: model.String("tag_key", "tag_string_value"), expected: "tag_key=tag_string_value"},
-		"true value":         {kv: model.Bool("tag_key", true), expected: "tag_key=true"},
-		"false value":        {kv: model.Bool("tag_key", false), expected: "tag_key=false"},
-		"positive int value": {kv: model.Int64("tag_key", 1203912), expected: "tag_key=1203912"},
-		"negative int value": {kv: model.Int64("tag_key", -1203912), expected: "tag_key=-1203912"},
-		"float value":        {kv: model.Float64("tag_key", 0.005009), expected: "tag_key=0.005009"},
-	}
-	for name, test := range tests {
-		t.Run(name, func(t *testing.T) {
-			assert.Equal(t, test.expected, tagString(&test.kv), "Incorrect tag string")
-		})
-	}
-}
-
-func TestSpanWriter_UniqueTagsForSpan(t *testing.T) {
-	spans := generateRandomSpans()
-	for _, span := range spans {
-		uniqueTags := make(map[string]struct{}, len(span.Tags)+len(span.Process.Tags))
-		for i := range span.Tags {
-			uniqueTags[tagString(&span.Tags[i])] = struct{}{}
-		}
-		for i := range span.Process.Tags {
-			uniqueTags[tagString(&span.Process.Tags[i])] = struct{}{}
-		}
-		for _, log := range span.Logs {
-			for i := range log.Fields {
-				uniqueTags[tagString(&log.Fields[i])] = struct{}{}
-			}
-		}
-		want := make([]string, 0, len(uniqueTags))
-		for tag := range uniqueTags {
-			want = append(want, tag)
-		}
-		sort.Strings(want)
-
-		assert.Equal(t, want, uniqueTagsForSpan(span))
-	}
-}
-
-func TestSpanWriter_WriteBatchNoIndexJSON(t *testing.T) {
-	testSpanWriterWriteBatchNoIndex(t, EncodingJSON, func(span *model.Span) ([]byte, error) { return json.Marshal(span) })
-}
-
-func TestSpanWriter_WriteBatchNoIndexProto(t *testing.T) {
-	testSpanWriterWriteBatchNoIndex(t, EncodingProto, func(span *model.Span) ([]byte, error) { return proto.Marshal(span) })
-}
-
-func testSpanWriterWriteBatchNoIndex(t *testing.T, encoding Encoding, marshal func(span *model.Span) ([]byte, error)) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := NewSpanWriter(
-		spyLogger,
-		db,
-		"",
-		testSpansTable,
-		encoding,
-		0,
-		0,
-	)
-
-	spans := generateRandomSpans()
-	require.NoError(t, expectModelWritten(mock, spans, marshal, spanWriter), "could not expect queries due to %s", err)
-	assert.NoError(t, spanWriter.writeBatch(spans), "Could not write spans")
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-
-	spyLogger.AssertLogsOfLevelEqual(
-		t,
-		hclog.Debug,
-		[]mocks.LogMock{
-			{Msg: "Writing spans", Args: []interface{}{"size", testSpanCount}},
-		},
-	)
-}
-
-func TestSpanWriter_WriteBatchJSON(t *testing.T) {
-	testSpanWriterWriteBatch(t, EncodingJSON, func(span *model.Span) ([]byte, error) { return json.Marshal(span) })
-}
-
-func TestSpanWriter_WriteBatchProto(t *testing.T) {
-	testSpanWriterWriteBatch(t, EncodingProto, func(span *model.Span) ([]byte, error) { return proto.Marshal(span) })
-}
-
-func testSpanWriterWriteBatch(t *testing.T, encoding Encoding, marshal func(span *model.Span) ([]byte, error)) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := getSpanWriter(spyLogger, db, encoding)
-	spans := generateRandomSpans()
-	require.NoError(t, expectModelWritten(mock, spans, marshal, spanWriter), "could not expect queries due to %s", err)
-	expectIndexWritten(mock, spans, spanWriter)
-	assert.NoError(t, spanWriter.writeBatch(spans), "Could not write spans")
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-
-	spyLogger.AssertLogsOfLevelEqual(
-		t,
-		hclog.Debug,
-		[]mocks.LogMock{
-			{Msg: "Writing spans", Args: []interface{}{"size", testSpanCount}},
-		},
-	)
-}
-
-func TestSpanWriter_WriteBatchModelError(t *testing.T) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := getSpanWriter(spyLogger, db, EncodingJSON)
-
-	mock.ExpectBegin()
-	prep := mock.ExpectPrepare(fmt.Sprintf("INSERT INTO %s (timestamp, traceID, model) VALUES (?, ?, ?)", spanWriter.spansTable))
-
-	span := generateRandomSpan()
-	serializedSpan, err := json.Marshal(span)
-	require.NoError(t, err, "could not marshal span", span)
-
-	prep.
-		ExpectExec().
-		WithArgs(
-			span.StartTime,
-			span.TraceID.String(),
-			serializedSpan,
-		).
-		WillReturnError(errorMock)
-	mock.ExpectRollback()
-
-	assert.EqualError(t, spanWriter.writeBatch([]*model.Span{&span}), errorMock.Error())
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-	spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, []mocks.LogMock{{Msg: "Writing spans", Args: []interface{}{"size", 1}}})
-}
-
-func TestSpanWriter_WriteBatchIndexError(t *testing.T) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := getSpanWriter(spyLogger, db, EncodingJSON)
-
-	mock.ExpectBegin()
-	prep := mock.ExpectPrepare(fmt.Sprintf("INSERT INTO %s (timestamp, traceID, model) VALUES (?, ?, ?)", spanWriter.spansTable))
-
-	span := generateRandomSpan()
-	serializedSpan, err := json.Marshal(span)
-	require.NoError(t, err, "could not marshal span", span)
-
-	prep.
-		ExpectExec().
-		WithArgs(
-			span.StartTime,
-			span.TraceID.String(),
-			serializedSpan,
-		).
-		WillReturnResult(sqlmock.NewResult(1, 1))
-	mock.ExpectCommit()
-
-	mock.ExpectBegin()
-	prep = mock.ExpectPrepare(fmt.Sprintf(
-		"INSERT INTO %s (timestamp, traceID, service, operation, durationUs, tags) VALUES (?, ?, ?, ?, ?, ?)",
-		spanWriter.indexTable,
-	))
-	prep.ExpectExec().WithArgs(
-		span.StartTime,
-		span.TraceID,
-		span.Process.ServiceName,
-		span.OperationName,
-		span.Duration.Microseconds(),
-		fmt.Sprint(uniqueTagsForSpan(&span)),
-	).WillReturnError(errorMock)
-	mock.ExpectRollback()
-
-	assert.EqualError(t, spanWriter.writeBatch([]*model.Span{&span}), errorMock.Error())
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-	spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, []mocks.LogMock{{Msg: "Writing spans", Args: []interface{}{"size", 1}}})
-}
-
-func TestSpanWriter_WriteModelBatchJSON(t *testing.T) {
-	testSpanWriterWriteModelBatch(t, EncodingJSON, func(span *model.Span) ([]byte, error) { return json.Marshal(span) })
-}
-
-func TestSpanWriter_WriteModelBatchProtobuf(t *testing.T) {
-	testSpanWriterWriteModelBatch(t, EncodingProto, func(span *model.Span) ([]byte, error) { return proto.Marshal(span) })
-}
-
-func testSpanWriterWriteModelBatch(t *testing.T, encoding Encoding, marshal func(span *model.Span) ([]byte, error)) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := getSpanWriter(spyLogger, db, encoding)
-
-	spans := generateRandomSpans()
-	require.NoError(t, expectModelWritten(mock, spans, marshal, spanWriter), "could not expect queries due to %s", err)
-	assert.NoError(t, spanWriter.writeModelBatch(spans), "Could not write spans")
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-	spyLogger.AssertLogsEmpty(t)
-}
-
-func TestSpanWriter_WriteModelBatchBeginError(t *testing.T) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := getSpanWriter(spyLogger, db, EncodingJSON)
-
-	mock.ExpectBegin().WillReturnError(errorMock)
-
-	span := generateRandomSpan()
-
-	assert.EqualError(t, spanWriter.writeIndexBatch([]*model.Span{&span}), errorMock.Error())
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-}
-
-func TestSpanWriter_WriteModelBatchPrepareError(t *testing.T) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := getSpanWriter(spyLogger, db, EncodingJSON)
-
-	mock.ExpectBegin()
-
-	span := generateRandomSpan()
-	_ = mock.ExpectPrepare(fmt.Sprintf("INSERT INTO %s (timestamp, traceID, model) VALUES (?, ?, ?)", spanWriter.spansTable)).WillReturnError(errorMock)
-
-	mock.ExpectRollback()
-
-	assert.EqualError(t, spanWriter.writeModelBatch([]*model.Span{&span}), errorMock.Error())
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-	spyLogger.AssertLogsEmpty(t)
-}
-
-func TestSpanWriter_WriteModelBatchJSONExecuteError(t *testing.T) {
-	testSpanWriterWriteModelBatchExecuteError(t, EncodingJSON, func(span *model.Span) ([]byte, error) { return json.Marshal(span) })
-}
-
-func TestSpanWriter_WriteModelBatchProtobufExecuteError(t *testing.T) {
-	testSpanWriterWriteModelBatchExecuteError(t, EncodingProto, func(span *model.Span) ([]byte, error) { return proto.Marshal(span) })
-}
-
-func testSpanWriterWriteModelBatchExecuteError(t *testing.T, encoding Encoding, marshal func(span *model.Span) ([]byte, error)) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := getSpanWriter(spyLogger, db, encoding)
-
-	mock.ExpectBegin()
-	prep := mock.ExpectPrepare(fmt.Sprintf("INSERT INTO %s (timestamp, traceID, model) VALUES (?, ?, ?)", spanWriter.spansTable))
-
-	span := generateRandomSpan()
-	serializedSpan, err := marshal(&span)
-	require.NoError(t, err, "could not marshal span", span)
-
-	prep.
-		ExpectExec().
-		WithArgs(
-			span.StartTime,
-			span.TraceID.String(),
-			serializedSpan,
-		).
-		WillReturnError(errorMock)
-	mock.ExpectRollback()
-
-	assert.EqualError(t, spanWriter.writeModelBatch([]*model.Span{&span}), errorMock.Error())
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-	spyLogger.AssertLogsEmpty(t)
-}
-
-func TestSpanWriter_WriteIndexBatch(t *testing.T) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := getSpanWriter(spyLogger, db, EncodingJSON)
-
-	spans := generateRandomSpans()
-	expectIndexWritten(mock, spans, spanWriter)
-	assert.NoError(t, spanWriter.writeIndexBatch(spans), "Could not write spans")
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-	spyLogger.AssertLogsEmpty(t)
-}
-
-func TestSpanWriter_WriteIndexBatchBeginError(t *testing.T) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := getSpanWriter(spyLogger, db, EncodingJSON)
-
-	mock.ExpectBegin().WillReturnError(errorMock)
-
-	span := generateRandomSpan()
-
-	assert.EqualError(t, spanWriter.writeIndexBatch([]*model.Span{&span}), errorMock.Error())
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-}
-
-func TestSpanWriter_WriteIndexBatchPrepareError(t *testing.T) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := getSpanWriter(spyLogger, db, EncodingJSON)
-
-	mock.ExpectBegin()
-
-	span := generateRandomSpan()
-	_ = mock.ExpectPrepare(fmt.Sprintf(
-		"INSERT INTO %s (timestamp, traceID, service, operation, durationUs, tags) VALUES (?, ?, ?, ?, ?, ?)",
-		spanWriter.indexTable,
-	)).WillReturnError(errorMock)
-
-	mock.ExpectRollback()
-
-	assert.EqualError(t, spanWriter.writeIndexBatch([]*model.Span{&span}), errorMock.Error())
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-	spyLogger.AssertLogsEmpty(t)
-}
-
-func TestSpanWriter_WriteIndexBatchExecuteError(t *testing.T) {
-	db, mock, err := getDbMock()
-	require.NoError(t, err, "an error was not expected when opening a stub database connection")
-	defer db.Close()
-
-	spyLogger := mocks.NewSpyLogger()
-	spanWriter := getSpanWriter(spyLogger, db, EncodingJSON)
-
-	mock.ExpectBegin()
-	prep := mock.ExpectPrepare(fmt.Sprintf(
-		"INSERT INTO %s (timestamp, traceID, service, operation, durationUs, tags) VALUES (?, ?, ?, ?, ?, ?)",
-		spanWriter.indexTable,
-	))
-
-	span := generateRandomSpan()
-	prep.ExpectExec().WithArgs(
-		span.StartTime,
-		span.TraceID,
-		span.Process.ServiceName,
-		span.OperationName,
-		span.Duration.Microseconds(),
-		fmt.Sprint(uniqueTagsForSpan(&span)),
-	).WillReturnError(errorMock)
-	mock.ExpectRollback()
-
-	assert.EqualError(t, spanWriter.writeIndexBatch([]*model.Span{&span}), errorMock.Error())
-	assert.NoError(t, mock.ExpectationsWereMet(), "Not all expected queries were made")
-	spyLogger.AssertLogsEmpty(t)
-}
-
-func expectModelWritten(
-	mock sqlmock.Sqlmock,
-	spans []*model.Span,
-	marshal func(span *model.Span) ([]byte, error),
-	spanWriter *SpanWriter,
-) error {
-	mock.ExpectBegin()
-	prep := mock.ExpectPrepare(fmt.Sprintf("INSERT INTO %s (timestamp, traceID, model) VALUES (?, ?, ?)", spanWriter.spansTable))
-	for _, span := range spans {
-		serializedSpan, err := marshal(span)
-		if err != nil {
-			return fmt.Errorf("could not marshal %s due to %s", fmt.Sprint(span), err)
-		}
-
-		prep.
-			ExpectExec().
-			WithArgs(
-				span.StartTime,
-				span.TraceID.String(),
-				serializedSpan,
-			).
-			WillReturnResult(sqlmock.NewResult(1, 1))
-	}
-	mock.ExpectCommit()
-	return nil
-}
-
-func expectIndexWritten(
-	mock sqlmock.Sqlmock,
-	spans []*model.Span,
-	spanWriter *SpanWriter,
-) {
-	mock.ExpectBegin()
-	prep := mock.ExpectPrepare(fmt.Sprintf(
-		"INSERT INTO %s (timestamp, traceID, service, operation, durationUs, tags) VALUES (?, ?, ?, ?, ?, ?)",
-		spanWriter.indexTable,
-	))
-	for _, span := range spans {
-		prep.
-			ExpectExec().
-			WithArgs(
-				span.StartTime,
-				span.TraceID,
-				span.Process.ServiceName,
-				span.OperationName,
-				span.Duration.Microseconds(),
-				fmt.Sprint(uniqueTagsForSpan(span)),
-			).
-			WillReturnResult(sqlmock.NewResult(1, 1))
-	}
-	mock.ExpectCommit()
-}
-
-func getDbMock() (*sql.DB, sqlmock.Sqlmock, error) {
-	return sqlmock.New(
-		sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual),
-		sqlmock.ValueConverterOption(mocks.ConverterMock{}),
-	)
-}
-
-func getSpanWriter(spyLogger mocks.SpyLogger, db *sql.DB, encoding Encoding) *SpanWriter {
-	return NewSpanWriter(
-		spyLogger,
-		db,
-		testIndexTable,
-		testSpansTable,
-		encoding,
-		0,
-		0,
-	)
-}
-
-func generateRandomSpans() []*model.Span {
-	spans := make([]*model.Span, testSpanCount)
-	for i := 0; i < testSpanCount; i++ {
-		span := generateRandomSpan()
-		spans[i] = &span
-	}
-	return spans
-}
-
-func generateRandomSpan() model.Span {
-	processTags := generateRandomKeyValues(testTagCount)
-	process := model.Process{
-		ServiceName: "service" + strconv.FormatUint(rand.Uint64(), 10),
-		Tags:        processTags,
-	}
-	span := model.Span{
-		TraceID:       model.NewTraceID(rand.Uint64(), rand.Uint64()),
-		SpanID:        model.NewSpanID(rand.Uint64()),
-		OperationName: "operation" + strconv.FormatUint(rand.Uint64(), 10),
-		StartTime:     getRandomTime(),
-		Process:       &process,
-		Tags:          generateRandomKeyValues(testTagCount),
-		Logs:          generateRandomLogs(),
-		Duration:      time.Unix(rand.Int63n(1<<32), 0).Sub(time.Unix(0, 0)),
-	}
-	return span
-}
-
-func generateRandomLogs() []model.Log {
-	logs := make([]model.Log, 0, testLogCount)
-	for i := 0; i < testLogCount; i++ {
-		timestamp := getRandomTime()
-		logs = append(logs, model.Log{Timestamp: timestamp, Fields: generateRandomKeyValues(testLogFieldCount)})
-	}
-	return logs
-}
-
-func getRandomTime() time.Time {
-	return time.Unix(rand.Int63n(time.Now().Unix()), 0)
-}
-
-func generateRandomKeyValues(count int) []model.KeyValue {
-	tags := make([]model.KeyValue, 0, count)
-	for i := 0; i < count; i++ {
-		key := "key" + strconv.FormatUint(rand.Uint64(), 16)
-		value := "key" + strconv.FormatUint(rand.Uint64(), 16)
-		kv := model.KeyValue{Key: key, VType: model.ValueType_STRING, VStr: value}
-		tags = append(tags, kv)
-	}
-
-	return tags
-}

storage/config.go

@@ -3,7 +3,7 @@ package storage
 import (
 	"time"
 
-	"github.com/pavolloffay/jaeger-clickhouse/storage/clickhousespanstore"
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore"
 )
 
 type EncodingType string
@@ -12,15 +12,17 @@ const (
 	defaultEncoding                     = JSONEncoding
 	JSONEncoding           EncodingType = "json"
 	ProtobufEncoding       EncodingType = "protobuf"
+	defaultMaxSpanCount                 = int(1e7)
 	defaultBatchSize                    = 10_000
 	defaultBatchDelay                   = time.Second * 5
 	defaultUsername                     = "default"
 	defaultDatabaseName                 = "default"
 	defaultMetricsEndpoint              = "localhost:9090"
+	defaultMaxNumSpans                  = 0
 
-	defaultSpansTable      = "jaeger_spans_local"
-	defaultSpansIndexTable = "jaeger_index_local"
-	defaultOperationsTable = "jaeger_operations_local"
+	defaultSpansTable      clickhousespanstore.TableName = "jaeger_spans"
+	defaultSpansIndexTable clickhousespanstore.TableName = "jaeger_index"
+	defaultOperationsTable clickhousespanstore.TableName = "jaeger_operations"
 )
 
 type Configuration struct {
@@ -28,12 +30,25 @@ type Configuration struct {
 	BatchWriteSize int64 `yaml:"batch_write_size"`
 	// Batch flush interval. Default is 5s.
 	BatchFlushInterval time.Duration `yaml:"batch_flush_interval"`
+	// Maximal amount of spans that can be pending writes at a time.
+	// New spans exceeding this limit will be discarded,
+	// keeping memory in check if there are issues writing to ClickHouse.
+	// Check the "jaeger_clickhouse_discarded_spans" metric to keep track of discards.
+	// Default 10_000_000, or disable the limit entirely by setting to 0.
+	MaxSpanCount int `yaml:"max_span_count"`
 	// Encoding either json or protobuf. Default is json.
 	Encoding EncodingType `yaml:"encoding"`
-	// ClickHouse address e.g. tcp://localhost:9000.
+	// ClickHouse address e.g. localhost:9000.
 	Address string `yaml:"address"`
-	// Directory with .sql files that are run at plugin startup.
+	// Directory with .sql files to run at plugin startup, mainly for integration tests.
+	// Depending on the value of init_tables, this can be run as a
+	// replacement or supplement to creating default tables for span storage.
+	// If init_tables is also enabled, the scripts in this directory will be run first.
 	InitSQLScriptsDir string `yaml:"init_sql_scripts_dir"`
+	// Whether to automatically attempt to create tables in ClickHouse.
+	// By default, this is enabled if init_sql_scripts_dir is empty,
+	// or disabled if init_sql_scripts_dir is provided.
+	InitTables *bool `yaml:"init_tables"`
 	// Indicates location of TLS certificate used to connect to database.
 	CaFile string `yaml:"ca_file"`
 	// Username for connection to database. Default is "default".
@@ -46,12 +61,27 @@ type Configuration struct {
 	MetricsEndpoint string `yaml:"metrics_endpoint"`
 	// Whether to use SQL scripts supporting replication and sharding. Default false.
 	Replication bool `yaml:"replication"`
+	// If non-empty, enables multitenancy in SQL scripts, and assigns the tenant name for this instance.
+	Tenant string `yaml:"tenant"`
 	// Table with spans. Default "jaeger_spans_local" or "jaeger_spans" when replication is enabled.
 	SpansTable clickhousespanstore.TableName `yaml:"spans_table"`
 	// Span index table. Default "jaeger_index_local" or "jaeger_index" when replication is enabled.
 	SpansIndexTable clickhousespanstore.TableName `yaml:"spans_index_table"`
 	// Operations table. Default "jaeger_operations_local" or "jaeger_operations" when replication is enabled.
-	OperationsTable clickhousespanstore.TableName `yaml:"operations_table"`
+	OperationsTable   clickhousespanstore.TableName `yaml:"operations_table"`
+	spansArchiveTable clickhousespanstore.TableName
+	// TTL for data in tables in days. If 0, no TTL is set. Default 0.
+	TTLDays uint `yaml:"ttl"`
+	// The maximum number of spans to fetch per trace. If 0, no limits is set. Default 0.
+	MaxNumSpans uint `yaml:"max_num_spans"`
+	// The maximum number of open connections to the database. Default is unlimited (see: https://pkg.go.dev/database/sql#DB.SetMaxOpenConns)
+	MaxOpenConns *uint `yaml:"max_open_conns"`
+	// The maximum number of database connections in the idle connection pool. Default 2. (see: https://pkg.go.dev/database/sql#DB.SetMaxIdleConns)
+	MaxIdleConns *uint `yaml:"max_idle_conns"`
+	// The maximum amount of milliseconds a database connection may be reused. Default = connections are never closed due to age (see: https://pkg.go.dev/database/sql#DB.SetConnMaxLifetime)
+	ConnMaxLifetimeMillis *uint `yaml:"conn_max_lifetime_millis"`
+	// The maximum amount of milliseconds a database connection may be idle. Default = connections are never closed due to idle time (see: https://pkg.go.dev/database/sql#DB.SetConnMaxIdleTime)
+	ConnMaxIdleTimeMillis *uint `yaml:"conn_max_idle_time_millis"`
 }
 
 func (cfg *Configuration) setDefaults() {
@@ -61,9 +91,22 @@ func (cfg *Configuration) setDefaults() {
 	if cfg.BatchFlushInterval == 0 {
 		cfg.BatchFlushInterval = defaultBatchDelay
 	}
+	if cfg.MaxSpanCount == 0 {
+		cfg.MaxSpanCount = defaultMaxSpanCount
+	}
 	if cfg.Encoding == "" {
 		cfg.Encoding = defaultEncoding
 	}
+	if cfg.InitTables == nil {
+		// Decide whether to init tables based on whether a custom script path was provided
+		var defaultInitTables bool
+		if cfg.InitSQLScriptsDir == "" {
+			defaultInitTables = true
+		} else {
+			defaultInitTables = false
+		}
+		cfg.InitTables = &defaultInitTables
+	}
 	if cfg.Username == "" {
 		cfg.Username = defaultUsername
 	}
@@ -73,17 +116,36 @@ func (cfg *Configuration) setDefaults() {
 	if cfg.MetricsEndpoint == "" {
 		cfg.MetricsEndpoint = defaultMetricsEndpoint
 	}
+	if cfg.MaxNumSpans == 0 {
+		cfg.MaxNumSpans = defaultMaxNumSpans
+	}
 	if cfg.SpansTable == "" {
-		cfg.SpansTable = defaultSpansTable
+		if cfg.Replication {
+			cfg.SpansTable = defaultSpansTable
+			cfg.spansArchiveTable = defaultSpansTable + "_archive"
+		} else {
+			cfg.SpansTable = defaultSpansTable.ToLocal()
+			cfg.spansArchiveTable = (defaultSpansTable + "_archive").ToLocal()
+		}
+	} else {
+		cfg.spansArchiveTable = cfg.SpansTable + "_archive"
 	}
 	if cfg.SpansIndexTable == "" {
-		cfg.SpansIndexTable = defaultSpansIndexTable
+		if cfg.Replication {
+			cfg.SpansIndexTable = defaultSpansIndexTable
+		} else {
+			cfg.SpansIndexTable = defaultSpansIndexTable.ToLocal()
+		}
 	}
 	if cfg.OperationsTable == "" {
-		cfg.OperationsTable = defaultOperationsTable
+		if cfg.Replication {
+			cfg.OperationsTable = defaultOperationsTable
+		} else {
+			cfg.OperationsTable = defaultOperationsTable.ToLocal()
+		}
 	}
 }
 
 func (cfg *Configuration) GetSpansArchiveTable() clickhousespanstore.TableName {
-	return cfg.SpansTable + "_archive"
+	return cfg.spansArchiveTable
 }

storage/config_test.go

@@ -4,32 +4,83 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/pavolloffay/jaeger-clickhouse/storage/clickhousespanstore"
-
 	"github.com/stretchr/testify/assert"
+
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore"
 )
 
 func TestSetDefaults(t *testing.T) {
-	config := Configuration{}
-	config.setDefaults()
 	tests := map[string]struct {
-		field    interface{}
-		expected interface{}
+		replication bool
+		getField    func(Configuration) interface{}
+		expected    interface{}
 	}{
-		"username":              {field: config.Username, expected: defaultUsername},
-		"database name":         {field: config.Database, expected: defaultDatabaseName},
-		"encoding":              {field: config.Encoding, expected: defaultEncoding},
-		"batch write size":      {field: config.BatchWriteSize, expected: defaultBatchSize},
-		"batch flush interval":  {field: config.BatchFlushInterval, expected: defaultBatchDelay},
-		"metrics endpoint":      {field: config.MetricsEndpoint, expected: defaultMetricsEndpoint},
-		"spans table name":      {field: config.SpansTable, expected: defaultSpansTable},
-		"index table name":      {field: config.SpansIndexTable, expected: defaultSpansIndexTable},
-		"operations table name": {field: config.OperationsTable, expected: defaultOperationsTable},
+		"username": {
+			getField: func(config Configuration) interface{} { return config.Username },
+			expected: defaultUsername,
+		},
+		"database name": {
+			getField: func(config Configuration) interface{} { return config.Database },
+			expected: defaultDatabaseName,
+		},
+		"encoding": {
+			getField: func(config Configuration) interface{} { return config.Encoding },
+			expected: defaultEncoding,
+		},
+		"batch write size": {
+			getField: func(config Configuration) interface{} { return config.BatchWriteSize },
+			expected: defaultBatchSize,
+		},
+		"batch flush interval": {
+			getField: func(config Configuration) interface{} { return config.BatchFlushInterval },
+			expected: defaultBatchDelay,
+		},
+		"max span count": {
+			getField: func(config Configuration) interface{} { return config.MaxSpanCount },
+			expected: defaultMaxSpanCount,
+		},
+		"metrics endpoint": {
+			getField: func(config Configuration) interface{} { return config.MetricsEndpoint },
+			expected: defaultMetricsEndpoint,
+		},
+		"spans table name local": {
+			getField: func(config Configuration) interface{} { return config.SpansTable },
+			expected: defaultSpansTable.ToLocal(),
+		},
+		"spans table name replication": {
+			replication: true,
+			getField:    func(config Configuration) interface{} { return config.SpansTable },
+			expected:    defaultSpansTable,
+		},
+		"index table name local": {
+			getField: func(config Configuration) interface{} { return config.SpansIndexTable },
+			expected: defaultSpansIndexTable.ToLocal(),
+		},
+		"index table name replication": {
+			replication: true,
+			getField:    func(config Configuration) interface{} { return config.SpansIndexTable },
+			expected:    defaultSpansIndexTable,
+		},
+		"operations table name local": {
+			getField: func(config Configuration) interface{} { return config.OperationsTable },
+			expected: defaultOperationsTable.ToLocal(),
+		},
+		"operations table name replication": {
+			replication: true,
+			getField:    func(config Configuration) interface{} { return config.OperationsTable },
+			expected:    defaultOperationsTable,
+		},
+		"max number spans": {
+			getField: func(config Configuration) interface{} { return config.MaxNumSpans },
+			expected: defaultMaxNumSpans,
+		},
 	}
 
 	for name, test := range tests {
 		t.Run(fmt.Sprintf("default %s", name), func(t *testing.T) {
-			assert.EqualValues(t, test.expected, test.field)
+			config := Configuration{Replication: test.replication}
+			config.setDefaults()
+			assert.EqualValues(t, test.expected, test.getField(config))
 		})
 	}
 }
@@ -39,8 +90,9 @@ func TestConfiguration_GetSpansArchiveTable(t *testing.T) {
 		config                        Configuration
 		expectedSpansArchiveTableName clickhousespanstore.TableName
 	}{
-		"default_config":     {config: Configuration{}, expectedSpansArchiveTableName: defaultSpansTable + "_archive"},
-		"custom_spans_table": {config: Configuration{SpansTable: "custom_table_name"}, expectedSpansArchiveTableName: "custom_table_name_archive"},
+		"default_config_local":       {config: Configuration{}, expectedSpansArchiveTableName: (defaultSpansTable + "_archive").ToLocal()},
+		"default_config_replication": {config: Configuration{Replication: true}, expectedSpansArchiveTableName: defaultSpansTable + "_archive"},
+		"custom_spans_table":         {config: Configuration{SpansTable: "custom_table_name"}, expectedSpansArchiveTableName: "custom_table_name_archive"},
 	}
 
 	for name, test := range tests {
@@ -50,3 +102,27 @@ func TestConfiguration_GetSpansArchiveTable(t *testing.T) {
 		})
 	}
 }
+
+func TestConfiguration_InitTables(test *testing.T) {
+	// for pointers below
+	t := true
+	f := false
+	tests := map[string]struct {
+		config             Configuration
+		expectedInitTables bool
+	}{
+		"scriptsempty_initnil":      {config: Configuration{}, expectedInitTables: true},
+		"scriptsprovided_initnil":   {config: Configuration{InitSQLScriptsDir: "hello"}, expectedInitTables: false},
+		"scriptsempty_inittrue":     {config: Configuration{InitTables: &t}, expectedInitTables: true},
+		"scriptsprovided_inittrue":  {config: Configuration{InitSQLScriptsDir: "hello", InitTables: &t}, expectedInitTables: true},
+		"scriptsempty_initfalse":    {config: Configuration{InitTables: &f}, expectedInitTables: false},
+		"scriptsprovided_initfalse": {config: Configuration{InitSQLScriptsDir: "hello", InitTables: &f}, expectedInitTables: false},
+	}
+
+	for name, testcase := range tests {
+		test.Run(name, func(t *testing.T) {
+			testcase.config.setDefaults()
+			assert.Equal(t, testcase.expectedInitTables, *(testcase.config.InitTables))
+		})
+	}
+}

storage/store.go

@@ -4,24 +4,24 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"database/sql"
-	"embed"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
+	"text/template"
+	"time"
 
-	jaegerclickhouse "github.com/pavolloffay/jaeger-clickhouse"
-
-	"github.com/ClickHouse/clickhouse-go"
-	"github.com/hashicorp/go-hclog"
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
+	hclog "github.com/hashicorp/go-hclog"
 	"github.com/jaegertracing/jaeger/plugin/storage/grpc/shared"
 	"github.com/jaegertracing/jaeger/storage/dependencystore"
 	"github.com/jaegertracing/jaeger/storage/spanstore"
 
-	"github.com/pavolloffay/jaeger-clickhouse/storage/clickhousedependencystore"
-	"github.com/pavolloffay/jaeger-clickhouse/storage/clickhousespanstore"
+	jaegerclickhouse "github.com/jaegertracing/jaeger-clickhouse"
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousedependencystore"
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore"
 )
 
 type Store struct {
@@ -32,14 +32,11 @@ type Store struct {
 	archiveReader spanstore.Reader
 }
 
-const (
-	tlsConfigKey = "clickhouse_tls_config_key"
-)
-
 var (
-	_ shared.StoragePlugin        = (*Store)(nil)
-	_ shared.ArchiveStoragePlugin = (*Store)(nil)
-	_ io.Closer                   = (*Store)(nil)
+	_ shared.StoragePlugin             = (*Store)(nil)
+	_ shared.ArchiveStoragePlugin      = (*Store)(nil)
+	_ shared.StreamingSpanWriterPlugin = (*Store)(nil)
+	_ io.Closer                        = (*Store)(nil)
 )
 
 func NewStore(logger hclog.Logger, cfg Configuration) (*Store, error) {
@@ -54,156 +51,242 @@ func NewStore(logger hclog.Logger, cfg Configuration) (*Store, error) {
 		return nil, err
 	}
 	if cfg.Replication {
-		var (
-			globalIndexTable        = cfg.SpansIndexTable.ToGlobal()
-			globalSpansTable        = cfg.SpansTable.ToGlobal()
-			globalSpansArchiveTable = cfg.GetSpansArchiveTable().ToGlobal()
-		)
 		return &Store{
-			db:            db,
-			writer:        clickhousespanstore.NewSpanWriter(logger, db, globalIndexTable, globalSpansTable, clickhousespanstore.Encoding(cfg.Encoding), cfg.BatchFlushInterval, cfg.BatchWriteSize),
-			reader:        clickhousespanstore.NewTraceReader(db, cfg.OperationsTable.ToGlobal(), globalIndexTable, globalSpansTable),
-			archiveWriter: clickhousespanstore.NewSpanWriter(logger, db, "", globalSpansArchiveTable, clickhousespanstore.Encoding(cfg.Encoding), cfg.BatchFlushInterval, cfg.BatchWriteSize),
-			archiveReader: clickhousespanstore.NewTraceReader(db, "", "", globalSpansArchiveTable),
+			db: db,
+			writer: clickhousespanstore.NewSpanWriter(
+				logger,
+				db,
+				cfg.SpansIndexTable,
+				cfg.SpansTable,
+				cfg.Tenant,
+				clickhousespanstore.Encoding(cfg.Encoding),
+				cfg.BatchFlushInterval,
+				cfg.BatchWriteSize,
+				cfg.MaxSpanCount,
+			),
+			reader: clickhousespanstore.NewTraceReader(
+				db,
+				cfg.OperationsTable,
+				cfg.SpansIndexTable,
+				cfg.SpansTable,
+				cfg.Tenant,
+				cfg.MaxNumSpans,
+			),
+			archiveWriter: clickhousespanstore.NewSpanWriter(
+				logger,
+				db,
+				"",
+				cfg.GetSpansArchiveTable(),
+				cfg.Tenant,
+				clickhousespanstore.Encoding(cfg.Encoding),
+				cfg.BatchFlushInterval,
+				cfg.BatchWriteSize,
+				cfg.MaxSpanCount,
+			),
+			archiveReader: clickhousespanstore.NewTraceReader(
+				db,
+				"",
+				"",
+				cfg.GetSpansArchiveTable(),
+				cfg.Tenant,
+				cfg.MaxNumSpans,
+			),
 		}, nil
 	}
 	return &Store{
-		db:            db,
-		writer:        clickhousespanstore.NewSpanWriter(logger, db, cfg.SpansIndexTable, cfg.SpansTable, clickhousespanstore.Encoding(cfg.Encoding), cfg.BatchFlushInterval, cfg.BatchWriteSize),
-		reader:        clickhousespanstore.NewTraceReader(db, cfg.OperationsTable, cfg.SpansIndexTable, cfg.SpansTable),
-		archiveWriter: clickhousespanstore.NewSpanWriter(logger, db, "", cfg.GetSpansArchiveTable(), clickhousespanstore.Encoding(cfg.Encoding), cfg.BatchFlushInterval, cfg.BatchWriteSize),
-		archiveReader: clickhousespanstore.NewTraceReader(db, "", "", cfg.GetSpansArchiveTable()),
+		db: db,
+		writer: clickhousespanstore.NewSpanWriter(
+			logger,
+			db,
+			cfg.SpansIndexTable,
+			cfg.SpansTable,
+			cfg.Tenant,
+			clickhousespanstore.Encoding(cfg.Encoding),
+			cfg.BatchFlushInterval,
+			cfg.BatchWriteSize,
+			cfg.MaxSpanCount,
+		),
+		reader: clickhousespanstore.NewTraceReader(
+			db,
+			cfg.OperationsTable,
+			cfg.SpansIndexTable,
+			cfg.SpansTable,
+			cfg.Tenant,
+			cfg.MaxNumSpans,
+		),
+		archiveWriter: clickhousespanstore.NewSpanWriter(
+			logger,
+			db,
+			"",
+			cfg.GetSpansArchiveTable(),
+			cfg.Tenant,
+			clickhousespanstore.Encoding(cfg.Encoding),
+			cfg.BatchFlushInterval,
+			cfg.BatchWriteSize,
+			cfg.MaxSpanCount,
+		),
+		archiveReader: clickhousespanstore.NewTraceReader(
+			db,
+			"",
+			"",
+			cfg.GetSpansArchiveTable(),
+			cfg.Tenant,
+			cfg.MaxNumSpans,
+		),
 	}, nil
 }
 
 func connector(cfg Configuration) (*sql.DB, error) {
-	params := fmt.Sprintf("%s?database=%s&username=%s&password=%s",
-		cfg.Address,
-		cfg.Database,
-		cfg.Username,
-		cfg.Password,
-	)
+	var conn *sql.DB
+
+	options := clickhouse.Options{
+		Addr: []string{sanitize(cfg.Address)},
+		Auth: clickhouse.Auth{
+			Database: cfg.Database,
+			Username: cfg.Username,
+			Password: cfg.Password,
+		},
+		Compression: &clickhouse.Compression{
+			Method: clickhouse.CompressionLZ4,
+		},
+	}
 
 	if cfg.CaFile != "" {
-		caCert, err := ioutil.ReadFile(cfg.CaFile)
+		caCert, err := os.ReadFile(cfg.CaFile)
 		if err != nil {
 			return nil, err
 		}
 		caCertPool := x509.NewCertPool()
 		caCertPool.AppendCertsFromPEM(caCert)
-		err = clickhouse.RegisterTLSConfig(tlsConfigKey, &tls.Config{RootCAs: caCertPool})
-		if err != nil {
-			return nil, err
+		options.TLS = &tls.Config{
+			RootCAs: caCertPool,
 		}
-		params += fmt.Sprintf(
-			"&secure=true&tls_config=%s",
-			tlsConfigKey,
-		)
 	}
-	return clickhouseConnector(params)
+	conn = clickhouse.OpenDB(&options)
+
+	if cfg.MaxOpenConns != nil {
+		conn.SetMaxIdleConns(int(*cfg.MaxOpenConns))
+	}
+	if cfg.MaxIdleConns != nil {
+		conn.SetMaxIdleConns(int(*cfg.MaxIdleConns))
+	}
+	if cfg.ConnMaxLifetimeMillis != nil {
+		conn.SetConnMaxLifetime(time.Millisecond * time.Duration(*cfg.ConnMaxLifetimeMillis))
+	}
+	if cfg.ConnMaxIdleTimeMillis != nil {
+		conn.SetConnMaxIdleTime(time.Millisecond * time.Duration(*cfg.ConnMaxIdleTimeMillis))
+	}
+
+	if err := conn.Ping(); err != nil {
+		return nil, err
+	}
+	return conn, nil
+}
+
+type tableArgs struct {
+	Database string
+
+	SpansIndexTable   clickhousespanstore.TableName
+	SpansTable        clickhousespanstore.TableName
+	OperationsTable   clickhousespanstore.TableName
+	SpansArchiveTable clickhousespanstore.TableName
+
+	TTLTimestamp string
+	TTLDate      string
+
+	Multitenant bool
+	Replication bool
+}
+
+type distributedTableArgs struct {
+	Database string
+	Table    clickhousespanstore.TableName
+	Hash     string
+}
+
+func render(templates *template.Template, filename string, args interface{}) string {
+	var statement strings.Builder
+	err := templates.ExecuteTemplate(&statement, filename, args)
+	if err != nil {
+		panic(err)
+	}
+	return statement.String()
 }
 
 func runInitScripts(logger hclog.Logger, db *sql.DB, cfg Configuration) error {
-	var embeddedScripts embed.FS
-	if cfg.Replication {
-		embeddedScripts = jaegerclickhouse.EmbeddedFilesReplication
-	} else {
-		embeddedScripts = jaegerclickhouse.EmbeddedFilesNoReplication
+	var (
+		sqlStatements []string
+		ttlTimestamp  string
+		ttlDate       string
+	)
+	if cfg.TTLDays > 0 {
+		ttlTimestamp = fmt.Sprintf("TTL timestamp + INTERVAL %d DAY DELETE", cfg.TTLDays)
+		ttlDate = fmt.Sprintf("TTL date + INTERVAL %d DAY DELETE", cfg.TTLDays)
 	}
-
-	var sqlStatements []string
-	switch {
-	case cfg.InitSQLScriptsDir != "":
+	if cfg.InitSQLScriptsDir != "" {
 		filePaths, err := walkMatch(cfg.InitSQLScriptsDir, "*.sql")
 		if err != nil {
 			return fmt.Errorf("could not list sql files: %q", err)
 		}
 		sort.Strings(filePaths)
 		for _, f := range filePaths {
-			sqlStatement, err := ioutil.ReadFile(filepath.Clean(f))
+			sqlStatement, err := os.ReadFile(filepath.Clean(f))
 			if err != nil {
 				return err
 			}
 			sqlStatements = append(sqlStatements, string(sqlStatement))
 		}
-	case cfg.Replication:
-		f, err := embeddedScripts.ReadFile("sqlscripts/replication/0001-jaeger-index-local.sql")
-		if err != nil {
-			return err
+	}
+	if *cfg.InitTables {
+		templates := template.Must(template.ParseFS(jaegerclickhouse.SQLScripts, "sqlscripts/*.tmpl.sql"))
+
+		args := tableArgs{
+			Database: cfg.Database,
+
+			SpansIndexTable:   cfg.SpansIndexTable,
+			SpansTable:        cfg.SpansTable,
+			OperationsTable:   cfg.OperationsTable,
+			SpansArchiveTable: cfg.GetSpansArchiveTable(),
+
+			TTLTimestamp: ttlTimestamp,
+			TTLDate:      ttlDate,
+
+			Multitenant: cfg.Tenant != "",
+			Replication: cfg.Replication,
 		}
-		sqlStatements = append(sqlStatements, fmt.Sprintf(string(f), cfg.SpansIndexTable))
-		f, err = embeddedScripts.ReadFile("sqlscripts/replication/0002-jaeger-spans-local.sql")
-		if err != nil {
-			return err
+
+		if cfg.Replication {
+			// Add "_local" to the local table names, and omit it from the distributed tables below
+			args.SpansIndexTable = args.SpansIndexTable.ToLocal()
+			args.SpansTable = args.SpansTable.ToLocal()
+			args.OperationsTable = args.OperationsTable.ToLocal()
+			args.SpansArchiveTable = args.SpansArchiveTable.ToLocal()
 		}
-		sqlStatements = append(sqlStatements, fmt.Sprintf(string(f), cfg.SpansTable))
-		f, err = embeddedScripts.ReadFile("sqlscripts/replication/0003-jaeger-operations-local.sql")
-		if err != nil {
-			return err
+
+		sqlStatements = append(sqlStatements, render(templates, "jaeger-index.tmpl.sql", args))
+		sqlStatements = append(sqlStatements, render(templates, "jaeger-operations.tmpl.sql", args))
+		sqlStatements = append(sqlStatements, render(templates, "jaeger-spans.tmpl.sql", args))
+		sqlStatements = append(sqlStatements, render(templates, "jaeger-spans-archive.tmpl.sql", args))
+
+		if cfg.Replication {
+			// Now these tables omit the "_local" suffix
+			distargs := distributedTableArgs{
+				Table:    cfg.SpansTable,
+				Database: cfg.Database,
+				Hash:     "cityHash64(traceID)",
+			}
+			sqlStatements = append(sqlStatements, render(templates, "distributed-table.tmpl.sql", distargs))
+
+			distargs.Table = cfg.SpansIndexTable
+			sqlStatements = append(sqlStatements, render(templates, "distributed-table.tmpl.sql", distargs))
+
+			distargs.Table = cfg.GetSpansArchiveTable()
+			sqlStatements = append(sqlStatements, render(templates, "distributed-table.tmpl.sql", distargs))
+
+			distargs.Table = cfg.OperationsTable
+			distargs.Hash = "rand()"
+			sqlStatements = append(sqlStatements, render(templates, "distributed-table.tmpl.sql", distargs))
 		}
-		sqlStatements = append(sqlStatements, fmt.Sprintf(string(f), cfg.OperationsTable, cfg.SpansIndexTable.AddDbName(cfg.Database)))
-		f, err = embeddedScripts.ReadFile("sqlscripts/replication/0004-jaeger-spans-archive-local.sql")
-		if err != nil {
-			return err
-		}
-		sqlStatements = append(sqlStatements, fmt.Sprintf(string(f), cfg.GetSpansArchiveTable()))
-		f, err = embeddedScripts.ReadFile("sqlscripts/replication/0005-distributed-city-hash.sql")
-		if err != nil {
-			return err
-		}
-		sqlStatements = append(sqlStatements, fmt.Sprintf(
-			string(f),
-			cfg.SpansTable.ToGlobal(),
-			cfg.SpansTable.AddDbName(cfg.Database),
-			cfg.Database,
-			cfg.SpansTable,
-		))
-		sqlStatements = append(sqlStatements, fmt.Sprintf(
-			string(f),
-			cfg.SpansIndexTable.ToGlobal(),
-			cfg.SpansIndexTable.AddDbName(cfg.Database),
-			cfg.Database,
-			cfg.SpansIndexTable,
-		))
-		sqlStatements = append(sqlStatements, fmt.Sprintf(
-			string(f),
-			cfg.GetSpansArchiveTable().ToGlobal(),
-			cfg.GetSpansArchiveTable().AddDbName(cfg.Database),
-			cfg.Database,
-			cfg.GetSpansArchiveTable(),
-		))
-		f, err = embeddedScripts.ReadFile("sqlscripts/replication/0006-distributed-rand.sql")
-		if err != nil {
-			return err
-		}
-		sqlStatements = append(sqlStatements, fmt.Sprintf(
-			string(f),
-			cfg.OperationsTable.ToGlobal(),
-			cfg.OperationsTable.AddDbName(cfg.Database),
-			cfg.Database,
-			cfg.OperationsTable,
-		))
-	default:
-		f, err := embeddedScripts.ReadFile("sqlscripts/local/0001-jaeger-index.sql")
-		if err != nil {
-			return err
-		}
-		sqlStatements = append(sqlStatements, fmt.Sprintf(string(f), cfg.SpansIndexTable))
-		f, err = embeddedScripts.ReadFile("sqlscripts/local/0002-jaeger-spans.sql")
-		if err != nil {
-			return err
-		}
-		sqlStatements = append(sqlStatements, fmt.Sprintf(string(f), cfg.SpansTable))
-		f, err = embeddedScripts.ReadFile("sqlscripts/local/0003-jaeger-operations.sql")
-		if err != nil {
-			return err
-		}
-		sqlStatements = append(sqlStatements, fmt.Sprintf(string(f), cfg.OperationsTable, cfg.SpansIndexTable))
-		f, err = embeddedScripts.ReadFile("sqlscripts/local/0004-jaeger-spans-archive.sql")
-		if err != nil {
-			return err
-		}
-		sqlStatements = append(sqlStatements, fmt.Sprintf(string(f), cfg.GetSpansArchiveTable()))
 	}
 	return executeScripts(logger, sqlStatements, db)
 }
@@ -228,27 +311,18 @@ func (s *Store) ArchiveSpanWriter() spanstore.Writer {
 	return s.archiveWriter
 }
 
-func (s *Store) Close() error {
-	return s.db.Close()
+func (s *Store) StreamingSpanWriter() spanstore.Writer {
+	return s.writer
 }
 
-func clickhouseConnector(params string) (*sql.DB, error) {
-	db, err := sql.Open("clickhouse", params)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := db.Ping(); err != nil {
-		return nil, err
-	}
-
-	return db, nil
+func (s *Store) Close() error {
+	return s.db.Close()
 }
 
 func executeScripts(logger hclog.Logger, sqlStatements []string, db *sql.DB) error {
 	tx, err := db.Begin()
 	if err != nil {
-		return nil
+		return err
 	}
 	committed := false
 	defer func() {
@@ -289,3 +363,10 @@ func walkMatch(root, pattern string) ([]string, error) {
 	}
 	return matches, nil
 }
+
+// Earlier version of clickhouse-go used to expect address as tcp://host:port
+// while newer version of clickhouse-go expect address as host:port (without scheme)
+// so to maintain backward compatibility we clean it up
+func sanitize(addr string) string {
+	return strings.TrimPrefix(addr, "tcp://")
+}

storage/store_test.go

@@ -0,0 +1,187 @@
+package storage
+
+import (
+	"database/sql"
+	"fmt"
+	"testing"
+
+	sqlmock "github.com/DATA-DOG/go-sqlmock"
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousedependencystore"
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore"
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore/mocks"
+)
+
+const (
+	testIndexTable        = "test_index_table"
+	testSpansTable        = "test_spans_table"
+	testOperationsTable   = "test_operation_table"
+	testSpansArchiveTable = "test_spans_archive_table"
+)
+
+var errorMock = fmt.Errorf("error mock")
+
+func TestStore_SpanWriter(t *testing.T) {
+	writer := clickhousespanstore.SpanWriter{}
+	store := Store{
+		writer: &writer,
+	}
+	assert.Equal(t, &writer, store.SpanWriter())
+}
+
+func TestStore_ArchiveSpanWriter(t *testing.T) {
+	writer := clickhousespanstore.SpanWriter{}
+	store := Store{
+		archiveWriter: &writer,
+	}
+	assert.Equal(t, &writer, store.ArchiveSpanWriter())
+}
+
+func TestStore_SpanReader(t *testing.T) {
+	reader := clickhousespanstore.TraceReader{}
+	store := Store{
+		reader: &reader,
+	}
+	assert.Equal(t, &reader, store.SpanReader())
+}
+
+func TestStore_ArchiveSpanReader(t *testing.T) {
+	reader := clickhousespanstore.TraceReader{}
+	store := Store{
+		archiveReader: &reader,
+	}
+	assert.Equal(t, &reader, store.ArchiveSpanReader())
+}
+
+func TestStore_DependencyReader(t *testing.T) {
+	store := Store{}
+	assert.Equal(t, &clickhousedependencystore.DependencyStore{}, store.DependencyReader())
+}
+
+func TestStore_Close(t *testing.T) {
+	db, mock, err := mocks.GetDbMock()
+	require.NoError(t, err)
+	defer db.Close()
+
+	logger := mocks.NewSpyLogger()
+	store := newStore(db, logger)
+
+	mock.ExpectClose()
+	require.NoError(t, store.Close())
+	assert.NoError(t, mock.ExpectationsWereMet())
+	logger.AssertLogsEmpty(t)
+}
+
+func newStore(db *sql.DB, logger mocks.SpyLogger) Store {
+	return Store{
+		db: db,
+		writer: clickhousespanstore.NewSpanWriter(
+			logger,
+			db,
+			testIndexTable,
+			testSpansTable,
+			"",
+			clickhousespanstore.EncodingJSON,
+			0,
+			0,
+			0,
+		),
+		reader: clickhousespanstore.NewTraceReader(
+			db,
+			testOperationsTable,
+			testIndexTable,
+			testSpansTable,
+			"",
+			0,
+		),
+		archiveWriter: clickhousespanstore.NewSpanWriter(
+			logger,
+			db,
+			testIndexTable,
+			testSpansArchiveTable,
+			"",
+			clickhousespanstore.EncodingJSON,
+			0,
+			0,
+			0,
+		),
+		archiveReader: clickhousespanstore.NewTraceReader(
+			db,
+			testOperationsTable,
+			testIndexTable,
+			testSpansArchiveTable,
+			"",
+			0,
+		),
+	}
+}
+
+func TestStore_executeScripts(t *testing.T) {
+	db, mock, err := mocks.GetDbMock()
+	require.NoError(t, err)
+	defer db.Close()
+
+	spyLogger := mocks.NewSpyLogger()
+	scripts := []string{
+		"first SQL script",
+		"second_SQL_script",
+	}
+
+	mock.ExpectBegin()
+	for _, script := range scripts {
+		mock.ExpectExec(script).WillReturnResult(sqlmock.NewResult(1, 1))
+	}
+	mock.ExpectCommit()
+	err = executeScripts(spyLogger, scripts, db)
+	require.NoError(t, err)
+	assert.NoError(t, mock.ExpectationsWereMet())
+	spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, func() []mocks.LogMock {
+		res := make([]mocks.LogMock, len(scripts))
+		for i, script := range scripts {
+			res[i] = mocks.LogMock{Msg: "Running SQL statement", Args: []interface{}{"statement", script}}
+		}
+		return res
+	}())
+}
+
+func TestStore_executeScriptsExecuteError(t *testing.T) {
+	db, mock, err := mocks.GetDbMock()
+	require.NoError(t, err)
+	defer db.Close()
+
+	spyLogger := mocks.NewSpyLogger()
+	scripts := []string{
+		"first SQL script",
+		"second_SQL_script",
+	}
+
+	mock.ExpectBegin()
+	mock.ExpectExec(scripts[0]).WillReturnError(errorMock)
+	mock.ExpectRollback()
+	err = executeScripts(spyLogger, scripts, db)
+	assert.EqualError(t, err, fmt.Sprintf("could not run sql %q: %q", scripts[0], errorMock))
+	spyLogger.AssertLogsOfLevelEqual(
+		t,
+		hclog.Debug,
+		[]mocks.LogMock{{Msg: "Running SQL statement", Args: []interface{}{"statement", scripts[0]}}},
+	)
+}
+
+func TestStore_executeScriptBeginError(t *testing.T) {
+	db, mock, err := mocks.GetDbMock()
+	require.NoError(t, err)
+	defer db.Close()
+
+	spyLogger := mocks.NewSpyLogger()
+	scripts := []string{
+		"first SQL script",
+		"second_SQL_script",
+	}
+
+	mock.ExpectBegin().WillReturnError(errorMock)
+	err = executeScripts(spyLogger, scripts, db)
+	assert.EqualError(t, err, errorMock.Error())
+}