Add deprecation warning

Signed-off-by: Yuri Shkuro <yurishkuro@users.noreply.github.com>
Use jaeger storage integration helpers from upstream (#119 )
2024-12-16 23:48:22 -04:00 · 2022-10-12 19:42:03 +13:00 · 2022-10-10 21:39:01 +13:00 · 2022-10-10 21:37:28 +13:00 · 2022-09-14 21:18:29 +12:00 · 2022-08-23 19:16:16 +12:00
65 changed files with 5401 additions and 1056 deletions
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@ -1,4 +1,4 @@
-name: Build, format and lint
+name: Build, test, format and lint

 on:
  push:
@ -7,7 +7,18 @@ on:
 jobs:
  build-binaries:
    runs-on: ubuntu-latest
-    name: Build binaries
+    name: Build binary for ${{ matrix.platform.name }}
+    strategy:
+      matrix:
+        platform:
+        - name: linux on amd64
+          task: build-linux-amd64
+        - name: linux on arm64
+          task: build-linux-arm64
+        - name: osx on amd64
+          task: build-darwin-amd64
+        - name: osx on arm64
+          task: build-darwin-arm64
    steps:
      - uses: actions/checkout@v2.3.4
        with:
@ -15,10 +26,10 @@ jobs:

      - uses: actions/setup-go@v2
        with:
-          go-version: ^1.16
+          go-version: ^1.19

      - name: Build binaries
-        run: make build
+        run: make ${{ matrix.platform.task }}

  format-lint:
    runs-on: ubuntu-latest
@ -30,7 +41,7 @@ jobs:

      - uses: actions/setup-go@v2
        with:
-          go-version: ^1.16
+          go-version: ^1.19

      - name: Install tools
        run: make install-tools
@ -40,3 +51,51 @@ jobs:

      - name: Lint
        run: make lint
+
+  e2e-test:
+    runs-on: ubuntu-latest
+    name: E2E Test
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          submodules: true
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ^1.19
+
+      - name: Run e2e test
+        run: make e2e-tests
+
+  unit-tests:
+    runs-on: ubuntu-latest
+    name: Unit tests
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          submodules: true
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ^1.19
+
+      - name: Run unit test
+        run: make test
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    name: Integration tests
+    steps:
+      - uses: actions/checkout@v2.3.4
+        with:
+          submodules: true
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ^1.19
+
+      - name: Setup database
+        run: docker run --rm -d -p9000:9000 --name test-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server:22
+
+      - name: Run integration tests
+        run: make integration-test
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@ -15,10 +15,10 @@ jobs:

      - uses: actions/setup-go@v2
        with:
-          go-version: ^1.16
+          go-version: ^1.19

      - name: Create release distribution
-        run: make build tar
+        run: make build-all-platforms tar-all-platforms

      - name: Log in to the Container registry
        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
--- a/.golangci.yml
+++ b/.golangci.yml
@ -125,6 +125,10 @@ issues:
    - text: "G402:"
      linters:
        - gosec
+    - path: grpc_test.go
+      linters:
+        # See https://github.com/golangci/golangci-lint/issues/2286
+        - typecheck

  # The list of ids of default excludes to include or disable. By default it's empty.
  # See the list of default excludes here https://golangci-lint.run/usage/configuration.
--- a/2
+++ b/2
@ -1,4 +1,4 @@
-FROM alpine:3.13
+FROM docker.io/library/alpine:3.16

 ADD jaeger-clickhouse-linux-amd64 /go/bin/jaeger-clickhouse

--- a/62
+++ b/62
@ -3,15 +3,39 @@ GOARCH ?= $(shell go env GOARCH)
 GOBUILD=CGO_ENABLED=0 installsuffix=cgo go build -trimpath

 TOOLS_MOD_DIR = ./internal/tools
-JAEGER_VERSION ?= 1.24.0
+JAEGER_VERSION ?= 1.32.0

-DOCKER_REPO ?= ghcr.io/pavolloffay/jaeger-clickhouse
+DOCKER_REPO ?= ghcr.io/jaegertracing/jaeger-clickhouse
 DOCKER_TAG ?= latest

 .PHONY: build
 build:
 	${GOBUILD} -o jaeger-clickhouse-$(GOOS)-$(GOARCH) ./cmd/jaeger-clickhouse/main.go

+.PHONY: build-linux-amd64
+build-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) build
+
+.PHONY: build-linux-arm64
+build-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) build
+
+.PHONY: build-darwin-amd64
+build-darwin-amd64:
+	GOOS=darwin GOARCH=amd64 $(MAKE) build
+
+.PHONY: build-darwin-arm64
+build-darwin-arm64:
+	GOOS=darwin GOARCH=arm64 $(MAKE) build
+
+.PHONY: build-all-platforms
+build-all-platforms: build-linux-amd64 build-linux-arm64 build-darwin-amd64 build-darwin-arm64
+
+.PHONY: e2e-tests
+e2e-tests:
+	GOOS=linux GOARCH=amd64 $(MAKE) build
+	E2E_TEST=true go test ./e2etests... -v
+
 .PHONY: run
 run:
 	docker run --rm --name jaeger -e JAEGER_DISABLED=false --link some-clickhouse-server -it -u ${shell id -u} -p16686:16686 -p14250:14250 -p14268:14268 -p6831:6831/udp -v "${PWD}:/data" -e SPAN_STORAGE_TYPE=grpc-plugin jaegertracing/all-in-one:${JAEGER_VERSION} --query.ui-config=/data/jaeger-ui.json --grpc-storage-plugin.binary=/data/jaeger-clickhouse-$(GOOS)-$(GOARCH) --grpc-storage-plugin.configuration-file=/data/config.yaml --grpc-storage-plugin.log-level=debug
@ -23,16 +47,46 @@ run-hotrod:
 .PHONY: fmt
 fmt:
 	go fmt ./...
-	goimports -w  -local github.com/pavolloffay/jaeger-clickhouse ./
+	goimports -w -local github.com/jaegertracing/jaeger-clickhouse ./

 .PHONY: lint
 lint:
-	golangci-lint run --allow-parallel-runners ./...
+	golangci-lint -v run --allow-parallel-runners ./...
+
+.PHONY: test
+test:
+	go test ./...
+
+.PHONY: integration-test
+integration-test: build
+	STORAGE=grpc-plugin \
+	PLUGIN_BINARY_PATH=$(PWD)/jaeger-clickhouse-linux-amd64 \
+	PLUGIN_CONFIG_PATH=$(PWD)/integration/config-local.yaml \
+	go test ./integration

 .PHONY: tar
 tar:
 	tar -czvf jaeger-clickhouse-$(GOOS)-$(GOARCH).tar.gz  jaeger-clickhouse-$(GOOS)-$(GOARCH) config.yaml

+.PHONY: tar-linux-amd64
+tar-linux-amd64:
+	GOOS=linux GOARCH=amd64 $(MAKE) tar
+
+.PHONY: tar-linux-arm64
+tar-linux-arm64:
+	GOOS=linux GOARCH=arm64 $(MAKE) tar
+
+.PHONY: tar-darwin-amd64
+tar-darwin-amd64:
+	GOOS=darwin GOARCH=amd64 $(MAKE) tar
+
+.PHONY: tar-darwin-arm64
+tar-darwin-arm64:
+	GOOS=darwin GOARCH=arm64 $(MAKE) tar
+
+.PHONY: tar-all-platforms
+tar-all-platforms: tar-linux-amd64 tar-linux-arm64 tar-darwin-amd64 tar-darwin-arm64
+
 .PHONY: docker
 docker: build
 	docker build -t ${DOCKER_REPO}:${DOCKER_TAG} -f Dockerfile .
--- a/README.md
+++ b/README.md
@ -1,16 +1,48 @@
-# Jaeger ClickHouse
+# Jaeger ClickHouse (experimental)

-Jaeger ClickHouse gRPC [storage plugin](https://github.com/jaegertracing/jaeger/tree/master/plugin/storage/grpc).
+⚠️ This module only implements grpc-plugin API that has been deprecated in Jaeger (https://github.com/jaegertracing/jaeger/issues/4647).

-This is WIP and it is based on https://github.com/bobrik/jaeger/tree/ivan/clickhouse/plugin/storage/clickhouse. 
-See as well [jaegertracing/jaeger/issues/1438](https://github.com/jaegertracing/jaeger/issues/1438) for ClickHouse plugin.
+This is a [Jaeger gRPC storage plugin](https://github.com/jaegertracing/jaeger/tree/master/plugin/storage/grpc) implementation for storing traces in ClickHouse.
+
+## Project status
+
+This is a community-driven project, and we would love to hear your issues and feature requests.
+Pull requests are also greatly appreciated.
+
+## Why use ClickHouse for Jaeger?
+
+[ClickHouse](https://clickhouse.com) is an analytical column-oriented database management system.
+It is designed to analyze streams of events which are kind of resemblant to spans.
+It's open-source, optimized for performance, and actively developed.
+
+## How it works
+
+Jaeger spans are stored in 2 tables. The first contains the whole span encoded either in JSON or Protobuf.
+The second stores key information about spans for searching. This table is indexed by span duration and tags.
+Also, info about operations is stored in the materialized view. There are not indexes for archived spans.
+Storing data in replicated local tables with distributed global tables is natively supported. Spans are bufferized.
+Span buffers are flushed to DB either by timer or after reaching max batch size. Timer interval and batch size can be
+set in [config file](./config.yaml).
+
+Database schema generated by JetBrains DataGrip
+![Picture of tables](./pictures/tables.png)
+
+# How to start using Jaeger over ClickHouse
+
+## Documentation
+
+Refer to the [config.yaml](./config.yaml) for all supported configuration options.
+
+* [Kubernetes deployment](./guide-kubernetes.md)
+* [Sharding and replication](./guide-sharding-and-replication.md)
+* [Multi-tenancy](./guide-multitenancy.md)

 ## Build & Run

 ### Docker database example

 ```bash
-docker run --rm -it -p9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server:21
+docker run --rm -it -p9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server:22
 GOOS=linux make build run
 make run-hotrod
 ```
@ -19,14 +51,15 @@ Open [localhost:16686](http://localhost:16686) and [localhost:8080](http://local

 ### Custom database

-You need to specify connection options in config.yaml file, then you can run 
+You need to specify connection options in `config.yaml`, then you can run

 ```bash
 make build
 SPAN_STORAGE_TYPE=grpc-plugin {Jaeger binary adress} --query.ui-config=jaeger-ui.json --grpc-storage-plugin.binary=./{name of built binary} --grpc-storage-plugin.configuration-file=config.yaml --grpc-storage-plugin.log-level=debug
 ```

-### Using TLS connection
+## Credits

-For TLS connection, you need to put CA-certificate to project directory and specify path to it in ``config.yaml``
-file using ``/data/{path to certificate from project directory}``
+This project is originally based on [this clickhouse plugin implementation](https://github.com/bobrik/jaeger/tree/ivan/clickhouse/plugin/storage/clickhouse).
+
+See also [jaegertracing/jaeger/issues/1438](https://github.com/jaegertracing/jaeger/issues/1438) for historical discussion regarding the implementation of a ClickHouse plugin.
--- a/blog/post1.md
+++ b/blog/post1.md
@ -0,0 +1,96 @@
+# Jaeger ClickHouse
+This is an implementation of Jaeger's [storage plugin](https://github.com/jaegertracing/jaeger/tree/master/plugin/storage/grpc) for ClickHouse.
+See as well [jaegertracing/jaeger/issues/1438](https://github.com/jaegertracing/jaeger/issues/1438) for historical discussion regarding Clickhouse plugin.
+
+## Project status
+
+Jaeger ClickHouse is a community-driven project, we would love to hear your feature requests.
+Pull requests also will be greatly appreciated.
+
+## Why use ClickHouse for Jaeger?
+
+[ClickHouse](https://github.com/clickhouse/clickhouse) is an analytical column-oriented database management system. It is designed to analyze streams of clicks which are kind of resemblant to spans. It's open-source, optimized for performance, and actively developed.
+
+## How does it work?
+
+Jaeger spans are stored in 2 tables. First one contains whole span encoded either in JSON or Protobuf.
+Second stores key information about spans for searching. This table is indexed by span duration and tags.
+Also, info about operations is stored in the materialized view. There are no indexes for archived spans.
+Storing data in replicated local tables with distributed global tables is natively supported. Spans are buffered.
+Span buffers are flushed to DB either by timer or after reaching max batch size.
+Timer interval and batch size can be set in [config file](../config.yaml).
+
+![Picture of tables](post1_pics/tables.png)
+
+## Benchmarks
+
+10^8 traces were flushed using [jaeger-tracegen](https://www.jaegertracing.io/docs/1.25/tools/) to Clickhouse and ElasticSearch servers.
+Clickhouse server consisted of 3 shards, 2 hosts in each, and 3 Zookeeper hosts. Elasticsearch server consisted of 6 hosts,
+with 5 shards for primary index and 1 replica. All hosts were equal(8 vCPU, 32 GiB RAM, 20 GiB SSD).
+
+### General stats
+
+Cpu usage, [% of 1 host CPU]
+![CPU usage](post1_pics/cpu-usage.png)
+
+Memory usage, [bytes]
+![Memory usage](post1_pics/memory-usage.png)
+
+IO write, [operations]
+![IO write](post1_pics/io-write.png)
+
+Disk usage, [bytes]
+![Disk usage](post1_pics/disk-usage.png)
+
+### Recorded
+
+#### ClickHouse
+
+```sql
+SELECT count()
+FROM jaeger_index
+WHERE service = 'tracegen'
+
+┌──count()─┐
+│ 57026426 │
+└──────────┘
+```
+
+#### Elasticsearch
+
+![Elasticsearch span count](post1_pics/elastic-count.png?raw=true)
+
+# How to start using Jaeger over ClickHouse
+
+## Documentation
+
+Refer to the [config.yaml](../config.yaml) for all supported configuration options.
+
+* [Kubernetes deployment](../guide-kubernetes.md)
+* [Sharding and replication](../guide-sharding-and-replication.md)
+* [Multi-tenancy](../guide-multitenancy.md)
+
+## Build & Run
+
+### Docker database example
+
+```bash
+docker run --rm -it -p9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server:22
+GOOS=linux make build run
+make run-hotrod
+```
+
+Open [localhost:16686](http://localhost:16686) and [localhost:8080](http://localhost:8080).
+
+### Custom database
+
+You need to specify connection options in config.yaml file, then you can run
+
+```bash
+make build
+SPAN_STORAGE_TYPE=grpc-plugin {Jaeger binary adress} --query.ui-config=jaeger-ui.json --grpc-storage-plugin.binary=./{name of built binary} --grpc-storage-plugin.configuration-file=config.yaml --grpc-storage-plugin.log-level=debug
+```
+
+## Credits
+
+This project is based on https://github.com/bobrik/jaeger/tree/ivan/clickhouse/plugin/storage/clickhouse.
--- a/blog/post1_pics/cpu-usage.png
+++ b/blog/post1_pics/cpu-usage.png
--- a/blog/post1_pics/disk-usage.png
+++ b/blog/post1_pics/disk-usage.png
--- a/blog/post1_pics/elastic-count.png
+++ b/blog/post1_pics/elastic-count.png
--- a/blog/post1_pics/io-write.png
+++ b/blog/post1_pics/io-write.png
--- a/blog/post1_pics/memory-usage.png
+++ b/blog/post1_pics/memory-usage.png
--- a/blog/post1_pics/tables.png
+++ b/blog/post1_pics/tables.png
--- a/cmd/jaeger-clickhouse/main.go
+++ b/cmd/jaeger-clickhouse/main.go
@ -2,7 +2,6 @@ package main

 import (
 	"flag"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@ -10,14 +9,13 @@ import (
 	// Package contains time zone info for connecting to ClickHouse servers with non-UTC time zone
 	_ "time/tzdata"

-	"github.com/hashicorp/go-hclog"
+	hclog "github.com/hashicorp/go-hclog"
 	"github.com/jaegertracing/jaeger/plugin/storage/grpc"
 	"github.com/jaegertracing/jaeger/plugin/storage/grpc/shared"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"gopkg.in/yaml.v3"
+	yaml "gopkg.in/yaml.v3"

-	jaegerclickhouse "github.com/pavolloffay/jaeger-clickhouse"
-	"github.com/pavolloffay/jaeger-clickhouse/storage"
+	"github.com/jaegertracing/jaeger-clickhouse/storage"
 )

 func main() {
@ -33,7 +31,7 @@ func main() {
 		JSONFormat: true,
 	})

-	cfgFile, err := ioutil.ReadFile(filepath.Clean(configPath))
+	cfgFile, err := os.ReadFile(filepath.Clean(configPath))
 	if err != nil {
 		logger.Error("Could not read config file", "config", configPath, "error", err)
 		os.Exit(1)
@ -53,13 +51,14 @@ func main() {
 	}()

 	var pluginServices shared.PluginServices
-	store, err := storage.NewStore(logger, cfg, jaegerclickhouse.EmbeddedFiles)
+	store, err := storage.NewStore(logger, cfg)
 	if err != nil {
 		logger.Error("Failed to create a storage", err)
 		os.Exit(1)
 	}
 	pluginServices.Store = store
 	pluginServices.ArchiveStore = store
+	pluginServices.StreamingSpanWriter = store

 	grpc.Serve(&pluginServices)
 	if err = store.Close(); err != nil {
--- a/config.yaml
+++ b/config.yaml
@ -1,6 +1,19 @@
-address: tcp://some-clickhouse-server:9000
-# When empty the embedded scripts from sqlscripts directory are used
+address: some-clickhouse-server:9000
+# Directory with .sql files to run at plugin startup, mainly for integration tests.
+# Depending on the value of "init_tables", this can be run as a
+# replacement or supplement to creating default tables for span storage.
+# If init_tables is also enabled, the scripts in this directory will be run first.
 init_sql_scripts_dir:
+# Whether to automatically attempt to create tables in ClickHouse.
+# By default, this is enabled if init_sql_scripts_dir is empty,
+# or disabled if init_sql_scripts_dir is provided.
+init_tables:
+# Maximal amount of spans that can be pending writes at a time.
+# New spans exceeding this limit will be discarded,
+# keeping memory in check if there are issues writing to ClickHouse.
+# Check the "jaeger_clickhouse_discarded_spans" metric to keep track of discards.
+# If 0, no limit is set. Default 10_000_000.
+max_span_count:
 # Batch write size. Default 10_000.
 batch_write_size:
 # Batch flush interval. Default 5s.
@ -9,17 +22,28 @@ batch_flush_interval:
 encoding:
 # Path to CA TLS certificate.
 ca_file:
-# Username for connection. Default is "default".
+# Username for connection to ClickHouse. Default is "default".
 username:
-# Password for connection.
+# Password for connection to ClickHouse.
 password:
-# Database name. Default is "default".
+# ClickHouse database name. The database must be created manually before Jaeger starts. Default is "default".
 database:
-# Endpoint for scraping prometheus metrics. Default localhost:9090.
+# If non-empty, enables a tenant column in tables, and uses the provided tenant name for this instance.
+# Default is empty. See guide-multitenancy.md for more information.
+tenant:
+# Endpoint for serving prometheus metrics. Default localhost:9090.
 metrics_endpoint: localhost:9090
-# Table with spans. Default "jaeger_spans_local".
+# Whether to use sql scripts supporting replication and sharding.
+# Replication can be used only on database with Atomic engine.
+# Default false.
+replication:
+# Table with spans. Default "jaeger_spans_local" or "jaeger_spans" when replication is enabled.
 spans_table:
-# Span index table. Default "jaeger_index_local".
+# Span index table. Default "jaeger_index_local" or "jaeger_index" when replication is enabled.
 spans_index_table:
-# Operations table. Default "jaeger_operations_local".
+# Operations table. Default "jaeger_operations_local" or "jaeger_operations" when replication is enabled.
 operations_table:
+# TTL for data in tables in days. If 0, no TTL is set. Default 0.
+ttl:
+# The maximum number of spans to fetch per trace. If 0, no limit is set. Default 0.
+max_num_spans:
--- a/e2etests/clickhouse-replicated.xml
+++ b/e2etests/clickhouse-replicated.xml
@ -0,0 +1,47 @@
+<!-- Minimal configuration to enable cluster mode in a single clickhouse process -->
+<yandex>
+    <macros>
+        <installation>cluster</installation>
+        <all-sharded-shard>0</all-sharded-shard>
+        <cluster>cluster</cluster>
+        <shard>0</shard>
+        <replica>cluster-0-0</replica>
+    </macros>
+
+    <remote_servers>
+        <cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster>
+    </remote_servers>
+
+    <keeper_server>
+        <tcp_port>2181</tcp_port>
+        <server_id>0</server_id>
+        <log_storage_path>/var/log/clickhouse-server/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <raft_configuration>
+            <server>
+                <id>0</id>
+                <hostname>localhost</hostname>
+                <port>9444</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+
+    <zookeeper>
+        <!-- Clickhouse Keeper -->
+        <node>
+            <host>localhost</host>
+            <port>2181</port>
+        </node>
+    </zookeeper>
+    <distributed_ddl>
+        <path>/clickhouse/cluster/task_queue/ddl</path>
+    </distributed_ddl>
+</yandex>
--- a/e2etests/config-local-multi1.yaml
+++ b/e2etests/config-local-multi1.yaml
@ -0,0 +1,4 @@
+address: chi:9000
+tenant: multi1
+# For test purposes flush on every write
+batch_write_size: 1
--- a/e2etests/config-local-multi2.yaml
+++ b/e2etests/config-local-multi2.yaml
@ -0,0 +1,4 @@
+address: chi:9000
+tenant: multi2
+# For test purposes flush on every write
+batch_write_size: 1
--- a/e2etests/config-local-single.yaml
+++ b/e2etests/config-local-single.yaml
@ -0,0 +1,3 @@
+address: chi:9000
+# For test purposes flush on every write
+batch_write_size: 1
--- a/e2etests/config-replication-multi1.yaml
+++ b/e2etests/config-replication-multi1.yaml
@ -0,0 +1,5 @@
+address: chi:9000
+replication: true
+tenant: multi1
+# For test purposes flush on every write
+batch_write_size: 1
--- a/e2etests/config-replication-multi2.yaml
+++ b/e2etests/config-replication-multi2.yaml
@ -0,0 +1,5 @@
+address: chi:9000
+replication: true
+tenant: multi2
+# For test purposes flush on every write
+batch_write_size: 1
--- a/e2etests/config-replication-single.yaml
+++ b/e2etests/config-replication-single.yaml
@ -0,0 +1,4 @@
+address: chi:9000
+replication: true
+# For test purposes flush on every write
+batch_write_size: 1
--- a/e2etests/e2e_test.go
+++ b/e2etests/e2e_test.go
@ -0,0 +1,203 @@
+package e2etests
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ecodia/golang-awaitility/awaitility"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	testcontainers "github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
+)
+
+const (
+	clickHouseImage = "clickhouse/clickhouse-server:22"
+	jaegerImage     = "jaegertracing/all-in-one:1.32.0"
+
+	networkName     = "chi-jaeger-test"
+	clickhousePort  = "9000/tcp"
+	jaegerQueryPort = "16686/tcp"
+	jaegerAdminPort = "14269/tcp"
+)
+
+type testCase struct {
+	configs []string
+	chiconf *string
+}
+
+func TestE2E(t *testing.T) {
+	if os.Getenv("E2E_TEST") == "" {
+		t.Skip("Set E2E_TEST=true to run the test")
+	}
+
+	// Minimal additional configuration (config.d) to enable cluster mode
+	chireplconf := "clickhouse-replicated.xml"
+
+	tests := map[string]testCase{
+		"local-single": {
+			configs: []string{"config-local-single.yaml"},
+			chiconf: nil,
+		},
+		"local-multi": {
+			configs: []string{"config-local-multi1.yaml", "config-local-multi2.yaml"},
+			chiconf: nil,
+		},
+		"replication-single": {
+			configs: []string{"config-replication-single.yaml"},
+			chiconf: &chireplconf,
+		},
+		"replication-multi": {
+			configs: []string{"config-replication-multi1.yaml", "config-replication-multi2.yaml"},
+			chiconf: &chireplconf,
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			testE2E(t, test)
+		})
+	}
+}
+
+func testE2E(t *testing.T, test testCase) {
+	ctx := context.Background()
+	workingDir, err := os.Getwd()
+	require.NoError(t, err)
+
+	network, err := testcontainers.GenericNetwork(ctx, testcontainers.GenericNetworkRequest{
+		NetworkRequest: testcontainers.NetworkRequest{Name: networkName},
+	})
+	require.NoError(t, err)
+	defer network.Remove(ctx)
+
+	var bindMounts map[string]string
+	if test.chiconf != nil {
+		bindMounts = map[string]string{
+			fmt.Sprintf("%s/%s", workingDir, *test.chiconf): "/etc/clickhouse-server/config.d/testconf.xml",
+		}
+	} else {
+		bindMounts = map[string]string{}
+	}
+	chReq := testcontainers.ContainerRequest{
+		Image:        clickHouseImage,
+		ExposedPorts: []string{clickhousePort},
+		WaitingFor:   &clickhouseWaitStrategy{test: t, pollInterval: time.Millisecond * 200, startupTimeout: time.Minute},
+		Networks:     []string{networkName},
+		Hostname:     "chi",
+		BindMounts:   bindMounts,
+	}
+	chContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: chReq,
+		Started:          true,
+	})
+	require.NoError(t, err)
+	defer chContainer.Terminate(ctx)
+
+	jaegerContainers := make([]testcontainers.Container, 0)
+	for _, pluginConfig := range test.configs {
+		jaegerReq := testcontainers.ContainerRequest{
+			Image:        jaegerImage,
+			ExposedPorts: []string{jaegerQueryPort, jaegerAdminPort},
+			WaitingFor:   wait.ForHTTP("/").WithPort(jaegerAdminPort).WithStartupTimeout(time.Second * 10),
+			Env: map[string]string{
+				"SPAN_STORAGE_TYPE": "grpc-plugin",
+			},
+			Cmd: []string{
+				"--grpc-storage-plugin.binary=/project-dir/jaeger-clickhouse-linux-amd64",
+				fmt.Sprintf("--grpc-storage-plugin.configuration-file=/project-dir/e2etests/%s", pluginConfig),
+				"--grpc-storage-plugin.log-level=debug",
+			},
+			BindMounts: map[string]string{
+				workingDir + "/..": "/project-dir",
+			},
+			Networks: []string{networkName},
+		}
+		// Call Start() manually here so that if it fails then we can still access the logs.
+		jaegerContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+			ContainerRequest: jaegerReq,
+		})
+		require.NoError(t, err)
+		defer func() {
+			logs, errLogs := jaegerContainer.Logs(ctx)
+			require.NoError(t, errLogs)
+			all, errLogs := ioutil.ReadAll(logs)
+			require.NoError(t, errLogs)
+			fmt.Printf("Jaeger logs:\n---->\n%s<----\n\n", string(all))
+			jaegerContainer.Terminate(ctx)
+		}()
+		err = jaegerContainer.Start(ctx)
+		require.NoError(t, err)
+
+		jaegerContainers = append(jaegerContainers, jaegerContainer)
+	}
+
+	for _, jaegerContainer := range jaegerContainers {
+		jaegerQueryPort, err := jaegerContainer.MappedPort(ctx, jaegerQueryPort)
+		require.NoError(t, err)
+
+		err = awaitility.Await(100*time.Millisecond, time.Second*3, func() bool {
+			// Jaeger traces itself so this request generates some spans
+			response, errHTTP := http.Get(fmt.Sprintf("http://localhost:%d/api/services", jaegerQueryPort.Int()))
+			require.NoError(t, errHTTP)
+			body, errHTTP := ioutil.ReadAll(response.Body)
+			require.NoError(t, errHTTP)
+			var r result
+			errHTTP = json.Unmarshal(body, &r)
+			require.NoError(t, errHTTP)
+			return len(r.Data) == 1 && r.Data[0] == "jaeger-query"
+		})
+		assert.NoError(t, err)
+	}
+}
+
+type result struct {
+	Data []string `json:"data"`
+}
+
+type clickhouseWaitStrategy struct {
+	test           *testing.T
+	pollInterval   time.Duration
+	startupTimeout time.Duration
+}
+
+var _ wait.Strategy = (*clickhouseWaitStrategy)(nil)
+
+func (c *clickhouseWaitStrategy) WaitUntilReady(ctx context.Context, target wait.StrategyTarget) error {
+	ctx, cancelContext := context.WithTimeout(ctx, c.startupTimeout)
+	defer cancelContext()
+
+	port, err := target.MappedPort(ctx, clickhousePort)
+	require.NoError(c.test, err)
+
+	db := clickhouse.OpenDB(&clickhouse.Options{
+		Addr: []string{
+			fmt.Sprintf("localhost:%d", port.Int()),
+		},
+		Auth: clickhouse.Auth{
+			Database: "default",
+		},
+		Compression: &clickhouse.Compression{
+			Method: clickhouse.CompressionLZ4,
+		},
+	})
+	require.NoError(c.test, err)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(c.pollInterval):
+			if err := db.Ping(); err != nil {
+				continue
+			}
+			return nil
+		}
+	}
+}
--- a/embed.go
+++ b/embed.go
@ -3,4 +3,4 @@ package jaegerclickhouse
 import "embed"

 //go:embed sqlscripts/*
-var EmbeddedFiles embed.FS
+var SQLScripts embed.FS
--- a/go.mod
+++ b/go.mod
@ -1,13 +1,102 @@
-module github.com/pavolloffay/jaeger-clickhouse
+module github.com/jaegertracing/jaeger-clickhouse

-go 1.16
+go 1.19

 require (
-	github.com/ClickHouse/clickhouse-go v1.4.5
+	github.com/ClickHouse/clickhouse-go/v2 v2.3.0
+	github.com/DATA-DOG/go-sqlmock v1.5.0
+	github.com/ecodia/golang-awaitility v0.0.0-20180710094957-fb55e59708c7
 	github.com/gogo/protobuf v1.3.2
-	github.com/hashicorp/go-hclog v0.16.1
-	github.com/jaegertracing/jaeger v1.24.0
+	github.com/hashicorp/go-hclog v1.3.1
+	github.com/jaegertracing/jaeger v1.38.2-0.20221007043206-b4c88ddf6cdd
 	github.com/opentracing/opentracing-go v1.2.0
-	github.com/prometheus/client_golang v1.11.0
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	github.com/prometheus/client_golang v1.13.0
+	github.com/stretchr/testify v1.8.0
+	github.com/testcontainers/testcontainers-go v0.11.1
+	go.uber.org/zap v1.23.0
+	gopkg.in/yaml.v3 v3.0.1
+)
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/ClickHouse/ch-go v0.47.3 // indirect
+	github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3 // indirect
+	github.com/Microsoft/hcsshim v0.8.16 // indirect
+	github.com/andybalholm/brotli v1.0.4 // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 // indirect
+	github.com/containerd/containerd v1.5.0-beta.4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/docker v20.10.7+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/fatih/color v1.13.0 // indirect
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.6.1 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
+	github.com/hashicorp/go-plugin v1.4.5 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/klauspost/compress v1.15.10 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/moby/sys/mountinfo v0.4.1 // indirect
+	github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
+	github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect
+	github.com/oklog/run v1.1.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opencontainers/runc v1.0.0-rc93 // indirect
+	github.com/paulmach/orb v0.7.1 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.1 // indirect
+	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/spf13/afero v1.8.2 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/cobra v1.5.0 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.13.0 // indirect
+	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
+	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
+	go.opencensus.io v0.23.0 // indirect
+	go.opentelemetry.io/otel v1.10.0 // indirect
+	go.opentelemetry.io/otel/trace v1.10.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	golang.org/x/net v0.0.0-20221002022538-bcab6841153b // indirect
+	golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
+	google.golang.org/grpc v1.50.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
--- a/go.sum
+++ b/go.sum
--- a/guide-kubernetes.md
+++ b/guide-kubernetes.md
@ -0,0 +1,87 @@
+# Kubernetes Deployment
+
+This is a guide to deploy Jaeger with Clickhouse storage on Kubernetes.
+
+## Prerequisites
+
+1. Deploy [Jaeger operator](https://github.com/jaegertracing/jaeger-operator). Note that `grpc-plugin` storage type is supported since version 1.25.0.
+2. Deploy [Clickhouse operator](https://github.com/Altinity/clickhouse-operator)
+3. Deploy [Zookeeper](https://github.com/Altinity/clickhouse-operator/blob/master/docs/replication_setup.md) (if replication is used)
+
+## Deploy
+
+Deploy Clickhouse:
+
+```yaml
+cat <<EOF | kubectl apply -f -
+apiVersion: clickhouse.altinity.com/v1
+kind: ClickHouseInstallation
+metadata:
+  name: jaeger
+  labels:
+    jaeger-clickhouse: demo
+spec:
+  configuration:
+    clusters:
+      - name: cluster1
+        layout:
+          shardsCount: 1
+EOF
+```
+
+Create config map for Jaeger Clickhouse plugin:
+
+```yaml
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: jaeger-clickhouse
+  labels:
+    jaeger-clickhouse: demo
+data:
+  config.yaml: |
+    address: clickhouse-jaeger:9000
+    username: clickhouse_operator
+    password: clickhouse_operator_password
+    spans_table:
+    spans_index_table:
+    operations_table:
+EOF
+```
+
+Deploy Jaeger:
+
+```yaml
+cat <<EOF | kubectl apply -f -
+apiVersion: jaegertracing.io/v1
+kind: Jaeger
+metadata:
+  name: jaeger-clickhouse
+  labels:
+    jaeger-clickhouse: demo
+spec:
+  storage:
+    type: grpc-plugin
+    grpcPlugin:
+      image: ghcr.io/jaegertracing/jaeger-clickhouse:0.7.0
+    options:
+      grpc-storage-plugin:
+        binary: /plugin/jaeger-clickhouse
+        configuration-file: /plugin-config/config.yaml
+        log-level: debug
+  volumeMounts:
+    - name: plugin-config
+      mountPath: /plugin-config
+  volumes:
+    - name: plugin-config
+      configMap:
+        name: jaeger-clickhouse
+EOF
+```
+
+## Delete all
+
+```bash
+kubectl delete jaeger,cm,chi -l jaeger-clickhouse=demo
+```
--- a/guide-multitenancy.md
+++ b/guide-multitenancy.md
@ -0,0 +1,78 @@
+# Multi-tenant deployment
+
+It may be desirable to share a common ClickHouse instance across multiple Jaeger instances.
+There are two ways of doing this, depending on whether spanning the tenants across separate databases is preferable.
+
+## Shared database/tables
+
+If you wish to reuse the same ClickHouse database/tables across all tenants, you can specify a different `tenant: "<name>"` in each jaeger-clickhouse instance config.
+
+When a non-empty `tenant` is specified, all tables will be created with a `tenant` column, and all reads/writes for a given Jaeger instance will be applied against the configured tenant name for that instance.
+
+1. Create a shared database:
+    ```sql
+    CREATE DATABASE shared ENGINE=Atomic
+    ```
+2. Configure the per-tenant jaeger-clickhouse clients to specify tenant names:
+    ```yaml
+    database: shared
+    tenant: tenant_1
+    ```
+    ```yaml
+    database: shared
+    tenant: tenant_2
+    ```
+
+Multitenant mode must be enabled when the deployment is first created and cannot be toggled later, except perhaps by manually adding/removing the `tenant` column from all tables.
+Multitenant/singletenant instances must not be mixed within the same database - the two modes are mutually exclusive of each other.
+
+## Separate databases
+
+If you wish to keep instances fully separate, you can configure one ClickHouse database per tenant.
+This may be useful when different per-database configuration across tenants is desirable.
+
+1. Create a database for each tenant:
+    ```sql
+    CREATE DATABASE tenant_1 ENGINE=Atomic;
+    CREATE DATABASE tenant_2 ENGINE=Atomic;
+    ```
+2. Configure the per-tenant jaeger-clickhouse plugins matching databases:
+    ```yaml
+    database: tenant_1
+    ```
+    ```yaml
+    database: tenant_2
+    ```
+
+## Mixing methods in the same ClickHouse instance
+
+Each of the methods applies on a per-database basis. The methods require different schemas and must not be mixed in a single database, but it is possible to have different databases using different methods in the same ClickHouse instance.
+
+For example, there could be a `shared` database where multiple tenants are sharing the same tables:
+
+    ```sql
+    CREATE DATABASE shared ENGINE=Atomic
+    ```
+    ```yaml
+    database: shared
+    tenant: tenant_1
+    ```
+    ```yaml
+    database: shared
+    tenant: tenant_2
+    ```
+
+Then there could be separate `isolated_x` databases for tenants that should be provided with their own dedicated tables, enabling e.g. better ACL isolation:
+
+    ```sql
+    CREATE DATABASE isolated_1 ENGINE=Atomic
+    CREATE DATABASE isolated_2 ENGINE=Atomic
+    ```
+    ```yaml
+    database: isolated_1
+    ```
+    ```yaml
+    database: isolated_2
+    ```
+
+
--- a/guide-sharding-and-replication.md
+++ b/guide-sharding-and-replication.md
@ -4,24 +4,33 @@ This is a guide how to setup sharding and replication for Jaeger data.
 This guide uses [clickhouse-operator](https://github.com/Altinity/clickhouse-operator) to deploy
 the storage.

+Note that the Jaeger ClickHouse plugin supports creating replicated schema out-of-the-box. Therefore,
+this guide is not necessary for setting up default replicated deployment. Also note that the
+ClickHouse operator uses by default `Ordinary` database engine, which does not work with the
+embedded replication scripts in Jaeger.
+Refer to the `config.yaml` how to setup replicated deployment.
+
 ## Sharding

 Sharding is a feature that allows splitting the data into multiple Clickhouse nodes to
 increase throughput and decrease latency.
 The sharding feature uses `Distributed` engine that is backed by local tables.
-The distributed engine is a "virtual" table that does not store any data. It is used as 
+The distributed engine is a "virtual" table that does not store any data. It is used as
 an interface to insert and query data.

 To setup sharding run the following statements on all nodes in the cluster.
 The "local" tables have to be created on the nodes before the distributed table.

 ```sql
+CREATE DATABASE jaeger ENGINE=Atomic;
+USE jaeger;
+
 CREATE TABLE IF NOT EXISTS jaeger_spans AS jaeger_spans_local ENGINE = Distributed('{cluster}', default, jaeger_spans_local, cityHash64(traceID));
 CREATE TABLE IF NOT EXISTS jaeger_index AS jaeger_index_local ENGINE = Distributed('{cluster}', default, jaeger_index_local, cityHash64(traceID));
 CREATE TABLE IF NOT EXISTS jaeger_operations AS jaeger_operations_local ENGINE = Distributed('{cluster}', default, jaeger_operations_local, rand());
 ```

-* The `AS <table-name>` statement creates table with the same schema as the specified one. 
+* The `AS <table-name>` statement creates table with the same schema as the specified one.
 * The `Distributed` engine takes as parameters cluster , database, table name and sharding key.

 If the distributed table is not created on all Clickhouse nodes the Jaeger query fails to get the data from the storage.
@ -32,8 +41,8 @@ Deploy Clickhouse with 2 shards:

 ```yaml
 cat <<EOF | kubectl apply -f -
-apiVersion: "clickhouse.altinity.com/v1"
-kind: "ClickHouseInstallation"
+apiVersion: clickhouse.altinity.com/v1
+kind: ClickHouseInstallation
 metadata:
  name: jaeger
 spec:
@ -47,7 +56,7 @@ EOF

 Use the following command to run `clickhouse-client` on Clickhouse nodes and create the distributed tables:
 ```bash
-kubectl exec -it statefulset.apps/chi-jaeger-cluster1-0-0 -- clickhouse-client  
+kubectl exec -it statefulset.apps/chi-jaeger-cluster1-0-0 -- clickhouse-client
 ```

 ### Plugin configuration
@ -55,7 +64,8 @@ kubectl exec -it statefulset.apps/chi-jaeger-cluster1-0-0 -- clickhouse-client
 The plugin has to be configured to write and read that from the global tables:

 ```yaml
-address: tcp://clickhouse-jaeger:9000
+address: clickhouse-jaeger:9000
+# database: jaeger
 spans_table: jaeger_spans
 spans_index_table: jaeger_index
 operations_table: jaeger_operations
@ -72,17 +82,20 @@ Zookeeper allows us to use `ON CLUSTER` to automatically replicate table creatio
 Therefore the following command can be run only on a single Clickhouse node:

 ```sql
-CREATE TABLE IF NOT EXISTS jaeger_spans_local ON CLUSTER '{cluster}'  (
-                                                                timestamp DateTime CODEC(Delta, ZSTD(1)),
+CREATE DATABASE IF NOT EXISTS jaeger ON CLUSTER '{cluster}' ENGINE=Atomic;
+USE jaeger;
+
+CREATE TABLE IF NOT EXISTS jaeger_spans_local ON CLUSTER '{cluster}' (
+    timestamp DateTime CODEC(Delta, ZSTD(1)),
    traceID String CODEC(ZSTD(1)),
    model String CODEC(ZSTD(3))
-    ) ENGINE ReplicatedMergeTree('/clickhouse/tables/{shard}/jaeger_spans', '{replica}')
-    PARTITION BY toDate(timestamp)
-    ORDER BY traceID
-    SETTINGS index_granularity=1024;
+) ENGINE ReplicatedMergeTree
+PARTITION BY toDate(timestamp)
+ORDER BY traceID
+SETTINGS index_granularity=1024;

 CREATE TABLE IF NOT EXISTS jaeger_index_local ON CLUSTER '{cluster}' (
-                                                               timestamp DateTime CODEC(Delta, ZSTD(1)),
+    timestamp DateTime CODEC(Delta, ZSTD(1)),
    traceID String CODEC(ZSTD(1)),
    service LowCardinality(String) CODEC(ZSTD(1)),
    operation LowCardinality(String) CODEC(ZSTD(1)),
@ -90,28 +103,28 @@ CREATE TABLE IF NOT EXISTS jaeger_index_local ON CLUSTER '{cluster}' (
    tags Array(String) CODEC(ZSTD(1)),
    INDEX idx_tags tags TYPE bloom_filter(0.01) GRANULARITY 64,
    INDEX idx_duration durationUs TYPE minmax GRANULARITY 1
-    ) ENGINE ReplicatedMergeTree('/clickhouse/tables/{shard}/jaeger_index', '{replica}')
-    PARTITION BY toDate(timestamp)
-    ORDER BY (service, -toUnixTimestamp(timestamp))
-    SETTINGS index_granularity=1024;
+) ENGINE ReplicatedMergeTree
+PARTITION BY toDate(timestamp)
+ORDER BY (service, -toUnixTimestamp(timestamp))
+SETTINGS index_granularity=1024;

 CREATE MATERIALIZED VIEW IF NOT EXISTS jaeger_operations_local ON CLUSTER '{cluster}'
-ENGINE ReplicatedMergeTree('/clickhouse/tables/{shard}/jaeger_operations', '{replica}')
+ENGINE ReplicatedMergeTree
 PARTITION BY toYYYYMM(date) ORDER BY (date, service, operation)
 SETTINGS index_granularity=32
 POPULATE
 AS SELECT
-              toDate(timestamp) AS date,
-  service,
-  operation,
-  count() as count
-   FROM jaeger_index_local
-   GROUP BY date, service, operation;
+    toDate(timestamp) AS date,
+    service,
+    operation,
+count() as count
+FROM jaeger.jaeger_index_local
+GROUP BY date, service, operation;


-CREATE TABLE IF NOT EXISTS jaeger_spans ON CLUSTER '{cluster}' AS jaeger_spans_local ENGINE = Distributed('{cluster}', default, jaeger_spans_local, cityHash64(traceID));
-CREATE TABLE IF NOT EXISTS jaeger_index ON CLUSTER '{cluster}' AS jaeger_index_local ENGINE = Distributed('{cluster}', default, jaeger_index_local, cityHash64(traceID));
-CREATE TABLE IF NOT EXISTS jaeger_operations on CLUSTER '{cluster}' AS jaeger_operations_local ENGINE = Distributed('{cluster}', default, jaeger_operations_local, rand());
+CREATE TABLE IF NOT EXISTS jaeger_spans ON CLUSTER '{cluster}' AS jaeger.jaeger_spans_local ENGINE = Distributed('{cluster}', jaeger, jaeger_spans_local, cityHash64(traceID));
+CREATE TABLE IF NOT EXISTS jaeger_index ON CLUSTER '{cluster}' AS jaeger.jaeger_index_local ENGINE = Distributed('{cluster}', jaeger, jaeger_index_local, cityHash64(traceID));
+CREATE TABLE IF NOT EXISTS jaeger_operations on CLUSTER '{cluster}' AS jaeger.jaeger_operations_local ENGINE = Distributed('{cluster}', jaeger, jaeger_operations_local, rand());
 ```

 ### Deploy Clickhouse
@ -122,11 +135,15 @@ Deploy Clickhouse with 3 shards and 2 replicas. In total Clickhouse operator wil

 ```yaml
 cat <<EOF | kubectl apply -f -
-apiVersion: "clickhouse.altinity.com/v1"
-kind: "ClickHouseInstallation"
+apiVersion: clickhouse.altinity.com/v1
+kind: ClickHouseInstallation
 metadata:
  name: jaeger
 spec:
+  defaults:
+    templates:
+      dataVolumeClaimTemplate: data-volume-template
+      logVolumeClaimTemplate: log-volume-template
  configuration:
    zookeeper:
      nodes:
@ -137,35 +154,55 @@ spec:
          shardsCount: 3
          replicasCount: 2
  templates:
-    podTemplates:
-      - name: clickhouse-with-empty-dir-volume-template
+    volumeClaimTemplates:
+      - name: data-volume-template
        spec:
-          containers:
-            - name: clickhouse-pod
-              image: yandex/clickhouse-server:20.7
-              volumeMounts:
-                - name: clickhouse-storage
-                  mountPath: /var/lib/clickhouse
-          volumes:
-            - name: clickhouse-storage
-              emptyDir:
-                medium: "" # accepted values:  empty str (means node's default medium) or "Memory"
-                sizeLimit: 1Gi
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 1Gi
+      - name: log-volume-template
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 100Mi
 EOF
 ```

+The Clickhouse deployment will look like this:
+```bash
+k get statefulsets
+NAME                      READY   AGE
+chi-jaeger-cluster1-0-0   1/1     17m    # shard 0
+chi-jaeger-cluster1-0-1   1/1     17m    # shard 0, replica 1
+chi-jaeger-cluster1-1-0   1/1     16m    # shard 1
+chi-jaeger-cluster1-1-1   1/1     16m    # shard 1, replica 1
+chi-jaeger-cluster1-2-0   1/1     7m43s  # shard 2
+chi-jaeger-cluster1-2-1   1/1     7m26s  # shard 2, replica 1
+```
+
+#### Scaling up
+
+Just increase `shardsCount` number and new Clickhouse node will come up. It will have initialized Jaeger tables so
+no other steps are required. Note that the old data are not re-balanced, only new writes take into the account
+the new node.
+
 ## Useful Commands

 ### SQL

 ```sql
-show tables
-select count() from jaeger_spans
+show tables;
+select count() from jaeger_spans;
 ```

 ### Kubectl

 ```bash
+kubectl get chi -o wide
 kubectl port-forward service/clickhouse-jaeger 9000:9000
-kubectl delete clickhouseinstallations.clickhouse.altinity.com jaeger
+kubectl delete chi jaeger
 ```
--- a/integration/config-local.yaml
+++ b/integration/config-local.yaml
@ -0,0 +1,3 @@
+address: localhost:9000
+init_sql_scripts_dir: init_sql_scripts
+init_tables: true
--- a/integration/grpc_test.go
+++ b/integration/grpc_test.go
@ -0,0 +1,107 @@
+// Copyright (c) 2019 The Jaeger Authors.
+// Copyright (c) 2018 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+	"os"
+	"testing"
+
+	"github.com/jaegertracing/jaeger/pkg/config"
+	"github.com/jaegertracing/jaeger/pkg/metrics"
+	"github.com/jaegertracing/jaeger/pkg/testutils"
+	"github.com/jaegertracing/jaeger/plugin/storage/grpc"
+	"github.com/jaegertracing/jaeger/plugin/storage/integration"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+)
+
+const defaultPluginBinaryPath = "../../../examples/memstore-plugin/memstore-plugin"
+
+type GRPCStorageIntegrationTestSuite struct {
+	integration.StorageIntegration
+	logger           *zap.Logger
+	pluginBinaryPath string
+	pluginConfigPath string
+}
+
+func (s *GRPCStorageIntegrationTestSuite) initialize() error {
+	s.logger, _ = testutils.NewLogger()
+
+	f := grpc.NewFactory()
+	v, command := config.Viperize(f.AddFlags)
+	flags := []string{
+		"--grpc-storage-plugin.binary",
+		s.pluginBinaryPath,
+		"--grpc-storage-plugin.log-level",
+		"debug",
+	}
+	if s.pluginConfigPath != "" {
+		flags = append(flags,
+			"--grpc-storage-plugin.configuration-file",
+			s.pluginConfigPath,
+		)
+	}
+	err := command.ParseFlags(flags)
+	if err != nil {
+		return err
+	}
+	f.InitFromViper(v, zap.NewNop())
+	if err = f.Initialize(metrics.NullFactory, s.logger); err != nil {
+		return err
+	}
+
+	if s.SpanWriter, err = f.CreateSpanWriter(); err != nil {
+		return err
+	}
+	if s.SpanReader, err = f.CreateSpanReader(); err != nil {
+		return err
+	}
+
+	// TODO DependencyWriter is not implemented in grpc store
+
+	s.Refresh = s.refresh
+	s.CleanUp = s.cleanUp
+	return nil
+}
+
+func (s *GRPCStorageIntegrationTestSuite) refresh() error {
+	return nil
+}
+
+func (s *GRPCStorageIntegrationTestSuite) cleanUp() error {
+	return s.initialize()
+}
+
+func TestGRPCStorage(t *testing.T) {
+	if os.Getenv("STORAGE") != "grpc-plugin" {
+		t.Skip("Integration test against grpc skipped; set STORAGE env var to grpc-plugin to run this")
+	}
+	binaryPath := os.Getenv("PLUGIN_BINARY_PATH")
+	if binaryPath == "" {
+		t.Logf("PLUGIN_BINARY_PATH env var not set, using %s", defaultPluginBinaryPath)
+		binaryPath = defaultPluginBinaryPath
+	}
+	configPath := os.Getenv("PLUGIN_CONFIG_PATH")
+	if configPath == "" {
+		t.Log("PLUGIN_CONFIG_PATH env var not set")
+	}
+	s := &GRPCStorageIntegrationTestSuite{
+		pluginBinaryPath: binaryPath,
+		pluginConfigPath: configPath,
+	}
+	require.NoError(t, s.initialize())
+	s.IntegrationTestAll(t)
+}
--- a/integration/init_sql_scripts/0001-drop-database.sql
+++ b/integration/init_sql_scripts/0001-drop-database.sql
@ -0,0 +1 @@
+DROP DATABASE IF EXISTS default;
--- a/integration/init_sql_scripts/0002-create-database.sql
+++ b/integration/init_sql_scripts/0002-create-database.sql
@ -0,0 +1 @@
+CREATE DATABASE IF NOT EXISTS default;
--- a/internal/tools/go.mod
+++ b/internal/tools/go.mod
@ -1,19 +1,145 @@
-module github.com/pavolloffay/jaeger-clickhouse/internal/tools
+module github.com/jaegertracing/jaeger-clickhouse/internal/tools

-go 1.16
+go 1.19

 require (
-	github.com/go-lintpack/lintpack v0.5.2 // indirect
-	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
-	github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 // indirect
-	github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
-	github.com/golangci/golangci-lint v1.41.1 // indirect
-	github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc // indirect
-	github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 // indirect
-	github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5 // indirect
-	github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada // indirect
-	github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa // indirect
-	github.com/ugorji/go v1.1.4 // indirect
+	github.com/golangci/golangci-lint v1.41.1
 	golang.org/x/tools v0.1.5
-	sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4 // indirect
+)
+
+require (
+	4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a // indirect
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/OpenPeeDeeP/depguard v1.0.1 // indirect
+	github.com/alexkohler/prealloc v1.0.0 // indirect
+	github.com/ashanbrown/forbidigo v1.2.0 // indirect
+	github.com/ashanbrown/makezero v0.0.0-20210520155254-b6261585ddde // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bkielbasa/cyclop v1.2.0 // indirect
+	github.com/bombsimon/wsl/v3 v3.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/charithe/durationcheck v0.0.8 // indirect
+	github.com/chavacava/garif v0.0.0-20210405164556-e8a0a408d6af // indirect
+	github.com/daixiang0/gci v0.2.8 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/denis-tingajkin/go-header v0.4.2 // indirect
+	github.com/esimonov/ifshort v1.0.2 // indirect
+	github.com/ettle/strcase v0.1.1 // indirect
+	github.com/fatih/color v1.12.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/fzipp/gocyclo v0.3.1 // indirect
+	github.com/go-critic/go-critic v0.5.6 // indirect
+	github.com/go-toolsmith/astcast v1.0.0 // indirect
+	github.com/go-toolsmith/astcopy v1.0.0 // indirect
+	github.com/go-toolsmith/astequal v1.0.0 // indirect
+	github.com/go-toolsmith/astfmt v1.0.0 // indirect
+	github.com/go-toolsmith/astp v1.0.0 // indirect
+	github.com/go-toolsmith/strparse v1.0.0 // indirect
+	github.com/go-toolsmith/typep v1.0.2 // indirect
+	github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gofrs/flock v0.8.0 // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
+	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
+	github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 // indirect
+	github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a // indirect
+	github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 // indirect
+	github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca // indirect
+	github.com/golangci/misspell v0.3.5 // indirect
+	github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5 // indirect
+	github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
+	github.com/google/go-cmp v0.5.4 // indirect
+	github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254 // indirect
+	github.com/gostaticanalysis/analysisutil v0.4.1 // indirect
+	github.com/gostaticanalysis/comment v1.4.1 // indirect
+	github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5 // indirect
+	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jgautheron/goconst v1.5.1 // indirect
+	github.com/jingyugao/rowserrcheck v1.1.0 // indirect
+	github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
+	github.com/julz/importas v0.0.0-20210419104244-841f0c0fe66d // indirect
+	github.com/kisielk/errcheck v1.6.0 // indirect
+	github.com/kisielk/gotool v1.0.0 // indirect
+	github.com/kulti/thelper v0.4.0 // indirect
+	github.com/kunwardeep/paralleltest v1.0.2 // indirect
+	github.com/kyoh86/exportloopref v0.1.8 // indirect
+	github.com/ldez/gomoddirectives v0.2.1 // indirect
+	github.com/ldez/tagliatelle v0.2.0 // indirect
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/maratori/testpackage v1.0.1 // indirect
+	github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mbilski/exhaustivestruct v1.2.0 // indirect
+	github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 // indirect
+	github.com/mgechev/revive v1.0.7 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/moricho/tparallel v0.2.1 // indirect
+	github.com/nakabonne/nestif v0.3.0 // indirect
+	github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
+	github.com/nishanths/exhaustive v0.1.0 // indirect
+	github.com/nishanths/predeclared v0.2.1 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pelletier/go-toml v1.2.0 // indirect
+	github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/polyfloyd/go-errorlint v0.0.0-20210510181950-ab96adb96fea // indirect
+	github.com/prometheus/client_golang v1.7.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.10.0 // indirect
+	github.com/prometheus/procfs v0.1.3 // indirect
+	github.com/quasilyte/go-ruleguard v0.3.4 // indirect
+	github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect
+	github.com/ryancurrah/gomodguard v1.2.2 // indirect
+	github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect
+	github.com/sanposhiho/wastedassign/v2 v2.0.6 // indirect
+	github.com/securego/gosec/v2 v2.8.0 // indirect
+	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/sonatard/noctx v0.0.1 // indirect
+	github.com/sourcegraph/go-diff v0.6.1 // indirect
+	github.com/spf13/afero v1.1.2 // indirect
+	github.com/spf13/cast v1.3.0 // indirect
+	github.com/spf13/cobra v1.1.3 // indirect
+	github.com/spf13/jwalterweatherman v1.0.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.7.1 // indirect
+	github.com/ssgreg/nlreturn/v2 v2.1.0 // indirect
+	github.com/stretchr/objx v0.1.1 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b // indirect
+	github.com/tetafro/godot v1.4.7 // indirect
+	github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 // indirect
+	github.com/tomarrell/wrapcheck/v2 v2.1.0 // indirect
+	github.com/tommy-muehle/go-mnd/v2 v2.4.0 // indirect
+	github.com/ultraware/funlen v0.0.3 // indirect
+	github.com/ultraware/whitespace v0.0.4 // indirect
+	github.com/uudashr/gocognit v1.0.1 // indirect
+	github.com/yeya24/promlinter v0.1.0 // indirect
+	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
+	golang.org/x/text v0.3.5 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/ini.v1 v1.51.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	honnef.co/go/tools v0.2.0 // indirect
+	mvdan.cc/gofumpt v0.1.1 // indirect
+	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
+	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
+	mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 // indirect
 )
--- a/internal/tools/go.sum
+++ b/internal/tools/go.sum
@ -50,7 +50,6 @@ github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuN
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
 github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
-github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@ -83,7 +82,6 @@ github.com/bkielbasa/cyclop v1.2.0/go.mod h1:qOI0yy6A7dYC4Zgsa72Ppm9kONl0RoIlPbz
 github.com/bombsimon/wsl/v3 v3.3.0 h1:Mka/+kRLoQJq7g2rggtgQsjuI/K5Efd87WX96EWFxjM=
 github.com/bombsimon/wsl/v3 v3.3.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@ -155,10 +153,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
@ -169,17 +165,14 @@ github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR
 github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
 github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
 github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
-github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
 github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
 github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
-github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
 github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
 github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
 github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
-github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
 github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
 github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
-github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
+github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
 github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
 github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
 github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
@ -229,23 +222,18 @@ github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5
 github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
 github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
 github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
-github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
 github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
 github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
-github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
-github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
 github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
 github.com/golangci/golangci-lint v1.41.1 h1:KH28pTSqRu6DTXIAANl1sPXNCmqg4VEH21z6G9Wj4SM=
 github.com/golangci/golangci-lint v1.41.1/go.mod h1:LPtcY3aAAU8wydHrKpnanx9Og8K/cblZSyGmI5CJZUk=
-github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
 github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
 github.com/golangci/misspell v0.3.5 h1:pLzmVdl3VxTOncgzHcvLOKirdvcx/TydsClUQXTehjo=
 github.com/golangci/misspell v0.3.5/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
-github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
 github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5 h1:c9Mqqrm/Clj5biNaG7rABrmwUq88nHh0uABo2b/WYmc=
 github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5/go.mod h1:LK+zW4MpyytAWQRz0M4xnzEk50lSvqDQKfx304apFkY=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
@ -280,6 +268,7 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU=
 github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254 h1:Nb2aRlC404yz7gQIfRZxX9/MLvQiqXyiBTJtgAy6yrI=
@ -353,7 +342,6 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
-github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.2.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@ -361,6 +349,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@ -382,6 +371,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kulti/thelper v0.4.0 h1:2Nx7XbdbE/BYZeoip2mURKUdtHQRuy6Ug+wR7K9ywNM=
 github.com/kulti/thelper v0.4.0/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5exl2U=
@ -466,6 +456,7 @@ github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaP
 github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
 github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 h1:4kuARK6Y6FxaNu/BnU2OAaLF86eTVhP2hjTB6iMvItA=
 github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nishanths/exhaustive v0.1.0 h1:kVlMw8h2LHPMGUVqUj6230oQjjTMFjwcZrnkhXzFfl8=
 github.com/nishanths/exhaustive v0.1.0/go.mod h1:S1j9110vxV1ECdCudXRkeMnFQ/DQk9ajLT0Uf2MYZQQ=
@ -473,6 +464,7 @@ github.com/nishanths/predeclared v0.0.0-20190419143655-18a43bb90ffc/go.mod h1:62
 github.com/nishanths/predeclared v0.2.1 h1:1TXtjmy4f3YCFjTxRd8zcFHOmoUir+gp0ESzjFzG2sw=
 github.com/nishanths/predeclared v0.2.1/go.mod h1:HvkGJcA3naj4lOwnFXFDkFxVtSqQMB9sbB1usJ+xjQE=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
@ -483,9 +475,11 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54=
 github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug=
 github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@ -558,9 +552,7 @@ github.com/securego/gosec/v2 v2.8.0/go.mod h1:hJZ6NT5TqoY+jmOsaxAV4cXoEdrMRLVaNP
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
 github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
-github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/gopsutil/v3 v3.21.5/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
-github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@ -570,7 +562,9 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/sonatard/noctx v0.0.1 h1:VC1Qhl6Oxx9vvWo3UDgrGXYCeKCe3Wbw7qAWL6FrmTY=
@ -626,11 +620,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1
 github.com/tomarrell/wrapcheck/v2 v2.1.0 h1:LTzwrYlgBUwi9JldazhbJN84fN9nS2UNGrZIo2syqxE=
 github.com/tomarrell/wrapcheck/v2 v2.1.0/go.mod h1:crK5eI4RGSUrb9duDTQ5GqcukbKZvi85vX6nbhsBAeI=
 github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
-github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa h1:RC4maTWLKKwb7p1cnoygsbKIgNlJqSYBeAFON3Ar8As=
-github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
 github.com/tommy-muehle/go-mnd/v2 v2.4.0 h1:1t0f8Uiaq+fqKteUR4N9Umr6E99R+lDnLnq7PwX2PPE=
 github.com/tommy-muehle/go-mnd/v2 v2.4.0/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw=
-github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA=
 github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
@ -762,6 +753,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -850,10 +842,8 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190307163923-6a08e3108db3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@ -1023,6 +1013,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
@ -1032,6 +1023,7 @@ gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
@ -1069,4 +1061,3 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
-sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
--- a/pictures/tables.png
+++ b/pictures/tables.png
--- a/sqlscripts/0001-jaeger-index.sql
+++ b/sqlscripts/0001-jaeger-index.sql
@ -1,13 +0,0 @@
-CREATE TABLE IF NOT EXISTS jaeger_index_local (
-                                 timestamp DateTime CODEC(Delta, ZSTD(1)),
-                                 traceID String CODEC(ZSTD(1)),
-                                 service LowCardinality(String) CODEC(ZSTD(1)),
-                                 operation LowCardinality(String) CODEC(ZSTD(1)),
-                                 durationUs UInt64 CODEC(ZSTD(1)),
-                                 tags Array(String) CODEC(ZSTD(1)),
-                                 INDEX idx_tags tags TYPE bloom_filter(0.01) GRANULARITY 64,
-                                 INDEX idx_duration durationUs TYPE minmax GRANULARITY 1
-) ENGINE MergeTree()
-PARTITION BY toDate(timestamp)
-ORDER BY (service, -toUnixTimestamp(timestamp))
-SETTINGS index_granularity=1024
--- a/sqlscripts/0002-jaeger-spans.sql
+++ b/sqlscripts/0002-jaeger-spans.sql
@ -1,8 +0,0 @@
-CREATE TABLE IF NOT EXISTS jaeger_spans_local (
-                                 timestamp DateTime CODEC(Delta, ZSTD(1)),
-                                 traceID String CODEC(ZSTD(1)),
-                                 model String CODEC(ZSTD(3))
-) ENGINE MergeTree()
-PARTITION BY toDate(timestamp)
-ORDER BY traceID
-SETTINGS index_granularity=1024
--- a/sqlscripts/0003-jaeger-operations.sql
+++ b/sqlscripts/0003-jaeger-operations.sql
@ -1,12 +0,0 @@
-CREATE MATERIALIZED VIEW IF NOT EXISTS jaeger_operations_local
-ENGINE SummingMergeTree
-PARTITION BY toYYYYMM(date) ORDER BY (date, service, operation)
-SETTINGS index_granularity=32
-POPULATE
-AS SELECT
-              toDate(timestamp) AS date,
-  service,
-  operation,
-  count() as count
-   FROM jaeger_index
-   GROUP BY date, service, operation
--- a/sqlscripts/0004-jaeger-spans-archive.sql
+++ b/sqlscripts/0004-jaeger-spans-archive.sql
@ -1,8 +0,0 @@
-CREATE TABLE IF NOT EXISTS jaeger_archive_spans_local (
-                                         timestamp DateTime CODEC(Delta, ZSTD(1)),
-                                         traceID String CODEC(ZSTD(1)),
-                                         model String CODEC(ZSTD(3))
-) ENGINE MergeTree()
-PARTITION BY toYYYYMM(timestamp)
-ORDER BY traceID
-SETTINGS index_granularity=1024
--- a/sqlscripts/distributed-table.tmpl.sql
+++ b/sqlscripts/distributed-table.tmpl.sql
@ -0,0 +1,3 @@
+CREATE TABLE IF NOT EXISTS {{.Table}}
+    ON CLUSTER '{cluster}' AS {{.Database}}.{{.Table}}_local
+    ENGINE = Distributed('{cluster}', {{.Database}}, {{.Table}}_local, {{.Hash}})
--- a/sqlscripts/jaeger-index.tmpl.sql
+++ b/sqlscripts/jaeger-index.tmpl.sql
@ -0,0 +1,28 @@
+CREATE TABLE IF NOT EXISTS {{.SpansIndexTable}}
+{{if .Replication}}ON CLUSTER '{cluster}'{{end}}
+(
+    {{if .Multitenant -}}
+    tenant     LowCardinality(String) CODEC (ZSTD(1)),
+    {{- end -}}
+    timestamp  DateTime CODEC (Delta, ZSTD(1)),
+    traceID    String CODEC (ZSTD(1)),
+    service    LowCardinality(String) CODEC (ZSTD(1)),
+    operation  LowCardinality(String) CODEC (ZSTD(1)),
+    durationUs UInt64 CODEC (ZSTD(1)),
+    tags Nested
+    (
+        key LowCardinality(String),
+        value String
+    ) CODEC (ZSTD(1)),
+    INDEX idx_tag_keys tags.key TYPE bloom_filter(0.01) GRANULARITY 64,
+    INDEX idx_duration durationUs TYPE minmax GRANULARITY 1
+) ENGINE {{if .Replication}}ReplicatedMergeTree{{else}}MergeTree(){{end}}
+    {{.TTLTimestamp}}
+    PARTITION BY (
+        {{if .Multitenant -}}
+        tenant,
+        {{- end -}}
+        toDate(timestamp)
+    )
+    ORDER BY (service, -toUnixTimestamp(timestamp))
+    SETTINGS index_granularity = 1024
--- a/sqlscripts/jaeger-operations.tmpl.sql
+++ b/sqlscripts/jaeger-operations.tmpl.sql
@ -0,0 +1,43 @@
+CREATE MATERIALIZED VIEW IF NOT EXISTS {{.OperationsTable}}
+{{if .Replication}}ON CLUSTER '{cluster}'{{end}}
+    ENGINE {{if .Replication}}ReplicatedSummingMergeTree{{else}}SummingMergeTree{{end}}
+    {{.TTLDate}}
+    PARTITION BY (
+        {{if .Multitenant -}}
+        tenant,
+        {{- end -}}
+        toYYYYMM(date)
+    )
+    ORDER BY (
+        {{if .Multitenant -}}
+        tenant,
+        {{- end -}}
+        date,
+        service,
+        operation
+    )
+    SETTINGS index_granularity = 32
+    POPULATE
+AS SELECT
+    {{if .Multitenant -}}
+    tenant,
+    {{- end -}}
+    toDate(timestamp) AS date,
+    service,
+    operation,
+    count() AS count,
+    if(
+        has(tags.key, 'span.kind'),
+        tags.value[indexOf(tags.key, 'span.kind')],
+        ''
+    ) AS spankind
+FROM {{.Database}}.{{.SpansIndexTable}}
+GROUP BY
+    {{if .Multitenant -}}
+    tenant,
+    {{- end -}}
+    date,
+    service,
+    operation,
+    tags.key,
+    tags.value
--- a/sqlscripts/jaeger-spans-archive.tmpl.sql
+++ b/sqlscripts/jaeger-spans-archive.tmpl.sql
@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS {{.SpansArchiveTable}}
+{{if .Replication}}ON CLUSTER '{cluster}'{{end}}
+(
+    {{if .Multitenant -}}
+    tenant    LowCardinality(String) CODEC (ZSTD(1)),
+    {{- end -}}
+    timestamp DateTime CODEC (Delta, ZSTD(1)),
+    traceID   String CODEC (ZSTD(1)),
+    model     String CODEC (ZSTD(3))
+) ENGINE {{if .Replication}}ReplicatedMergeTree{{else}}MergeTree(){{end}}
+    {{.TTLTimestamp}}
+    PARTITION BY (
+        {{if .Multitenant -}}
+        tenant,
+        {{- end -}}
+        toYYYYMM(timestamp)
+    )
+    ORDER BY traceID
+    SETTINGS index_granularity = 1024
--- a/sqlscripts/jaeger-spans.tmpl.sql
+++ b/sqlscripts/jaeger-spans.tmpl.sql
@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS {{.SpansTable}}
+{{if .Replication}}ON CLUSTER '{cluster}'{{end}}
+(
+    {{if .Multitenant -}}
+    tenant    LowCardinality(String) CODEC (ZSTD(1)),
+    {{- end -}}
+    timestamp DateTime CODEC (Delta, ZSTD(1)),
+    traceID   String CODEC (ZSTD(1)),
+    model     String CODEC (ZSTD(3))
+) ENGINE {{if .Replication}}ReplicatedMergeTree{{else}}MergeTree(){{end}}
+    {{.TTLTimestamp}}
+    PARTITION BY (
+        {{if .Multitenant -}}
+        tenant,
+        {{- end -}}
+        toDate(timestamp)
+    )
+    ORDER BY traceID
+    SETTINGS index_granularity = 1024
--- a/storage/clickhousedependencystore/dependencystore_test.go
+++ b/storage/clickhousedependencystore/dependencystore_test.go
@ -0,0 +1,18 @@
+package clickhousedependencystore
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDependencyStore_GetDependencies(t *testing.T) {
+	dependencyStore := NewDependencyStore()
+
+	dependencies, err := dependencyStore.GetDependencies(context.Background(), time.Now(), time.Hour)
+
+	assert.EqualError(t, err, errNotImplemented.Error())
+	assert.Nil(t, dependencies)
+}
--- a/storage/clickhousespanstore/heap.go
+++ b/storage/clickhousespanstore/heap.go
@ -0,0 +1,85 @@
+package clickhousespanstore
+
+import (
+	"container/heap"
+	"fmt"
+	"time"
+)
+
+var (
+	_                 heap.Interface = workerHeap{}
+	errWorkerNotFound                = fmt.Errorf("worker not found in heap")
+)
+
+type heapItem struct {
+	pushTime time.Time
+	worker   *WriteWorker
+}
+
+// workerHeap is a heap for WriteWorkers where worker's push time is the key.
+type workerHeap struct {
+	elems   *[]*heapItem
+	indexes map[*WriteWorker]int
+}
+
+func newWorkerHeap(cap int) workerHeap {
+	elems := make([]*heapItem, 0, cap)
+	return workerHeap{
+		elems:   &elems,
+		indexes: make(map[*WriteWorker]int),
+	}
+}
+
+func (workerHeap workerHeap) AddWorker(worker *WriteWorker) {
+	heap.Push(workerHeap, heapItem{
+		worker:   worker,
+		pushTime: time.Now(),
+	})
+}
+
+func (workerHeap *workerHeap) RemoveWorker(worker *WriteWorker) error {
+	idx, ok := workerHeap.indexes[worker]
+	if !ok {
+		return errWorkerNotFound
+	}
+	heap.Remove(workerHeap, idx)
+	return nil
+}
+
+func (workerHeap *workerHeap) CloseWorkers() {
+	for _, item := range *workerHeap.elems {
+		item.worker.Close()
+	}
+}
+
+func (workerHeap workerHeap) Len() int {
+	return len(*workerHeap.elems)
+}
+
+func (workerHeap workerHeap) Less(i, j int) bool {
+	return (*workerHeap.elems)[i].pushTime.Before((*workerHeap.elems)[j].pushTime)
+}
+
+func (workerHeap workerHeap) Swap(i, j int) {
+	(*workerHeap.elems)[i], (*workerHeap.elems)[j] = (*workerHeap.elems)[j], (*workerHeap.elems)[i]
+	workerHeap.indexes[(*workerHeap.elems)[i].worker] = i
+	workerHeap.indexes[(*workerHeap.elems)[j].worker] = j
+}
+
+func (workerHeap workerHeap) Push(x interface{}) {
+	switch t := x.(type) {
+	case heapItem:
+		*workerHeap.elems = append(*workerHeap.elems, &t)
+		workerHeap.indexes[t.worker] = len(*workerHeap.elems) - 1
+	default:
+		panic("Unknown type")
+	}
+}
+
+func (workerHeap workerHeap) Pop() interface{} {
+	lastInd := len(*workerHeap.elems) - 1
+	last := (*workerHeap.elems)[lastInd]
+	delete(workerHeap.indexes, last.worker)
+	*workerHeap.elems = (*workerHeap.elems)[:lastInd]
+	return last.worker
+}
--- a/storage/clickhousespanstore/mocks/converter.go
+++ b/storage/clickhousespanstore/mocks/converter.go
@ -0,0 +1,40 @@
+package mocks
+
+import (
+	"database/sql/driver"
+	"fmt"
+	"time"
+
+	"github.com/jaegertracing/jaeger/model"
+)
+
+var _ driver.ValueConverter = ConverterMock{}
+
+type ConverterMock struct{}
+
+func (conv ConverterMock) ConvertValue(v interface{}) (driver.Value, error) {
+	switch t := v.(type) {
+	case model.TraceID:
+		return driver.Value(t.String()), nil
+	case time.Time:
+		return driver.Value(t), nil
+	case time.Duration:
+		return driver.Value(t.Nanoseconds()), nil
+	case model.SpanID:
+		return driver.Value(t), nil
+	case string:
+		return driver.Value(t), nil
+	case []uint8:
+		return driver.Value(t), nil
+	case int64:
+		return driver.Value(t), nil
+	case uint64:
+		return driver.Value(t), nil
+	case int:
+		return driver.Value(t), nil
+	case []string:
+		return driver.Value(fmt.Sprint(t)), nil
+	default:
+		return nil, fmt.Errorf("unknown type %T", t)
+	}
+}
--- a/storage/clickhousespanstore/mocks/converter_test.go
+++ b/storage/clickhousespanstore/mocks/converter_test.go
@ -0,0 +1,63 @@
+package mocks
+
+import (
+	"database/sql/driver"
+	"testing"
+	"time"
+
+	"github.com/jaegertracing/jaeger/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConverterMock_ConvertValue(t *testing.T) {
+	converter := ConverterMock{}
+
+	testCases := map[string]struct {
+		valueToConvert interface{}
+		expectedResult driver.Value
+	}{
+		"string value":       {valueToConvert: "some string value", expectedResult: driver.Value("some string value")},
+		"string slice value": {valueToConvert: []string{"some", "slice", "of", "strings"}, expectedResult: driver.Value("[some slice of strings]")},
+		"time value": {
+			valueToConvert: time.Date(2002, time.February, 19, 14, 43, 51, 0, time.UTC),
+			expectedResult: driver.Value(time.Date(2002, time.February, 19, 14, 43, 51, 0, time.UTC)),
+		},
+		"duration value": {
+			valueToConvert: time.Unix(12340, 123456789).Sub(time.Unix(0, 0)),
+			expectedResult: driver.Value(int64(12340123456789)),
+		},
+		"int64 value":         {valueToConvert: int64(1823), expectedResult: driver.Value(int64(1823))},
+		"int value":           {valueToConvert: 1823, expectedResult: driver.Value(1823)},
+		"model.SpanID value":  {valueToConvert: model.SpanID(318148), expectedResult: driver.Value(model.SpanID(318148))},
+		"model.TraceID value": {valueToConvert: model.TraceID{Low: 0xabd5, High: 0xa31}, expectedResult: driver.Value("0000000000000a31000000000000abd5")},
+		"uint8 slice value":   {valueToConvert: []uint8("asdkja"), expectedResult: driver.Value([]uint8{0x61, 0x73, 0x64, 0x6b, 0x6a, 0x61})},
+	}
+
+	for name, test := range testCases {
+		t.Run(name, func(t *testing.T) {
+			converted, err := converter.ConvertValue(test.valueToConvert)
+			assert.NoError(t, err)
+			assert.Equal(t, test.expectedResult, converted)
+		})
+	}
+}
+
+func TestConverterMock_Fail(t *testing.T) {
+	converter := ConverterMock{}
+
+	tests := map[string]struct {
+		valueToConvert   interface{}
+		expectedErrorMsg string
+	}{
+		"float64 value": {valueToConvert: float64(1e-4), expectedErrorMsg: "unknown type float64"},
+		"int32 value":   {valueToConvert: int32(12831), expectedErrorMsg: "unknown type int32"},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			val, err := converter.ConvertValue(test.valueToConvert)
+			assert.Equal(t, nil, val)
+			assert.EqualError(t, err, test.expectedErrorMsg)
+		})
+	}
+}
--- a/storage/clickhousespanstore/mocks/db.go
+++ b/storage/clickhousespanstore/mocks/db.go
@ -0,0 +1,14 @@
+package mocks
+
+import (
+	"database/sql"
+
+	sqlmock "github.com/DATA-DOG/go-sqlmock"
+)
+
+func GetDbMock() (*sql.DB, sqlmock.Sqlmock, error) {
+	return sqlmock.New(
+		sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual),
+		sqlmock.ValueConverterOption(ConverterMock{}),
+	)
+}
--- a/storage/clickhousespanstore/mocks/spylogger.go
+++ b/storage/clickhousespanstore/mocks/spylogger.go
@ -0,0 +1,115 @@
+package mocks
+
+import (
+	"io"
+	"log"
+	"testing"
+
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/stretchr/testify/assert"
+)
+
+const levelCount = 5
+
+var _ hclog.Logger = SpyLogger{}
+
+type LogMock struct {
+	Msg  string
+	Args []interface{}
+}
+
+type SpyLogger struct {
+	logs [][]LogMock
+}
+
+func NewSpyLogger() SpyLogger {
+	return SpyLogger{logs: make([][]LogMock, levelCount)}
+}
+
+func (logger *SpyLogger) AssertLogsOfLevelEqual(t *testing.T, level hclog.Level, want []LogMock) {
+	assert.Equal(t, want, logger.getLogs(level))
+}
+
+func (logger *SpyLogger) getLogs(level hclog.Level) []LogMock {
+	return logger.logs[level-1]
+}
+
+func (logger *SpyLogger) AssertLogsEmpty(t *testing.T) {
+	assert.Equal(t, logger.logs, make([][]LogMock, levelCount))
+}
+
+func (logger SpyLogger) Log(level hclog.Level, msg string, args ...interface{}) {
+	logger.logs[level-1] = append(logger.getLogs(level), LogMock{msg, args})
+}
+
+func (logger SpyLogger) Trace(msg string, args ...interface{}) {
+	logger.Log(hclog.Trace, msg, args...)
+}
+
+func (logger SpyLogger) Debug(msg string, args ...interface{}) {
+	logger.Log(hclog.Debug, msg, args...)
+}
+
+func (logger SpyLogger) Info(msg string, args ...interface{}) {
+	logger.Log(hclog.Info, msg, args...)
+}
+
+func (logger SpyLogger) Warn(msg string, args ...interface{}) {
+	logger.Log(hclog.Warn, msg, args...)
+}
+
+func (logger SpyLogger) Error(msg string, args ...interface{}) {
+	logger.Log(hclog.Error, msg, args...)
+}
+
+func (logger SpyLogger) IsTrace() bool {
+	panic("implement me")
+}
+
+func (logger SpyLogger) IsDebug() bool {
+	panic("implement me")
+}
+
+func (logger SpyLogger) IsInfo() bool {
+	panic("implement me")
+}
+
+func (logger SpyLogger) IsWarn() bool {
+	panic("implement me")
+}
+
+func (logger SpyLogger) IsError() bool {
+	panic("implement me")
+}
+
+func (logger SpyLogger) ImpliedArgs() []interface{} {
+	panic("implement me")
+}
+
+func (logger SpyLogger) With(args ...interface{}) hclog.Logger {
+	panic("implement me")
+}
+
+func (logger SpyLogger) Name() string {
+	return "spy logger"
+}
+
+func (logger SpyLogger) Named(name string) hclog.Logger {
+	panic("implement me")
+}
+
+func (logger SpyLogger) ResetNamed(name string) hclog.Logger {
+	panic("implement me")
+}
+
+func (logger SpyLogger) SetLevel(level hclog.Level) {
+	panic("implement me")
+}
+
+func (logger SpyLogger) StandardLogger(opts *hclog.StandardLoggerOptions) *log.Logger {
+	panic("implement me")
+}
+
+func (logger SpyLogger) StandardWriter(opts *hclog.StandardLoggerOptions) io.Writer {
+	panic("implement me")
+}
--- a/storage/clickhousespanstore/mocks/spylogger_test.go
+++ b/storage/clickhousespanstore/mocks/spylogger_test.go
@ -0,0 +1,170 @@
+package mocks
+
+import (
+	"math/rand"
+	"strconv"
+	"testing"
+
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	maxLogCount = 80
+	maxArgCount = 10
+)
+
+func TestSpyLogger_AssertLogsEmpty(t *testing.T) {
+	logger := NewSpyLogger()
+	logger.AssertLogsEmpty(t)
+}
+
+func TestSpyLogger_AssertLogsOfLevelEqualNoArgs(t *testing.T) {
+	logger := NewSpyLogger()
+	var logs = make([][]LogMock, levelCount)
+	for level, levelLogs := range logs {
+		logsCount := rand.Intn(maxLogCount)
+		for i := 0; i < logsCount; i++ {
+			msg := "msg" + strconv.FormatUint(rand.Uint64(), 10)
+			levelLogs = append(levelLogs, LogMock{Msg: msg})
+			logger.Log(hclog.Level(level+1), msg)
+		}
+		logs[level] = levelLogs
+	}
+
+	for level, levelLogs := range logs {
+		logger.AssertLogsOfLevelEqual(t, hclog.Level(level+1), levelLogs)
+	}
+}
+
+func TestSpyLogger_AssertLogsOfLevelEqualArgs(t *testing.T) {
+	logger := NewSpyLogger()
+	var logs = make([][]LogMock, levelCount)
+	for level, levelLogs := range logs {
+		logsCount := rand.Intn(maxLogCount)
+		for i := 0; i < logsCount; i++ {
+			msg := "msg" + strconv.FormatUint(rand.Uint64(), 10)
+			args := generateArgs(rand.Intn(maxArgCount))
+			levelLogs = append(levelLogs, LogMock{Msg: msg, Args: args})
+			logger.Log(hclog.Level(level+1), msg, args...)
+		}
+		logs[level] = levelLogs
+	}
+
+	for level, levelLogs := range logs {
+		logger.AssertLogsOfLevelEqual(t, hclog.Level(level+1), levelLogs)
+	}
+}
+
+func TestSpyLogger_Trace(t *testing.T) {
+	logger := NewSpyLogger()
+	logsCount := rand.Intn(maxLogCount)
+	logs := make([]LogMock, 0, logsCount)
+	for i := 0; i < logsCount; i++ {
+		msg := "msg" + strconv.FormatUint(rand.Uint64(), 10)
+		args := generateArgs(rand.Intn(maxArgCount))
+		logs = append(logs, LogMock{Msg: msg, Args: args})
+		logger.Trace(msg, args...)
+	}
+
+	logger.AssertLogsOfLevelEqual(t, hclog.Trace, logs)
+}
+
+func TestSpyLogger_Debug(t *testing.T) {
+	logger := NewSpyLogger()
+	logsCount := rand.Intn(maxLogCount)
+	logs := make([]LogMock, 0, logsCount)
+	for i := 0; i < logsCount; i++ {
+		msg := "msg" + strconv.FormatUint(rand.Uint64(), 10)
+		args := generateArgs(rand.Intn(maxArgCount))
+		logs = append(logs, LogMock{Msg: msg, Args: args})
+		logger.Debug(msg, args...)
+	}
+
+	logger.AssertLogsOfLevelEqual(t, hclog.Debug, logs)
+}
+
+func TestSpyLogger_Info(t *testing.T) {
+	logger := NewSpyLogger()
+	logsCount := rand.Intn(maxLogCount)
+	logs := make([]LogMock, 0, logsCount)
+	for i := 0; i < logsCount; i++ {
+		msg := "msg" + strconv.FormatUint(rand.Uint64(), 10)
+		args := generateArgs(rand.Intn(maxArgCount))
+		logs = append(logs, LogMock{Msg: msg, Args: args})
+		logger.Info(msg, args...)
+	}
+
+	logger.AssertLogsOfLevelEqual(t, hclog.Info, logs)
+}
+
+func TestSpyLogger_Warn(t *testing.T) {
+	logger := NewSpyLogger()
+	logsCount := rand.Intn(maxLogCount)
+	logs := make([]LogMock, 0, logsCount)
+	for i := 0; i < logsCount; i++ {
+		msg := "msg" + strconv.FormatUint(rand.Uint64(), 10)
+		args := generateArgs(rand.Intn(maxArgCount))
+		logs = append(logs, LogMock{Msg: msg, Args: args})
+		logger.Warn(msg, args...)
+	}
+
+	logger.AssertLogsOfLevelEqual(t, hclog.Warn, logs)
+}
+
+func TestSpyLogger_Error(t *testing.T) {
+	logger := NewSpyLogger()
+	logsCount := rand.Intn(maxLogCount)
+	logs := make([]LogMock, 0, logsCount)
+	for i := 0; i < logsCount; i++ {
+		msg := "msg" + strconv.FormatUint(rand.Uint64(), 10)
+		args := generateArgs(rand.Intn(maxArgCount))
+		logs = append(logs, LogMock{Msg: msg, Args: args})
+		logger.Error(msg, args...)
+	}
+
+	logger.AssertLogsOfLevelEqual(t, hclog.Error, logs)
+}
+
+func TestSpyLogger_Name(t *testing.T) {
+	assert.Equal(t, "spy logger", NewSpyLogger().Name())
+}
+
+func TestNotImplemented(t *testing.T) {
+	logger := NewSpyLogger()
+
+	tests := map[string]struct {
+		function assert.PanicTestFunc
+	}{
+		"is_trace":        {function: func() { _ = logger.IsTrace() }},
+		"is_debug":        {function: func() { _ = logger.IsDebug() }},
+		"is_info":         {function: func() { _ = logger.IsInfo() }},
+		"is_warn":         {function: func() { _ = logger.IsWarn() }},
+		"is_error":        {function: func() { _ = logger.IsError() }},
+		"implied_args":    {function: func() { _ = logger.ImpliedArgs() }},
+		"with":            {function: func() { _ = logger.With() }},
+		"named":           {function: func() { _ = logger.Named("") }},
+		"reset_named":     {function: func() { _ = logger.ResetNamed("") }},
+		"set_level":       {function: func() { logger.SetLevel(hclog.NoLevel) }},
+		"standard_logger": {function: func() { _ = logger.StandardLogger(nil) }},
+		"standard_writer": {function: func() { _ = logger.StandardWriter(nil) }},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			assert.Panics(t, test.function, "implement me")
+		})
+	}
+}
+
+func generateArgs(count int) []interface{} {
+	args := make([]interface{}, 0, 2*count)
+	for j := 0; j < count; j++ {
+		args = append(
+			args,
+			"key"+strconv.FormatUint(rand.Uint64(), 10),
+			"value"+strconv.FormatUint(rand.Uint64(), 10),
+		)
+	}
+	return args
+}
--- a/storage/clickhousespanstore/params.go
+++ b/storage/clickhousespanstore/params.go
@ -0,0 +1,19 @@
+package clickhousespanstore
+
+import (
+	"database/sql"
+	"time"
+
+	hclog "github.com/hashicorp/go-hclog"
+)
+
+// WorkerParams contains parameters that are shared between WriteWorkers
+type WorkerParams struct {
+	logger     hclog.Logger
+	db         *sql.DB
+	indexTable TableName
+	spansTable TableName
+	tenant     string
+	encoding   Encoding
+	delay      time.Duration
+}
--- a/storage/clickhousespanstore/pool.go
+++ b/storage/clickhousespanstore/pool.go
@ -0,0 +1,131 @@
+package clickhousespanstore
+
+import (
+	"math"
+	"sync"
+
+	"github.com/jaegertracing/jaeger/model"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	numDiscardedSpans = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "jaeger_clickhouse_discarded_spans",
+		Help: "Count of spans that have been discarded due to pending writes exceeding max_span_count",
+	})
+	numPendingSpans = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "jaeger_clickhouse_pending_spans",
+		Help: "Number of spans that are currently pending, counts against max_span_count",
+	})
+)
+
+// WriteWorkerPool is a worker pool for writing batches of spans.
+// Given a new batch, WriteWorkerPool creates a new WriteWorker.
+// If the number of currently processed spans if more than maxSpanCount, then the oldest worker is removed.
+type WriteWorkerPool struct {
+	params *WorkerParams
+
+	finish  chan bool
+	done    sync.WaitGroup
+	batches chan []*model.Span
+
+	maxSpanCount int
+	mutex        sync.Mutex
+	workers      workerHeap
+	workerDone   chan *WriteWorker
+}
+
+var registerPoolMetrics sync.Once
+
+func NewWorkerPool(params *WorkerParams, maxSpanCount int) WriteWorkerPool {
+	registerPoolMetrics.Do(func() {
+		prometheus.MustRegister(numDiscardedSpans, numPendingSpans)
+	})
+
+	return WriteWorkerPool{
+		params:  params,
+		finish:  make(chan bool),
+		done:    sync.WaitGroup{},
+		batches: make(chan []*model.Span),
+
+		mutex:      sync.Mutex{},
+		workers:    newWorkerHeap(100),
+		workerDone: make(chan *WriteWorker),
+
+		maxSpanCount: maxSpanCount,
+	}
+}
+
+func (pool *WriteWorkerPool) Work() {
+	finish := false
+	nextWorkerID := int32(1)
+	pendingSpanCount := 0
+	for {
+		// Initialize to zero, or update value from previous loop
+		numPendingSpans.Set(float64(pendingSpanCount))
+
+		pool.done.Add(1)
+		select {
+		case batch := <-pool.batches:
+			batchSize := len(batch)
+			if pool.checkLimit(pendingSpanCount, batchSize) {
+				// Limit disabled or batch fits within limit, write the batch.
+				worker := WriteWorker{
+					workerID: nextWorkerID,
+
+					params: pool.params,
+					batch:  batch,
+
+					finish:     make(chan bool),
+					workerDone: pool.workerDone,
+					done:       sync.WaitGroup{},
+				}
+				if nextWorkerID == math.MaxInt32 {
+					nextWorkerID = 1
+				} else {
+					nextWorkerID++
+				}
+				pool.workers.AddWorker(&worker)
+				pendingSpanCount += batchSize
+				go worker.Work()
+			} else {
+				// Limit exceeded, complain
+				numDiscardedSpans.Add(float64(batchSize))
+				pool.params.logger.Error("Discarding batch of spans due to exceeding pending span count", "batch_size", batchSize, "pending_span_count", pendingSpanCount, "max_span_count", pool.maxSpanCount)
+			}
+		case worker := <-pool.workerDone:
+			// The worker has finished, subtract its work from the count and clean it from the heap.
+			pendingSpanCount -= len(worker.batch)
+			if err := pool.workers.RemoveWorker(worker); err != nil {
+				pool.params.logger.Error("could not remove worker", "worker", worker, "error", err)
+			}
+		case <-pool.finish:
+			pool.workers.CloseWorkers()
+			finish = true
+		}
+		pool.done.Done()
+
+		if finish {
+			break
+		}
+	}
+}
+
+func (pool *WriteWorkerPool) WriteBatch(batch []*model.Span) {
+	pool.batches <- batch
+}
+
+func (pool *WriteWorkerPool) Close() {
+	pool.finish <- true
+	pool.done.Wait()
+}
+
+// checkLimit returns whether batchSize fits within the maxSpanCount
+func (pool *WriteWorkerPool) checkLimit(pendingSpanCount int, batchSize int) bool {
+	if pool.maxSpanCount <= 0 {
+		return true
+	}
+
+	// Check limit, add batchSize if within limit
+	return pendingSpanCount+batchSize <= pool.maxSpanCount
+}
--- a/storage/clickhousespanstore/reader.go
+++ b/storage/clickhousespanstore/reader.go
@ -10,10 +10,9 @@ import (
 	"time"

 	"github.com/gogo/protobuf/proto"
-	"github.com/opentracing/opentracing-go"
-
 	"github.com/jaegertracing/jaeger/model"
 	"github.com/jaegertracing/jaeger/storage/spanstore"
+	opentracing "github.com/opentracing/opentracing-go"
 )

 const (
@ -31,20 +30,24 @@ var (
 // TraceReader for reading spans from ClickHouse
 type TraceReader struct {
 	db              *sql.DB
-	operationsTable string
-	indexTable      string
-	spansTable      string
+	operationsTable TableName
+	indexTable      TableName
+	spansTable      TableName
+	tenant          string
+	maxNumSpans     uint
 }

 var _ spanstore.Reader = (*TraceReader)(nil)

 // NewTraceReader returns a TraceReader for the database
-func NewTraceReader(db *sql.DB, operationsTable, indexTable, spansTable string) *TraceReader {
+func NewTraceReader(db *sql.DB, operationsTable, indexTable, spansTable TableName, tenant string, maxNumSpans uint) *TraceReader {
 	return &TraceReader{
 		db:              db,
 		operationsTable: operationsTable,
 		indexTable:      indexTable,
 		spansTable:      spansTable,
+		tenant:          tenant,
+		maxNumSpans:     maxNumSpans,
 	}
 }

@ -58,20 +61,29 @@ func (r *TraceReader) getTraces(ctx context.Context, traceIDs []model.TraceID) (
 	span, _ := opentracing.StartSpanFromContext(ctx, "getTraces")
 	defer span.Finish()

-	values := make([]interface{}, len(traceIDs))
+	args := make([]interface{}, len(traceIDs))
 	for i, traceID := range traceIDs {
-		values[i] = traceID.String()
+		args[i] = traceID.String()
 	}

-	// It's more efficient to do PREWHERE on traceID to then only read needed models:
+	// It's more efficient to do PREWHERE on traceID to the only read needed models:
 	// * https://clickhouse.tech/docs/en/sql-reference/statements/select/prewhere/
 	//nolint:gosec  , G201: SQL string formatting
-	query := fmt.Sprintf("SELECT model FROM %s PREWHERE traceID IN (%s)", r.spansTable, "?"+strings.Repeat(",?", len(values)-1))
+	query := fmt.Sprintf("SELECT model FROM %s PREWHERE traceID IN (%s)", r.spansTable, "?"+strings.Repeat(",?", len(traceIDs)-1))
+
+	if r.tenant != "" {
+		query += " AND tenant = ?"
+		args = append(args, r.tenant)
+	}
+
+	if r.maxNumSpans > 0 {
+		query += fmt.Sprintf(" ORDER BY timestamp LIMIT %d BY traceID", r.maxNumSpans)
+	}

 	span.SetTag("db.statement", query)
-	span.SetTag("db.args", values)
+	span.SetTag("db.args", args)

-	rows, err := r.db.QueryContext(ctx, query, values...)
+	rows, err := r.db.QueryContext(ctx, query, args...)
 	if err != nil {
 		return nil, err
 	}
@ -145,7 +157,7 @@ func (r *TraceReader) getStrings(ctx context.Context, sql string, args ...interf

 	defer rows.Close()

-	values := []string{}
+	values := make([]string, 0)

 	for rows.Next() {
 		var value string
@ -171,11 +183,19 @@ func (r *TraceReader) GetServices(ctx context.Context) ([]string, error) {
 		return nil, errNoOperationsTable
 	}

-	query := fmt.Sprintf("SELECT service FROM %s GROUP BY service", r.operationsTable)
+	query := fmt.Sprintf("SELECT service FROM %s", r.operationsTable)
+	args := make([]interface{}, 0)

+	if r.tenant != "" {
+		query += " WHERE tenant = ?"
+		args = append(args, r.tenant)
+	}
+
+	query += " GROUP BY service"
 	span.SetTag("db.statement", query)
+	span.SetTag("db.args", args)

-	return r.getStrings(ctx, query)
+	return r.getStrings(ctx, query, args...)
 }

 // GetOperations fetches operations in the service and empty slice if service does not exists
@ -190,20 +210,44 @@ func (r *TraceReader) GetOperations(
 		return nil, errNoOperationsTable
 	}

-	query := fmt.Sprintf("SELECT operation FROM %s WHERE service = ? GROUP BY operation", r.operationsTable)
-	args := []interface{}{params.ServiceName}
+	//nolint:gosec  , G201: SQL string formatting
+	query := fmt.Sprintf("SELECT operation, spankind FROM %s WHERE", r.operationsTable)
+	args := make([]interface{}, 0)
+
+	if r.tenant != "" {
+		query += " tenant = ? AND"
+		args = append(args, r.tenant)
+	}
+
+	query += " service = ? GROUP BY operation, spankind ORDER BY operation"
+	args = append(args, params.ServiceName)

 	span.SetTag("db.statement", query)
 	span.SetTag("db.args", args)

-	names, err := r.getStrings(ctx, query, args...)
+	rows, err := r.db.QueryContext(ctx, query, args...)
 	if err != nil {
 		return nil, err
 	}

-	operations := make([]spanstore.Operation, len(names))
-	for i, name := range names {
-		operations[i].Name = name
+	defer rows.Close()
+
+	operations := make([]spanstore.Operation, 0)
+
+	for rows.Next() {
+		var name, spanKind string
+		if err := rows.Scan(&name, &spanKind); err != nil {
+			return nil, err
+		}
+		operation := spanstore.Operation{Name: name}
+		if spanKind != "" {
+			operation.SpanKind = spanKind
+		}
+		operations = append(operations, operation)
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, err
 	}

 	return operations, nil
@ -251,7 +295,7 @@ func (r *TraceReader) FindTraceIDs(ctx context.Context, params *spanstore.TraceQ
 		timeSpan = minTimespanForProgressiveSearch
 	}

-	found := []model.TraceID{}
+	found := make([]model.TraceID, 0)

 	for step := 0; step < maxProgressiveSteps; step++ {
 		if len(found) >= params.NumTraces {
@ -290,7 +334,7 @@ func (r *TraceReader) findTraceIDsInRange(ctx context.Context, params *spanstore
 	span, ctx := opentracing.StartSpanFromContext(ctx, "findTraceIDsInRange")
 	defer span.Finish()

-	if end.Before(start) || end.UTC() == start.UTC() {
+	if end.Before(start) || end == start {
 		return []model.TraceID{}, nil
 	}

@ -303,16 +347,18 @@ func (r *TraceReader) findTraceIDsInRange(ctx context.Context, params *spanstore
 	query := fmt.Sprintf("SELECT DISTINCT traceID FROM %s WHERE service = ?", r.indexTable)
 	args := []interface{}{params.ServiceName}

+	if r.tenant != "" {
+		query += " AND tenant = ?"
+		args = append(args, r.tenant)
+	}
+
 	if params.OperationName != "" {
 		query += " AND operation = ?"
 		args = append(args, params.OperationName)
 	}

-	query += " AND -toUnixTimestamp(timestamp) <= -toUnixTimestamp(?)"
-	args = append(args, start)
-
-	query += " AND -toUnixTimestamp(timestamp) >= -toUnixTimestamp(?)"
-	args = append(args, end)
+	query += " AND timestamp >= ? AND timestamp <= ?"
+	args = append(args, start, end)

 	if params.DurationMin != 0 {
 		query += " AND durationUs >= ?"
@ -325,8 +371,8 @@ func (r *TraceReader) findTraceIDsInRange(ctx context.Context, params *spanstore
 	}

 	for key, value := range params.Tags {
-		query += " AND has(tags, ?)"
-		args = append(args, fmt.Sprintf("%s=%s", key, value))
+		query += " AND has(tags.key, ?) AND has(splitByChar(',', tags.value[indexOf(tags.key, ?)]), ?)"
+		args = append(args, key, key, value)
 	}

 	if len(skip) > 0 {
@ -338,7 +384,7 @@ func (r *TraceReader) findTraceIDsInRange(ctx context.Context, params *spanstore

 	// Sorting by service is required for early termination of primary key scan:
 	// * https://github.com/ClickHouse/ClickHouse/issues/7102
-	query += " ORDER BY service, -toUnixTimestamp(timestamp) LIMIT ?"
+	query += " ORDER BY service, timestamp DESC LIMIT ?"
 	args = append(args, params.NumTraces-len(skip))

 	span.SetTag("db.statement", query)
--- a/storage/clickhousespanstore/reader_test.go
+++ b/storage/clickhousespanstore/reader_test.go
--- a/storage/clickhousespanstore/tablename.go
+++ b/storage/clickhousespanstore/tablename.go
@ -0,0 +1,7 @@
+package clickhousespanstore
+
+type TableName string
+
+func (tableName TableName) ToLocal() TableName {
+	return tableName + "_local"
+}
--- a/storage/clickhousespanstore/tablename_test.go
+++ b/storage/clickhousespanstore/tablename_test.go
@ -0,0 +1,13 @@
+package clickhousespanstore
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTableName_ToLocal(t *testing.T) {
+	tableName := TableName("some_table")
+	assert.Equal(t, tableName+"_local", tableName.ToLocal())
+
+}
--- a/storage/clickhousespanstore/worker.go
+++ b/storage/clickhousespanstore/worker.go
@ -0,0 +1,274 @@
+package clickhousespanstore
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/jaegertracing/jaeger/model"
+)
+
+var delays = []int{2, 3, 5, 8}
+
+// WriteWorker writes spans to CLickHouse.
+// Given a batch of spans, WriteWorker attempts to write them to database.
+// Interval in seconds between attempts changes due to delays slice, then it remains the same as the last value in delays.
+type WriteWorker struct {
+	// workerID is an arbitrary identifier for keeping track of this worker in logs
+	workerID   int32
+	params     *WorkerParams
+	batch      []*model.Span
+	finish     chan bool
+	workerDone chan *WriteWorker
+	done       sync.WaitGroup
+}
+
+func (worker *WriteWorker) Work() {
+	worker.done.Add(1)
+
+	defer worker.done.Done()
+
+	// TODO: look for specific error(connection refused | database error)
+	if err := worker.writeBatch(worker.batch); err != nil {
+		worker.params.logger.Error("Could not write a batch of spans", "error", err, "worker_id", worker.workerID)
+	} else {
+		worker.close()
+		return
+	}
+	attempt := 0
+	for {
+		currentDelay := worker.getCurrentDelay(&attempt, worker.params.delay)
+		timer := time.After(currentDelay)
+		select {
+		case <-worker.finish:
+			worker.close()
+			return
+		case <-timer:
+			if err := worker.writeBatch(worker.batch); err != nil {
+				worker.params.logger.Error("Could not write a batch of spans", "error", err, "worker_id", worker.workerID)
+			} else {
+				worker.close()
+				return
+			}
+		}
+	}
+}
+
+func (worker *WriteWorker) Close() {
+	worker.finish <- true
+	worker.done.Wait()
+}
+
+func (worker *WriteWorker) getCurrentDelay(attempt *int, delay time.Duration) time.Duration {
+	if *attempt < len(delays) {
+		*attempt++
+	}
+	return time.Duration(int64(delays[*attempt-1]) * delay.Nanoseconds())
+}
+
+func (worker *WriteWorker) close() {
+	worker.workerDone <- worker
+}
+
+func (worker *WriteWorker) writeBatch(batch []*model.Span) error {
+	worker.params.logger.Debug("Writing spans", "size", len(batch))
+	if err := worker.writeModelBatch(batch); err != nil {
+		return err
+	}
+
+	if worker.params.indexTable != "" {
+		if err := worker.writeIndexBatch(batch); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (worker *WriteWorker) writeModelBatch(batch []*model.Span) error {
+	tx, err := worker.params.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	committed := false
+
+	defer func() {
+		if !committed {
+			// Clickhouse does not support real rollback
+			_ = tx.Rollback()
+		}
+	}()
+
+	var query string
+	if worker.params.tenant == "" {
+		query = fmt.Sprintf("INSERT INTO %s (timestamp, traceID, model) VALUES (?, ?, ?)", worker.params.spansTable)
+	} else {
+		query = fmt.Sprintf("INSERT INTO %s (tenant, timestamp, traceID, model) VALUES (?, ?, ?, ?)", worker.params.spansTable)
+	}
+
+	statement, err := tx.Prepare(query)
+	if err != nil {
+		return err
+	}
+
+	defer statement.Close()
+
+	for _, span := range batch {
+		var serialized []byte
+
+		if worker.params.encoding == EncodingJSON {
+			serialized, err = json.Marshal(span)
+		} else {
+			serialized, err = proto.Marshal(span)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if worker.params.tenant == "" {
+			_, err = statement.Exec(span.StartTime, span.TraceID.String(), serialized)
+		} else {
+			_, err = statement.Exec(worker.params.tenant, span.StartTime, span.TraceID.String(), serialized)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	committed = true
+
+	return tx.Commit()
+}
+
+func (worker *WriteWorker) writeIndexBatch(batch []*model.Span) error {
+	tx, err := worker.params.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	committed := false
+
+	defer func() {
+		if !committed {
+			// Clickhouse does not support real rollback
+			_ = tx.Rollback()
+		}
+	}()
+
+	var query string
+	if worker.params.tenant == "" {
+		query = fmt.Sprintf(
+			"INSERT INTO %s (timestamp, traceID, service, operation, durationUs, tags.key, tags.value) VALUES (?, ?, ?, ?, ?, ?, ?)",
+			worker.params.indexTable,
+		)
+	} else {
+		query = fmt.Sprintf(
+			"INSERT INTO %s (tenant, timestamp, traceID, service, operation, durationUs, tags.key, tags.value) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+			worker.params.indexTable,
+		)
+	}
+
+	statement, err := tx.Prepare(query)
+	if err != nil {
+		return err
+	}
+
+	defer statement.Close()
+
+	for _, span := range batch {
+		keys, values := uniqueTagsForSpan(span)
+		if worker.params.tenant == "" {
+			_, err = statement.Exec(
+				span.StartTime,
+				span.TraceID.String(),
+				span.Process.ServiceName,
+				span.OperationName,
+				uint64(span.Duration.Microseconds()),
+				keys,
+				values,
+			)
+		} else {
+			_, err = statement.Exec(
+				worker.params.tenant,
+				span.StartTime,
+				span.TraceID.String(),
+				span.Process.ServiceName,
+				span.OperationName,
+				uint64(span.Duration.Microseconds()),
+				keys,
+				values,
+			)
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	committed = true
+
+	return tx.Commit()
+}
+
+func uniqueTagsForSpan(span *model.Span) (keys, values []string) {
+	uniqueTags := make(map[string][]string, len(span.Tags)+len(span.Process.Tags))
+
+	for i := range span.Tags {
+		key := tagKey(&span.GetTags()[i])
+		uniqueTags[key] = append(uniqueTags[key], tagValue(&span.GetTags()[i]))
+	}
+
+	for i := range span.Process.Tags {
+		key := tagKey(&span.GetProcess().GetTags()[i])
+		uniqueTags[key] = append(uniqueTags[key], tagValue(&span.GetProcess().GetTags()[i]))
+	}
+
+	for _, event := range span.Logs {
+		for i := range event.Fields {
+			key := tagKey(&event.GetFields()[i])
+			uniqueTags[key] = append(uniqueTags[key], tagValue(&event.GetFields()[i]))
+		}
+	}
+
+	keys = make([]string, 0, len(uniqueTags))
+	for k := range uniqueTags {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	values = make([]string, 0, len(uniqueTags))
+	for _, key := range keys {
+		values = append(values, strings.Join(unique(uniqueTags[key]), ","))
+	}
+
+	return keys, values
+}
+
+func tagKey(kv *model.KeyValue) string {
+	return kv.Key
+}
+
+func tagValue(kv *model.KeyValue) string {
+	return kv.AsString()
+}
+
+func unique(slice []string) []string {
+	if len(slice) == 1 {
+		return slice
+	}
+
+	keys := make(map[string]bool)
+	list := []string{}
+	for _, entry := range slice {
+		if _, value := keys[entry]; !value {
+			keys[entry] = true
+			list = append(list, entry)
+		}
+	}
+	return list
+}
--- a/storage/clickhousespanstore/worker_test.go
+++ b/storage/clickhousespanstore/worker_test.go
@ -0,0 +1,552 @@
+package clickhousespanstore
+
+import (
+	"database/sql"
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"strconv"
+	"testing"
+	"time"
+
+	sqlmock "github.com/DATA-DOG/go-sqlmock"
+	"github.com/gogo/protobuf/proto"
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/jaegertracing/jaeger/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore/mocks"
+)
+
+const (
+	testTagCount      = 10
+	testLogCount      = 5
+	testLogFieldCount = 5
+	testIndexTable    = "test_index_table"
+	testSpansTable    = "test_spans_table"
+	testTenant        = "test_tenant"
+)
+
+type expectation struct {
+	preparation string
+	execArgs    [][]driver.Value
+}
+
+var (
+	errorMock = fmt.Errorf("error mock")
+	process   = model.NewProcess("test_service", []model.KeyValue{model.String("test_process_key", "test_process_value")})
+	testSpan  = model.Span{
+		TraceID:       model.NewTraceID(1, 2),
+		SpanID:        model.NewSpanID(3),
+		OperationName: "GET /unit_test",
+		StartTime:     testStartTime,
+		Process:       process,
+		Tags:          []model.KeyValue{model.String("test_string_key", "test_string_value"), model.Int64("test_int64_key", 4)},
+		Logs:          []model.Log{{Timestamp: testStartTime, Fields: []model.KeyValue{model.String("test_log_key", "test_log_value")}}},
+		Duration:      time.Minute,
+	}
+	testSpans             = []*model.Span{&testSpan}
+	keys, values          = uniqueTagsForSpan(&testSpan)
+	indexWriteExpectation = expectation{
+		preparation: fmt.Sprintf("INSERT INTO %s (timestamp, traceID, service, operation, durationUs, tags.key, tags.value) VALUES (?, ?, ?, ?, ?, ?, ?)", testIndexTable),
+		execArgs: [][]driver.Value{{
+			testSpan.StartTime,
+			testSpan.TraceID.String(),
+			testSpan.Process.GetServiceName(),
+			testSpan.OperationName,
+			uint64(testSpan.Duration.Microseconds()),
+			keys,
+			values,
+		}}}
+	indexWriteExpectationTenant = expectation{
+		preparation: fmt.Sprintf("INSERT INTO %s (tenant, timestamp, traceID, service, operation, durationUs, tags.key, tags.value) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", testIndexTable),
+		execArgs: [][]driver.Value{{
+			testTenant,
+			testSpan.StartTime,
+			testSpan.TraceID.String(),
+			testSpan.Process.GetServiceName(),
+			testSpan.OperationName,
+			uint64(testSpan.Duration.Microseconds()),
+			keys,
+			values,
+		}}}
+	writeBatchLogs = []mocks.LogMock{{Msg: "Writing spans", Args: []interface{}{"size", len(testSpans)}}}
+)
+
+func TestSpanWriter_TagKeyValue(t *testing.T) {
+	tests := map[string]struct {
+		kv       model.KeyValue
+		expected string
+	}{
+		"string value":       {kv: model.String("tag_key", "tag_string_value"), expected: "tag_string_value"},
+		"true value":         {kv: model.Bool("tag_key", true), expected: "true"},
+		"false value":        {kv: model.Bool("tag_key", false), expected: "false"},
+		"positive int value": {kv: model.Int64("tag_key", 1203912), expected: "1203912"},
+		"negative int value": {kv: model.Int64("tag_key", -1203912), expected: "-1203912"},
+		"float value":        {kv: model.Float64("tag_key", 0.005009), expected: "0.005009"},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			assert.Equal(t, test.expected, tagValue(&test.kv), "Incorrect tag value string")
+		})
+	}
+}
+
+func TestSpanWriter_UniqueTagsForSpan(t *testing.T) {
+	tests := map[string]struct {
+		tags           []model.KeyValue
+		processTags    []model.KeyValue
+		logs           []model.Log
+		expectedKeys   []string
+		expectedValues []string
+	}{
+		"default": {
+			tags:           []model.KeyValue{model.String("key2", "value")},
+			processTags:    []model.KeyValue{model.Int64("key3", 412)},
+			logs:           []model.Log{{Fields: []model.KeyValue{model.Float64("key1", .5)}}},
+			expectedKeys:   []string{"key1", "key2", "key3"},
+			expectedValues: []string{"0.5", "value", "412"},
+		},
+		"repeating tags": {
+			tags:           []model.KeyValue{model.String("key2", "value"), model.String("key2", "value")},
+			processTags:    []model.KeyValue{model.Int64("key3", 412)},
+			logs:           []model.Log{{Fields: []model.KeyValue{model.Float64("key1", .5)}}},
+			expectedKeys:   []string{"key1", "key2", "key3"},
+			expectedValues: []string{"0.5", "value", "412"},
+		},
+		"repeating keys": {
+			tags:           []model.KeyValue{model.String("key2", "value_a"), model.String("key2", "value_b")},
+			processTags:    []model.KeyValue{model.Int64("key3", 412)},
+			logs:           []model.Log{{Fields: []model.KeyValue{model.Float64("key1", .5)}}},
+			expectedKeys:   []string{"key1", "key2", "key3"},
+			expectedValues: []string{"0.5", "value_a,value_b", "412"},
+		},
+		"repeating values": {
+			tags:           []model.KeyValue{model.String("key2", "value"), model.Int64("key4", 412)},
+			processTags:    []model.KeyValue{model.Int64("key3", 412)},
+			logs:           []model.Log{{Fields: []model.KeyValue{model.Float64("key1", .5)}}},
+			expectedKeys:   []string{"key1", "key2", "key3", "key4"},
+			expectedValues: []string{"0.5", "value", "412", "412"},
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			process := model.Process{Tags: test.processTags}
+			span := model.Span{Tags: test.tags, Process: &process, Logs: test.logs}
+			actualKeys, actualValues := uniqueTagsForSpan(&span)
+			assert.Equal(t, test.expectedKeys, actualKeys)
+			assert.Equal(t, test.expectedValues, actualValues)
+		})
+	}
+}
+
+func TestSpanWriter_General(t *testing.T) {
+	spanJSON, err := json.Marshal(&testSpan)
+	require.NoError(t, err)
+	modelWriteExpectationJSON := getModelWriteExpectation(spanJSON, "")
+	modelWriteExpectationJSONTenant := getModelWriteExpectation(spanJSON, testTenant)
+	spanProto, err := proto.Marshal(&testSpan)
+	require.NoError(t, err)
+	modelWriteExpectationProto := getModelWriteExpectation(spanProto, "")
+	modelWriteExpectationProtoTenant := getModelWriteExpectation(spanProto, testTenant)
+	tests := map[string]struct {
+		encoding     Encoding
+		indexTable   TableName
+		tenant       string
+		spans        []*model.Span
+		expectations []expectation
+		action       func(writeWorker *WriteWorker, spans []*model.Span) error
+		expectedLogs []mocks.LogMock
+	}{
+		"write index batch": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			spans:        testSpans,
+			expectations: []expectation{indexWriteExpectation},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeIndexBatch(spans) },
+		},
+		"write index tenant batch": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			spans:        testSpans,
+			expectations: []expectation{indexWriteExpectationTenant},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeIndexBatch(spans) },
+		},
+		"write model batch JSON": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationJSON},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeModelBatch(spans) },
+		},
+		"write model tenant batch JSON": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationJSONTenant},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeModelBatch(spans) },
+		},
+		"write model batch Proto": {
+			encoding:     EncodingProto,
+			indexTable:   testIndexTable,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationProto},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeModelBatch(spans) },
+		},
+		"write model tenant batch Proto": {
+			encoding:     EncodingProto,
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationProtoTenant},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeModelBatch(spans) },
+		},
+		"write batch no index JSON": {
+			encoding:     EncodingJSON,
+			indexTable:   "",
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationJSON},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch no index Proto": {
+			encoding:     EncodingProto,
+			indexTable:   "",
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationProto},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch JSON": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationJSON, indexWriteExpectation},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch tenant JSON": {
+			encoding:     EncodingJSON,
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationJSONTenant, indexWriteExpectationTenant},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch Proto": {
+			encoding:     EncodingProto,
+			indexTable:   testIndexTable,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationProto, indexWriteExpectation},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch tenant Proto": {
+			encoding:     EncodingProto,
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			spans:        testSpans,
+			expectations: []expectation{modelWriteExpectationProtoTenant, indexWriteExpectationTenant},
+			action:       func(writeWorker *WriteWorker, spans []*model.Span) error { return writeWorker.writeBatch(spans) },
+			expectedLogs: writeBatchLogs,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			db, mock, err := mocks.GetDbMock()
+			require.NoError(t, err, "an error was not expected when opening a stub database connection")
+			defer db.Close()
+
+			spyLogger := mocks.NewSpyLogger()
+			worker := getWriteWorker(spyLogger, db, test.encoding, test.indexTable, test.tenant)
+
+			for _, expectation := range test.expectations {
+				mock.ExpectBegin()
+				prep := mock.ExpectPrepare(expectation.preparation)
+				for _, args := range expectation.execArgs {
+					prep.ExpectExec().WithArgs(args...).WillReturnResult(sqlmock.NewResult(1, 1))
+				}
+				mock.ExpectCommit()
+			}
+
+			assert.NoError(t, test.action(&worker, test.spans))
+			assert.NoError(t, mock.ExpectationsWereMet())
+			spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, test.expectedLogs)
+		})
+	}
+}
+
+func TestSpanWriter_BeginError(t *testing.T) {
+	tests := map[string]struct {
+		action       func(writeWorker *WriteWorker) error
+		expectedLogs []mocks.LogMock
+	}{
+		"write model batch": {action: func(writeWorker *WriteWorker) error { return writeWorker.writeModelBatch(testSpans) }},
+		"write index batch": {action: func(writeWorker *WriteWorker) error { return writeWorker.writeIndexBatch(testSpans) }},
+		"write batch": {
+			action:       func(writeWorker *WriteWorker) error { return writeWorker.writeBatch(testSpans) },
+			expectedLogs: writeBatchLogs,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			db, mock, err := mocks.GetDbMock()
+			require.NoError(t, err, "an error was not expected when opening a stub database connection")
+			defer db.Close()
+
+			spyLogger := mocks.NewSpyLogger()
+			writeWorker := getWriteWorker(spyLogger, db, EncodingJSON, testIndexTable, "")
+
+			mock.ExpectBegin().WillReturnError(errorMock)
+
+			assert.ErrorIs(t, test.action(&writeWorker), errorMock)
+			assert.NoError(t, mock.ExpectationsWereMet())
+			spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, test.expectedLogs)
+		})
+	}
+}
+
+func TestSpanWriter_PrepareError(t *testing.T) {
+	spanJSON, err := json.Marshal(&testSpan)
+	require.NoError(t, err)
+	modelWriteExpectation := getModelWriteExpectation(spanJSON, "")
+	modelWriteExpectationTenant := getModelWriteExpectation(spanJSON, testTenant)
+
+	tests := map[string]struct {
+		action       func(writeWorker *WriteWorker) error
+		tenant       string
+		expectation  expectation
+		expectedLogs []mocks.LogMock
+	}{
+		"write model batch": {
+			action:      func(writeWorker *WriteWorker) error { return writeWorker.writeModelBatch(testSpans) },
+			expectation: modelWriteExpectation,
+		},
+		"write model tenant batch": {
+			action:      func(writeWorker *WriteWorker) error { return writeWorker.writeModelBatch(testSpans) },
+			tenant:      testTenant,
+			expectation: modelWriteExpectationTenant,
+		},
+		"write index batch": {
+			action:      func(writeWorker *WriteWorker) error { return writeWorker.writeIndexBatch(testSpans) },
+			expectation: indexWriteExpectation,
+		},
+		"write index tenant batch": {
+			action:      func(writeWorker *WriteWorker) error { return writeWorker.writeIndexBatch(testSpans) },
+			tenant:      testTenant,
+			expectation: indexWriteExpectationTenant,
+		},
+		"write batch": {
+			action:       func(writeWorker *WriteWorker) error { return writeWorker.writeBatch(testSpans) },
+			expectation:  modelWriteExpectation,
+			expectedLogs: writeBatchLogs,
+		},
+		"write tenant batch": {
+			action:       func(writeWorker *WriteWorker) error { return writeWorker.writeBatch(testSpans) },
+			tenant:       testTenant,
+			expectation:  modelWriteExpectationTenant,
+			expectedLogs: writeBatchLogs,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			db, mock, err := mocks.GetDbMock()
+			require.NoError(t, err, "an error was not expected when opening a stub database connection")
+			defer db.Close()
+
+			spyLogger := mocks.NewSpyLogger()
+			spanWriter := getWriteWorker(spyLogger, db, EncodingJSON, testIndexTable, test.tenant)
+
+			mock.ExpectBegin()
+			mock.ExpectPrepare(test.expectation.preparation).WillReturnError(errorMock)
+			mock.ExpectRollback()
+
+			assert.ErrorIs(t, test.action(&spanWriter), errorMock)
+			assert.NoError(t, mock.ExpectationsWereMet())
+			spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, test.expectedLogs)
+		})
+	}
+}
+
+func TestSpanWriter_ExecError(t *testing.T) {
+	spanJSON, err := json.Marshal(&testSpan)
+	require.NoError(t, err)
+	modelWriteExpectation := getModelWriteExpectation(spanJSON, "")
+	modelWriteExpectationTenant := getModelWriteExpectation(spanJSON, testTenant)
+	tests := map[string]struct {
+		indexTable   TableName
+		tenant       string
+		expectations []expectation
+		action       func(writer *WriteWorker) error
+		expectedLogs []mocks.LogMock
+	}{
+		"write model batch": {
+			indexTable:   testIndexTable,
+			expectations: []expectation{modelWriteExpectation},
+			action:       func(writer *WriteWorker) error { return writer.writeModelBatch(testSpans) },
+		},
+		"write model tenant batch": {
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			expectations: []expectation{modelWriteExpectationTenant},
+			action:       func(writer *WriteWorker) error { return writer.writeModelBatch(testSpans) },
+		},
+		"write index batch": {
+			indexTable:   testIndexTable,
+			expectations: []expectation{indexWriteExpectation},
+			action:       func(writer *WriteWorker) error { return writer.writeIndexBatch(testSpans) },
+		},
+		"write index tenant batch": {
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			expectations: []expectation{indexWriteExpectationTenant},
+			action:       func(writer *WriteWorker) error { return writer.writeIndexBatch(testSpans) },
+		},
+		"write batch no index": {
+			indexTable:   "",
+			expectations: []expectation{modelWriteExpectation},
+			action:       func(writer *WriteWorker) error { return writer.writeBatch(testSpans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write batch": {
+			indexTable:   testIndexTable,
+			expectations: []expectation{modelWriteExpectation, indexWriteExpectation},
+			action:       func(writer *WriteWorker) error { return writer.writeBatch(testSpans) },
+			expectedLogs: writeBatchLogs,
+		},
+		"write tenant batch": {
+			indexTable:   testIndexTable,
+			tenant:       testTenant,
+			expectations: []expectation{modelWriteExpectationTenant, indexWriteExpectationTenant},
+			action:       func(writer *WriteWorker) error { return writer.writeBatch(testSpans) },
+			expectedLogs: writeBatchLogs,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			db, mock, err := mocks.GetDbMock()
+			require.NoError(t, err, "an error was not expected when opening a stub database connection")
+			defer db.Close()
+
+			spyLogger := mocks.NewSpyLogger()
+			writeWorker := getWriteWorker(spyLogger, db, EncodingJSON, testIndexTable, test.tenant)
+
+			for i, expectation := range test.expectations {
+				mock.ExpectBegin()
+				prep := mock.ExpectPrepare(expectation.preparation)
+				if i < len(test.expectations)-1 {
+					for _, args := range expectation.execArgs {
+						prep.ExpectExec().WithArgs(args...).WillReturnResult(sqlmock.NewResult(1, 1))
+					}
+					mock.ExpectCommit()
+				} else {
+					prep.ExpectExec().WithArgs(expectation.execArgs[0]...).WillReturnError(errorMock)
+					mock.ExpectRollback()
+				}
+			}
+
+			assert.ErrorIs(t, test.action(&writeWorker), errorMock)
+			assert.NoError(t, mock.ExpectationsWereMet())
+			spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, test.expectedLogs)
+		})
+	}
+}
+
+func getWriteWorker(spyLogger mocks.SpyLogger, db *sql.DB, encoding Encoding, indexTable TableName, tenant string) WriteWorker {
+	return WriteWorker{
+		params: &WorkerParams{
+			logger:     spyLogger,
+			db:         db,
+			spansTable: testSpansTable,
+			indexTable: indexTable,
+			tenant:     tenant,
+			encoding:   encoding,
+		},
+		workerDone: make(chan *WriteWorker),
+	}
+}
+
+func generateRandomSpans(count int) []*model.Span {
+	spans := make([]*model.Span, count)
+	for i := 0; i < count; i++ {
+		span := generateRandomSpan()
+		spans[i] = &span
+	}
+	return spans
+}
+
+func generateRandomSpan() model.Span {
+	processTags := generateRandomKeyValues(testTagCount)
+	process := model.Process{
+		ServiceName: "service" + strconv.FormatUint(rand.Uint64(), 10),
+		Tags:        processTags,
+	}
+	span := model.Span{
+		TraceID:       model.NewTraceID(rand.Uint64(), rand.Uint64()),
+		SpanID:        model.NewSpanID(rand.Uint64()),
+		OperationName: "operation" + strconv.FormatUint(rand.Uint64(), 10),
+		StartTime:     getRandomTime(),
+		Process:       &process,
+		Tags:          generateRandomKeyValues(testTagCount),
+		Logs:          generateRandomLogs(),
+		Duration:      time.Unix(rand.Int63n(1<<32), 0).Sub(time.Unix(0, 0)),
+	}
+	return span
+}
+
+func generateRandomLogs() []model.Log {
+	logs := make([]model.Log, 0, testLogCount)
+	for i := 0; i < testLogCount; i++ {
+		timestamp := getRandomTime()
+		logs = append(logs, model.Log{Timestamp: timestamp, Fields: generateRandomKeyValues(testLogFieldCount)})
+	}
+	return logs
+}
+
+func getRandomTime() time.Time {
+	return time.Unix(rand.Int63n(time.Now().Unix()), 0)
+}
+
+func generateRandomKeyValues(count int) []model.KeyValue {
+	tags := make([]model.KeyValue, 0, count)
+	for i := 0; i < count; i++ {
+		key := "key" + strconv.FormatUint(rand.Uint64(), 16)
+		value := "key" + strconv.FormatUint(rand.Uint64(), 16)
+		kv := model.KeyValue{Key: key, VType: model.ValueType_STRING, VStr: value}
+		tags = append(tags, kv)
+	}
+
+	return tags
+}
+
+func getModelWriteExpectation(spanJSON []byte, tenant string) expectation {
+	if tenant == "" {
+		return expectation{
+			preparation: fmt.Sprintf("INSERT INTO %s (timestamp, traceID, model) VALUES (?, ?, ?)", testSpansTable),
+			execArgs: [][]driver.Value{{
+				testSpan.StartTime,
+				testSpan.TraceID.String(),
+				spanJSON,
+			}},
+		}
+	} else {
+		return expectation{
+			preparation: fmt.Sprintf("INSERT INTO %s (tenant, timestamp, traceID, model) VALUES (?, ?, ?, ?)", testSpansTable),
+			execArgs: [][]driver.Value{{
+				tenant,
+				testSpan.StartTime,
+				testSpan.TraceID.String(),
+				spanJSON,
+			}},
+		}
+	}
+}
--- a/storage/clickhousespanstore/writer.go
+++ b/storage/clickhousespanstore/writer.go
@ -3,15 +3,10 @@ package clickhousespanstore
 import (
 	"context"
 	"database/sql"
-	"encoding/json"
-	"fmt"
-	"sort"
-	"strings"
 	"sync"
 	"time"

-	"github.com/gogo/protobuf/proto"
-	"github.com/hashicorp/go-hclog"
+	hclog "github.com/hashicorp/go-hclog"
 	"github.com/jaegertracing/jaeger/model"
 	"github.com/jaegertracing/jaeger/storage/spanstore"
 	"github.com/prometheus/client_golang/prometheus"
@ -39,52 +34,63 @@ var (

 // SpanWriter for writing spans to ClickHouse
 type SpanWriter struct {
-	logger     hclog.Logger
-	db         *sql.DB
-	indexTable string
-	spansTable string
-	encoding   Encoding
-	delay      time.Duration
-	size       int64
-	spans      chan *model.Span
-	finish     chan bool
-	done       sync.WaitGroup
+	workerParams WorkerParams
+
+	size   int64
+	spans  chan *model.Span
+	finish chan bool
+	done   sync.WaitGroup
 }

-var registerMetrics sync.Once
+var registerWriterMetrics sync.Once
 var _ spanstore.Writer = (*SpanWriter)(nil)

 // NewSpanWriter returns a SpanWriter for the database
-func NewSpanWriter(logger hclog.Logger, db *sql.DB, indexTable string, spansTable string, encoding Encoding, delay time.Duration, size int64) *SpanWriter {
+func NewSpanWriter(
+	logger hclog.Logger,
+	db *sql.DB,
+	indexTable,
+	spansTable TableName,
+	tenant string,
+	encoding Encoding,
+	delay time.Duration,
+	size int64,
+	maxSpanCount int,
+) *SpanWriter {
 	writer := &SpanWriter{
-		logger:     logger,
-		db:         db,
-		indexTable: indexTable,
-		spansTable: spansTable,
-		encoding:   encoding,
-		delay:      delay,
-		size:       size,
-		spans:      make(chan *model.Span, size),
-		finish:     make(chan bool),
+		workerParams: WorkerParams{
+			logger:     logger,
+			db:         db,
+			indexTable: indexTable,
+			spansTable: spansTable,
+			tenant:     tenant,
+			encoding:   encoding,
+			delay:      delay,
+		},
+		size:   size,
+		spans:  make(chan *model.Span, size),
+		finish: make(chan bool),
 	}

 	writer.registerMetrics()
-	go writer.backgroundWriter()
+	go writer.backgroundWriter(maxSpanCount)

 	return writer
 }

 func (w *SpanWriter) registerMetrics() {
-	registerMetrics.Do(func() {
+	registerWriterMetrics.Do(func() {
 		prometheus.MustRegister(numWritesWithBatchSize)
 		prometheus.MustRegister(numWritesWithFlushInterval)
 	})
 }

-func (w *SpanWriter) backgroundWriter() {
+func (w *SpanWriter) backgroundWriter(maxSpanCount int) {
+	pool := NewWorkerPool(&w.workerParams, maxSpanCount)
+	go pool.Work()
 	batch := make([]*model.Span, 0, w.size)

-	timer := time.After(w.delay)
+	timer := time.After(w.workerParams.delay)
 	last := time.Now()

 	for {
@ -98,31 +104,32 @@ func (w *SpanWriter) backgroundWriter() {
 			batch = append(batch, span)
 			flush = len(batch) == cap(batch)
 			if flush {
-				w.logger.Debug("Flush due to batch size", "size", len(batch))
+				w.workerParams.logger.Debug("Flush due to batch size", "size", len(batch))
 				numWritesWithBatchSize.Inc()
 			}
 		case <-timer:
-			timer = time.After(w.delay)
-			flush = time.Since(last) > w.delay && len(batch) > 0
+			timer = time.After(w.workerParams.delay)
+			flush = time.Since(last) > w.workerParams.delay && len(batch) > 0
 			if flush {
-				w.logger.Debug("Flush due to timer")
+				w.workerParams.logger.Debug("Flush due to timer")
 				numWritesWithFlushInterval.Inc()
 			}
 		case <-w.finish:
 			finish = true
 			flush = len(batch) > 0
-			w.logger.Debug("Finish channel")
+			w.workerParams.logger.Debug("Finish channel")
 		}

 		if flush {
-			if err := w.writeBatch(batch); err != nil {
-				w.logger.Error("Could not write a batch of spans", "error", err)
-			}
+			pool.WriteBatch(batch)

 			batch = make([]*model.Span, 0, w.size)
 			last = time.Now()
 		}

+		if finish {
+			pool.Close()
+		}
 		w.done.Done()

 		if finish {
@ -131,108 +138,6 @@ func (w *SpanWriter) backgroundWriter() {
 	}
 }

-func (w *SpanWriter) writeBatch(batch []*model.Span) error {
-	w.logger.Debug("Writing spans", "size", len(batch))
-	if err := w.writeModelBatch(batch); err != nil {
-		return err
-	}
-
-	if w.indexTable != "" {
-		if err := w.writeIndexBatch(batch); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (w *SpanWriter) writeModelBatch(batch []*model.Span) error {
-	tx, err := w.db.Begin()
-	if err != nil {
-		return err
-	}
-
-	committed := false
-
-	defer func() {
-		if !committed {
-			// Clickhouse does not support real rollback
-			_ = tx.Rollback()
-		}
-	}()
-
-	statement, err := tx.Prepare(fmt.Sprintf("INSERT INTO %s (timestamp, traceID, model) VALUES (?, ?, ?)", w.spansTable))
-	if err != nil {
-		return nil
-	}
-
-	defer statement.Close()
-
-	for _, span := range batch {
-		var serialized []byte
-
-		if w.encoding == EncodingJSON {
-			serialized, err = json.Marshal(span)
-		} else {
-			serialized, err = proto.Marshal(span)
-		}
-
-		if err != nil {
-			return err
-		}
-
-		_, err = statement.Exec(span.StartTime, span.TraceID.String(), serialized)
-		if err != nil {
-			return err
-		}
-	}
-
-	committed = true
-
-	return tx.Commit()
-}
-
-func (w *SpanWriter) writeIndexBatch(batch []*model.Span) error {
-	tx, err := w.db.Begin()
-	if err != nil {
-		return err
-	}
-
-	committed := false
-
-	defer func() {
-		if !committed {
-			// Clickhouse does not support real rollback
-			_ = tx.Rollback()
-		}
-	}()
-
-	statement, err := tx.Prepare(fmt.Sprintf("INSERT INTO %s (timestamp, traceID, service, operation, durationUs, tags) VALUES (?, ?, ?, ?, ?, ?)", w.indexTable))
-	if err != nil {
-		return err
-	}
-
-	defer statement.Close()
-
-	for _, span := range batch {
-		_, err = statement.Exec(
-			span.StartTime,
-			span.TraceID.String(),
-			span.Process.ServiceName,
-			span.OperationName,
-			span.Duration.Microseconds(),
-			uniqueTagsForSpan(span),
-		)
-		if err != nil {
-			return err
-		}
-	}
-
-	committed = true
-
-	return tx.Commit()
-}
-
 // WriteSpan writes the encoded span
 func (w *SpanWriter) WriteSpan(_ context.Context, span *model.Span) error {
 	w.spans <- span
@ -245,43 +150,3 @@ func (w *SpanWriter) Close() error {
 	w.done.Wait()
 	return nil
 }
-
-func uniqueTagsForSpan(span *model.Span) []string {
-	uniqueTags := make(map[string]struct{}, len(span.Tags)+len(span.Process.Tags))
-
-	buf := &strings.Builder{}
-
-	for i := range span.Tags {
-		uniqueTags[tagString(buf, &span.GetTags()[i])] = struct{}{}
-	}
-
-	for i := range span.Process.Tags {
-		uniqueTags[tagString(buf, &span.GetProcess().GetTags()[i])] = struct{}{}
-	}
-
-	for _, event := range span.Logs {
-		for i := range event.Fields {
-			uniqueTags[tagString(buf, &event.GetFields()[i])] = struct{}{}
-		}
-	}
-
-	tags := make([]string, 0, len(uniqueTags))
-
-	for kv := range uniqueTags {
-		tags = append(tags, kv)
-	}
-
-	sort.Strings(tags)
-
-	return tags
-}
-
-func tagString(buf *strings.Builder, kv *model.KeyValue) string {
-	buf.Reset()
-
-	buf.WriteString(kv.Key)
-	buf.WriteByte('=')
-	buf.WriteString(kv.AsString())
-
-	return buf.String()
-}
--- a/storage/config.go
+++ b/storage/config.go
@ -1,21 +1,28 @@
 package storage

-import "time"
+import (
+	"time"
+
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore"
+)

 type EncodingType string

 const (
+	defaultEncoding                     = JSONEncoding
 	JSONEncoding           EncodingType = "json"
 	ProtobufEncoding       EncodingType = "protobuf"
+	defaultMaxSpanCount                 = int(1e7)
 	defaultBatchSize                    = 10_000
 	defaultBatchDelay                   = time.Second * 5
 	defaultUsername                     = "default"
 	defaultDatabaseName                 = "default"
 	defaultMetricsEndpoint              = "localhost:9090"
+	defaultMaxNumSpans                  = 0

-	defaultSpansTable      = "jaeger_spans_local"
-	defaultSpansIndexTable = "jaeger_index_local"
-	defaultOperationsTable = "jaeger_operations_local"
+	defaultSpansTable      clickhousespanstore.TableName = "jaeger_spans"
+	defaultSpansIndexTable clickhousespanstore.TableName = "jaeger_index"
+	defaultOperationsTable clickhousespanstore.TableName = "jaeger_operations"
 )

 type Configuration struct {
@ -23,12 +30,25 @@ type Configuration struct {
 	BatchWriteSize int64 `yaml:"batch_write_size"`
 	// Batch flush interval. Default is 5s.
 	BatchFlushInterval time.Duration `yaml:"batch_flush_interval"`
+	// Maximal amount of spans that can be pending writes at a time.
+	// New spans exceeding this limit will be discarded,
+	// keeping memory in check if there are issues writing to ClickHouse.
+	// Check the "jaeger_clickhouse_discarded_spans" metric to keep track of discards.
+	// Default 10_000_000, or disable the limit entirely by setting to 0.
+	MaxSpanCount int `yaml:"max_span_count"`
 	// Encoding either json or protobuf. Default is json.
 	Encoding EncodingType `yaml:"encoding"`
-	// ClickHouse address e.g. tcp://localhost:9000.
+	// ClickHouse address e.g. localhost:9000.
 	Address string `yaml:"address"`
-	// Directory with .sql files that are run at plugin startup.
+	// Directory with .sql files to run at plugin startup, mainly for integration tests.
+	// Depending on the value of init_tables, this can be run as a
+	// replacement or supplement to creating default tables for span storage.
+	// If init_tables is also enabled, the scripts in this directory will be run first.
 	InitSQLScriptsDir string `yaml:"init_sql_scripts_dir"`
+	// Whether to automatically attempt to create tables in ClickHouse.
+	// By default, this is enabled if init_sql_scripts_dir is empty,
+	// or disabled if init_sql_scripts_dir is provided.
+	InitTables *bool `yaml:"init_tables"`
 	// Indicates location of TLS certificate used to connect to database.
 	CaFile string `yaml:"ca_file"`
 	// Username for connection to database. Default is "default".
@ -39,12 +59,29 @@ type Configuration struct {
 	Database string `yaml:"database"`
 	// Endpoint for scraping prometheus metrics e.g. localhost:9090.
 	MetricsEndpoint string `yaml:"metrics_endpoint"`
-	// Table with spans. Default "jaeger_spans_local".
-	SpansTable string `yaml:"spans_table"`
-	// Span index table. Default "jaeger_index_local".
-	SpansIndexTable string `yaml:"spans_index_table"`
-	// Operations table. Default "jaeger_operations_local.
-	OperationsTable string `yaml:"operations_table"`
+	// Whether to use SQL scripts supporting replication and sharding. Default false.
+	Replication bool `yaml:"replication"`
+	// If non-empty, enables multitenancy in SQL scripts, and assigns the tenant name for this instance.
+	Tenant string `yaml:"tenant"`
+	// Table with spans. Default "jaeger_spans_local" or "jaeger_spans" when replication is enabled.
+	SpansTable clickhousespanstore.TableName `yaml:"spans_table"`
+	// Span index table. Default "jaeger_index_local" or "jaeger_index" when replication is enabled.
+	SpansIndexTable clickhousespanstore.TableName `yaml:"spans_index_table"`
+	// Operations table. Default "jaeger_operations_local" or "jaeger_operations" when replication is enabled.
+	OperationsTable   clickhousespanstore.TableName `yaml:"operations_table"`
+	spansArchiveTable clickhousespanstore.TableName
+	// TTL for data in tables in days. If 0, no TTL is set. Default 0.
+	TTLDays uint `yaml:"ttl"`
+	// The maximum number of spans to fetch per trace. If 0, no limits is set. Default 0.
+	MaxNumSpans uint `yaml:"max_num_spans"`
+	// The maximum number of open connections to the database. Default is unlimited (see: https://pkg.go.dev/database/sql#DB.SetMaxOpenConns)
+	MaxOpenConns *uint `yaml:"max_open_conns"`
+	// The maximum number of database connections in the idle connection pool. Default 2. (see: https://pkg.go.dev/database/sql#DB.SetMaxIdleConns)
+	MaxIdleConns *uint `yaml:"max_idle_conns"`
+	// The maximum amount of milliseconds a database connection may be reused. Default = connections are never closed due to age (see: https://pkg.go.dev/database/sql#DB.SetConnMaxLifetime)
+	ConnMaxLifetimeMillis *uint `yaml:"conn_max_lifetime_millis"`
+	// The maximum amount of milliseconds a database connection may be idle. Default = connections are never closed due to idle time (see: https://pkg.go.dev/database/sql#DB.SetConnMaxIdleTime)
+	ConnMaxIdleTimeMillis *uint `yaml:"conn_max_idle_time_millis"`
 }

 func (cfg *Configuration) setDefaults() {
@ -54,8 +91,21 @@ func (cfg *Configuration) setDefaults() {
 	if cfg.BatchFlushInterval == 0 {
 		cfg.BatchFlushInterval = defaultBatchDelay
 	}
+	if cfg.MaxSpanCount == 0 {
+		cfg.MaxSpanCount = defaultMaxSpanCount
+	}
 	if cfg.Encoding == "" {
-		cfg.Encoding = JSONEncoding
+		cfg.Encoding = defaultEncoding
+	}
+	if cfg.InitTables == nil {
+		// Decide whether to init tables based on whether a custom script path was provided
+		var defaultInitTables bool
+		if cfg.InitSQLScriptsDir == "" {
+			defaultInitTables = true
+		} else {
+			defaultInitTables = false
+		}
+		cfg.InitTables = &defaultInitTables
 	}
 	if cfg.Username == "" {
 		cfg.Username = defaultUsername
@ -66,13 +116,36 @@ func (cfg *Configuration) setDefaults() {
 	if cfg.MetricsEndpoint == "" {
 		cfg.MetricsEndpoint = defaultMetricsEndpoint
 	}
+	if cfg.MaxNumSpans == 0 {
+		cfg.MaxNumSpans = defaultMaxNumSpans
+	}
 	if cfg.SpansTable == "" {
-		cfg.SpansTable = defaultSpansTable
+		if cfg.Replication {
+			cfg.SpansTable = defaultSpansTable
+			cfg.spansArchiveTable = defaultSpansTable + "_archive"
+		} else {
+			cfg.SpansTable = defaultSpansTable.ToLocal()
+			cfg.spansArchiveTable = (defaultSpansTable + "_archive").ToLocal()
+		}
+	} else {
+		cfg.spansArchiveTable = cfg.SpansTable + "_archive"
 	}
 	if cfg.SpansIndexTable == "" {
-		cfg.SpansIndexTable = defaultSpansIndexTable
+		if cfg.Replication {
+			cfg.SpansIndexTable = defaultSpansIndexTable
+		} else {
+			cfg.SpansIndexTable = defaultSpansIndexTable.ToLocal()
+		}
 	}
 	if cfg.OperationsTable == "" {
-		cfg.OperationsTable = defaultOperationsTable
+		if cfg.Replication {
+			cfg.OperationsTable = defaultOperationsTable
+		} else {
+			cfg.OperationsTable = defaultOperationsTable.ToLocal()
+		}
 	}
 }
+
+func (cfg *Configuration) GetSpansArchiveTable() clickhousespanstore.TableName {
+	return cfg.spansArchiveTable
+}
--- a/storage/config_test.go
+++ b/storage/config_test.go
@ -0,0 +1,128 @@
+package storage
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore"
+)
+
+func TestSetDefaults(t *testing.T) {
+	tests := map[string]struct {
+		replication bool
+		getField    func(Configuration) interface{}
+		expected    interface{}
+	}{
+		"username": {
+			getField: func(config Configuration) interface{} { return config.Username },
+			expected: defaultUsername,
+		},
+		"database name": {
+			getField: func(config Configuration) interface{} { return config.Database },
+			expected: defaultDatabaseName,
+		},
+		"encoding": {
+			getField: func(config Configuration) interface{} { return config.Encoding },
+			expected: defaultEncoding,
+		},
+		"batch write size": {
+			getField: func(config Configuration) interface{} { return config.BatchWriteSize },
+			expected: defaultBatchSize,
+		},
+		"batch flush interval": {
+			getField: func(config Configuration) interface{} { return config.BatchFlushInterval },
+			expected: defaultBatchDelay,
+		},
+		"max span count": {
+			getField: func(config Configuration) interface{} { return config.MaxSpanCount },
+			expected: defaultMaxSpanCount,
+		},
+		"metrics endpoint": {
+			getField: func(config Configuration) interface{} { return config.MetricsEndpoint },
+			expected: defaultMetricsEndpoint,
+		},
+		"spans table name local": {
+			getField: func(config Configuration) interface{} { return config.SpansTable },
+			expected: defaultSpansTable.ToLocal(),
+		},
+		"spans table name replication": {
+			replication: true,
+			getField:    func(config Configuration) interface{} { return config.SpansTable },
+			expected:    defaultSpansTable,
+		},
+		"index table name local": {
+			getField: func(config Configuration) interface{} { return config.SpansIndexTable },
+			expected: defaultSpansIndexTable.ToLocal(),
+		},
+		"index table name replication": {
+			replication: true,
+			getField:    func(config Configuration) interface{} { return config.SpansIndexTable },
+			expected:    defaultSpansIndexTable,
+		},
+		"operations table name local": {
+			getField: func(config Configuration) interface{} { return config.OperationsTable },
+			expected: defaultOperationsTable.ToLocal(),
+		},
+		"operations table name replication": {
+			replication: true,
+			getField:    func(config Configuration) interface{} { return config.OperationsTable },
+			expected:    defaultOperationsTable,
+		},
+		"max number spans": {
+			getField: func(config Configuration) interface{} { return config.MaxNumSpans },
+			expected: defaultMaxNumSpans,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(fmt.Sprintf("default %s", name), func(t *testing.T) {
+			config := Configuration{Replication: test.replication}
+			config.setDefaults()
+			assert.EqualValues(t, test.expected, test.getField(config))
+		})
+	}
+}
+
+func TestConfiguration_GetSpansArchiveTable(t *testing.T) {
+	tests := map[string]struct {
+		config                        Configuration
+		expectedSpansArchiveTableName clickhousespanstore.TableName
+	}{
+		"default_config_local":       {config: Configuration{}, expectedSpansArchiveTableName: (defaultSpansTable + "_archive").ToLocal()},
+		"default_config_replication": {config: Configuration{Replication: true}, expectedSpansArchiveTableName: defaultSpansTable + "_archive"},
+		"custom_spans_table":         {config: Configuration{SpansTable: "custom_table_name"}, expectedSpansArchiveTableName: "custom_table_name_archive"},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			test.config.setDefaults()
+			assert.Equal(t, test.expectedSpansArchiveTableName, test.config.GetSpansArchiveTable())
+		})
+	}
+}
+
+func TestConfiguration_InitTables(test *testing.T) {
+	// for pointers below
+	t := true
+	f := false
+	tests := map[string]struct {
+		config             Configuration
+		expectedInitTables bool
+	}{
+		"scriptsempty_initnil":      {config: Configuration{}, expectedInitTables: true},
+		"scriptsprovided_initnil":   {config: Configuration{InitSQLScriptsDir: "hello"}, expectedInitTables: false},
+		"scriptsempty_inittrue":     {config: Configuration{InitTables: &t}, expectedInitTables: true},
+		"scriptsprovided_inittrue":  {config: Configuration{InitSQLScriptsDir: "hello", InitTables: &t}, expectedInitTables: true},
+		"scriptsempty_initfalse":    {config: Configuration{InitTables: &f}, expectedInitTables: false},
+		"scriptsprovided_initfalse": {config: Configuration{InitSQLScriptsDir: "hello", InitTables: &f}, expectedInitTables: false},
+	}
+
+	for name, testcase := range tests {
+		test.Run(name, func(t *testing.T) {
+			testcase.config.setDefaults()
+			assert.Equal(t, testcase.expectedInitTables, *(testcase.config.InitTables))
+		})
+	}
+}
--- a/storage/store.go
+++ b/storage/store.go
@ -4,22 +4,24 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"database/sql"
-	"embed"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
+	"text/template"
+	"time"

-	"github.com/ClickHouse/clickhouse-go"
-	"github.com/hashicorp/go-hclog"
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
+	hclog "github.com/hashicorp/go-hclog"
 	"github.com/jaegertracing/jaeger/plugin/storage/grpc/shared"
 	"github.com/jaegertracing/jaeger/storage/dependencystore"
 	"github.com/jaegertracing/jaeger/storage/spanstore"

-	"github.com/pavolloffay/jaeger-clickhouse/storage/clickhousedependencystore"
-	"github.com/pavolloffay/jaeger-clickhouse/storage/clickhousespanstore"
+	jaegerclickhouse "github.com/jaegertracing/jaeger-clickhouse"
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousedependencystore"
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore"
 )

 type Store struct {
@ -30,99 +32,263 @@ type Store struct {
 	archiveReader spanstore.Reader
 }

-const (
-	tlsConfigKey = "clickhouse_tls_config_key"
+var (
+	_ shared.StoragePlugin             = (*Store)(nil)
+	_ shared.ArchiveStoragePlugin      = (*Store)(nil)
+	_ shared.StreamingSpanWriterPlugin = (*Store)(nil)
+	_ io.Closer                        = (*Store)(nil)
 )

-var _ shared.StoragePlugin = (*Store)(nil)
-var _ shared.ArchiveStoragePlugin = (*Store)(nil)
-var _ io.Closer = (*Store)(nil)
-
-func NewStore(logger hclog.Logger, cfg Configuration, embeddedSQLScripts embed.FS) (*Store, error) {
+func NewStore(logger hclog.Logger, cfg Configuration) (*Store, error) {
 	cfg.setDefaults()
 	db, err := connector(cfg)
 	if err != nil {
 		return nil, fmt.Errorf("could not connect to database: %q", err)
 	}

-	if err := initializeDB(db, cfg.InitSQLScriptsDir, embeddedSQLScripts); err != nil {
+	if err := runInitScripts(logger, db, cfg); err != nil {
 		_ = db.Close()
 		return nil, err
 	}
+	if cfg.Replication {
+		return &Store{
+			db: db,
+			writer: clickhousespanstore.NewSpanWriter(
+				logger,
+				db,
+				cfg.SpansIndexTable,
+				cfg.SpansTable,
+				cfg.Tenant,
+				clickhousespanstore.Encoding(cfg.Encoding),
+				cfg.BatchFlushInterval,
+				cfg.BatchWriteSize,
+				cfg.MaxSpanCount,
+			),
+			reader: clickhousespanstore.NewTraceReader(
+				db,
+				cfg.OperationsTable,
+				cfg.SpansIndexTable,
+				cfg.SpansTable,
+				cfg.Tenant,
+				cfg.MaxNumSpans,
+			),
+			archiveWriter: clickhousespanstore.NewSpanWriter(
+				logger,
+				db,
+				"",
+				cfg.GetSpansArchiveTable(),
+				cfg.Tenant,
+				clickhousespanstore.Encoding(cfg.Encoding),
+				cfg.BatchFlushInterval,
+				cfg.BatchWriteSize,
+				cfg.MaxSpanCount,
+			),
+			archiveReader: clickhousespanstore.NewTraceReader(
+				db,
+				"",
+				"",
+				cfg.GetSpansArchiveTable(),
+				cfg.Tenant,
+				cfg.MaxNumSpans,
+			),
+		}, nil
+	}
 	return &Store{
-		db:            db,
-		writer:        clickhousespanstore.NewSpanWriter(logger, db, cfg.SpansIndexTable, cfg.SpansTable, clickhousespanstore.Encoding(cfg.Encoding), cfg.BatchFlushInterval, cfg.BatchWriteSize),
-		reader:        clickhousespanstore.NewTraceReader(db, cfg.OperationsTable, cfg.SpansIndexTable, cfg.SpansTable),
-		archiveWriter: clickhousespanstore.NewSpanWriter(logger, db, "", "jaeger_archive_spans", clickhousespanstore.Encoding(cfg.Encoding), cfg.BatchFlushInterval, cfg.BatchWriteSize),
-		archiveReader: clickhousespanstore.NewTraceReader(db, "", "", "jaeger_archive_spans"),
+		db: db,
+		writer: clickhousespanstore.NewSpanWriter(
+			logger,
+			db,
+			cfg.SpansIndexTable,
+			cfg.SpansTable,
+			cfg.Tenant,
+			clickhousespanstore.Encoding(cfg.Encoding),
+			cfg.BatchFlushInterval,
+			cfg.BatchWriteSize,
+			cfg.MaxSpanCount,
+		),
+		reader: clickhousespanstore.NewTraceReader(
+			db,
+			cfg.OperationsTable,
+			cfg.SpansIndexTable,
+			cfg.SpansTable,
+			cfg.Tenant,
+			cfg.MaxNumSpans,
+		),
+		archiveWriter: clickhousespanstore.NewSpanWriter(
+			logger,
+			db,
+			"",
+			cfg.GetSpansArchiveTable(),
+			cfg.Tenant,
+			clickhousespanstore.Encoding(cfg.Encoding),
+			cfg.BatchFlushInterval,
+			cfg.BatchWriteSize,
+			cfg.MaxSpanCount,
+		),
+		archiveReader: clickhousespanstore.NewTraceReader(
+			db,
+			"",
+			"",
+			cfg.GetSpansArchiveTable(),
+			cfg.Tenant,
+			cfg.MaxNumSpans,
+		),
 	}, nil
 }

 func connector(cfg Configuration) (*sql.DB, error) {
-	params := fmt.Sprintf("%s?database=%s&username=%s&password=%s",
-		cfg.Address,
-		cfg.Database,
-		cfg.Username,
-		cfg.Password,
-	)
+	var conn *sql.DB
+
+	options := clickhouse.Options{
+		Addr: []string{sanitize(cfg.Address)},
+		Auth: clickhouse.Auth{
+			Database: cfg.Database,
+			Username: cfg.Username,
+			Password: cfg.Password,
+		},
+		Compression: &clickhouse.Compression{
+			Method: clickhouse.CompressionLZ4,
+		},
+	}

 	if cfg.CaFile != "" {
-		caCert, err := ioutil.ReadFile(cfg.CaFile)
+		caCert, err := os.ReadFile(cfg.CaFile)
 		if err != nil {
 			return nil, err
 		}
 		caCertPool := x509.NewCertPool()
 		caCertPool.AppendCertsFromPEM(caCert)
-		err = clickhouse.RegisterTLSConfig(tlsConfigKey, &tls.Config{RootCAs: caCertPool})
-		if err != nil {
-			return nil, err
+		options.TLS = &tls.Config{
+			RootCAs: caCertPool,
 		}
-		params += fmt.Sprintf(
-			"&secure=true&tls_config=%s",
-			tlsConfigKey,
-		)
 	}
-	return clickhouseConnector(params)
+	conn = clickhouse.OpenDB(&options)
+
+	if cfg.MaxOpenConns != nil {
+		conn.SetMaxIdleConns(int(*cfg.MaxOpenConns))
+	}
+	if cfg.MaxIdleConns != nil {
+		conn.SetMaxIdleConns(int(*cfg.MaxIdleConns))
+	}
+	if cfg.ConnMaxLifetimeMillis != nil {
+		conn.SetConnMaxLifetime(time.Millisecond * time.Duration(*cfg.ConnMaxLifetimeMillis))
+	}
+	if cfg.ConnMaxIdleTimeMillis != nil {
+		conn.SetConnMaxIdleTime(time.Millisecond * time.Duration(*cfg.ConnMaxIdleTimeMillis))
+	}
+
+	if err := conn.Ping(); err != nil {
+		return nil, err
+	}
+	return conn, nil
 }

-func initializeDB(db *sql.DB, initSQLScriptsDir string, embeddedScripts embed.FS) error {
-	var sqlStatements []string
-	if initSQLScriptsDir != "" {
-		filePaths, err := walkMatch(initSQLScriptsDir, "*.sql")
+type tableArgs struct {
+	Database string
+
+	SpansIndexTable   clickhousespanstore.TableName
+	SpansTable        clickhousespanstore.TableName
+	OperationsTable   clickhousespanstore.TableName
+	SpansArchiveTable clickhousespanstore.TableName
+
+	TTLTimestamp string
+	TTLDate      string
+
+	Multitenant bool
+	Replication bool
+}
+
+type distributedTableArgs struct {
+	Database string
+	Table    clickhousespanstore.TableName
+	Hash     string
+}
+
+func render(templates *template.Template, filename string, args interface{}) string {
+	var statement strings.Builder
+	err := templates.ExecuteTemplate(&statement, filename, args)
+	if err != nil {
+		panic(err)
+	}
+	return statement.String()
+}
+
+func runInitScripts(logger hclog.Logger, db *sql.DB, cfg Configuration) error {
+	var (
+		sqlStatements []string
+		ttlTimestamp  string
+		ttlDate       string
+	)
+	if cfg.TTLDays > 0 {
+		ttlTimestamp = fmt.Sprintf("TTL timestamp + INTERVAL %d DAY DELETE", cfg.TTLDays)
+		ttlDate = fmt.Sprintf("TTL date + INTERVAL %d DAY DELETE", cfg.TTLDays)
+	}
+	if cfg.InitSQLScriptsDir != "" {
+		filePaths, err := walkMatch(cfg.InitSQLScriptsDir, "*.sql")
 		if err != nil {
 			return fmt.Errorf("could not list sql files: %q", err)
 		}
 		sort.Strings(filePaths)
 		for _, f := range filePaths {
-			sqlStatement, err := ioutil.ReadFile(filepath.Clean(f))
+			sqlStatement, err := os.ReadFile(filepath.Clean(f))
 			if err != nil {
 				return err
 			}
 			sqlStatements = append(sqlStatements, string(sqlStatement))
 		}
-	} else {
-		f, err := embeddedScripts.ReadFile("sqlscripts/0001-jaeger-index.sql")
-		if err != nil {
-			return err
-		}
-		sqlStatements = append(sqlStatements, string(f))
-		f, err = embeddedScripts.ReadFile("sqlscripts/0002-jaeger-spans.sql")
-		if err != nil {
-			return err
-		}
-		sqlStatements = append(sqlStatements, string(f))
-		f, err = embeddedScripts.ReadFile("sqlscripts/0003-jaeger-operations.sql")
-		if err != nil {
-			return err
-		}
-		sqlStatements = append(sqlStatements, string(f))
-		f, err = embeddedScripts.ReadFile("sqlscripts/0004-jaeger-spans-archive.sql")
-		if err != nil {
-			return err
-		}
-		sqlStatements = append(sqlStatements, string(f))
 	}
-	return executeScripts(sqlStatements, db)
+	if *cfg.InitTables {
+		templates := template.Must(template.ParseFS(jaegerclickhouse.SQLScripts, "sqlscripts/*.tmpl.sql"))
+
+		args := tableArgs{
+			Database: cfg.Database,
+
+			SpansIndexTable:   cfg.SpansIndexTable,
+			SpansTable:        cfg.SpansTable,
+			OperationsTable:   cfg.OperationsTable,
+			SpansArchiveTable: cfg.GetSpansArchiveTable(),
+
+			TTLTimestamp: ttlTimestamp,
+			TTLDate:      ttlDate,
+
+			Multitenant: cfg.Tenant != "",
+			Replication: cfg.Replication,
+		}
+
+		if cfg.Replication {
+			// Add "_local" to the local table names, and omit it from the distributed tables below
+			args.SpansIndexTable = args.SpansIndexTable.ToLocal()
+			args.SpansTable = args.SpansTable.ToLocal()
+			args.OperationsTable = args.OperationsTable.ToLocal()
+			args.SpansArchiveTable = args.SpansArchiveTable.ToLocal()
+		}
+
+		sqlStatements = append(sqlStatements, render(templates, "jaeger-index.tmpl.sql", args))
+		sqlStatements = append(sqlStatements, render(templates, "jaeger-operations.tmpl.sql", args))
+		sqlStatements = append(sqlStatements, render(templates, "jaeger-spans.tmpl.sql", args))
+		sqlStatements = append(sqlStatements, render(templates, "jaeger-spans-archive.tmpl.sql", args))
+
+		if cfg.Replication {
+			// Now these tables omit the "_local" suffix
+			distargs := distributedTableArgs{
+				Table:    cfg.SpansTable,
+				Database: cfg.Database,
+				Hash:     "cityHash64(traceID)",
+			}
+			sqlStatements = append(sqlStatements, render(templates, "distributed-table.tmpl.sql", distargs))
+
+			distargs.Table = cfg.SpansIndexTable
+			sqlStatements = append(sqlStatements, render(templates, "distributed-table.tmpl.sql", distargs))
+
+			distargs.Table = cfg.GetSpansArchiveTable()
+			sqlStatements = append(sqlStatements, render(templates, "distributed-table.tmpl.sql", distargs))
+
+			distargs.Table = cfg.OperationsTable
+			distargs.Hash = "rand()"
+			sqlStatements = append(sqlStatements, render(templates, "distributed-table.tmpl.sql", distargs))
+		}
+	}
+	return executeScripts(logger, sqlStatements, db)
 }

 func (s *Store) SpanReader() spanstore.Reader {
@ -145,27 +311,18 @@ func (s *Store) ArchiveSpanWriter() spanstore.Writer {
 	return s.archiveWriter
 }

+func (s *Store) StreamingSpanWriter() spanstore.Writer {
+	return s.writer
+}
+
 func (s *Store) Close() error {
 	return s.db.Close()
 }

-func clickhouseConnector(params string) (*sql.DB, error) {
-	db, err := sql.Open("clickhouse", params)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := db.Ping(); err != nil {
-		return nil, err
-	}
-
-	return db, nil
-}
-
-func executeScripts(sqlStatements []string, db *sql.DB) error {
+func executeScripts(logger hclog.Logger, sqlStatements []string, db *sql.DB) error {
 	tx, err := db.Begin()
 	if err != nil {
-		return nil
+		return err
 	}
 	committed := false
 	defer func() {
@ -174,10 +331,11 @@ func executeScripts(sqlStatements []string, db *sql.DB) error {
 		}
 	}()

-	for _, file := range sqlStatements {
-		_, err = tx.Exec(file)
+	for _, statement := range sqlStatements {
+		logger.Debug("Running SQL statement", "statement", statement)
+		_, err = tx.Exec(statement)
 		if err != nil {
-			return fmt.Errorf("could not run sql %q: %q", file, err)
+			return fmt.Errorf("could not run sql %q: %q", statement, err)
 		}
 	}
 	committed = true
@ -205,3 +363,10 @@ func walkMatch(root, pattern string) ([]string, error) {
 	}
 	return matches, nil
 }
+
+// Earlier version of clickhouse-go used to expect address as tcp://host:port
+// while newer version of clickhouse-go expect address as host:port (without scheme)
+// so to maintain backward compatibility we clean it up
+func sanitize(addr string) string {
+	return strings.TrimPrefix(addr, "tcp://")
+}
--- a/storage/store_test.go
+++ b/storage/store_test.go
@ -0,0 +1,187 @@
+package storage
+
+import (
+	"database/sql"
+	"fmt"
+	"testing"
+
+	sqlmock "github.com/DATA-DOG/go-sqlmock"
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousedependencystore"
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore"
+	"github.com/jaegertracing/jaeger-clickhouse/storage/clickhousespanstore/mocks"
+)
+
+const (
+	testIndexTable        = "test_index_table"
+	testSpansTable        = "test_spans_table"
+	testOperationsTable   = "test_operation_table"
+	testSpansArchiveTable = "test_spans_archive_table"
+)
+
+var errorMock = fmt.Errorf("error mock")
+
+func TestStore_SpanWriter(t *testing.T) {
+	writer := clickhousespanstore.SpanWriter{}
+	store := Store{
+		writer: &writer,
+	}
+	assert.Equal(t, &writer, store.SpanWriter())
+}
+
+func TestStore_ArchiveSpanWriter(t *testing.T) {
+	writer := clickhousespanstore.SpanWriter{}
+	store := Store{
+		archiveWriter: &writer,
+	}
+	assert.Equal(t, &writer, store.ArchiveSpanWriter())
+}
+
+func TestStore_SpanReader(t *testing.T) {
+	reader := clickhousespanstore.TraceReader{}
+	store := Store{
+		reader: &reader,
+	}
+	assert.Equal(t, &reader, store.SpanReader())
+}
+
+func TestStore_ArchiveSpanReader(t *testing.T) {
+	reader := clickhousespanstore.TraceReader{}
+	store := Store{
+		archiveReader: &reader,
+	}
+	assert.Equal(t, &reader, store.ArchiveSpanReader())
+}
+
+func TestStore_DependencyReader(t *testing.T) {
+	store := Store{}
+	assert.Equal(t, &clickhousedependencystore.DependencyStore{}, store.DependencyReader())
+}
+
+func TestStore_Close(t *testing.T) {
+	db, mock, err := mocks.GetDbMock()
+	require.NoError(t, err)
+	defer db.Close()
+
+	logger := mocks.NewSpyLogger()
+	store := newStore(db, logger)
+
+	mock.ExpectClose()
+	require.NoError(t, store.Close())
+	assert.NoError(t, mock.ExpectationsWereMet())
+	logger.AssertLogsEmpty(t)
+}
+
+func newStore(db *sql.DB, logger mocks.SpyLogger) Store {
+	return Store{
+		db: db,
+		writer: clickhousespanstore.NewSpanWriter(
+			logger,
+			db,
+			testIndexTable,
+			testSpansTable,
+			"",
+			clickhousespanstore.EncodingJSON,
+			0,
+			0,
+			0,
+		),
+		reader: clickhousespanstore.NewTraceReader(
+			db,
+			testOperationsTable,
+			testIndexTable,
+			testSpansTable,
+			"",
+			0,
+		),
+		archiveWriter: clickhousespanstore.NewSpanWriter(
+			logger,
+			db,
+			testIndexTable,
+			testSpansArchiveTable,
+			"",
+			clickhousespanstore.EncodingJSON,
+			0,
+			0,
+			0,
+		),
+		archiveReader: clickhousespanstore.NewTraceReader(
+			db,
+			testOperationsTable,
+			testIndexTable,
+			testSpansArchiveTable,
+			"",
+			0,
+		),
+	}
+}
+
+func TestStore_executeScripts(t *testing.T) {
+	db, mock, err := mocks.GetDbMock()
+	require.NoError(t, err)
+	defer db.Close()
+
+	spyLogger := mocks.NewSpyLogger()
+	scripts := []string{
+		"first SQL script",
+		"second_SQL_script",
+	}
+
+	mock.ExpectBegin()
+	for _, script := range scripts {
+		mock.ExpectExec(script).WillReturnResult(sqlmock.NewResult(1, 1))
+	}
+	mock.ExpectCommit()
+	err = executeScripts(spyLogger, scripts, db)
+	require.NoError(t, err)
+	assert.NoError(t, mock.ExpectationsWereMet())
+	spyLogger.AssertLogsOfLevelEqual(t, hclog.Debug, func() []mocks.LogMock {
+		res := make([]mocks.LogMock, len(scripts))
+		for i, script := range scripts {
+			res[i] = mocks.LogMock{Msg: "Running SQL statement", Args: []interface{}{"statement", script}}
+		}
+		return res
+	}())
+}
+
+func TestStore_executeScriptsExecuteError(t *testing.T) {
+	db, mock, err := mocks.GetDbMock()
+	require.NoError(t, err)
+	defer db.Close()
+
+	spyLogger := mocks.NewSpyLogger()
+	scripts := []string{
+		"first SQL script",
+		"second_SQL_script",
+	}
+
+	mock.ExpectBegin()
+	mock.ExpectExec(scripts[0]).WillReturnError(errorMock)
+	mock.ExpectRollback()
+	err = executeScripts(spyLogger, scripts, db)
+	assert.EqualError(t, err, fmt.Sprintf("could not run sql %q: %q", scripts[0], errorMock))
+	spyLogger.AssertLogsOfLevelEqual(
+		t,
+		hclog.Debug,
+		[]mocks.LogMock{{Msg: "Running SQL statement", Args: []interface{}{"statement", scripts[0]}}},
+	)
+}
+
+func TestStore_executeScriptBeginError(t *testing.T) {
+	db, mock, err := mocks.GetDbMock()
+	require.NoError(t, err)
+	defer db.Close()
+
+	spyLogger := mocks.NewSpyLogger()
+	scripts := []string{
+		"first SQL script",
+		"second_SQL_script",
+	}
+
+	mock.ExpectBegin().WillReturnError(errorMock)
+	err = executeScripts(spyLogger, scripts, db)
+	assert.EqualError(t, err, errorMock.Error())
+}