From edfd74eaa37e4de0156fa1ee72850105c38e088a Mon Sep 17 00:00:00 2001 From: Simon Leet Date: Fri, 16 Jul 2021 11:12:53 -0700 Subject: [PATCH] Update Kafka pubsub & bindings component specs - Add missing `ConsumerGroup` and `ClientID` metadata properties to Kafka pubsub component spec. - Converged documentation of common properties between Kafka pubsub and bindings components. - Changed default use of `saslPassword` property to `secretKeyRef` instead of insecure plaintext string in sample yaml. - Cleaned up markdown lint issues in Kafka component specs. --- .../supported-bindings/kafka.md | 56 +++++++++---------- .../supported-pubsub/setup-apache-kafka.md | 47 ++++++++-------- 2 files changed, 51 insertions(+), 52 deletions(-) diff --git a/daprdocs/content/en/reference/components-reference/supported-bindings/kafka.md b/daprdocs/content/en/reference/components-reference/supported-bindings/kafka.md index 72018ab86..52702e7e6 100644 --- a/daprdocs/content/en/reference/components-reference/supported-bindings/kafka.md +++ b/daprdocs/content/en/reference/components-reference/supported-bindings/kafka.md @@ -9,53 +9,50 @@ aliases: ## Component format -To setup Kafka binding create a component of type `bindings.kafka`. See [this guide]({{< ref "howto-bindings.md#1-create-a-binding" >}}) on how to create and apply a binding configuration. - +To setup Kafka binding create a component of type `bindings.kafka`. See [this guide]({{< ref "howto-bindings.md#1-create-a-binding" >}}) on how to create and apply a binding configuration. For details on using `secretKeyRef`, see the guide on [how to reference secrets in components]({{< ref component-secrets.md >}}). ```yaml apiVersion: dapr.io/v1alpha1 kind: Component metadata: - name: - namespace: + name: kafka-binding + namespace: default spec: type: bindings.kafka version: v1 metadata: - - name: topics # Optional. in use for input bindings - value: topic1,topic2 - - name: brokers - value: localhost:9092,localhost:9093 - - name: consumerGroup - value: group1 - - name: publishTopic # Optional. in use for output bindings - value: topic3 - - name: authRequired # Required. default: "true" - value: "false" - - name: saslUsername # Optional. + - name: topics # Optional. Used for input bindings. + value: "topic1,topic2" + - name: brokers # Required. + value: "localhost:9092,localhost:9093" + - name: consumerGroup # Optional. Used for input bindings. + value: "group1" + - name: publishTopic # Optional. Used for output bindings. + value: "topic3" + - name: authRequired # Required. + value: "true" + - name: saslUsername # Required if authRequired is `true`. value: "user" - - name: saslPassword # Optional. - value: "password" + - name: saslPassword # Required if authRequired is `true`. + secretKeyRef: + name: kafka-secrets + key: saslPasswordSecret - name: maxMessageBytes # Optional. value: 1024 ``` -{{% alert title="Warning" color="warning" %}} -The above example uses secrets as plain strings. It is recommended to use a secret store for the secrets as described [here]({{< ref component-secrets.md >}}). -{{% /alert %}} ## Spec metadata fields | Field | Required | Binding support | Details | Example | |--------------------|:--------:|------------|-----|---------| -| topics | N | Input | A comma separated string of topics | `"mytopic1,topic2"` | -| brokers | Y | Input/Output | A comma separated string of kafka brokers | `"localhost:9092,localhost:9093"` | -| consumerGroup | N | Input | A kafka consumer group to listen on | `"group1"` | -| publishTopic | Y | Output | The topic to publish to | `"mytopic"` | -| authRequired | Y | Input/Output | Determines whether to use SASL authentication or not. Defaults to `"true"` | `"true"`, `"false"` | -| saslUsername | N | Input/Output | The SASL username for authentication. Only used if `authRequired` is set to - `"true"` | `"user"` | -| saslPassword | N | Input/Output | The SASL password for authentication. Only used if `authRequired` is set to - `"true"` | `"password"` | -| maxMessageBytes | N | Input/Output | The maximum size allowed for a single Kafka message. Defaults to 1024 | `2048` | - +| topics | N | Input | A comma-separated string of topics. | `"mytopic1,topic2"` | +| brokers | Y | Input/Output | A comma-separated string of Kafka brokers. | `"localhost:9092,dapr-kafka.myapp.svc.cluster.local:9093"` | +| consumerGroup | N | Input | A kafka consumer group to listen on. Each record published to a topic is delivered to one consumer within each consumer group subscribed to the topic. | `"group1"` | +| publishTopic | Y | Output | The topic to publish to. | `"mytopic"` | +| authRequired | Y | Input/Output | Enable [SASL](https://en.wikipedia.org/wiki/Simple_Authentication_and_Security_Layer) authentication with the Kafka brokers. | `"true"`, `"false"` | +| saslUsername | N | Input/Output | The SASL username used for authentication. Only required if `authRequired` is set to `"true"`. | `"adminuser"` | +| saslPassword | N | Input/Output | The SASL password used for authentication. Can be `secretKeyRef` to use a [secret reference]({{< ref component-secrets.md >}}). Only required if `authRequired` is set to `"true"`. | `""`, `"KeFg23!"` | +| maxMessageBytes | N | Input/Output | The maximum size in bytes allowed for a single Kafka message. Defaults to 1024. | `2048` | ## Binding support @@ -87,7 +84,6 @@ curl -X POST http://localhost:3500/v1.0/bindings/myKafka \ }' ``` - ## Related links - [Basic schema for a Dapr component]({{< ref component-schema >}}) diff --git a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-apache-kafka.md b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-apache-kafka.md index c20eded86..cad93f7d5 100644 --- a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-apache-kafka.md +++ b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-apache-kafka.md @@ -9,7 +9,7 @@ aliases: ## Component format -To setup Apache Kafka pubsub create a component of type `pubsub.kafka`. See [this guide]({{< ref "howto-publish-subscribe.md#step-1-setup-the-pubsub-component" >}}) on how to create and apply a pubsub configuration. +To setup Apache Kafka pubsub create a component of type `pubsub.kafka`. See [this guide]({{< ref "howto-publish-subscribe.md#step-1-setup-the-pubsub-component" >}}) on how to create and apply a pubsub configuration. For details on using `secretKeyRef`, see the guide on [how to reference secrets in components]({{< ref component-secrets.md >}}). ```yaml apiVersion: dapr.io/v1alpha1 @@ -21,32 +21,35 @@ spec: type: pubsub.kafka version: v1 metadata: - # Kafka broker connection setting - - name: brokers - value: "dapr-kafka.myapp.svc.cluster.local:9092" - - name: authRequired - value: "true" - - name: saslUsername - value: "adminuser" - - name: saslPassword - value: "KeFg23!" - - name: maxMessageBytes - value: 1024 + - name: brokers # Required. Kafka broker connection setting + value: "dapr-kafka.myapp.svc.cluster.local:9092" + - name: consumerGroup # Optional. Used for input bindings. + value: "group1" + - name: clientID # Optional. Used as client tracing ID by Kafka brokers. + value: "my-dapr-app-id" + - name: authRequired # Required. + value: "true" + - name: saslUsername # Required if authRequired is `true`. + value: "adminuser" + - name: saslPassword # Required if authRequired is `true`. + secretKeyRef: + name: kafka-secrets + key: saslPasswordSecret + - name: maxMessageBytes # Optional. + value: 1024 ``` -{{% alert title="Warning" color="warning" %}} -The above example uses secrets as plain strings. It is recommended to use a secret store for the secrets as described [here]({{< ref component-secrets.md >}}). -{{% /alert %}} - ## Spec metadata fields | Field | Required | Details | Example | |--------------------|:--------:|---------|---------| -| brokers | Y | Comma separated list of kafka brokers | `localhost:9092`, `dapr-kafka.myapp.svc.cluster.local:9092` -| authRequired | N | Enable authentication on the Kafka broker. Defaults to `"false"`. |`"true"`, `"false"` -| saslUsername | N | Username used for authentication. Only required if authRequired is set to true. | `"adminuser"` -| saslPassword | N | Password used for authentication. Can be `secretKeyRef` to use a secret reference. Only required if authRequired is set to true. Can be `secretKeyRef` to use a [secret reference]({{< ref component-secrets.md >}}) | `""`, `"KeFg23!"` -| maxMessageBytes | N | The maximum message size allowed for a single Kafka message. Default is 1024. | `2048` +| brokers | Y | A comma-separated list of Kafka brokers. | `"localhost:9092,dapr-kafka.myapp.svc.cluster.local:9093"` +| consumerGroup | N | A kafka consumer group to listen on. Each record published to a topic is delivered to one consumer within each consumer group subscribed to the topic. | `"group1"` +| clientID | N | A user-provided string sent with every request to the Kafka brokers for logging, debugging, and auditing purposes. Defaults to `"sarama"`. | `"my-dapr-app"` +| authRequired | Y | Enable [SASL](https://en.wikipedia.org/wiki/Simple_Authentication_and_Security_Layer) authentication with the Kafka brokers. | `"true"`, `"false"` +| saslUsername | N | The SASL username used for authentication. Only required if `authRequired` is set to `"true"`. | `"adminuser"` +| saslPassword | N | The SASL password used for authentication. Can be `secretKeyRef` to use a [secret reference]({{< ref component-secrets.md >}}). Only required if `authRequired` is set to `"true"`. | `""`, `"KeFg23!"` +| maxMessageBytes | N | The maximum size in bytes allowed for a single Kafka message. Defaults to 1024. | `2048` ## Per-call metadata fields @@ -69,6 +72,7 @@ curl -X POST http://localhost:3500/v1.0/publish/myKafka/myTopic?metadata.partiti ``` ## Create a Kafka instance + {{< tabs "Self-Hosted" "Kubernetes">}} {{% codetab %}} @@ -82,7 +86,6 @@ To run Kafka on Kubernetes, you can use any Kafka operator, such as [Strimzi](ht {{< /tabs >}} - ## Related links - [Basic schema for a Dapr component]({{< ref component-schema >}}) - Read [this guide]({{< ref "howto-publish-subscribe.md##step-1-setup-the-pubsub-component" >}}) for instructions on configuring pub/sub components