From 5777270109aa4c311d6216a28d59b08446418d8f Mon Sep 17 00:00:00 2001 From: binnn6 <734819342@qq.com> Date: Wed, 14 Jun 2023 00:52:26 +0800 Subject: [PATCH 1/4] add new metadata field for pulsar Signed-off-by: binnn6 <734819342@qq.com> --- .../supported-pubsub/setup-pulsar.md | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md index d7e53c008..9f052b2b4 100644 --- a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md +++ b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md @@ -70,7 +70,7 @@ spec: | tenant | N | The topic tenant within the instance. Tenants are essential to multi-tenancy in Pulsar, and spread across clusters. Default: `"public"` | `"public"` | | consumerID | N | Used to set the subscription name or consumer ID. | `"topic1"` | namespace | N | The administrative unit of the topic, which acts as a grouping mechanism for related topics. Default: `"default"` | `"default"` -| persistent | N | Pulsar supports two kinds of topics: [persistent](https://pulsar.apache.org/docs/en/concepts-architecture-overview#persistent-storage) and [non-persistent](https://pulsar.apache.org/docs/en/concepts-messaging/#non-persistent-topics). With persistent topics, all messages are durably persisted on disks (if the broker is not standalone, messages are durably persisted on multiple disks), whereas data for non-persistent topics is not persisted to storage disks. +| persistent | N | Pulsar supports two kinds of topics: [persistent](https://pulsar.apache.org/docs/en/concepts-architecture-overview#persistent-storage) and [non-persistent](https://pulsar.apache.org/docs/en/concepts-messaging/#non-persistent-topics). With persistent topics, all messages are durably persisted on disks (if the broker is not standalone, messages are durably persisted on multiple disks), whereas data for non-persistent topics is not persisted to storage disks. | disableBatching | N | disable batching.When batching enabled default batch delay is set to 10 ms and default batch size is 1000 messages,Setting `disableBatching: true` will make the producer to send messages individually. Default: `"false"` | `"true"`, `"false"`| | batchingMaxPublishDelay | N | batchingMaxPublishDelay set the time period within which the messages sent will be batched,if batch messages are enabled. If set to a non zero value, messages will be queued until this time interval or batchingMaxMessages (see below) or batchingMaxSize (see below). There are two valid formats, one is the fraction with a unit suffix format, and the other is the pure digital format that is processed as milliseconds. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Default: `"10ms"` | `"10ms"`, `"10"`| | batchingMaxMessages | N | batchingMaxMessages set the maximum number of messages permitted in a batch.If set to a value greater than 1, messages will be queued until this threshold is reached or batchingMaxSize (see below) has been reached or the batch interval has elapsed. Default: `"1000"` | `"1000"`| @@ -80,6 +80,9 @@ spec: | publicKey | N | A public key to be used for publisher and consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | privateKey | N | A private key to be used for consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | keys | N | A comma delimited string containing names of [Pulsar session keys](https://pulsar.apache.org/docs/3.0.x/security-encryption/#how-it-works-in-pulsar). Used in conjunction with `publicKey` for publisher encryption | +| processMode | N | Eable proccess multiple messages at once. Default: `"async"` | `"async"`, `"sync"`| +| subscribeType | N | Pulsar supports four kinds of [subscription types](https://pulsar.apache.org/docs/3.0.x/concepts-messaging/#subscription-types). Default: `"shared"` | `"shared"`, `"exclusive"`, `"failover"`, `"key_shared"`| +| partitionKey | N | partitionKey sets the key of the message for routing policy. Default: `""` | | ### Enabling message delivery retries @@ -202,6 +205,46 @@ spec: value: "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA1KDAM4L8RtJ+nLaXBrBhzVpvTemsKVZoAct8A+ShepOHT9lg\nHOCGLFGWNla6K6j+b3AV/P/fAAhwj82vwTDdruXSflvSdmYeFAw3Ypphc1A5oM53\nwSRWhg63potBNWqdDzj8ApYgqjpmjYSQdL5/a3golb36GYFrY0MLFTv7wZ87pmMI\nPsOgGIcPbCHker2fRZ34WXYLb1hkeUpwx4eKjpwcg35gccvR6o/UhbKAuc60V1J9\nWof2sNgtlRaQej45wnpjWYzZrIyk5qUbn0QiCdpIrXvYtANq0Id6gP8zJvUEdPIg\nNuYxEmVCl9jI+8eGI6peD0qIt8U80hf9axhJ3QIDAQABAoIBAQCKuHnM4ac/eXM7\nQPDVX1vfgyHc3hgBPCtNCHnXfGFRvFBqavKGxIElBvGOcBS0CWQ+Rg1Ca5kMx3TQ\njSweSYhH5A7pe3Sa5FK5V6MGxJvRhMSkQi/lJZUBjzaIBJA9jln7pXzdHx8ekE16\nBMPONr6g2dr4nuI9o67xKrtfViwRDGaG6eh7jIMlEqMMc6WqyhvI67rlVDSTHFKX\njlMcozJ3IT8BtTzKg2Tpy7ReVuJEpehum8yn1ZVdAnotBDJxI07DC1cbOP4M2fHM\ngfgPYWmchauZuTeTFu4hrlY5jg0/WLs6by8r/81+vX3QTNvejX9UdTHMSIfQdX82\nAfkCKUVhAoGBAOvGv+YXeTlPRcYC642x5iOyLQm+BiSX4jKtnyJiTU2s/qvvKkIu\nxAOk3OtniT9NaUAHEZE9tI71dDN6IgTLQlAcPCzkVh6Sc5eG0MObqOO7WOMCWBkI\nlaAKKBbd6cGDJkwGCJKnx0pxC9f8R4dw3fmXWgWAr8ENiekMuvjSfjZ5AoGBAObd\ns2L5uiUPTtpyh8WZ7rEvrun3djBhzi+d7rgxEGdditeiLQGKyZbDPMSMBuus/5wH\nwfi0xUq50RtYDbzQQdC3T/C20oHmZbjWK5mDaLRVzWS89YG/NT2Q8eZLBstKqxkx\ngoT77zoUDfRy+CWs1xvXzgxagD5Yg8/OrCuXOqWFAoGAPIw3r6ELknoXEvihASxU\nS4pwInZYIYGXpygLG8teyrnIVOMAWSqlT8JAsXtPNaBtjPHDwyazfZrvEmEk51JD\nX0tA8M5ah1NYt+r5JaKNxp3P/8wUT6lyszyoeubWJsnFRfSusuq/NRC+1+KDg/aq\nKnSBu7QGbm9JoT2RrmBv5RECgYBRn8Lj1I1muvHTNDkiuRj2VniOSirkUkA2/6y+\nPMKi+SS0tqcY63v4rNCYYTW1L7Yz8V44U5mJoQb4lvpMbolGhPljjxAAU3hVkItb\nvGVRlSCIZHKczADD4rJUDOS7DYxO3P1bjUN4kkyYx+lKUMDBHFzCa2D6Kgt4dobS\n5qYajQKBgQC7u7MFPkkEMqNqNGu5erytQkBq1v1Ipmf9rCi3iIj4XJLopxMgw0fx\n6jwcwNInl72KzoUBLnGQ9PKGVeBcgEgdI+a+tq+1TJo6Ta+hZSx+4AYiKY18eRKG\neNuER9NOcSVJ7Eqkcw4viCGyYDm2vgNV9HJ0VlAo3RDh8x5spEN+mg==\n-----END RSA PRIVATE KEY-----\n" ``` +### Partition Key + +When invoking the plusar pub/sub, its possible to provide an optional partition key by using the `metadata` query param in the request url. + +The param name is `partitionKey`. + +Example: + +```shell +curl -X POST http://localhost:3500/v1.0/publish/myPlusar/myTopic?metadata.partitionKey=key1 \ + -H "Content-Type: application/json" \ + -d '{ + "data": { + "message": "Hi" + } + }' +``` + +### Message headers + +All other metadata key/value pairs (that are not `partitionKey`) are set as headers in the Pulsar message. Here is an example setting a `correlationId` for the message. + +```shell +curl -X POST http://localhost:3500/v1.0/publish/myPlusar/myTopic?metadata.correlationId=myCorrelationID&metadata.partitionKey=key1 \ + -H "Content-Type: application/json" \ + -d '{ + "data": { + "message": "Hi" + } + }' +``` + +## Order guarantee + +To ensure that messages arrive in order for each consumer subscribed to a specific key, 3 conditions must be met. + +1. subscribeType should be set to `key_shared`. +2. partitionKey must be set. +3. processMode should be set to `sync`. + ## Create a Pulsar instance {{< tabs "Self-Hosted" "Kubernetes">}} From 0d25e77685b0cd38c48c7b534bc21af2359e1350 Mon Sep 17 00:00:00 2001 From: binnn6 <734819342@qq.com> Date: Wed, 14 Jun 2023 10:50:19 +0800 Subject: [PATCH 2/4] few edits on pubsub plusar doc Signed-off-by: binnn6 <734819342@qq.com> --- .../supported-pubsub/setup-pulsar.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md index 9f052b2b4..ed363c3a8 100644 --- a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md +++ b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md @@ -80,9 +80,9 @@ spec: | publicKey | N | A public key to be used for publisher and consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | privateKey | N | A private key to be used for consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | keys | N | A comma delimited string containing names of [Pulsar session keys](https://pulsar.apache.org/docs/3.0.x/security-encryption/#how-it-works-in-pulsar). Used in conjunction with `publicKey` for publisher encryption | -| processMode | N | Eable proccess multiple messages at once. Default: `"async"` | `"async"`, `"sync"`| +| processMode | N | Eable processing multiple messages at once. Default: `"async"` | `"async"`, `"sync"`| | subscribeType | N | Pulsar supports four kinds of [subscription types](https://pulsar.apache.org/docs/3.0.x/concepts-messaging/#subscription-types). Default: `"shared"` | `"shared"`, `"exclusive"`, `"failover"`, `"key_shared"`| -| partitionKey | N | partitionKey sets the key of the message for routing policy. Default: `""` | | +| partitionKey | N | Sets the key of the message for routing policy. Default: `""` | | ### Enabling message delivery retries @@ -207,9 +207,9 @@ spec: ### Partition Key -When invoking the plusar pub/sub, its possible to provide an optional partition key by using the `metadata` query param in the request url. +When invoking the Plusar pub/sub, its possible to provide an optional partition key by using the `metadata` query parameter in the request url. -The param name is `partitionKey`. +The parameter name is `partitionKey`. Example: @@ -225,7 +225,7 @@ curl -X POST http://localhost:3500/v1.0/publish/myPlusar/myTopic?metadata.partit ### Message headers -All other metadata key/value pairs (that are not `partitionKey`) are set as headers in the Pulsar message. Here is an example setting a `correlationId` for the message. +All other metadata key/value pairs (that are not `partitionKey`) are set as headers in the Pulsar message. For example, set a `correlationId` for the message: ```shell curl -X POST http://localhost:3500/v1.0/publish/myPlusar/myTopic?metadata.correlationId=myCorrelationID&metadata.partitionKey=key1 \ @@ -239,11 +239,11 @@ curl -X POST http://localhost:3500/v1.0/publish/myPlusar/myTopic?metadata.correl ## Order guarantee -To ensure that messages arrive in order for each consumer subscribed to a specific key, 3 conditions must be met. +To ensure that messages arrive in order for each consumer subscribed to a specific key, three conditions must be met. -1. subscribeType should be set to `key_shared`. -2. partitionKey must be set. -3. processMode should be set to `sync`. +1. `subscribeType` should be set to `key_shared`. +2. `partitionKey` must be set. +3. `processMode` should be set to `sync`. ## Create a Pulsar instance From 664236e0f80720e7906df15e6c742aa840963ad0 Mon Sep 17 00:00:00 2001 From: Hannah Hunter <94493363+hhunter-ms@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:20:05 -0400 Subject: [PATCH 3/4] typo Signed-off-by: Hannah Hunter <94493363+hhunter-ms@users.noreply.github.com> --- .../components-reference/supported-pubsub/setup-pulsar.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md index ed363c3a8..30375aab5 100644 --- a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md +++ b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md @@ -80,7 +80,7 @@ spec: | publicKey | N | A public key to be used for publisher and consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | privateKey | N | A private key to be used for consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | keys | N | A comma delimited string containing names of [Pulsar session keys](https://pulsar.apache.org/docs/3.0.x/security-encryption/#how-it-works-in-pulsar). Used in conjunction with `publicKey` for publisher encryption | -| processMode | N | Eable processing multiple messages at once. Default: `"async"` | `"async"`, `"sync"`| +| processMode | N | Enable processing multiple messages at once. Default: `"async"` | `"async"`, `"sync"`| | subscribeType | N | Pulsar supports four kinds of [subscription types](https://pulsar.apache.org/docs/3.0.x/concepts-messaging/#subscription-types). Default: `"shared"` | `"shared"`, `"exclusive"`, `"failover"`, `"key_shared"`| | partitionKey | N | Sets the key of the message for routing policy. Default: `""` | | From f1e3112cb9eb1a647b690dbfa3b378d2c8ce33d0 Mon Sep 17 00:00:00 2001 From: Hannah Hunter <94493363+hhunter-ms@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:21:14 -0400 Subject: [PATCH 4/4] typo Signed-off-by: Hannah Hunter <94493363+hhunter-ms@users.noreply.github.com> --- .../components-reference/supported-pubsub/setup-pulsar.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md index 30375aab5..bd23f8848 100644 --- a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md +++ b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md @@ -207,7 +207,7 @@ spec: ### Partition Key -When invoking the Plusar pub/sub, its possible to provide an optional partition key by using the `metadata` query parameter in the request url. +When invoking the Pulsar pub/sub, it's possible to provide an optional partition key by using the `metadata` query parameter in the request url. The parameter name is `partitionKey`.