From 5777270109aa4c311d6216a28d59b08446418d8f Mon Sep 17 00:00:00 2001 From: binnn6 <734819342@qq.com> Date: Wed, 14 Jun 2023 00:52:26 +0800 Subject: [PATCH 1/5] add new metadata field for pulsar Signed-off-by: binnn6 <734819342@qq.com> --- .../supported-pubsub/setup-pulsar.md | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md index d7e53c008..9f052b2b4 100644 --- a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md +++ b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md @@ -70,7 +70,7 @@ spec: | tenant | N | The topic tenant within the instance. Tenants are essential to multi-tenancy in Pulsar, and spread across clusters. Default: `"public"` | `"public"` | | consumerID | N | Used to set the subscription name or consumer ID. | `"topic1"` | namespace | N | The administrative unit of the topic, which acts as a grouping mechanism for related topics. Default: `"default"` | `"default"` -| persistent | N | Pulsar supports two kinds of topics: [persistent](https://pulsar.apache.org/docs/en/concepts-architecture-overview#persistent-storage) and [non-persistent](https://pulsar.apache.org/docs/en/concepts-messaging/#non-persistent-topics). With persistent topics, all messages are durably persisted on disks (if the broker is not standalone, messages are durably persisted on multiple disks), whereas data for non-persistent topics is not persisted to storage disks. +| persistent | N | Pulsar supports two kinds of topics: [persistent](https://pulsar.apache.org/docs/en/concepts-architecture-overview#persistent-storage) and [non-persistent](https://pulsar.apache.org/docs/en/concepts-messaging/#non-persistent-topics). With persistent topics, all messages are durably persisted on disks (if the broker is not standalone, messages are durably persisted on multiple disks), whereas data for non-persistent topics is not persisted to storage disks. | disableBatching | N | disable batching.When batching enabled default batch delay is set to 10 ms and default batch size is 1000 messages,Setting `disableBatching: true` will make the producer to send messages individually. Default: `"false"` | `"true"`, `"false"`| | batchingMaxPublishDelay | N | batchingMaxPublishDelay set the time period within which the messages sent will be batched,if batch messages are enabled. If set to a non zero value, messages will be queued until this time interval or batchingMaxMessages (see below) or batchingMaxSize (see below). There are two valid formats, one is the fraction with a unit suffix format, and the other is the pure digital format that is processed as milliseconds. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Default: `"10ms"` | `"10ms"`, `"10"`| | batchingMaxMessages | N | batchingMaxMessages set the maximum number of messages permitted in a batch.If set to a value greater than 1, messages will be queued until this threshold is reached or batchingMaxSize (see below) has been reached or the batch interval has elapsed. Default: `"1000"` | `"1000"`| @@ -80,6 +80,9 @@ spec: | publicKey | N | A public key to be used for publisher and consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | privateKey | N | A private key to be used for consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | keys | N | A comma delimited string containing names of [Pulsar session keys](https://pulsar.apache.org/docs/3.0.x/security-encryption/#how-it-works-in-pulsar). Used in conjunction with `publicKey` for publisher encryption | +| processMode | N | Eable proccess multiple messages at once. Default: `"async"` | `"async"`, `"sync"`| +| subscribeType | N | Pulsar supports four kinds of [subscription types](https://pulsar.apache.org/docs/3.0.x/concepts-messaging/#subscription-types). Default: `"shared"` | `"shared"`, `"exclusive"`, `"failover"`, `"key_shared"`| +| partitionKey | N | partitionKey sets the key of the message for routing policy. Default: `""` | | ### Enabling message delivery retries @@ -202,6 +205,46 @@ spec: value: "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA1KDAM4L8RtJ+nLaXBrBhzVpvTemsKVZoAct8A+ShepOHT9lg\nHOCGLFGWNla6K6j+b3AV/P/fAAhwj82vwTDdruXSflvSdmYeFAw3Ypphc1A5oM53\nwSRWhg63potBNWqdDzj8ApYgqjpmjYSQdL5/a3golb36GYFrY0MLFTv7wZ87pmMI\nPsOgGIcPbCHker2fRZ34WXYLb1hkeUpwx4eKjpwcg35gccvR6o/UhbKAuc60V1J9\nWof2sNgtlRaQej45wnpjWYzZrIyk5qUbn0QiCdpIrXvYtANq0Id6gP8zJvUEdPIg\nNuYxEmVCl9jI+8eGI6peD0qIt8U80hf9axhJ3QIDAQABAoIBAQCKuHnM4ac/eXM7\nQPDVX1vfgyHc3hgBPCtNCHnXfGFRvFBqavKGxIElBvGOcBS0CWQ+Rg1Ca5kMx3TQ\njSweSYhH5A7pe3Sa5FK5V6MGxJvRhMSkQi/lJZUBjzaIBJA9jln7pXzdHx8ekE16\nBMPONr6g2dr4nuI9o67xKrtfViwRDGaG6eh7jIMlEqMMc6WqyhvI67rlVDSTHFKX\njlMcozJ3IT8BtTzKg2Tpy7ReVuJEpehum8yn1ZVdAnotBDJxI07DC1cbOP4M2fHM\ngfgPYWmchauZuTeTFu4hrlY5jg0/WLs6by8r/81+vX3QTNvejX9UdTHMSIfQdX82\nAfkCKUVhAoGBAOvGv+YXeTlPRcYC642x5iOyLQm+BiSX4jKtnyJiTU2s/qvvKkIu\nxAOk3OtniT9NaUAHEZE9tI71dDN6IgTLQlAcPCzkVh6Sc5eG0MObqOO7WOMCWBkI\nlaAKKBbd6cGDJkwGCJKnx0pxC9f8R4dw3fmXWgWAr8ENiekMuvjSfjZ5AoGBAObd\ns2L5uiUPTtpyh8WZ7rEvrun3djBhzi+d7rgxEGdditeiLQGKyZbDPMSMBuus/5wH\nwfi0xUq50RtYDbzQQdC3T/C20oHmZbjWK5mDaLRVzWS89YG/NT2Q8eZLBstKqxkx\ngoT77zoUDfRy+CWs1xvXzgxagD5Yg8/OrCuXOqWFAoGAPIw3r6ELknoXEvihASxU\nS4pwInZYIYGXpygLG8teyrnIVOMAWSqlT8JAsXtPNaBtjPHDwyazfZrvEmEk51JD\nX0tA8M5ah1NYt+r5JaKNxp3P/8wUT6lyszyoeubWJsnFRfSusuq/NRC+1+KDg/aq\nKnSBu7QGbm9JoT2RrmBv5RECgYBRn8Lj1I1muvHTNDkiuRj2VniOSirkUkA2/6y+\nPMKi+SS0tqcY63v4rNCYYTW1L7Yz8V44U5mJoQb4lvpMbolGhPljjxAAU3hVkItb\nvGVRlSCIZHKczADD4rJUDOS7DYxO3P1bjUN4kkyYx+lKUMDBHFzCa2D6Kgt4dobS\n5qYajQKBgQC7u7MFPkkEMqNqNGu5erytQkBq1v1Ipmf9rCi3iIj4XJLopxMgw0fx\n6jwcwNInl72KzoUBLnGQ9PKGVeBcgEgdI+a+tq+1TJo6Ta+hZSx+4AYiKY18eRKG\neNuER9NOcSVJ7Eqkcw4viCGyYDm2vgNV9HJ0VlAo3RDh8x5spEN+mg==\n-----END RSA PRIVATE KEY-----\n" ``` +### Partition Key + +When invoking the plusar pub/sub, its possible to provide an optional partition key by using the `metadata` query param in the request url. + +The param name is `partitionKey`. + +Example: + +```shell +curl -X POST http://localhost:3500/v1.0/publish/myPlusar/myTopic?metadata.partitionKey=key1 \ + -H "Content-Type: application/json" \ + -d '{ + "data": { + "message": "Hi" + } + }' +``` + +### Message headers + +All other metadata key/value pairs (that are not `partitionKey`) are set as headers in the Pulsar message. Here is an example setting a `correlationId` for the message. + +```shell +curl -X POST http://localhost:3500/v1.0/publish/myPlusar/myTopic?metadata.correlationId=myCorrelationID&metadata.partitionKey=key1 \ + -H "Content-Type: application/json" \ + -d '{ + "data": { + "message": "Hi" + } + }' +``` + +## Order guarantee + +To ensure that messages arrive in order for each consumer subscribed to a specific key, 3 conditions must be met. + +1. subscribeType should be set to `key_shared`. +2. partitionKey must be set. +3. processMode should be set to `sync`. + ## Create a Pulsar instance {{< tabs "Self-Hosted" "Kubernetes">}} From 669bc0cc2d84261b21c62e2b430a9b904127a5ec Mon Sep 17 00:00:00 2001 From: Hannah Hunter Date: Tue, 13 Jun 2023 13:42:36 -0400 Subject: [PATCH 2/5] updates to workflow docs Signed-off-by: Hannah Hunter --- .../building-blocks/workflow/workflow-features-concepts.md | 6 ++---- .../building-blocks/workflow/workflow-overview.md | 7 +++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/daprdocs/content/en/developing-applications/building-blocks/workflow/workflow-features-concepts.md b/daprdocs/content/en/developing-applications/building-blocks/workflow/workflow-features-concepts.md index 28b1a36b6..c8e3de131 100644 --- a/daprdocs/content/en/developing-applications/building-blocks/workflow/workflow-features-concepts.md +++ b/daprdocs/content/en/developing-applications/building-blocks/workflow/workflow-features-concepts.md @@ -141,12 +141,10 @@ Sometimes workflows will need to wait for events that are raised by external sys External events have a _name_ and a _payload_ and are delivered to a single workflow instance. Workflows can create "_wait for external event_" tasks that subscribe to external events and _await_ those tasks to block execution until the event is received. The workflow can then read the payload of these events and make decisions about which next steps to take. External events can be processed serially or in parallel. External events can be raised by other workflows or by workflow code. -{{% alert title="Note" color="primary" %}} -The ability to raise external events to workflows is not included in the alpha version of Dapr's workflow API. -{{% /alert %}} - Workflows can also wait for multiple external event signals of the same name, in which case they are dispatched to the corresponding workflow tasks in a first-in, first-out (FIFO) manner. If a workflow receives an external event signal but has not yet created a "wait for external event" task, the event will be saved into the workflow's history and consumed immediately after the workflow requests the event. +Learn more about [external system interaction.]({{< ref "workflow-patterns.md#external-system-interaction" >}}) + ## Limitations ### Workflow determinism and code restraints diff --git a/daprdocs/content/en/developing-applications/building-blocks/workflow/workflow-overview.md b/daprdocs/content/en/developing-applications/building-blocks/workflow/workflow-overview.md index f5113c695..9f70500c0 100644 --- a/daprdocs/content/en/developing-applications/building-blocks/workflow/workflow-overview.md +++ b/daprdocs/content/en/developing-applications/building-blocks/workflow/workflow-overview.md @@ -16,7 +16,7 @@ The durable, resilient Dapr Workflow capability: - Offers a built-in workflow runtime for driving Dapr Workflow execution. - Provides SDKs for authoring workflows in code, using any language. -- Provides HTTP and gRPC APIs for managing workflows (start, query, suspend/resume, terminate). +- Provides HTTP and gRPC APIs for managing workflows (start, query, pause/resume, raise event, terminate, purge). - Integrates with any other workflow runtime via workflow components. Diagram showing basics of Dapr Workflow @@ -56,7 +56,10 @@ Same as Dapr actors, you can schedule reminder-like durable delays for any time When you create an application with workflow code and run it with Dapr, you can call specific workflows that reside in the application. Each individual workflow can be: - Started or terminated through a POST request -- Queried through a GET request +- Triggered to deliver a named event through a POST request +- Paused and then resumed through a POST request +- Purged from your state store through a POST request +- Queried for workflow status through a GET request [Learn more about how manage a workflow using HTTP calls.]({{< ref workflow_api.md >}}) From 0d25e77685b0cd38c48c7b534bc21af2359e1350 Mon Sep 17 00:00:00 2001 From: binnn6 <734819342@qq.com> Date: Wed, 14 Jun 2023 10:50:19 +0800 Subject: [PATCH 3/5] few edits on pubsub plusar doc Signed-off-by: binnn6 <734819342@qq.com> --- .../supported-pubsub/setup-pulsar.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md index 9f052b2b4..ed363c3a8 100644 --- a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md +++ b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md @@ -80,9 +80,9 @@ spec: | publicKey | N | A public key to be used for publisher and consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | privateKey | N | A private key to be used for consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | keys | N | A comma delimited string containing names of [Pulsar session keys](https://pulsar.apache.org/docs/3.0.x/security-encryption/#how-it-works-in-pulsar). Used in conjunction with `publicKey` for publisher encryption | -| processMode | N | Eable proccess multiple messages at once. Default: `"async"` | `"async"`, `"sync"`| +| processMode | N | Eable processing multiple messages at once. Default: `"async"` | `"async"`, `"sync"`| | subscribeType | N | Pulsar supports four kinds of [subscription types](https://pulsar.apache.org/docs/3.0.x/concepts-messaging/#subscription-types). Default: `"shared"` | `"shared"`, `"exclusive"`, `"failover"`, `"key_shared"`| -| partitionKey | N | partitionKey sets the key of the message for routing policy. Default: `""` | | +| partitionKey | N | Sets the key of the message for routing policy. Default: `""` | | ### Enabling message delivery retries @@ -207,9 +207,9 @@ spec: ### Partition Key -When invoking the plusar pub/sub, its possible to provide an optional partition key by using the `metadata` query param in the request url. +When invoking the Plusar pub/sub, its possible to provide an optional partition key by using the `metadata` query parameter in the request url. -The param name is `partitionKey`. +The parameter name is `partitionKey`. Example: @@ -225,7 +225,7 @@ curl -X POST http://localhost:3500/v1.0/publish/myPlusar/myTopic?metadata.partit ### Message headers -All other metadata key/value pairs (that are not `partitionKey`) are set as headers in the Pulsar message. Here is an example setting a `correlationId` for the message. +All other metadata key/value pairs (that are not `partitionKey`) are set as headers in the Pulsar message. For example, set a `correlationId` for the message: ```shell curl -X POST http://localhost:3500/v1.0/publish/myPlusar/myTopic?metadata.correlationId=myCorrelationID&metadata.partitionKey=key1 \ @@ -239,11 +239,11 @@ curl -X POST http://localhost:3500/v1.0/publish/myPlusar/myTopic?metadata.correl ## Order guarantee -To ensure that messages arrive in order for each consumer subscribed to a specific key, 3 conditions must be met. +To ensure that messages arrive in order for each consumer subscribed to a specific key, three conditions must be met. -1. subscribeType should be set to `key_shared`. -2. partitionKey must be set. -3. processMode should be set to `sync`. +1. `subscribeType` should be set to `key_shared`. +2. `partitionKey` must be set. +3. `processMode` should be set to `sync`. ## Create a Pulsar instance From 664236e0f80720e7906df15e6c742aa840963ad0 Mon Sep 17 00:00:00 2001 From: Hannah Hunter <94493363+hhunter-ms@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:20:05 -0400 Subject: [PATCH 4/5] typo Signed-off-by: Hannah Hunter <94493363+hhunter-ms@users.noreply.github.com> --- .../components-reference/supported-pubsub/setup-pulsar.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md index ed363c3a8..30375aab5 100644 --- a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md +++ b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md @@ -80,7 +80,7 @@ spec: | publicKey | N | A public key to be used for publisher and consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | privateKey | N | A private key to be used for consumer encryption. Value can be one of two options: file path for a local PEM cert, or the cert data string value | | keys | N | A comma delimited string containing names of [Pulsar session keys](https://pulsar.apache.org/docs/3.0.x/security-encryption/#how-it-works-in-pulsar). Used in conjunction with `publicKey` for publisher encryption | -| processMode | N | Eable processing multiple messages at once. Default: `"async"` | `"async"`, `"sync"`| +| processMode | N | Enable processing multiple messages at once. Default: `"async"` | `"async"`, `"sync"`| | subscribeType | N | Pulsar supports four kinds of [subscription types](https://pulsar.apache.org/docs/3.0.x/concepts-messaging/#subscription-types). Default: `"shared"` | `"shared"`, `"exclusive"`, `"failover"`, `"key_shared"`| | partitionKey | N | Sets the key of the message for routing policy. Default: `""` | | From f1e3112cb9eb1a647b690dbfa3b378d2c8ce33d0 Mon Sep 17 00:00:00 2001 From: Hannah Hunter <94493363+hhunter-ms@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:21:14 -0400 Subject: [PATCH 5/5] typo Signed-off-by: Hannah Hunter <94493363+hhunter-ms@users.noreply.github.com> --- .../components-reference/supported-pubsub/setup-pulsar.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md index 30375aab5..bd23f8848 100644 --- a/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md +++ b/daprdocs/content/en/reference/components-reference/supported-pubsub/setup-pulsar.md @@ -207,7 +207,7 @@ spec: ### Partition Key -When invoking the Plusar pub/sub, its possible to provide an optional partition key by using the `metadata` query parameter in the request url. +When invoking the Pulsar pub/sub, it's possible to provide an optional partition key by using the `metadata` query parameter in the request url. The parameter name is `partitionKey`.