From c29cc11c70f9d1d2ca209ba88c5426e01c74399f Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Mon, 2 Dec 2019 21:36:34 +0100 Subject: [PATCH] Backport changes to CamelSources to the 0.10 branch (#1999) * Update documentation about CamelSources (#1937) * Update documentation about CamelSources * Fix version * Apply suggestions from code review Applying suggestions Co-Authored-By: Sam O'Dell <31352624+samodell@users.noreply.github.com> * Update camel examples to reflect recent changes (#1976) * fix links * wrong filename --- .../samples/apache-camel-source/README.md | 138 +++++++++++------- .../display_resources.yaml | 12 +- .../apache-camel-source/source_camel_k.yaml | 24 --- .../source_http_poller.yaml | 26 ++++ .../apache-camel-source/source_mqtt.yaml | 32 ++++ .../apache-camel-source/source_telegram.yaml | 37 +++-- .../apache-camel-source/source_timer.yaml | 25 +++- 7 files changed, 190 insertions(+), 104 deletions(-) delete mode 100644 docs/eventing/samples/apache-camel-source/source_camel_k.yaml create mode 100644 docs/eventing/samples/apache-camel-source/source_http_poller.yaml create mode 100644 docs/eventing/samples/apache-camel-source/source_mqtt.yaml diff --git a/docs/eventing/samples/apache-camel-source/README.md b/docs/eventing/samples/apache-camel-source/README.md index eed12a53d..666473b1c 100644 --- a/docs/eventing/samples/apache-camel-source/README.md +++ b/docs/eventing/samples/apache-camel-source/README.md @@ -5,10 +5,11 @@ weight: 5 type: "docs" --- -These samples show how to configure a Camel Source. It is an Event Source that -can leverage one of the -[250+ Apache Camel components](https://github.com/apache/camel/tree/master/components) -for generating events. +These samples show how to configure Camel Sources. These event sources are highly dynamic and allow you to +generate events from a variety of systems (cloud platforms, social networks, datastores, message brokers, legacy systems, etc.), +leveraging all the [300+ components provided by Apache Camel](https://camel.apache.org/components/latest/). + +All Camel Sources use [Apache Camel K](https://github.com/apache/camel-k) as the runtime engine. ## Prerequisites @@ -18,17 +19,13 @@ for generating events. any namespace where you want to run Camel sources. The preferred version that is compatible with Camel sources is - [Camel K v0.3.3](https://github.com/apache/camel-k/releases/tag/0.3.3). + [Camel K v1.0.0-M4](https://github.com/apache/camel-k/releases). Installation instructions are provided in the - [Apache Camel K GitHub repository](https://github.com/apache/camel-k#installation). + [Apache Camel K Manual](https://camel.apache.org/camel-k/latest/installation/installation.html). Documentation includes specific instructions for common Kubernetes environments, including development clusters. -1.(Optional) Download Kail binaries for Linux or OSX, which can be found on the -[latest release](https://github.com/boz/kail/releases/latest) page. You can use -`kail` instead of `kubectl logs` to tail the logs of the subscriber. - 1. Install the Camel Source from the `camel.yaml` in the [Eventing Sources release page](https://github.com/knative/eventing-contrib/releases): @@ -36,9 +33,10 @@ for generating events. kubectl apply --filename camel.yaml ``` -### Create a Channel and a Subscriber +### Create Test Resources -To check if a `CamelSource` is fully working, create: +All the `CamelSource` examples use some test resources for the purpose of displaying the generated events. +The following resources need to be created: - a simple Knative event display service that prints incoming events to its log - an in-memory channel named `camel-test` that buffers events created by the @@ -52,21 +50,23 @@ Deploy the [`display_resources.yaml`](./display_resources.yaml): kubectl apply --filename display_resources.yaml ``` -### Run a CamelSource using the Timer component +### Run a Timer CamelSource The samples directory contains some sample sources that can be used to generate events. -The simplest example of the `CamelSource`, that does not require additional -configuration, is the "timer" source. +The simplest example of `CamelSource`, that does not require additional +configuration, is the timer source. + +The timer source periodically generates "Hello world!" events and forwards them to the provided destination. If you want, you can customize the source behavior using options available in the Apache Camel documentation for the -[timer component](https://github.com/apache/camel/blob/master/components/camel-timer/src/main/docs/timer-component.adoc). +[timer component](https://camel.apache.org/components/latest/timer-component.html). All Camel components are documented in the -[Apache Camel GitHub repository](https://github.com/apache/camel/tree/master/components). +[Apache Camel Website](https://camel.apache.org/components/latest/). -Install the [timer CamelSource](source_timer.yaml) from source: +Install the [timer CamelSource](./source_timer.yaml) from source: ```shell kubectl apply -f source_timer.yaml @@ -79,19 +79,53 @@ looking at what is downstream of the `CamelSource`. kubectl logs --selector serving.knative.dev/service=camel-event-display -c user-container ``` -or - -You can also use [`kail`](https://github.com/boz/kail) to tail the logs of the -subscriber. - -```shell -kail -d camel-event-display --since=10m -``` - -If you have deployed the timer source, you should see log lines appearing every +If you have deployed the timer source, you should see new log lines appearing every 3 seconds. -### Run a CamelSource using the Telegram component +### Run a MQTT CamelSource + +One of the 300+ Camel components that you can leverage is [Camel-Paho](https://camel.apache.org/components/latest/paho-component.html), +based on the [Eclipse Paho](https://www.eclipse.org/paho/) open source project. + +A source based on Paho (like the provided [MQTT CamelSource](./source_mqtt.yaml)) allows to bridge any MQTT broker to a Knative resource, +automatically converting IoT messages to Cloudevents. + +To use the MQTT source, you need a MQTT broker running and reachable from your cluster. +For example, it's possible to run a [Mosquitto MQTT Broker](https://mosquitto.org/) for testing purposes. + +First, edit the [MQTT CamelSource](./source_mqtt.yaml) and put the +correct address of the MQTT broker in the `brokerUrl` field. +You also need to provide the name of the topic that you want to subscribe to: just change `paho:mytopic` to match +the topic that you want to use. + +You can also scale this source out, in order to obtain more throughput, by changing the value of the `replicas` field. +By default it creates *2* replicas for demonstration purposes. + +To reduce noise in the event display, you can remove all previously created +CamelSources from the namespace: + +```shell +kubectl delete camelsource --all +``` + +Install the [mqtt CamelSource](./source_telegram.yaml): + +```shell +kubectl apply -f source_mqtt.yaml +``` + +You can now send MQTT messages to your broker using your favourite client (you can even use Camel K for sending test events). + +Each message you send to the MQTT broker will be printed by the event display as a Cloudevent. + +You can verify that your messages reach the event display by checking its logs: + +```shell +kubectl logs --selector serving.knative.dev/service=camel-event-display -c user-container +``` + + +### Run a Telegram CamelSource Another useful component available with Camel is the Telegram component. It can be used to forward messages of a [Telegram](https://telegram.org/) chat into @@ -105,37 +139,42 @@ Bot, using your preferred Telegram client (mobile or web). After you create the bot, you will receive an **authorization token** that is needed for the source to work. -First, edit the [telegram CamelSource](source_telegram.yaml) and put the +First, edit the [telegram CamelSource](./source_telegram.yaml) and put the authorization token, replacing the `` placeholder. -To reduce noise in the event display, you can remove the previously created -timer CamelSource from the namespace: +To reduce noise in the event display, you can remove all previously created +CamelSources from the namespace: ```shell -kubectl delete camelsource camel-timer-source +kubectl delete camelsource --all ``` -Install the [telegram CamelSource](source_telegram.yaml) from source: +Install the [telegram CamelSource](./source_telegram.yaml): ```shell kubectl apply -f source_telegram.yaml ``` -Start `kail` again and keep it open on the event display: +Now, you can contact your bot with any Telegram client. Each message you send to +the bot will be printed by the event display as a Cloudevent. + +You can verify that your messages reach the event display by checking its logs: ```shell -kail -d camel-event-display --since=10m +kubectl logs --selector serving.knative.dev/service=camel-event-display -c user-container ``` -Now, you can contact your bot with any Telegram client. Each message you send to -the bot will be printed by the event display as a cloudevent. -### Run a Camel K Source +### Run an HTTP Poller CamelSource -For complex use cases that require multiple steps to be executed before event -data is ready to be published, you can use Camel K sources. Camel K lets you use -Camel DSL to design one or more routes that can define complex workflows before -sending events to the target sink. +CamelSources are not limited to using a single Camel component. For example, +you can combine the [Camel Timer component](https://camel.apache.org/components/latest/timer-component.html) +with the [Camel HTTP component](https://camel.apache.org/components/latest/http-component.html) +to periodically fetch an external API, transform the result into a Cloudevent and forward it to a +given destination. + +The example will retrieve a static JSON file from a remote URL, but you can edit the +[HTTP poller CamelSource](./source_http_poller.yaml) to add your own API. If you have previously deployed other CamelSources, to reduce noise in the event display, you can remove them all from the namespace: @@ -144,18 +183,15 @@ display, you can remove them all from the namespace: kubectl delete camelsource --all ``` -Install the [Camel K Source](source_camel_k.yaml) from source: +Install the [HTTP poller CamelSource](./source_http_poller.yaml): ```shell -kubectl apply -f source_camel_k.yaml +kubectl apply -f source_http_poller.yaml ``` -Start `kail` again and keep it open on the event display: +The event display will show some JSON data periodically pulled from the external +REST API. To check the logs: ```shell -kail -d camel-event-display --since=10m +kubectl logs --selector serving.knative.dev/service=camel-event-display -c user-container ``` - -The event display will show some JSON data periodically pulled from an external -REST API. The API in the example is static, but you can use your own dynamic API -by replacing the endpoint. diff --git a/docs/eventing/samples/apache-camel-source/display_resources.yaml b/docs/eventing/samples/apache-camel-source/display_resources.yaml index 33383d85f..ce0258e55 100644 --- a/docs/eventing/samples/apache-camel-source/display_resources.yaml +++ b/docs/eventing/samples/apache-camel-source/display_resources.yaml @@ -1,26 +1,22 @@ # Channel for testing events. apiVersion: messaging.knative.dev/v1alpha1 -kind: Channel +kind: InMemoryChannel metadata: name: camel-test -spec: - channelTemplate: - apiVersion: messaging.knative.dev/v1alpha1 - kind: InMemoryChannel --- # Subscription from the CamelSource's output Channel to the Knative Service below. -apiVersion: eventing.knative.dev/v1alpha1 +apiVersion: messaging.knative.dev/v1alpha1 kind: Subscription metadata: name: camel-source-display spec: channel: apiVersion: messaging.knative.dev/v1alpha1 - kind: Channel + kind: InMemoryChannel name: camel-test subscriber: ref: @@ -40,4 +36,4 @@ spec: template: spec: containers: - - image: gcr.io/knative-releases/github.com/knative/eventing-sources/cmd/event_display@sha256:bf45b3eb1e7fc4cb63d6a5a6416cf696295484a7662e0cf9ccdf5c080542c21d + - image: gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display@sha256:f4628e97a836c77ed38bd3b6fd3d0b06de4d5e7db6704772fe674d48b20bd477 diff --git a/docs/eventing/samples/apache-camel-source/source_camel_k.yaml b/docs/eventing/samples/apache-camel-source/source_camel_k.yaml deleted file mode 100644 index 0faade924..000000000 --- a/docs/eventing/samples/apache-camel-source/source_camel_k.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Apache Camel K Source -# -# Camel K documentation: https://github.com/apache/camel-k -# -# -apiVersion: sources.eventing.knative.dev/v1alpha1 -kind: CamelSource -metadata: - name: camel-k-source -spec: - source: - integration: - sources: - - name: file.groovy - content: - from("timer:tick?period=5s") - .log("Fetching quote of the day") - .to("https4://gist.githubusercontent.com/nicolaferraro/e3c72ace3c751f9f88273896611ce5fe/raw/3b6f54060bacb56b6719b7386a4645cb59ad6cc1/quote.json") - .setHeader("Content-Type").constant("application/json") - .to("knative:endpoint/sink?cloudEventsType=org.apache.camel.quote") - sink: - apiVersion: messaging.knative.dev/v1alpha1 - kind: Channel - name: camel-test diff --git a/docs/eventing/samples/apache-camel-source/source_http_poller.yaml b/docs/eventing/samples/apache-camel-source/source_http_poller.yaml new file mode 100644 index 000000000..5e41e2ec4 --- /dev/null +++ b/docs/eventing/samples/apache-camel-source/source_http_poller.yaml @@ -0,0 +1,26 @@ +# Apache Camel Http Poller Source +# +# Timer Component documentation: https://camel.apache.org/components/latest/timer-component.html +# Http Component documentation: https://camel.apache.org/components/latest/http-component.html +# +# List of available Apache Camel components: https://camel.apache.org/components/latest/ +# +apiVersion: sources.eventing.knative.dev/v1alpha1 +kind: CamelSource +metadata: + name: camel-http-poller-source +spec: + source: + flow: + from: + uri: timer:tick?period=10s + steps: + - to: https://gist.githubusercontent.com/nicolaferraro/e3c72ace3c751f9f88273896611ce5fe/raw/3b6f54060bacb56b6719b7386a4645cb59ad6cc1/quote.json + - set-header: + name: Content-Type + constant: application/json + sink: + ref: + apiVersion: messaging.knative.dev/v1alpha1 + kind: InMemoryChannel + name: camel-test diff --git a/docs/eventing/samples/apache-camel-source/source_mqtt.yaml b/docs/eventing/samples/apache-camel-source/source_mqtt.yaml new file mode 100644 index 000000000..d924efa4e --- /dev/null +++ b/docs/eventing/samples/apache-camel-source/source_mqtt.yaml @@ -0,0 +1,32 @@ +# Apache Camel MQTT Source +# +# Paho Component documentation: https://camel.apache.org/components/latest/paho-component.html +# +# List of available Apache Camel components: https://camel.apache.org/components/latest/ +# +apiVersion: sources.eventing.knative.dev/v1alpha1 +kind: CamelSource +metadata: + name: camel-mqtt-source +spec: + source: + # To increase throughput, you can (optionally) read using multiple pods in parallel + integration: + replicas: 2 + flow: + from: + # Specify here the topic to subscribe to + uri: paho:mytopic + parameters: + # URL of the MQTT source + brokerUrl: tcp://mosquitto:1883 + # A unique client ID for the source + clientId: mqtt-knative-bridge + steps: + - log: + message: "Forwarding: ${body}" + sink: + ref: + apiVersion: messaging.knative.dev/v1alpha1 + kind: InMemoryChannel + name: camel-test diff --git a/docs/eventing/samples/apache-camel-source/source_telegram.yaml b/docs/eventing/samples/apache-camel-source/source_telegram.yaml index f040fb57f..4016514da 100644 --- a/docs/eventing/samples/apache-camel-source/source_telegram.yaml +++ b/docs/eventing/samples/apache-camel-source/source_telegram.yaml @@ -1,8 +1,9 @@ # Apache Camel Telegram Source # -# Telegram Component documentation: https://github.com/apache/camel/blob/master/components/camel-telegram/src/main/docs/telegram-component.adoc +# Telegram Component documentation: https://camel.apache.org/components/latest/telegram-component.html +# Simple language documentation: https://camel.apache.org/manual/latest/simple-language.html # -# List of available Apache Camel components: https://github.com/apache/camel/tree/master/components +# List of available Apache Camel components: https://camel.apache.org/components/latest/ # apiVersion: sources.eventing.knative.dev/v1alpha1 kind: CamelSource @@ -10,15 +11,25 @@ metadata: name: camel-telegram-source spec: source: - component: - uri: telegram:bots - properties: - # Camel Telegram component option (ask it to the Botfather: https://telegram.me/botfather) - camel.component.telegram.authorizationToken: "" - - # Camel K option to enable serialization of the component output - camel.component.knative.jsonSerializationEnabled: "true" + flow: + from: + uri: telegram:bots + parameters: + authorizationToken: "" + steps: + - set-header: + name: ce-author + simple: "${body.from.firstName} ${body.from.lastName}" + - set-header: + name: ce-chat + simple: "${body.chat.id}" + - set-header: + name: Content-Type + constant: text/plain + - transform: + simple: "${body.text}" sink: - apiVersion: messaging.knative.dev/v1alpha1 - kind: Channel - name: camel-test + ref: + apiVersion: messaging.knative.dev/v1alpha1 + kind: InMemoryChannel + name: camel-test diff --git a/docs/eventing/samples/apache-camel-source/source_timer.yaml b/docs/eventing/samples/apache-camel-source/source_timer.yaml index 9ef339130..89c4705df 100644 --- a/docs/eventing/samples/apache-camel-source/source_timer.yaml +++ b/docs/eventing/samples/apache-camel-source/source_timer.yaml @@ -1,8 +1,8 @@ # Apache Camel Timer Source # -# Timer Component documentation: https://github.com/apache/camel/blob/master/camel-core/src/main/docs/timer-component.adoc +# Timer Component documentation: https://camel.apache.org/components/latest/timer-component.html # -# List of available Apache Camel components: https://github.com/apache/camel/tree/master/components +# List of available Apache Camel components: https://camel.apache.org/components/latest/ # apiVersion: sources.eventing.knative.dev/v1alpha1 kind: CamelSource @@ -10,10 +10,19 @@ metadata: name: camel-timer-source spec: source: - component: - # Using 'period' URI option (see component documentation) - uri: timer:tick?period=3s + flow: + from: + uri: timer:tick + parameters: + period: 3s + steps: + - set-header: + name: Content-Type + constant: text/plain + - set-body: + constant: Hello world! sink: - apiVersion: messaging.knative.dev/v1alpha1 - kind: Channel - name: camel-test + ref: + apiVersion: messaging.knative.dev/v1alpha1 + kind: InMemoryChannel + name: camel-test