pub/sub rocketmq:upgrade with v2 (#1383)
* pub/sub rocketmq:upgrade with client v2 Signed-off-by: zach <zachchou016@gmail.com> * pubsub/rocketmq:remove cache and add start with setup Signed-off-by: zach <zachchou016@gmail.com> * pubsub/rocketmq:fix variable golint Signed-off-by: zach <zachchou016@gmail.com> * pubsub/rocketmq:change license and fix goimports Signed-off-by: zach <zachchou016@gmail.com> Co-authored-by: Looong Dai <long.dai@intel.com>
This commit is contained in:
Zach
GitHub
parent
897c2a4db5
commit
5c9365b314
|
|
@ -0,0 +1,84 @@
|
|||
// ------------------------------------------------------------
|
||||
// Copyright (c) Microsoft Corporation and Dapr Contributors.
|
||||
// Licensed under the Apache License.
|
||||
// ------------------------------------------------------------
|
||||
|
||||
package rocketmq
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/dapr/components-contrib/pubsub"
|
||||
"github.com/dapr/kit/config"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrRocketmqPublishMsg = errors.New("rocketmq publish msg error")
|
||||
ErrRocketmqValidPublishMsgTyp = errors.New("rocketmq publish msg error, invalid msg type")
|
||||
)
|
||||
|
||||
const (
|
||||
metadataRocketmqTag = "rocketmq-tag"
|
||||
metadataRocketmqKey = "rocketmq-key"
|
||||
metadataRocketmqShardingKey = "rocketmq-shardingkey"
|
||||
metadataRocketmqConsumerGroup = "rocketmq-consumerGroup"
|
||||
metadataRocketmqType = "rocketmq-sub-type"
|
||||
metadataRocketmqExpression = "rocketmq-sub-expression"
|
||||
metadataRocketmqBrokerName = "rocketmq-broker-name"
|
||||
)
|
||||
|
||||
type rocketMQMetaData struct {
|
||||
AccessProto string `mapstructure:"accessProto"`
|
||||
// rocketmq Credentials
|
||||
AccessKey string `mapstructure:"accessKey"`
|
||||
SecretKey string `mapstructure:"secretKey"`
|
||||
NameServer string `mapstructure:"nameServer"`
|
||||
GroupName string `mapstructure:"groupName"`
|
||||
NameSpace string `mapstructure:"nameSpace"`
|
||||
// consumer group rocketmq's subscribers
|
||||
ConsumerGroup string `mapstructure:"consumerGroup"`
|
||||
ConsumerBatchSize int `mapstructure:"consumerBatchSize"`
|
||||
// rocketmq's name server domain
|
||||
NameServerDomain string `mapstructure:"nameServerDomain"`
|
||||
// msg's content-type
|
||||
ContentType string `mapstructure:"content-type"`
|
||||
// retry times to connect rocketmq's broker
|
||||
Retries int `mapstructure:"retries"`
|
||||
SendTimeOut int `mapstructure:"sendTimeOut"`
|
||||
}
|
||||
|
||||
func getDefaultRocketMQMetaData() *rocketMQMetaData {
|
||||
return &rocketMQMetaData{
|
||||
AccessProto: "",
|
||||
AccessKey: "",
|
||||
SecretKey: "",
|
||||
NameServer: "",
|
||||
GroupName: "",
|
||||
NameSpace: "",
|
||||
ConsumerGroup: "",
|
||||
ConsumerBatchSize: 0,
|
||||
NameServerDomain: "",
|
||||
ContentType: pubsub.DefaultCloudEventDataContentType,
|
||||
Retries: 3,
|
||||
SendTimeOut: 10,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *rocketMQMetaData) Decode(in interface{}) error {
|
||||
if err := config.Decode(in, &s); err != nil {
|
||||
return fmt.Errorf("decode failed. %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseRocketMQMetaData(metadata pubsub.Metadata) (*rocketMQMetaData, error) {
|
||||
rMetaData := getDefaultRocketMQMetaData()
|
||||
if metadata.Properties != nil {
|
||||
err := rMetaData.Decode(metadata.Properties)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rocketmq configuration error: %w", err)
|
||||
}
|
||||
}
|
||||
return rMetaData, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
// ------------------------------------------------------------
|
||||
// Copyright (c) Microsoft Corporation and Dapr Contributors.
|
||||
// Licensed under the Apache License.
|
||||
// ------------------------------------------------------------
|
||||
|
||||
package rocketmq
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/dapr/components-contrib/pubsub"
|
||||
)
|
||||
|
||||
func TestMetaDataDecode(t *testing.T) {
|
||||
props := map[string]string{
|
||||
"accessProto": "http",
|
||||
"accessKey": "**",
|
||||
"secretKey": "***",
|
||||
"nameServer": "http://test.nameserver",
|
||||
"consumerGroup": "defaultGroup",
|
||||
"nameSpace": "defaultNamespace",
|
||||
}
|
||||
pubsubMeta := pubsub.Metadata{Properties: props}
|
||||
metaData, err := parseRocketMQMetaData(pubsubMeta)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "http", metaData.AccessProto)
|
||||
assert.Equal(t, "**", metaData.AccessKey)
|
||||
assert.Equal(t, "***", metaData.SecretKey)
|
||||
assert.Equal(t, "defaultGroup", metaData.ConsumerGroup)
|
||||
}
|
||||
|
|
@ -1,20 +1,22 @@
|
|||
// ------------------------------------------------------------
|
||||
// Copyright (c) Microsoft Corporation and Dapr Contributors.
|
||||
// Licensed under the MIT License.
|
||||
// Licensed under the Apache License.
|
||||
// ------------------------------------------------------------
|
||||
|
||||
package rocketmq
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
mq "github.com/apache/rocketmq-client-go/v2"
|
||||
mqc "github.com/apache/rocketmq-client-go/v2/consumer"
|
||||
"github.com/apache/rocketmq-client-go/v2/primitive"
|
||||
mqw "github.com/cinience/go_rocketmq"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
mqp "github.com/apache/rocketmq-client-go/v2/producer"
|
||||
|
||||
"github.com/dapr/components-contrib/pubsub"
|
||||
"github.com/dapr/kit/logger"
|
||||
|
|
@ -22,240 +24,145 @@ import (
|
|||
)
|
||||
|
||||
type rocketMQ struct {
|
||||
name string
|
||||
settings Settings
|
||||
producer mqw.Producer
|
||||
consumer mqw.PushConsumer
|
||||
logger logger.Logger
|
||||
json jsoniter.API
|
||||
topics map[string]mqc.MessageSelector
|
||||
name string
|
||||
metadata *rocketMQMetaData
|
||||
pushConsumer mq.PushConsumer
|
||||
|
||||
logger logger.Logger
|
||||
lock sync.Mutex
|
||||
topics map[string]mqc.MessageSelector
|
||||
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
backOffConfig retry.Config
|
||||
}
|
||||
|
||||
// NewRocketMQ creates a new RocketMQ pub/sub.
|
||||
func NewRocketMQ(logger logger.Logger) pubsub.PubSub {
|
||||
return &rocketMQ{ //nolint:exhaustivestruct
|
||||
name: "rocketmq",
|
||||
consumer: nil,
|
||||
logger: logger,
|
||||
json: nil,
|
||||
topics: nil,
|
||||
func NewRocketMQ(l logger.Logger) pubsub.PubSub {
|
||||
return &rocketMQ{
|
||||
name: "rocketmq",
|
||||
logger: l,
|
||||
topics: make(map[string]mqc.MessageSelector),
|
||||
}
|
||||
}
|
||||
|
||||
// Init does metadata parsing and connection creation.
|
||||
func (r *rocketMQ) Init(md pubsub.Metadata) error {
|
||||
// Settings default values
|
||||
r.settings = Settings{ //nolint:exhaustivestruct
|
||||
ContentType: pubsub.DefaultCloudEventDataContentType,
|
||||
}
|
||||
err := r.settings.Decode(md.Properties)
|
||||
func (r *rocketMQ) Init(metadata pubsub.Metadata) error {
|
||||
var err error
|
||||
r.metadata, err = parseRocketMQMetaData(metadata)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rocketmq configuration error: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
r.ctx, r.cancel = context.WithCancel(context.Background())
|
||||
|
||||
// Default retry configuration is used if no
|
||||
// backOff properties are set.
|
||||
if err = retry.DecodeConfigWithPrefix(
|
||||
&r.backOffConfig,
|
||||
md.Properties,
|
||||
metadata.Properties,
|
||||
"backOff"); err != nil {
|
||||
return fmt.Errorf("retry configuration error: %w", err)
|
||||
}
|
||||
|
||||
r.producer, err = r.setupPublisher()
|
||||
if err != nil {
|
||||
return fmt.Errorf("setupPublisher error: %w", err)
|
||||
}
|
||||
|
||||
r.json = jsoniter.ConfigFastest
|
||||
r.topics = make(map[string]mqc.MessageSelector)
|
||||
|
||||
r.consumer, err = r.setupConsumer()
|
||||
if err != nil {
|
||||
r.logger.Errorf("rocketmq init consumer failed: %v", err)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *rocketMQ) setupPublisher() (mqw.Producer, error) {
|
||||
if producer, ok := mqw.Producers[r.settings.AccessProto]; ok {
|
||||
md := r.settings.ToRocketMQMetadata()
|
||||
if err := producer.Init(md); err != nil {
|
||||
r.logger.Debugf("rocketmq producer init failed: %v", err)
|
||||
|
||||
return nil, fmt.Errorf("setupPublisher failed. %w", err)
|
||||
}
|
||||
r.logger.Infof("rocketmq proto: %s", r.settings.AccessProto)
|
||||
|
||||
return producer, nil
|
||||
func (r *rocketMQ) setUpConsumer() (mq.PushConsumer, error) {
|
||||
opts := make([]mqc.Option, 0)
|
||||
if r.metadata.ConsumerGroup != "" {
|
||||
opts = append(opts, mqc.WithGroupName(r.metadata.ConsumerGroup))
|
||||
}
|
||||
|
||||
return nil, errors.New("rocketmq error: cannot found rocketmq producer")
|
||||
if r.metadata.NameSpace != "" {
|
||||
opts = append(opts, mqc.WithNamespace(r.metadata.NameSpace))
|
||||
}
|
||||
if r.metadata.Retries != 0 {
|
||||
opts = append(opts, mqc.WithRetry(r.metadata.Retries))
|
||||
}
|
||||
if r.metadata.NameServerDomain != "" {
|
||||
opts = append(opts, mqc.WithNameServerDomain(r.metadata.NameServerDomain))
|
||||
}
|
||||
if r.metadata.NameServer != "" {
|
||||
opts = append(opts, mqc.WithNameServer(primitive.NamesrvAddr{r.metadata.NameServer}))
|
||||
}
|
||||
if r.metadata.AccessKey != "" && r.metadata.SecretKey != "" {
|
||||
opts = append(opts, mqc.WithCredentials(primitive.Credentials{
|
||||
AccessKey: r.metadata.AccessKey,
|
||||
SecretKey: r.metadata.SecretKey,
|
||||
}))
|
||||
}
|
||||
return mq.NewPushConsumer(opts...)
|
||||
}
|
||||
|
||||
func (r *rocketMQ) setupConsumer() (mqw.PushConsumer, error) {
|
||||
if consumer, ok := mqw.Consumers[r.settings.AccessProto]; ok {
|
||||
md := r.settings.ToRocketMQMetadata()
|
||||
if err := consumer.Init(md); err != nil {
|
||||
r.logger.Errorf("rocketmq consumer init failed: %v", err)
|
||||
|
||||
return nil, fmt.Errorf("setupConsumer failed. %w", err)
|
||||
}
|
||||
r.logger.Infof("rocketmq access proto: %s", r.settings.AccessProto)
|
||||
|
||||
return consumer, nil
|
||||
func (r *rocketMQ) setUpProducer() (mq.Producer, error) {
|
||||
opts := make([]mqp.Option, 0)
|
||||
if r.metadata.Retries != 0 {
|
||||
opts = append(opts, mqp.WithRetry(r.metadata.Retries))
|
||||
}
|
||||
|
||||
return nil, errors.New("rocketmq error: cannot found consumer")
|
||||
}
|
||||
|
||||
func (r *rocketMQ) Publish(req *pubsub.PublishRequest) error {
|
||||
msg := primitive.NewMessage(req.Topic, req.Data).WithTag(req.Metadata[metadataRocketmqTag]).
|
||||
WithKeys([]string{req.Metadata[metadataRocketmqKey]})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
result, err := r.producer.SendSync(ctx, msg)
|
||||
|
||||
if result != nil {
|
||||
r.logger.Debugf("rocketmq send result topic:%s tag:%s status:%v", req.Topic, msg.GetTags(), result.Status)
|
||||
if r.metadata.GroupName != "" {
|
||||
opts = append(opts, mqp.WithGroupName(r.metadata.GroupName))
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
r.logger.Errorf("error send message topic:%s : %v", req.Topic, err)
|
||||
|
||||
return fmt.Errorf("publish message failed. %w", err)
|
||||
if r.metadata.NameServerDomain != "" {
|
||||
opts = append(opts, mqp.WithNameServerDomain(r.metadata.NameServerDomain))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *rocketMQ) addTopic(newTopic string, selector mqc.MessageSelector) []string {
|
||||
// Add topic to our map of topics
|
||||
r.topics[newTopic] = selector
|
||||
|
||||
topics := make([]string, len(r.topics))
|
||||
|
||||
i := 0
|
||||
for topic := range r.topics {
|
||||
topics[i] = topic
|
||||
i++
|
||||
if r.metadata.NameSpace != "" {
|
||||
opts = append(opts, mqp.WithNamespace(r.metadata.NameSpace))
|
||||
}
|
||||
|
||||
return topics
|
||||
}
|
||||
|
||||
// Close down consumer group resources, refresh once.
|
||||
func (r *rocketMQ) closeSubscripionResources() {
|
||||
if r.consumer != nil {
|
||||
if len(r.topics) > 0 {
|
||||
_ = r.consumer.Shutdown()
|
||||
}
|
||||
if r.metadata.NameServer != "" {
|
||||
opts = append(opts, mqp.WithNameServer(primitive.NamesrvAddr{r.metadata.NameServer}))
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rocketMQ) Subscribe(req pubsub.SubscribeRequest, handler pubsub.Handler) error {
|
||||
if req.Metadata == nil {
|
||||
req.Metadata = make(map[string]string, 1)
|
||||
if r.metadata.AccessKey != "" && r.metadata.SecretKey != "" {
|
||||
opts = append(opts, mqp.WithCredentials(primitive.Credentials{
|
||||
AccessKey: r.metadata.AccessKey,
|
||||
SecretKey: r.metadata.SecretKey,
|
||||
}))
|
||||
}
|
||||
|
||||
consumerGroup := req.Metadata[metadataRocketmqConsumerGroup]
|
||||
if len(consumerGroup) == 0 {
|
||||
consumerGroup = r.settings.ConsumerGroup
|
||||
}
|
||||
|
||||
mqType := req.Metadata[metadataRocketmqType]
|
||||
mqExpr := req.Metadata[metadataRocketmqExpression]
|
||||
if len(mqType) != 0 &&
|
||||
(mqType != string(mqc.SQL92) &&
|
||||
mqType != string(mqc.TAG)) {
|
||||
r.logger.Warnf("rocketmq subscribe to topic %s failed because some illegal type(%s).", req.Topic, req.Metadata[metadataRocketmqType])
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
r.closeSubscripionResources()
|
||||
|
||||
var err error
|
||||
if r.consumer, err = r.setupConsumer(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
topics := r.addTopic(req.Topic, mqc.MessageSelector{Type: mqc.ExpressionType(mqType), Expression: mqExpr})
|
||||
|
||||
err = r.subscribeAllTopics(topics, consumerGroup, handler)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rocketmq error: %w", err)
|
||||
}
|
||||
|
||||
err = r.consumer.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("consumer start failed. %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *rocketMQ) subscribeAllTopics(topics []string, consumerGroup string, handler pubsub.Handler) error {
|
||||
for _, topic := range topics {
|
||||
selector, ok := r.topics[topic]
|
||||
if !ok {
|
||||
return fmt.Errorf("cannot found topic:%s selector", topic)
|
||||
}
|
||||
|
||||
r.logger.Debugf("rocketmq start subscribe:%s group:%s type:%s expr:%s", topic, consumerGroup, string(selector.Type), selector.Expression)
|
||||
|
||||
err := r.consumer.Subscribe(topic, selector, r.adaptCallback(topic, consumerGroup, string(selector.Type), selector.Expression, handler))
|
||||
if err != nil {
|
||||
r.logger.Errorf("rocketmq error to subscribe: %s", err)
|
||||
|
||||
return fmt.Errorf("subscribe %s failed. %w", topic, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return mq.NewProducer(opts...)
|
||||
}
|
||||
|
||||
func (r *rocketMQ) Features() []pubsub.Feature {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *rocketMQ) Close() error {
|
||||
r.cancel()
|
||||
|
||||
if r.consumer != nil {
|
||||
_ = r.consumer.Shutdown()
|
||||
func (r *rocketMQ) Publish(req *pubsub.PublishRequest) error {
|
||||
r.logger.Debugf("rocketmq publish topic:%s with data:%v", req.Topic, req.Data)
|
||||
msg := newRocketMQMessage(req)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(r.metadata.SendTimeOut))
|
||||
defer cancel()
|
||||
var (
|
||||
producer mq.Producer
|
||||
result *primitive.SendResult
|
||||
err error
|
||||
)
|
||||
producer, err = r.setUpProducer()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if r.producer != nil {
|
||||
_ = r.producer.Shutdown()
|
||||
if err1 := producer.Start(); err1 != nil {
|
||||
return err1
|
||||
}
|
||||
|
||||
result, err = producer.SendSync(ctx, msg)
|
||||
if err != nil {
|
||||
r.logger.Errorf("error send message topic:%s : %v", req.Topic, err)
|
||||
return ErrRocketmqPublishMsg
|
||||
}
|
||||
r.logger.Debugf("rocketmq send result topic:%s tag:%s status:%v", req.Topic, msg.GetTags(), result.Status)
|
||||
return nil
|
||||
}
|
||||
|
||||
type mqCallback func(ctx context.Context, msgs ...*primitive.MessageExt) (mqc.ConsumeResult, error)
|
||||
func newRocketMQMessage(req *pubsub.PublishRequest) *primitive.Message {
|
||||
return primitive.NewMessage(req.Topic, req.Data).
|
||||
WithTag(req.Metadata[metadataRocketmqTag]).
|
||||
WithKeys([]string{req.Metadata[metadataRocketmqKey]}).
|
||||
WithShardingKey(req.Metadata[metadataRocketmqShardingKey])
|
||||
}
|
||||
|
||||
func (r *rocketMQ) adaptCallback(topic, consumerGroup, mqType, mqExpr string, handler pubsub.Handler) mqCallback {
|
||||
type mqSubscribeCallback func(ctx context.Context, msgs ...*primitive.MessageExt) (mqc.ConsumeResult, error)
|
||||
|
||||
func (r *rocketMQ) adaptCallback(topic, consumerGroup, mqType, mqExpr string, handler pubsub.Handler) mqSubscribeCallback {
|
||||
return func(ctx context.Context, msgs ...*primitive.MessageExt) (mqc.ConsumeResult, error) {
|
||||
success := true
|
||||
for _, v := range msgs {
|
||||
data := pubsub.NewCloudEventsEnvelope(v.MsgId, v.StoreHost, r.name,
|
||||
v.GetProperty(primitive.PropertyKeys), v.Topic, r.name, r.settings.ContentType, v.Body, "", "")
|
||||
dataBytes, err := r.json.Marshal(data)
|
||||
for _, msg := range msgs {
|
||||
cloudEventsMap := pubsub.NewCloudEventsEnvelope(msg.MsgId, msg.StoreHost, r.name, msg.GetProperty(primitive.PropertyKeys), msg.Topic, r.name, r.metadata.ContentType, msg.Body, "", "")
|
||||
dataBytes, err := json.Marshal(cloudEventsMap)
|
||||
if err != nil {
|
||||
r.logger.Warn("rocketmq fail to marshal data message, topic:%s data-length:%d err:%v ", v.Topic, len(v.Body), err)
|
||||
r.logger.Warn("rocketmq fail to marshal cloudEventsMap message, topic:%s cloudEventsMap-length:%d err:%newMessage ", msg.Topic, len(msg.Body), err)
|
||||
success = false
|
||||
|
||||
continue
|
||||
}
|
||||
metadata := map[string]string{
|
||||
|
|
@ -263,39 +170,122 @@ func (r *rocketMQ) adaptCallback(topic, consumerGroup, mqType, mqExpr string, ha
|
|||
metadataRocketmqExpression: mqExpr,
|
||||
metadataRocketmqConsumerGroup: consumerGroup,
|
||||
}
|
||||
if v.Queue != nil {
|
||||
metadata[metadataRocketmqBrokerName] = v.Queue.BrokerName
|
||||
if msg.Queue != nil {
|
||||
metadata[metadataRocketmqBrokerName] = msg.Queue.BrokerName
|
||||
}
|
||||
msg := pubsub.NewMessage{
|
||||
newMessage := pubsub.NewMessage{
|
||||
Topic: topic,
|
||||
Data: dataBytes,
|
||||
Metadata: metadata,
|
||||
}
|
||||
|
||||
b := r.backOffConfig.NewBackOffWithContext(r.ctx)
|
||||
|
||||
rerr := retry.NotifyRecover(func() error {
|
||||
herr := handler(ctx, &msg)
|
||||
retError := retry.NotifyRecover(func() error {
|
||||
herr := handler(ctx, &newMessage)
|
||||
if herr != nil {
|
||||
r.logger.Errorf("rocketmq error: fail to send message to dapr application. topic:%s data-length:%d err:%v ", v.Topic, len(v.Body), herr)
|
||||
r.logger.Errorf("rocketmq error: fail to send message to dapr application. topic:%s cloudEventsMap-length:%d err:%newMessage ", newMessage.Topic, len(msg.Body), herr)
|
||||
success = false
|
||||
}
|
||||
|
||||
return herr
|
||||
}, b, func(err error, d time.Duration) {
|
||||
r.logger.Errorf("rocketmq error: fail to processing message. topic:%s data-length:%d. Retrying...", v.Topic, len(v.Body))
|
||||
r.logger.Errorf("rocketmq error: fail to processing message. topic:%s cloudEventsMap-length:%d. Retrying...", newMessage.Topic, len(msg.Body))
|
||||
}, func() {
|
||||
r.logger.Infof("rocketmq successfully processed message after it previously failed. topic:%s data-length:%d.", v.Topic, len(v.Body))
|
||||
r.logger.Infof("rocketmq successfully processed message after it previously failed. topic:%s cloudEventsMap-length:%d.", newMessage.Topic, len(msg.Body))
|
||||
})
|
||||
if rerr != nil && !errors.Is(rerr, context.Canceled) {
|
||||
r.logger.Errorf("rocketmq error: processing message and retries are exhausted. topic:%s data-length:%d.", v.Topic, len(v.Body))
|
||||
if retError != nil && !errors.Is(retError, context.Canceled) {
|
||||
r.logger.Errorf("rocketmq error: processing message and retries are exhausted. topic:%s cloudEventsMap-length:%d.", newMessage.Topic, len(msg.Body))
|
||||
}
|
||||
}
|
||||
|
||||
if !success {
|
||||
return mqc.ConsumeRetryLater, nil
|
||||
}
|
||||
|
||||
return mqc.ConsumeSuccess, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rocketMQ) Subscribe(req pubsub.SubscribeRequest, handler pubsub.Handler) error {
|
||||
if req.Metadata == nil {
|
||||
req.Metadata = make(map[string]string)
|
||||
}
|
||||
consumerGroup := r.metadata.ConsumerGroup
|
||||
// get consumer group from request first
|
||||
if group, ok := req.Metadata[metadataRocketmqConsumerGroup]; ok {
|
||||
consumerGroup = group
|
||||
}
|
||||
var (
|
||||
mqExpr = req.Metadata[metadataRocketmqExpression]
|
||||
mqType = req.Metadata[metadataRocketmqType]
|
||||
err error
|
||||
)
|
||||
if !r.validMqTypeParams(mqType) {
|
||||
return ErrRocketmqValidPublishMsgTyp
|
||||
}
|
||||
r.closeSubscriptionResources()
|
||||
if r.pushConsumer, err = r.setUpConsumer(); err != nil {
|
||||
return err
|
||||
}
|
||||
topics := r.addTopic(req.Topic, mqc.MessageSelector{
|
||||
Type: mqc.ExpressionType(mqType),
|
||||
Expression: mqExpr,
|
||||
})
|
||||
r.subscribeAllTopics(topics, consumerGroup, handler)
|
||||
err = r.pushConsumer.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("consumer start failed. %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *rocketMQ) validMqTypeParams(mqType string) bool {
|
||||
if len(mqType) != 0 && (mqType != string(mqc.SQL92) && mqType != string(mqc.TAG)) {
|
||||
r.logger.Warnf("rocketmq subscribe failed because some illegal type(%s).", mqType)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Close down consumer group resources, refresh once.
|
||||
func (r *rocketMQ) closeSubscriptionResources() {
|
||||
if r.pushConsumer != nil {
|
||||
if len(r.topics) > 0 {
|
||||
_ = r.pushConsumer.Shutdown()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rocketMQ) subscribeAllTopics(topics []string, consumerGroup string, handler pubsub.Handler) {
|
||||
for _, topic := range topics {
|
||||
selector, ok := r.topics[topic]
|
||||
if !ok {
|
||||
r.logger.Errorf("no selector for topic:" + topic)
|
||||
continue
|
||||
}
|
||||
err := r.pushConsumer.Subscribe(topic, selector, r.adaptCallback(topic, consumerGroup, string(selector.Type), selector.Expression, handler))
|
||||
if err != nil {
|
||||
r.logger.Errorf("subscribe topic:%v failed,error:%v", topic, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rocketMQ) addTopic(topic string, selector mqc.MessageSelector) []string {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
r.topics[topic] = selector
|
||||
return r.getAllTopics()
|
||||
}
|
||||
|
||||
func (r *rocketMQ) getAllTopics() []string {
|
||||
topics := make([]string, 0, len(r.topics))
|
||||
for topic := range r.topics {
|
||||
topics = append(topics, topic)
|
||||
}
|
||||
return topics
|
||||
}
|
||||
|
||||
func (r *rocketMQ) Close() error {
|
||||
r.cancel()
|
||||
if r.pushConsumer != nil {
|
||||
_ = r.pushConsumer.Shutdown()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,83 +1,78 @@
|
|||
// ------------------------------------------------------------
|
||||
// Copyright (c) Microsoft Corporation and Dapr Contributors.
|
||||
// Licensed under the MIT License.
|
||||
// Licensed under the Apache License.
|
||||
// ------------------------------------------------------------
|
||||
|
||||
package rocketmq
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/dapr/components-contrib/pubsub"
|
||||
"github.com/dapr/kit/logger"
|
||||
)
|
||||
|
||||
func TestPubSub(t *testing.T) { //nolint:paralleltest
|
||||
if !isLiveTest() {
|
||||
return
|
||||
}
|
||||
m := pubsub.Metadata{Properties: getTestMetadata()}
|
||||
|
||||
r := NewRocketMQ(logger.NewLogger("test"))
|
||||
err := r.Init(m)
|
||||
require.NoError(t, err)
|
||||
|
||||
var count int32
|
||||
handler := func(_ context.Context, msg *pubsub.NewMessage) error {
|
||||
type TopicEvent struct {
|
||||
Data interface{} `json:"data"`
|
||||
}
|
||||
var in TopicEvent
|
||||
err = json.NewDecoder(bytes.NewReader(msg.Data)).Decode(&in)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, "hello", in.Data.(string))
|
||||
|
||||
atomic.AddInt32(&count, 1)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
err = r.Subscribe(pubsub.SubscribeRequest{Topic: "TOPIC_TEST", Metadata: map[string]string{}}, handler)
|
||||
require.NoError(t, err)
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
atomic.StoreInt32(&count, 0)
|
||||
err = r.Publish(&pubsub.PublishRequest{PubsubName: "test", Topic: "TOPIC_TEST", Metadata: map[string]string{}, Data: []byte("hello")})
|
||||
require.NoError(t, err)
|
||||
|
||||
time.Sleep(10 * time.Second)
|
||||
for i := 0; i < 30; i++ {
|
||||
if atomic.LoadInt32(&count) > 0 {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
assert.Equal(t, int32(1), atomic.LoadInt32(&count))
|
||||
}
|
||||
|
||||
func isLiveTest() bool {
|
||||
return os.Getenv("RUN_LIVE_ROCKETMQ_TEST") == "true"
|
||||
}
|
||||
|
||||
func getTestMetadata() map[string]string {
|
||||
return map[string]string{
|
||||
"accessProto": "tcp",
|
||||
"nameServer": "http://**.mq-internet-access.mq-internet.aliyuncs.com:80",
|
||||
"consumerGroup": "GID_DAPR-MQ-TCP",
|
||||
"accessKey": "**",
|
||||
"secretKey": "**",
|
||||
"instanceId": "MQ_INST_**",
|
||||
"nameServer": "127.0.0.1:9876",
|
||||
"consumerGroup": "dapr.rocketmq.producer",
|
||||
"accessKey": "RocketMQ",
|
||||
"secretKey": "12345",
|
||||
"consumerBatchSize": "1",
|
||||
"consumerThreadNums": "2",
|
||||
"retries": "2",
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRocketMQMetadata(t *testing.T) {
|
||||
t.Run("correct metadata", func(t *testing.T) {
|
||||
meta := getTestMetadata()
|
||||
_, err := parseRocketMQMetaData(pubsub.Metadata{Properties: meta})
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("correct init", func(t *testing.T) {
|
||||
meta := getTestMetadata()
|
||||
r := NewRocketMQ(logger.NewLogger("test"))
|
||||
err := r.Init(pubsub.Metadata{Properties: meta})
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("setup producer missing nameserver", func(t *testing.T) {
|
||||
meta := getTestMetadata()
|
||||
delete(meta, "nameServer")
|
||||
r := NewRocketMQ(logger.NewLogger("test"))
|
||||
err := r.Init(pubsub.Metadata{Properties: meta})
|
||||
assert.Nil(t, err)
|
||||
req := &pubsub.PublishRequest{
|
||||
Data: []byte("hello"),
|
||||
PubsubName: "rocketmq",
|
||||
Topic: "test",
|
||||
Metadata: map[string]string{},
|
||||
}
|
||||
err = r.Publish(req)
|
||||
assert.NotNil(t, err)
|
||||
})
|
||||
|
||||
t.Run("subscribe illegal type", func(t *testing.T) {
|
||||
meta := getTestMetadata()
|
||||
r := NewRocketMQ(logger.NewLogger("test"))
|
||||
err := r.Init(pubsub.Metadata{Properties: meta})
|
||||
assert.Nil(t, err)
|
||||
|
||||
req := pubsub.SubscribeRequest{
|
||||
Topic: "test",
|
||||
Metadata: map[string]string{
|
||||
metadataRocketmqType: "incorrect type",
|
||||
},
|
||||
}
|
||||
handler := func(ctx context.Context, msg *pubsub.NewMessage) error {
|
||||
return nil
|
||||
}
|
||||
err = r.Subscribe(req, handler)
|
||||
assert.NotNil(t, err)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,74 +0,0 @@
|
|||
// ------------------------------------------------------------
|
||||
// Copyright (c) Microsoft Corporation and Dapr Contributors.
|
||||
// Licensed under the MIT License.
|
||||
// ------------------------------------------------------------
|
||||
|
||||
package rocketmq
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
mqw "github.com/cinience/go_rocketmq"
|
||||
|
||||
"github.com/dapr/kit/config"
|
||||
)
|
||||
|
||||
const (
|
||||
metadataRocketmqTag = "rocketmq-tag"
|
||||
metadataRocketmqKey = "rocketmq-key"
|
||||
metadataRocketmqConsumerGroup = "rocketmq-consumerGroup"
|
||||
metadataRocketmqType = "rocketmq-sub-type"
|
||||
metadataRocketmqExpression = "rocketmq-sub-expression"
|
||||
metadataRocketmqBrokerName = "rocketmq-broker-name"
|
||||
)
|
||||
|
||||
type Settings struct {
|
||||
// sdk proto (tcp, tcp-cgo，http)
|
||||
AccessProto string `mapstructure:"accessProto"`
|
||||
// rocketmq Credentials
|
||||
AccessKey string `mapstructure:"accessKey"`
|
||||
// rocketmq Credentials
|
||||
SecretKey string `mapstructure:"secretKey"`
|
||||
// rocketmq's name server, optional
|
||||
NameServer string `mapstructure:"nameServer"`
|
||||
// rocketmq's endpoint, optional, just for http proto
|
||||
Endpoint string `mapstructure:"endpoint"`
|
||||
// rocketmq's instanceId, optional
|
||||
InstanceID string `mapstructure:"instanceId"`
|
||||
// consumer group for rocketmq's subscribers, suggested to provide
|
||||
ConsumerGroup string `mapstructure:"consumerGroup"`
|
||||
// consumer group for rocketmq's subscribers, suggested to provide
|
||||
ConsumerBatchSize int `mapstructure:"consumerBatchSize"`
|
||||
// consumer group for rocketmq's subscribers, suggested to provide, just for tcp-cgo proto
|
||||
ConsumerThreadNums int `mapstructure:"consumerThreadNums"`
|
||||
// rocketmq's name server domain, optional
|
||||
NameServerDomain string `mapstructure:"nameServerDomain"`
|
||||
// retry times to connect rocketmq's broker, optional
|
||||
Retries int `mapstructure:"retries"`
|
||||
// msg's content-type eg:"application/cloudevents+json; charset=utf-8", application/octet-stream
|
||||
ContentType string `mapstructure:"content-type"`
|
||||
}
|
||||
|
||||
func (s *Settings) Decode(in interface{}) error {
|
||||
if err := config.Decode(in, s); err != nil {
|
||||
return fmt.Errorf("decode failed. %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Settings) ToRocketMQMetadata() *mqw.Metadata {
|
||||
return &mqw.Metadata{
|
||||
AccessProto: s.AccessProto,
|
||||
AccessKey: s.AccessKey,
|
||||
SecretKey: s.SecretKey,
|
||||
NameServer: s.NameServer,
|
||||
Endpoint: s.Endpoint,
|
||||
InstanceId: s.InstanceID,
|
||||
ConsumerGroup: s.ConsumerGroup,
|
||||
ConsumerBatchSize: s.ConsumerBatchSize,
|
||||
ConsumerThreadNums: s.ConsumerThreadNums,
|
||||
NameServerDomain: s.NameServerDomain,
|
||||
Retries: s.Retries,
|
||||
}
|
||||
}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
// ------------------------------------------------------------
|
||||
// Copyright (c) Microsoft Corporation and Dapr Contributors.
|
||||
// Licensed under the MIT License.
|
||||
// ------------------------------------------------------------
|
||||
|
||||
package rocketmq
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSettingsDecode(t *testing.T) { //nolint:paralleltest
|
||||
props := map[string]string{
|
||||
"accessProto": "http",
|
||||
"accessKey": "**",
|
||||
"secretKey": "***",
|
||||
"endpoint": "http://test.endpoint",
|
||||
"nameServer": "http://test.nameserver",
|
||||
"consumerGroup": "defaultGroup",
|
||||
"instanceId": "defaultNamespace",
|
||||
"topics": "defaultTopic",
|
||||
}
|
||||
|
||||
var settings Settings
|
||||
err := settings.Decode(props)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "http", settings.AccessProto)
|
||||
assert.Equal(t, "**", settings.AccessKey)
|
||||
assert.Equal(t, "***", settings.SecretKey)
|
||||
assert.Equal(t, "http://test.endpoint", settings.Endpoint)
|
||||
assert.Equal(t, "defaultGroup", settings.ConsumerGroup)
|
||||
assert.Equal(t, "defaultNamespace", settings.InstanceID)
|
||||
}
|
||||
|
||||
func TestParseCommonMetadata(t *testing.T) { //nolint:paralleltest
|
||||
props := map[string]string{
|
||||
"accessProto": "http",
|
||||
"accessKey": "**",
|
||||
"secretKey": "***",
|
||||
"endpoint": "http://test.endpoint",
|
||||
"nameServer": "http://test.nameserver",
|
||||
"consumerGroup": "defaultGroup",
|
||||
"instanceId": "defaultNamespace",
|
||||
"topics": "defaultTopic",
|
||||
}
|
||||
|
||||
var settings Settings
|
||||
err := settings.Decode(props)
|
||||
require.NoError(t, err)
|
||||
|
||||
b := settings.ToRocketMQMetadata()
|
||||
|
||||
assert.Equal(t, "http", b.AccessProto)
|
||||
assert.Equal(t, "**", b.AccessKey)
|
||||
assert.Equal(t, "***", b.SecretKey)
|
||||
assert.Equal(t, "http://test.endpoint", b.Endpoint)
|
||||
assert.Equal(t, "defaultGroup", b.ConsumerGroup)
|
||||
assert.Equal(t, "defaultNamespace", b.InstanceId)
|
||||
}
|
||||
Loading…
Reference in New Issue