feat: manually mirror opencoze's code from bytedance

Change-Id: I09a73aadda978ad9511264a756b2ce51f5761adf

backend/infra/impl/eventbus/rmq/consumer.go

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rmq
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/apache/rocketmq-client-go/v2"
+	"github.com/apache/rocketmq-client-go/v2/consumer"
+	"github.com/apache/rocketmq-client-go/v2/primitive"
+	"github.com/apache/rocketmq-client-go/v2/rlog"
+
+	"github.com/coze-dev/coze-studio/backend/infra/contract/eventbus"
+	"github.com/coze-dev/coze-studio/backend/pkg/lang/conv"
+	"github.com/coze-dev/coze-studio/backend/pkg/lang/signal"
+	"github.com/coze-dev/coze-studio/backend/pkg/logs"
+	"github.com/coze-dev/coze-studio/backend/pkg/safego"
+	"github.com/coze-dev/coze-studio/backend/types/consts"
+)
+
+func RegisterConsumer(nameServer, topic, group string, consumerHandler eventbus.ConsumerHandler, opts ...eventbus.ConsumerOpt) error {
+	if nameServer == "" {
+		return fmt.Errorf("name server is empty")
+	}
+	if topic == "" {
+		return fmt.Errorf("topic is empty")
+	}
+
+	if group == "" {
+		return fmt.Errorf("group is empty")
+	}
+
+	if consumerHandler == nil {
+		return fmt.Errorf("consumer handler is nil")
+	}
+
+	rlog.SetLogLevel("error")
+
+	o := &eventbus.ConsumerOption{}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	defaultOptions := []consumer.Option{
+		consumer.WithGroupName(group),
+		consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{nameServer})),
+		consumer.WithConsumeFromWhere(consumer.ConsumeFromLastOffset),
+		consumer.WithCredentials(primitive.Credentials{
+			AccessKey: os.Getenv(consts.RMQAccessKey),
+			SecretKey: os.Getenv(consts.RMQSecretKey),
+		}),
+	}
+
+	if o.Orderly != nil {
+		defaultOptions = append(defaultOptions, consumer.WithConsumerOrder(*o.Orderly))
+	}
+
+	c, err := rocketmq.NewPushConsumer(defaultOptions...)
+	if err != nil {
+		return fmt.Errorf("[RegisterConsumer] nameServer: %s, topic: %s, group : %s, err: %w", nameServer, topic, group, err)
+	}
+
+	err = c.Subscribe(topic, consumer.MessageSelector{},
+		func(ctx context.Context, msgArr ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+			for i := range msgArr {
+
+				msg := &eventbus.Message{
+					Topic: msgArr[i].Topic,
+					Group: group,
+					Body:  msgArr[i].Body,
+				}
+
+				logs.CtxDebugf(ctx, "[Subscribe] receive msg : %v \n", conv.DebugJsonToStr(msg))
+				err = consumerHandler.HandleMessage(ctx, msg)
+				if err != nil {
+					logs.CtxErrorf(ctx, "[Subscribe] handle msg failed, topic : %s , group : %s, err: %v \n", msg.Topic, msg.Group, err)
+					return consumer.ConsumeRetryLater, err // TODO: 策略可以可以配置
+				}
+
+				fmt.Printf("subscribe callback: %v \n", msgArr[i])
+			}
+
+			return consumer.ConsumeSuccess, nil
+		})
+	if err != nil {
+		return fmt.Errorf("consumer Subscribe failed, err=%w", err)
+	}
+
+	if err = c.Start(); err != nil {
+		return fmt.Errorf("[RegisterConsumer-Start] nameServer: %s, topic: %s, group : %s, err: %w", nameServer, topic, group, err)
+	}
+
+	safego.Go(context.Background(), func() {
+		signal.WaitExit()
+		if err := c.Shutdown(); err != nil {
+			logs.Errorf("shutdown consumer error: %v", err)
+		}
+	})
+
+	return nil
+}

backend/infra/impl/eventbus/rmq/local_test.go

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rmq
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/apache/rocketmq-client-go/v2"
+	"github.com/apache/rocketmq-client-go/v2/consumer"
+	"github.com/apache/rocketmq-client-go/v2/primitive"
+	"github.com/apache/rocketmq-client-go/v2/producer"
+	"github.com/stretchr/testify/assert"
+)
+
+var endpoint = "127.0.0.1:9876"
+
+func TestProducer(t *testing.T) {
+	if os.Getenv("RMQ_LOCAL_TEST") != "true" {
+		return
+	}
+
+	p, err := rocketmq.NewProducer(
+		producer.WithNameServer([]string{endpoint}),
+		producer.WithRetry(2),
+		producer.WithGroupName("test_group"),
+	)
+	assert.NoError(t, err)
+	assert.NoError(t, p.Start())
+
+	result, err := p.SendSync(context.Background(), &primitive.Message{
+		Topic: "test_topic",
+		Body:  []byte("hello"),
+	})
+	assert.NoError(t, err)
+	fmt.Println(result)
+}
+
+func TestConsumer(t *testing.T) {
+	if os.Getenv("RMQ_LOCAL_TEST") != "true" {
+		return
+	}
+
+	c, err := rocketmq.NewPushConsumer(
+		consumer.WithNameServer([]string{endpoint}),
+		consumer.WithConsumerModel(consumer.Clustering),
+		consumer.WithConsumeFromWhere(consumer.ConsumeFromLastOffset),
+		consumer.WithConsumerOrder(true),
+		consumer.WithGroupName("test_group"),
+	)
+	assert.NoError(t, err)
+
+	wg := sync.WaitGroup{}
+	err = c.Subscribe("test_topic", consumer.MessageSelector{}, func(ctx context.Context, ext ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+		orderlyCtx, _ := primitive.GetOrderlyCtx(ctx)
+		fmt.Println(orderlyCtx)
+
+		for i, e := range ext {
+			fmt.Println(i, e.Body)
+		}
+
+		wg.Done()
+		return consumer.ConsumeSuccess, nil
+	})
+	assert.NoError(t, err)
+
+	err = c.Start()
+	assert.NoError(t, err)
+
+	wg.Wait()
+	time.Sleep(time.Second)
+	_ = c.Shutdown()
+}

backend/infra/impl/eventbus/rmq/producer.go

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rmq
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/apache/rocketmq-client-go/v2"
+	"github.com/apache/rocketmq-client-go/v2/primitive"
+	"github.com/apache/rocketmq-client-go/v2/producer"
+
+	"github.com/coze-dev/coze-studio/backend/infra/contract/eventbus"
+	"github.com/coze-dev/coze-studio/backend/pkg/lang/signal"
+	"github.com/coze-dev/coze-studio/backend/pkg/logs"
+	"github.com/coze-dev/coze-studio/backend/pkg/safego"
+	"github.com/coze-dev/coze-studio/backend/types/consts"
+)
+
+type producerImpl struct {
+	nameServer string
+	topic      string
+	p          rocketmq.Producer
+}
+
+func NewProducer(nameServer, topic, group string, retries int) (eventbus.Producer, error) {
+	if nameServer == "" {
+		return nil, fmt.Errorf("name server is empty")
+	}
+
+	if topic == "" {
+		return nil, fmt.Errorf("topic is empty")
+	}
+
+	p, err := rocketmq.NewProducer(
+		producer.WithNsResolver(primitive.NewPassthroughResolver([]string{nameServer})),
+		producer.WithRetry(retries),
+		producer.WithGroupName(group),
+		producer.WithCredentials(primitive.Credentials{
+			AccessKey: os.Getenv(consts.RMQAccessKey),
+			SecretKey: os.Getenv(consts.RMQSecretKey),
+		}),
+		// producer.WithNsResolver(primitive.NewGRPCCredentialsResolver(nil)),
+		// producer.WithInstanceName("rocketmq-cnngf291ea363b7a"),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("NewProducer failed, nameServer: %s, topic: %s, err: %w", nameServer, topic, err)
+	}
+
+	err = p.Start()
+	if err != nil {
+		return nil, fmt.Errorf("start producer error: %w", err)
+	}
+
+	safego.Go(context.Background(), func() {
+		signal.WaitExit()
+		if err := p.Shutdown(); err != nil {
+			logs.Errorf("shutdown producer error: %s", err.Error())
+		}
+	})
+
+	return &producerImpl{
+		nameServer: nameServer,
+		topic:      topic,
+		p:          p,
+	}, nil
+}
+
+func (r *producerImpl) Send(ctx context.Context, body []byte, opts ...eventbus.SendOpt) error {
+	_, err := r.p.SendSync(context.Background(), primitive.NewMessage(r.topic, body))
+	if err != nil {
+		return fmt.Errorf("[producerImpl] send message failed: %w", err)
+	}
+	return err
+}
+
+func (r *producerImpl) BatchSend(ctx context.Context, bodyArr [][]byte, opts ...eventbus.SendOpt) error {
+	option := eventbus.SendOption{}
+	for _, opt := range opts {
+		opt(&option)
+	}
+
+	var msgArr []*primitive.Message
+	for _, body := range bodyArr {
+		msg := primitive.NewMessage(r.topic, body)
+
+		if option.ShardingKey != nil {
+			msg.WithShardingKey(*option.ShardingKey)
+		}
+
+		msgArr = append(msgArr, msg)
+	}
+
+	_, err := r.p.SendSync(ctx, msgArr...)
+	return err
+}