feat: manually mirror opencoze's code from bytedance

Change-Id: I09a73aadda978ad9511264a756b2ce51f5761adf

backend/infra/impl/embedding/ark/ark.go

@@ -0,0 +1,97 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ark
+
+import (
+	"context"
+	"fmt"
+	"math"
+
+	"github.com/cloudwego/eino-ext/components/embedding/ark"
+	"github.com/cloudwego/eino/components/embedding"
+
+	contract "github.com/coze-dev/coze-studio/backend/infra/contract/embedding"
+	"github.com/coze-dev/coze-studio/backend/pkg/lang/slices"
+)
+
+func NewArkEmbedder(ctx context.Context, config *ark.EmbeddingConfig, dimensions int64) (contract.Embedder, error) {
+	emb, err := ark.NewEmbedder(ctx, config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &embWrap{dims: dimensions, Embedder: emb}, nil
+}
+
+type embWrap struct {
+	dims int64
+	embedding.Embedder
+}
+
+func (d embWrap) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) {
+	resp := make([][]float64, 0, len(texts))
+	for _, part := range slices.Chunks(texts, 100) {
+		partResult, err := d.Embedder.EmbedStrings(ctx, part, opts...)
+		if err != nil {
+			return nil, err
+		}
+		normed, err := d.slicedNormL2(partResult)
+		if err != nil {
+			return nil, err
+		}
+		resp = append(resp, normed...)
+	}
+	return resp, nil
+}
+
+func (d embWrap) EmbedStringsHybrid(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, []map[int]float64, error) {
+	return nil, nil, fmt.Errorf("[arkEmbedder] EmbedStringsHybrid not support")
+}
+
+func (d embWrap) Dimensions() int64 {
+	return d.dims
+}
+
+func (d embWrap) SupportStatus() contract.SupportStatus {
+	return contract.SupportDense
+}
+
+func (d embWrap) slicedNormL2(vectors [][]float64) ([][]float64, error) {
+	if len(vectors) == 0 {
+		return vectors, nil
+	}
+	if curDims := len(vectors[0]); curDims < int(d.dims) {
+		return nil, fmt.Errorf("[slicedNormL2] got dims=%d less than %d", curDims, d.dims)
+	} else if curDims == int(d.dims) {
+		return vectors, nil
+	}
+	result := make([][]float64, len(vectors))
+	for i, vec := range vectors {
+		sliced := vec[:d.dims]
+		sumSq := 0.0
+		for _, v := range sliced {
+			sumSq += v * v
+		}
+		norm := math.Sqrt(sumSq)
+		r := make([]float64, len(sliced))
+		for j, v := range sliced {
+			r[j] = v / norm
+		}
+		result[i] = r
+	}
+	return result, nil
+}

backend/infra/impl/embedding/http/http.go

@@ -0,0 +1,158 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package http
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+
+	opt "github.com/cloudwego/eino/components/embedding"
+
+	"github.com/coze-dev/coze-studio/backend/infra/contract/embedding"
+)
+
+const (
+	pathDim   = "/dimension"
+	pathEmbed = "/embedding"
+)
+
+type embedReq struct {
+	Texts      []string `json:"texts"`
+	NeedSparse bool     `json:"need_sparse"`
+}
+
+type embedResp struct {
+	Dense  [][]float64       `json:"dense"`
+	Sparse []map[int]float64 `json:"sparse"`
+}
+
+func NewEmbedding(addr string) (embedding.Embedder, error) {
+	cli := &http.Client{Timeout: time.Second * 30}
+	req, err := http.NewRequest(http.MethodGet, addr+pathDim, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := cli.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	dim, err := strconv.ParseInt(string(b), 10, 64)
+	if err != nil {
+		return nil, err
+	}
+
+	return &embedder{
+		cli:  cli,
+		addr: addr,
+		dim:  dim,
+	}, nil
+}
+
+type embedder struct {
+	cli  *http.Client
+	addr string
+	dim  int64
+}
+
+func (e *embedder) EmbedStrings(ctx context.Context, texts []string, opts ...opt.Option) ([][]float64, error) {
+	rb, err := json.Marshal(&embedReq{
+		Texts:      texts,
+		NeedSparse: false,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.addr+pathEmbed, bytes.NewReader(rb))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	resp, err := e.cli.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	r := &embedResp{}
+	if err = json.Unmarshal(b, r); err != nil {
+		return nil, err
+	}
+
+	return r.Dense, nil
+}
+
+func (e *embedder) EmbedStringsHybrid(ctx context.Context, texts []string, opts ...opt.Option) ([][]float64, []map[int]float64, error) {
+	rb, err := json.Marshal(&embedReq{
+		Texts:      texts,
+		NeedSparse: true,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.addr+pathEmbed, bytes.NewReader(rb))
+	if err != nil {
+		return nil, nil, err
+	}
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+
+	resp, err := e.cli.Do(req)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	defer resp.Body.Close()
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	r := &embedResp{}
+	if err = json.Unmarshal(b, r); err != nil {
+		return nil, nil, err
+	}
+
+	return r.Dense, r.Sparse, nil
+
+}
+
+func (e *embedder) Dimensions() int64 {
+	return e.dim
+}
+
+func (e *embedder) SupportStatus() embedding.SupportStatus {
+	return embedding.SupportDenseAndSparse
+}

backend/infra/impl/embedding/http/http_test.go

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package http
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHTTPEmbedding(t *testing.T) {
+	if os.Getenv("TEST_HTTP_EMBEDDING") != "true" {
+		return
+	}
+
+	ctx := context.Background()
+	emb, err := NewEmbedding("http://127.0.0.1:6543")
+	assert.NoError(t, err)
+	texts := []string{
+		"hello",
+		"Eiffel Tower: Located in Paris, France, it is one of the most famous landmarks in the world.",
+	}
+
+	dense, err := emb.EmbedStrings(ctx, texts)
+	assert.NoError(t, err)
+	fmt.Println(dense)
+
+	dense, sparse, err := emb.EmbedStringsHybrid(ctx, texts)
+	assert.NoError(t, err)
+	fmt.Println(dense, sparse)
+}

backend/infra/impl/embedding/wrap/dense_only.go

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package wrap
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cloudwego/eino/components/embedding"
+
+	contract "github.com/coze-dev/coze-studio/backend/infra/contract/embedding"
+	"github.com/coze-dev/coze-studio/backend/pkg/lang/slices"
+)
+
+type denseOnlyWrap struct {
+	dims int64
+	embedding.Embedder
+}
+
+func (d denseOnlyWrap) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) {
+	resp := make([][]float64, 0, len(texts))
+	for _, part := range slices.Chunks(texts, 100) {
+		partResult, err := d.Embedder.EmbedStrings(ctx, part, opts...)
+		if err != nil {
+			return nil, err
+		}
+		resp = append(resp, partResult...)
+	}
+	return resp, nil
+}
+
+func (d denseOnlyWrap) EmbedStringsHybrid(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, []map[int]float64, error) {
+	return nil, nil, fmt.Errorf("[denseOnlyWrap] EmbedStringsHybrid not support")
+}
+
+func (d denseOnlyWrap) Dimensions() int64 {
+	return d.dims
+}
+
+func (d denseOnlyWrap) SupportStatus() contract.SupportStatus {
+	return contract.SupportDense
+}

backend/infra/impl/embedding/wrap/openai.go

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package wrap
+
+import (
+	"context"
+
+	"github.com/cloudwego/eino-ext/components/embedding/openai"
+
+	contract "github.com/coze-dev/coze-studio/backend/infra/contract/embedding"
+)
+
+func NewOpenAIEmbedder(ctx context.Context, config *openai.EmbeddingConfig, dimensions int64) (contract.Embedder, error) {
+	emb, err := openai.NewEmbedder(ctx, config)
+	if err != nil {
+		return nil, err
+	}
+	return &denseOnlyWrap{dims: dimensions, Embedder: emb}, nil
+}