From 36923bd0a4ff2c64820c3a6da1566face3e549bb Mon Sep 17 00:00:00 2001 From: N3ko Date: Mon, 4 Aug 2025 18:06:38 +0800 Subject: [PATCH] feat: support ark embedding api_type (#519) --- backend/application/knowledge/init.go | 15 ++++++++-- backend/go.mod | 4 +-- backend/go.sum | 8 +++--- docker/.env.debug.example | 40 ++++++++++++++------------- docker/.env.example | 40 ++++++++++++++------------- 5 files changed, 61 insertions(+), 46 deletions(-) diff --git a/backend/application/knowledge/init.go b/backend/application/knowledge/init.go index 10c17ba7..09888d0b 100644 --- a/backend/application/knowledge/init.go +++ b/backend/application/knowledge/init.go @@ -338,8 +338,9 @@ func getEmbedding(ctx context.Context) (embedding.Embedder, error) { arkEmbeddingApiKey = os.Getenv("ARK_EMBEDDING_API_KEY") // deprecated: use ARK_EMBEDDING_API_KEY instead // ARK_EMBEDDING_AK will be removed in the future - arkEmbeddingAK = os.Getenv("ARK_EMBEDDING_AK") - arkEmbeddingDims = os.Getenv("ARK_EMBEDDING_DIMS") + arkEmbeddingAK = os.Getenv("ARK_EMBEDDING_AK") + arkEmbeddingDims = os.Getenv("ARK_EMBEDDING_DIMS") + arkEmbeddingAPIType = os.Getenv("ARK_EMBEDDING_API_TYPE") ) dims, err := strconv.ParseInt(arkEmbeddingDims, 10, 64) @@ -347,6 +348,15 @@ func getEmbedding(ctx context.Context) (embedding.Embedder, error) { return nil, fmt.Errorf("init ark embedding dims failed, err=%w", err) } + apiType := ark.APITypeText + if arkEmbeddingAPIType != "" { + if t := ark.APIType(arkEmbeddingAPIType); t != ark.APITypeText && t != ark.APITypeMultiModal { + return nil, fmt.Errorf("init ark embedding api_type failed, invalid api_type=%s", t) + } else { + apiType = t + } + } + emb, err = arkemb.NewArkEmbedder(ctx, &ark.EmbeddingConfig{ APIKey: func() string { if arkEmbeddingApiKey != "" { @@ -356,6 +366,7 @@ func getEmbedding(ctx context.Context) (embedding.Embedder, error) { }(), Model: arkEmbeddingModel, BaseURL: arkEmbeddingBaseURL, + APIType: &apiType, }, dims, batchSize) if err != nil { return nil, fmt.Errorf("init ark embedding client failed, err=%w", err) diff --git a/backend/go.mod b/backend/go.mod index 077002c1..a507d37e 100755 --- a/backend/go.mod +++ b/backend/go.mod @@ -39,7 +39,7 @@ require ( go.uber.org/mock v0.5.1 golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 golang.org/x/mod v0.25.0 - golang.org/x/sync v0.15.0 + golang.org/x/sync v0.16.0 gopkg.in/yaml.v2 v2.4.0 gorm.io/driver/mysql v1.5.7 gorm.io/driver/sqlite v1.4.3 @@ -55,7 +55,7 @@ require github.com/alicebob/miniredis/v2 v2.34.0 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.84.1 - github.com/cloudwego/eino-ext/components/embedding/ark v0.0.0-20250522060253-ddb617598b09 + github.com/cloudwego/eino-ext/components/embedding/ark v0.1.0 github.com/cloudwego/eino-ext/components/embedding/ollama v0.0.0-20250728060543-79ec300857b8 github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20250522060253-ddb617598b09 github.com/cloudwego/eino-ext/components/model/gemini v0.1.2 diff --git a/backend/go.sum b/backend/go.sum index 9c096e2b..24568a42 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -926,8 +926,8 @@ github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCy github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/eino v0.3.55 h1:lMZrGtEh0k3qykQTLNXSXuAa98OtF2tS43GMHyvN7nA= github.com/cloudwego/eino v0.3.55/go.mod h1:wUjz990apdsaOraOXdh6CdhVXq8DJsOvLsVlxNTcNfY= -github.com/cloudwego/eino-ext/components/embedding/ark v0.0.0-20250522060253-ddb617598b09 h1:hZScBE/Etiji2RqjlABcAkq6n1uzYPu+jo4GV5TF8Hc= -github.com/cloudwego/eino-ext/components/embedding/ark v0.0.0-20250522060253-ddb617598b09/go.mod h1:pLtH5BZKgb7/bB8+P3W5/f1d46gTl9K77+08j88Gb4k= +github.com/cloudwego/eino-ext/components/embedding/ark v0.1.0 h1:AuJsMdaTXc+dGUDQp82MifLYK8oiJf4gLQPUETmKISM= +github.com/cloudwego/eino-ext/components/embedding/ark v0.1.0/go.mod h1:0FZG/KRBl3hGWkNsm55UaXyVa6PDVIy5u+QvboAB+cY= github.com/cloudwego/eino-ext/components/embedding/ollama v0.0.0-20250728060543-79ec300857b8 h1:uJrs6SmfYnca8A+k9+3qJ4MYwYHMncUlGac1mYQT+Ak= github.com/cloudwego/eino-ext/components/embedding/ollama v0.0.0-20250728060543-79ec300857b8/go.mod h1:nav79aUcd+UR24dLA+7l7RcHCMlg26zbDAKvjONdrw0= github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20250522060253-ddb617598b09 h1:C8RjF193iguUuevkuv0q4SC+XGlM/DlJEgic7l8OUAI= @@ -2261,8 +2261,8 @@ golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= -golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/docker/.env.debug.example b/docker/.env.debug.example index d0ef95ff..fef45e7b 100644 --- a/docker/.env.debug.example +++ b/docker/.env.debug.example @@ -96,33 +96,35 @@ export VIKING_DB_MODEL_NAME="" # if vikingdb model name is not set, you need to # Settings for Embedding # The Embedding model relied on by knowledge base vectorization does not need to be configured # if the vector database comes with built-in Embedding functionality (such as VikingDB). Currently, -# Coze Studio supports three access methods: openai, ark, ollama, and custom http. Users can simply choose one of them when using +# Coze Studio supports four access methods: openai, ark, ollama, and custom http. Users can simply choose one of them when using # embedding type: openai / ark / ollama / http export EMBEDDING_TYPE="ark" export EMBEDDING_MAX_BATCH_SIZE=100 -# openai embedding -export OPENAI_EMBEDDING_BASE_URL="" # (string) OpenAI base_url -export OPENAI_EMBEDDING_MODEL="" # (string) OpenAI embedding model -export OPENAI_EMBEDDING_API_KEY="" # (string) OpenAI api_key -export OPENAI_EMBEDDING_BY_AZURE=false # (bool) OpenAI by_azure -export OPENAI_EMBEDDING_API_VERSION="" # OpenAI azure api version -export OPENAI_EMBEDDING_DIMS=1024 # (int) 向量维度 -export OPENAI_EMBEDDING_REQUEST_DIMS=1024 -# ark embedding -export ARK_EMBEDDING_MODEL="" -export ARK_EMBEDDING_API_KEY="" -export ARK_EMBEDDING_DIMS="2048" -export ARK_EMBEDDING_BASE_URL="" +# openai embedding +export OPENAI_EMBEDDING_BASE_URL="" # (string, required) OpenAI embedding base_url +export OPENAI_EMBEDDING_MODEL="" # (string, required) OpenAI embedding model +export OPENAI_EMBEDDING_API_KEY="" # (string, required) OpenAI embedding api_key +export OPENAI_EMBEDDING_BY_AZURE=false # (bool, optional) OpenAI embedding by_azure +export OPENAI_EMBEDDING_API_VERSION="" # (string, optional) OpenAI embedding azure api version +export OPENAI_EMBEDDING_DIMS=1024 # (int, required) OpenAI embedding dimensions +export OPENAI_EMBEDDING_REQUEST_DIMS=1024 # (int, optional) OpenAI embedding dimensions in requests, need to be empty if api doesn't support specifying dimensions. + +# ark embedding by volcengine / byteplus +export ARK_EMBEDDING_MODEL="" # (string, required) Ark embedding model +export ARK_EMBEDDING_API_KEY="" # (string, required) Ark embedding api_key +export ARK_EMBEDDING_DIMS="2048" # (int, required) Ark embedding dimensions +export ARK_EMBEDDING_BASE_URL="" # (string, required) Ark embedding base_url +export ARK_EMBEDDING_API_TYPE="" # (string, optional) Ark embedding api type, should be "text_api" / "multi_modal_api". Default "text_api". # ollama embedding -export OLLAMA_EMBEDDING_BASE_URL="" -export OLLAMA_EMBEDDING_MODEL="" -export OLLAMA_EMBEDDING_DIMS="" +export OLLAMA_EMBEDDING_BASE_URL="" # (string, required) Ollama embedding base_url +export OLLAMA_EMBEDDING_MODEL="" # (string, required) Ollama embedding model +export OLLAMA_EMBEDDING_DIMS="" # (int, required) Ollama embedding dimensions # http embedding -export HTTP_EMBEDDING_ADDR="" -export HTTP_EMBEDDING_DIMS=1024 +export HTTP_EMBEDDING_ADDR="" # (string, required) http embedding address +export HTTP_EMBEDDING_DIMS=1024 # (string, required) http embedding dimensions # Settings for OCR # If you want to use the OCR-related functions in the knowledge base feature，You need to set up the OCR configuration. diff --git a/docker/.env.example b/docker/.env.example index fbf0d86f..7ddd8a0e 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -96,33 +96,35 @@ export VIKING_DB_MODEL_NAME="" # if vikingdb model name is not set, you need to # Settings for Embedding # The Embedding model relied on by knowledge base vectorization does not need to be configured # if the vector database comes with built-in Embedding functionality (such as VikingDB). Currently, -# Coze Studio supports three access methods: openai, ark, ollama, and custom http. Users can simply choose one of them when using +# Coze Studio supports four access methods: openai, ark, ollama, and custom http. Users can simply choose one of them when using # embedding type: openai / ark / ollama / http export EMBEDDING_TYPE="ark" export EMBEDDING_MAX_BATCH_SIZE=100 -# openai embedding -export OPENAI_EMBEDDING_BASE_URL="" # (string) OpenAI base_url -export OPENAI_EMBEDDING_MODEL="" # (string) OpenAI embedding model -export OPENAI_EMBEDDING_API_KEY="" # (string) OpenAI api_key -export OPENAI_EMBEDDING_BY_AZURE=false # (bool) OpenAI by_azure -export OPENAI_EMBEDDING_API_VERSION="" # OpenAI azure api version -export OPENAI_EMBEDDING_DIMS=1024 # (int) 向量维度 -export OPENAI_EMBEDDING_REQUEST_DIMS=1024 -# ark embedding -export ARK_EMBEDDING_MODEL="" -export ARK_EMBEDDING_API_KEY="" -export ARK_EMBEDDING_DIMS="2048" -export ARK_EMBEDDING_BASE_URL="" +# openai embedding +export OPENAI_EMBEDDING_BASE_URL="" # (string, required) OpenAI embedding base_url +export OPENAI_EMBEDDING_MODEL="" # (string, required) OpenAI embedding model +export OPENAI_EMBEDDING_API_KEY="" # (string, required) OpenAI embedding api_key +export OPENAI_EMBEDDING_BY_AZURE=false # (bool, optional) OpenAI embedding by_azure +export OPENAI_EMBEDDING_API_VERSION="" # (string, optional) OpenAI embedding azure api version +export OPENAI_EMBEDDING_DIMS=1024 # (int, required) OpenAI embedding dimensions +export OPENAI_EMBEDDING_REQUEST_DIMS=1024 # (int, optional) OpenAI embedding dimensions in requests, need to be empty if api doesn't support specifying dimensions. + +# ark embedding by volcengine / byteplus +export ARK_EMBEDDING_MODEL="" # (string, required) Ark embedding model +export ARK_EMBEDDING_API_KEY="" # (string, required) Ark embedding api_key +export ARK_EMBEDDING_DIMS="2048" # (int, required) Ark embedding dimensions +export ARK_EMBEDDING_BASE_URL="" # (string, required) Ark embedding base_url +export ARK_EMBEDDING_API_TYPE="" # (string, optional) Ark embedding api type, should be "text_api" / "multi_modal_api". Default "text_api". # ollama embedding -export OLLAMA_EMBEDDING_BASE_URL="" -export OLLAMA_EMBEDDING_MODEL="" -export OLLAMA_EMBEDDING_DIMS="" +export OLLAMA_EMBEDDING_BASE_URL="" # (string, required) Ollama embedding base_url +export OLLAMA_EMBEDDING_MODEL="" # (string, required) Ollama embedding model +export OLLAMA_EMBEDDING_DIMS="" # (int, required) Ollama embedding dimensions # http embedding -export HTTP_EMBEDDING_ADDR="" -export HTTP_EMBEDDING_DIMS=1024 +export HTTP_EMBEDDING_ADDR="" # (string, required) http embedding address +export HTTP_EMBEDDING_DIMS=1024 # (string, required) http embedding dimensions # Settings for OCR # If you want to use the OCR-related functions in the knowledge base feature，You need to set up the OCR configuration.