refactor(knowledge): Move the all dependent components to app infra (#795)
This commit is contained in:
parent
23a468c72c
commit
f940edf585
|
|
@ -21,7 +21,6 @@ import (
|
|||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"net/http"
|
||||
"os"
|
||||
"reflect"
|
||||
|
|
@ -31,6 +30,8 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
|
||||
"github.com/bytedance/mockey"
|
||||
"github.com/cloudwego/eino/callbacks"
|
||||
model2 "github.com/cloudwego/eino/components/model"
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ import (
|
|||
variablesImpl "github.com/coze-dev/coze-studio/backend/crossdomain/impl/variables"
|
||||
workflowImpl "github.com/coze-dev/coze-studio/backend/crossdomain/impl/workflow"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/eventbus"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/chatmodel"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/checkpoint"
|
||||
implEventbus "github.com/coze-dev/coze-studio/backend/infra/impl/eventbus"
|
||||
)
|
||||
|
|
@ -191,7 +192,9 @@ func initPrimaryServices(ctx context.Context, basicServices *basicServices) (*pr
|
|||
|
||||
memorySVC := memory.InitService(basicServices.toMemoryServiceComponents())
|
||||
|
||||
knowledgeSVC, err := knowledge.InitService(basicServices.toKnowledgeServiceComponents(memorySVC))
|
||||
knowledgeSVC, err := knowledge.InitService(ctx,
|
||||
basicServices.toKnowledgeServiceComponents(memorySVC),
|
||||
basicServices.eventbus.resourceEventBus)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -256,14 +259,18 @@ func (b *basicServices) toPluginServiceComponents() *plugin.ServiceComponents {
|
|||
func (b *basicServices) toKnowledgeServiceComponents(memoryService *memory.MemoryApplicationServices) *knowledge.ServiceComponents {
|
||||
return &knowledge.ServiceComponents{
|
||||
DB: b.infra.DB,
|
||||
IDGenSVC: b.infra.IDGenSVC,
|
||||
Storage: b.infra.TOSClient,
|
||||
IDGen: b.infra.IDGenSVC,
|
||||
RDB: memoryService.RDBDomainSVC,
|
||||
Producer: b.infra.KnowledgeEventProducer,
|
||||
SearchStoreManagers: b.infra.SearchStoreManagers,
|
||||
EventBus: b.eventbus.resourceEventBus,
|
||||
CacheCli: b.infra.CacheCli,
|
||||
ParseManager: b.infra.ParserManager,
|
||||
Storage: b.infra.TOSClient,
|
||||
Rewriter: b.infra.Rewriter,
|
||||
Reranker: b.infra.Reranker,
|
||||
NL2Sql: b.infra.NL2SQL,
|
||||
OCR: b.infra.OCR,
|
||||
ParserManager: b.infra.ParserManager,
|
||||
CacheCli: b.infra.CacheCli,
|
||||
ModelFactory: chatmodel.NewDefaultFactory(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -292,6 +299,7 @@ func (b *basicServices) toWorkflowServiceComponents(pluginSVC *plugin.PluginAppl
|
|||
DomainNotifier: b.eventbus.resourceEventBus,
|
||||
CPStore: checkpoint.NewRedisStore(b.infra.CacheCli),
|
||||
CodeRunner: b.infra.CodeRunner,
|
||||
WorkflowBuildInChatModel: b.infra.WorkflowBuildInChatModel,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,9 +18,11 @@ package appinfra
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -31,26 +33,32 @@ import (
|
|||
"github.com/cloudwego/eino-ext/components/embedding/gemini"
|
||||
"github.com/cloudwego/eino-ext/components/embedding/ollama"
|
||||
"github.com/cloudwego/eino-ext/components/embedding/openai"
|
||||
"github.com/cloudwego/eino/components/prompt"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/milvus-io/milvus/client/v2/milvusclient"
|
||||
"github.com/volcengine/volc-sdk-golang/service/visual"
|
||||
|
||||
"github.com/coze-dev/coze-studio/backend/application/internal"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/cache"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/chatmodel"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/coderunner"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/document/nl2sql"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/document/ocr"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/document/parser"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/document/rerank"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/document/searchstore"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/embedding"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/imagex"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/messages2query"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/modelmgr"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/cache/redis"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/coderunner/direct"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/coderunner/sandbox"
|
||||
builtinNL2SQL "github.com/coze-dev/coze-studio/backend/infra/impl/document/nl2sql/builtin"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/ocr/ppocr"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/ocr/veocr"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/parser/builtin"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/parser/ppstructure"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/rerank/rrf"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/searchstore/elasticsearch"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/searchstore/milvus"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/searchstore/vikingdb"
|
||||
|
|
@ -61,6 +69,7 @@ import (
|
|||
"github.com/coze-dev/coze-studio/backend/infra/impl/eventbus"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/idgen"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/imagex/veimagex"
|
||||
builtinM2Q "github.com/coze-dev/coze-studio/backend/infra/impl/messages2query/builtin"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/mysql"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/storage"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/lang/conv"
|
||||
|
|
@ -78,11 +87,16 @@ type AppDependencies struct {
|
|||
TOSClient storage.Storage
|
||||
ResourceEventProducer eventbus.Producer
|
||||
AppEventProducer eventbus.Producer
|
||||
KnowledgeEventProducer eventbus.Producer
|
||||
ModelMgr modelmgr.Manager
|
||||
CodeRunner coderunner.Runner
|
||||
OCR ocr.OCR
|
||||
ParserManager parser.Manager
|
||||
SearchStoreManagers []searchstore.Manager
|
||||
Reranker rerank.Reranker
|
||||
Rewriter messages2query.MessagesToQuery
|
||||
NL2SQL nl2sql.NL2SQL
|
||||
WorkflowBuildInChatModel chatmodel.BaseChatModel
|
||||
}
|
||||
|
||||
func Init(ctx context.Context) (*AppDependencies, error) {
|
||||
|
|
@ -126,6 +140,23 @@ func Init(ctx context.Context) (*AppDependencies, error) {
|
|||
return nil, fmt.Errorf("init app event producer failed, err=%w", err)
|
||||
}
|
||||
|
||||
deps.KnowledgeEventProducer, err = initKnowledgeEventBusProducer()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init knowledge event bus producer failed, err=%w", err)
|
||||
}
|
||||
|
||||
deps.Reranker = rrf.NewRRFReranker(0)
|
||||
|
||||
deps.Rewriter, err = initRewriter(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init rewriter failed, err=%w", err)
|
||||
}
|
||||
|
||||
deps.NL2SQL, err = initNL2SQL(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init nl2sql failed, err=%w", err)
|
||||
}
|
||||
|
||||
deps.ModelMgr, err = initModelMgr()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init model manager failed, err=%w", err)
|
||||
|
|
@ -135,11 +166,21 @@ func Init(ctx context.Context) (*AppDependencies, error) {
|
|||
|
||||
deps.OCR = initOCR()
|
||||
|
||||
imageAnnotationModel, _, err := internal.GetBuiltinChatModel(ctx, "IA_")
|
||||
imageAnnotationModel, _, err := getBuiltinChatModel(ctx, "IA_")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get builtin chat model failed, err=%w", err)
|
||||
}
|
||||
|
||||
var ok bool
|
||||
deps.WorkflowBuildInChatModel, ok, err = getBuiltinChatModel(ctx, "WKR_")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get workflow builtin chat model failed, err=%w", err)
|
||||
}
|
||||
|
||||
if !ok {
|
||||
logs.CtxWarnf(ctx, "workflow builtin chat model for knowledge recall not configured")
|
||||
}
|
||||
|
||||
deps.ParserManager, err = initParserManager(deps.TOSClient, deps.OCR, imageAnnotationModel)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init parser manager failed, err=%w", err)
|
||||
|
|
@ -166,6 +207,71 @@ func initSearchStoreManagers(ctx context.Context, es es.Client) ([]searchstore.M
|
|||
return []searchstore.Manager{esSearchstoreManager, mgr}, nil
|
||||
}
|
||||
|
||||
func initRewriter(ctx context.Context) (messages2query.MessagesToQuery, error) {
|
||||
rewriterChatModel, _, err := getBuiltinChatModel(ctx, "M2Q_")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
filePath := filepath.Join(getWorkingDirectory(), "resources/conf/prompt/messages_to_query_template_jinja2.json")
|
||||
rewriterTemplate, err := readJinja2PromptTemplate(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rewriter, err := builtinM2Q.NewMessagesToQuery(ctx, rewriterChatModel, rewriterTemplate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return rewriter, nil
|
||||
}
|
||||
|
||||
func getWorkingDirectory() string {
|
||||
root, err := os.Getwd()
|
||||
if err != nil {
|
||||
logs.Warnf("[InitConfig] Failed to get current working directory: %v", err)
|
||||
root = os.Getenv("PWD")
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func readJinja2PromptTemplate(jsonFilePath string) (prompt.ChatTemplate, error) {
|
||||
b, err := os.ReadFile(jsonFilePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m2qMessages []*schema.Message
|
||||
if err = json.Unmarshal(b, &m2qMessages); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tpl := make([]schema.MessagesTemplate, len(m2qMessages))
|
||||
for i := range m2qMessages {
|
||||
tpl[i] = m2qMessages[i]
|
||||
}
|
||||
return prompt.FromMessages(schema.Jinja2, tpl...), nil
|
||||
}
|
||||
|
||||
func initNL2SQL(ctx context.Context) (nl2sql.NL2SQL, error) {
|
||||
n2sChatModel, _, err := getBuiltinChatModel(ctx, "NL2SQL_")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
filePath := filepath.Join(getWorkingDirectory(), "resources/conf/prompt/nl2sql_template_jinja2.json")
|
||||
n2sTemplate, err := readJinja2PromptTemplate(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
n2s, err := builtinNL2SQL.NewNL2SQL(ctx, n2sChatModel, n2sTemplate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return n2s, nil
|
||||
}
|
||||
|
||||
func initImageX(ctx context.Context) (imagex.ImageX, error) {
|
||||
uploadComponentType := os.Getenv(consts.FileUploadComponentType)
|
||||
if uploadComponentType != consts.FileUploadComponentTypeImagex {
|
||||
|
|
@ -206,6 +312,17 @@ func initAppEventProducer() (eventbus.Producer, error) {
|
|||
return appEventProducer, nil
|
||||
}
|
||||
|
||||
func initKnowledgeEventBusProducer() (eventbus.Producer, error) {
|
||||
nameServer := os.Getenv(consts.MQServer)
|
||||
|
||||
knowledgeProducer, err := eventbus.NewProducer(nameServer, consts.RMQTopicKnowledge, consts.RMQConsumeGroupKnowledge, 2)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init knowledge producer failed, err=%w", err)
|
||||
}
|
||||
|
||||
return knowledgeProducer, nil
|
||||
}
|
||||
|
||||
func initCodeRunner() coderunner.Runner {
|
||||
switch typ := os.Getenv(consts.CodeRunnerType); typ {
|
||||
case "sandbox":
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package internal
|
||||
package appinfra
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
|
@ -33,7 +33,7 @@ import (
|
|||
"github.com/coze-dev/coze-studio/backend/infra/contract/chatmodel"
|
||||
)
|
||||
|
||||
func GetBuiltinChatModel(ctx context.Context, envPrefix string) (bcm chatmodel.BaseChatModel, configured bool, err error) {
|
||||
func getBuiltinChatModel(ctx context.Context, envPrefix string) (bcm chatmodel.BaseChatModel, configured bool, err error) {
|
||||
getEnv := func(key string) string {
|
||||
if val := os.Getenv(envPrefix + key); val != "" {
|
||||
return val
|
||||
|
|
@ -99,7 +99,7 @@ func GetBuiltinChatModel(ctx context.Context, envPrefix string) (bcm chatmodel.B
|
|||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("knowledge init openai chat mode failed, %w", err)
|
||||
return nil, false, fmt.Errorf("builtin %s chat model init failed, %w", envPrefix, err)
|
||||
}
|
||||
if bcm != nil {
|
||||
configured = true
|
||||
|
|
@ -18,132 +18,27 @@ package knowledge
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/cloudwego/eino/components/prompt"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/coze-dev/coze-studio/backend/application/internal"
|
||||
"github.com/coze-dev/coze-studio/backend/application/search"
|
||||
knowledgeImpl "github.com/coze-dev/coze-studio/backend/domain/knowledge/service"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/cache"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/document/nl2sql"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/document/ocr"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/document/parser"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/document/searchstore"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/idgen"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/messages2query"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/rdb"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/storage"
|
||||
chatmodelImpl "github.com/coze-dev/coze-studio/backend/infra/impl/chatmodel"
|
||||
builtinNL2SQL "github.com/coze-dev/coze-studio/backend/infra/impl/document/nl2sql/builtin"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/rerank/rrf"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/eventbus"
|
||||
builtinM2Q "github.com/coze-dev/coze-studio/backend/infra/impl/messages2query/builtin"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/logs"
|
||||
"github.com/coze-dev/coze-studio/backend/types/consts"
|
||||
)
|
||||
|
||||
type ServiceComponents struct {
|
||||
DB *gorm.DB
|
||||
IDGenSVC idgen.IDGenerator
|
||||
Storage storage.Storage
|
||||
RDB rdb.RDB
|
||||
EventBus search.ResourceEventBus
|
||||
CacheCli cache.Cmdable
|
||||
OCR ocr.OCR
|
||||
ParserManager parser.Manager
|
||||
SearchStoreManagers []searchstore.Manager
|
||||
}
|
||||
type ServiceComponents = knowledgeImpl.KnowledgeSVCConfig
|
||||
|
||||
func InitService(c *ServiceComponents) (*KnowledgeApplicationService, error) {
|
||||
ctx := context.Background()
|
||||
func InitService(ctx context.Context, c *ServiceComponents, bus search.ResourceEventBus) (*KnowledgeApplicationService, error) {
|
||||
knowledgeDomainSVC, knowledgeEventHandler := knowledgeImpl.NewKnowledgeSVC(c)
|
||||
|
||||
nameServer := os.Getenv(consts.MQServer)
|
||||
|
||||
knowledgeProducer, err := eventbus.NewProducer(nameServer, consts.RMQTopicKnowledge, consts.RMQConsumeGroupKnowledge, 2)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("init knowledge producer failed, err=%w", err)
|
||||
}
|
||||
|
||||
root, err := os.Getwd()
|
||||
if err != nil {
|
||||
logs.Warnf("[InitConfig] Failed to get current working directory: %v", err)
|
||||
root = os.Getenv("PWD")
|
||||
}
|
||||
|
||||
var rewriter messages2query.MessagesToQuery
|
||||
if rewriterChatModel, _, err := internal.GetBuiltinChatModel(ctx, "M2Q_"); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
filePath := filepath.Join(root, "resources/conf/prompt/messages_to_query_template_jinja2.json")
|
||||
rewriterTemplate, err := readJinja2PromptTemplate(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rewriter, err = builtinM2Q.NewMessagesToQuery(ctx, rewriterChatModel, rewriterTemplate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var n2s nl2sql.NL2SQL
|
||||
if n2sChatModel, _, err := internal.GetBuiltinChatModel(ctx, "NL2SQL_"); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
filePath := filepath.Join(root, "resources/conf/prompt/nl2sql_template_jinja2.json")
|
||||
n2sTemplate, err := readJinja2PromptTemplate(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n2s, err = builtinNL2SQL.NewNL2SQL(ctx, n2sChatModel, n2sTemplate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
knowledgeDomainSVC, knowledgeEventHandler := knowledgeImpl.NewKnowledgeSVC(&knowledgeImpl.KnowledgeSVCConfig{
|
||||
DB: c.DB,
|
||||
IDGen: c.IDGenSVC,
|
||||
RDB: c.RDB,
|
||||
Producer: knowledgeProducer,
|
||||
SearchStoreManagers: c.SearchStoreManagers,
|
||||
ParseManager: c.ParserManager,
|
||||
Storage: c.Storage,
|
||||
Rewriter: rewriter,
|
||||
Reranker: rrf.NewRRFReranker(0), // default rrf
|
||||
NL2Sql: n2s,
|
||||
OCR: c.OCR,
|
||||
CacheCli: c.CacheCli,
|
||||
ModelFactory: chatmodelImpl.NewDefaultFactory(),
|
||||
})
|
||||
|
||||
if err = eventbus.DefaultSVC().RegisterConsumer(nameServer, consts.RMQTopicKnowledge, consts.RMQConsumeGroupKnowledge, knowledgeEventHandler); err != nil {
|
||||
if err := eventbus.DefaultSVC().RegisterConsumer(nameServer, consts.RMQTopicKnowledge, consts.RMQConsumeGroupKnowledge, knowledgeEventHandler); err != nil {
|
||||
return nil, fmt.Errorf("register knowledge consumer failed, err=%w", err)
|
||||
}
|
||||
|
||||
KnowledgeSVC.DomainSVC = knowledgeDomainSVC
|
||||
KnowledgeSVC.eventBus = c.EventBus
|
||||
KnowledgeSVC.eventBus = bus
|
||||
KnowledgeSVC.storage = c.Storage
|
||||
return KnowledgeSVC, nil
|
||||
}
|
||||
|
||||
func readJinja2PromptTemplate(jsonFilePath string) (prompt.ChatTemplate, error) {
|
||||
b, err := os.ReadFile(jsonFilePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m2qMessages []*schema.Message
|
||||
if err = json.Unmarshal(b, &m2qMessages); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tpl := make([]schema.MessagesTemplate, len(m2qMessages))
|
||||
for i := range m2qMessages {
|
||||
tpl[i] = m2qMessages[i]
|
||||
}
|
||||
return prompt.FromMessages(schema.Jinja2, tpl...), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ import (
|
|||
"github.com/cloudwego/eino/compose"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/coze-dev/coze-studio/backend/application/internal"
|
||||
"github.com/coze-dev/coze-studio/backend/crossdomain/impl/code"
|
||||
knowledge "github.com/coze-dev/coze-studio/backend/domain/knowledge/service"
|
||||
dbservice "github.com/coze-dev/coze-studio/backend/domain/memory/database/service"
|
||||
|
|
@ -35,11 +34,11 @@ import (
|
|||
"github.com/coze-dev/coze-studio/backend/domain/workflow/service"
|
||||
workflowservice "github.com/coze-dev/coze-studio/backend/domain/workflow/service"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/cache"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/chatmodel"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/coderunner"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/idgen"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/imagex"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/storage"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/logs"
|
||||
)
|
||||
|
||||
type ServiceComponents struct {
|
||||
|
|
@ -55,21 +54,14 @@ type ServiceComponents struct {
|
|||
ImageX imagex.ImageX
|
||||
CPStore compose.CheckPointStore
|
||||
CodeRunner coderunner.Runner
|
||||
WorkflowBuildInChatModel chatmodel.BaseChatModel
|
||||
}
|
||||
|
||||
func InitService(ctx context.Context, components *ServiceComponents) (*ApplicationService, error) {
|
||||
bcm, ok, err := internal.GetBuiltinChatModel(ctx, "WKR_")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
logs.CtxWarnf(ctx, "workflow builtin chat model for knowledge recall not configured")
|
||||
}
|
||||
|
||||
service.RegisterAllNodeAdaptors()
|
||||
|
||||
workflowRepo := service.NewWorkflowRepository(components.IDGen, components.DB, components.Cache,
|
||||
components.Tos, components.CPStore, bcm)
|
||||
components.Tos, components.CPStore, components.WorkflowBuildInChatModel)
|
||||
workflow.SetRepository(workflowRepo)
|
||||
|
||||
workflowDomainSVC := service.NewWorkflowService(workflowRepo)
|
||||
|
|
@ -83,5 +75,5 @@ func InitService(ctx context.Context, components *ServiceComponents) (*Applicati
|
|||
SVC.TosClient = components.Tos
|
||||
SVC.IDGenerator = components.IDGen
|
||||
|
||||
return SVC, err
|
||||
return SVC, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,9 +58,7 @@ import (
|
|||
"github.com/coze-dev/coze-studio/backend/infra/contract/rdb"
|
||||
rdbEntity "github.com/coze-dev/coze-studio/backend/infra/contract/rdb/entity"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/storage"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/parser/builtin"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/progressbar"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/rerank/rrf"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/errorx"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/lang/ptr"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/lang/slices"
|
||||
|
|
@ -87,12 +85,6 @@ func NewKnowledgeSVC(config *KnowledgeSVCConfig) (Knowledge, eventbus.ConsumerHa
|
|||
cacheCli: config.CacheCli,
|
||||
modelFactory: config.ModelFactory,
|
||||
}
|
||||
if svc.reranker == nil {
|
||||
svc.reranker = rrf.NewRRFReranker(0)
|
||||
}
|
||||
if svc.parseManager == nil {
|
||||
svc.parseManager = builtin.NewManager(config.Storage, config.OCR, nil)
|
||||
}
|
||||
|
||||
return svc, svc
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,8 @@ import (
|
|||
"github.com/coze-dev/coze-studio/backend/infra/contract/document/searchstore"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/storage"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/cache/redis"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/parser/builtin"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/rerank/rrf"
|
||||
sses "github.com/coze-dev/coze-studio/backend/infra/impl/document/searchstore/elasticsearch"
|
||||
ssmilvus "github.com/coze-dev/coze-studio/backend/infra/impl/document/searchstore/milvus"
|
||||
hembed "github.com/coze-dev/coze-studio/backend/infra/impl/embedding/http"
|
||||
|
|
@ -169,10 +171,10 @@ func (suite *KnowledgeTestSuite) SetupSuite() {
|
|||
RDB: rdbService,
|
||||
Producer: knowledgeProducer,
|
||||
SearchStoreManagers: mgrs,
|
||||
ParseManager: nil, // default builtin
|
||||
ParseManager: builtin.NewManager(tosClient, nil, nil), // default builtin
|
||||
Storage: tosClient,
|
||||
Rewriter: nil,
|
||||
Reranker: nil, // default rrf
|
||||
Reranker: rrf.NewRRFReranker(0), // default rrf
|
||||
EnableCompactTable: ptr.Of(true),
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ import (
|
|||
knowledgeModel "github.com/coze-dev/coze-studio/backend/api/model/crossdomain/knowledge"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/entity"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/contract/document"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/parser/builtin"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/document/rerank/rrf"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/impl/rdb"
|
||||
producerMock "github.com/coze-dev/coze-studio/backend/internal/mock/infra/contract/eventbus"
|
||||
mock "github.com/coze-dev/coze-studio/backend/internal/mock/infra/contract/idgen"
|
||||
|
|
@ -103,6 +105,9 @@ func MockKnowledgeSVC(t *testing.T) Knowledge {
|
|||
Storage: mockStorage,
|
||||
Producer: producer,
|
||||
RDB: rdb,
|
||||
Reranker: rrf.NewRRFReranker(0),
|
||||
ParseManager: builtin.NewManager(mockStorage, nil, nil), // default builtin
|
||||
|
||||
})
|
||||
return svc
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue