diff --git a/Makefile b/Makefile index 98a7eb14..e763934b 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ fe: @echo "Building frontend..." @bash $(BUILD_FE_SCRIPT) -server: env +server: env setup_es_index @if [ ! -d "$(STATIC_DIR)" ]; then \ echo "Static directory '$(STATIC_DIR)' not found, building frontend..."; \ $(MAKE) fe; \ diff --git a/backend/go.mod b/backend/go.mod index e1fa95bb..f40889ec 100755 --- a/backend/go.mod +++ b/backend/go.mod @@ -59,7 +59,7 @@ require ( github.com/cloudwego/eino-ext/components/embedding/ollama v0.0.0-20250728060543-79ec300857b8 github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20250522060253-ddb617598b09 github.com/cloudwego/eino-ext/components/model/gemini v0.1.2 - github.com/cloudwego/eino-ext/components/model/ollama v0.0.0-20250610035057-2c4e7c8488a5 + github.com/cloudwego/eino-ext/components/model/ollama v0.1.0 github.com/cloudwego/eino-ext/components/model/qwen v0.0.0-20250612061754-5a3deb091dc5 github.com/dimchansky/utfbom v1.1.1 github.com/elastic/go-elasticsearch/v7 v7.17.10 diff --git a/backend/go.sum b/backend/go.sum index 12e93bf7..a59a7b71 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -942,6 +942,8 @@ github.com/cloudwego/eino-ext/components/model/gemini v0.1.2 h1:bt9xftOQhP0Nuh1P github.com/cloudwego/eino-ext/components/model/gemini v0.1.2/go.mod h1:1tv89uZ9hR/4AyQ+9yxFWLn52GaJDKtPXdEY7WZdyZc= github.com/cloudwego/eino-ext/components/model/ollama v0.0.0-20250610035057-2c4e7c8488a5 h1:GkAAQHQkb1cOTwm6uRokj4lM//wrt/3AkMwTxyFJUg4= github.com/cloudwego/eino-ext/components/model/ollama v0.0.0-20250610035057-2c4e7c8488a5/go.mod h1:giNUFqA+V7xrm/EDvH7JFnDqoWI+e2m1SVAnReU+Fd8= +github.com/cloudwego/eino-ext/components/model/ollama v0.1.0 h1:FW067iMfg3EZbUaZIo8v3i2ILBAZDzY23/9pbprvE0M= +github.com/cloudwego/eino-ext/components/model/ollama v0.1.0/go.mod h1:+qA5kkUCM0mIrXGSNzxLcjxh6K1AghPNigtEyyMdkOc= github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250715055739-0d0e28441a2f h1:ovS39vuN2JW+C/O9jtEmOUuLEY4fw0yYh8//yhMfJNM= github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250715055739-0d0e28441a2f/go.mod h1:2mFQQnlhJrNgbW6YX1MOUUfXkGSbTz9Ylx37fbR0xBo= github.com/cloudwego/eino-ext/components/model/qwen v0.0.0-20250612061754-5a3deb091dc5 h1:4zAZiNo/PkeVc0Gw8YLpzpbR8zDzccL7H5PLqTuGhv4= diff --git a/backend/infra/impl/document/ocr/veocr/ve_ocr.go b/backend/infra/impl/document/ocr/veocr/ve_ocr.go index 56a01708..f23183ec 100644 --- a/backend/infra/impl/document/ocr/veocr/ve_ocr.go +++ b/backend/infra/impl/document/ocr/veocr/ve_ocr.go @@ -24,12 +24,12 @@ import ( "net/url" "strconv" - "github.com/coze-dev/coze-studio/backend/pkg/errorx" - "github.com/coze-dev/coze-studio/backend/types/errno" "github.com/volcengine/volc-sdk-golang/service/visual" "github.com/volcengine/volcengine-go-sdk/service/arkruntime/model" "github.com/coze-dev/coze-studio/backend/infra/contract/document/ocr" + "github.com/coze-dev/coze-studio/backend/pkg/errorx" + "github.com/coze-dev/coze-studio/backend/types/errno" ) type Config struct { diff --git a/backend/infra/impl/embedding/ark/ark.go b/backend/infra/impl/embedding/ark/ark.go index 8e850589..a3c0b0ed 100644 --- a/backend/infra/impl/embedding/ark/ark.go +++ b/backend/infra/impl/embedding/ark/ark.go @@ -25,12 +25,12 @@ import ( "github.com/cloudwego/eino-ext/components/embedding/ark" "github.com/cloudwego/eino/components/embedding" - "github.com/coze-dev/coze-studio/backend/pkg/errorx" - "github.com/coze-dev/coze-studio/backend/types/errno" "github.com/volcengine/volcengine-go-sdk/service/arkruntime/model" contract "github.com/coze-dev/coze-studio/backend/infra/contract/embedding" + "github.com/coze-dev/coze-studio/backend/pkg/errorx" "github.com/coze-dev/coze-studio/backend/pkg/lang/slices" + "github.com/coze-dev/coze-studio/backend/types/errno" ) func NewArkEmbedder(ctx context.Context, config *ark.EmbeddingConfig, dimensions int64) (contract.Embedder, error) { diff --git a/backend/infra/impl/embedding/wrap/ollama.go b/backend/infra/impl/embedding/wrap/ollama.go index bc062572..516ac075 100644 --- a/backend/infra/impl/embedding/wrap/ollama.go +++ b/backend/infra/impl/embedding/wrap/ollama.go @@ -20,6 +20,7 @@ import ( "context" "github.com/cloudwego/eino-ext/components/embedding/ollama" + contract "github.com/coze-dev/coze-studio/backend/infra/contract/embedding" ) diff --git a/backend/infra/impl/storage/minio/minio.go b/backend/infra/impl/storage/minio/minio.go index 85faad6f..a763e5ed 100644 --- a/backend/infra/impl/storage/minio/minio.go +++ b/backend/infra/impl/storage/minio/minio.go @@ -23,9 +23,7 @@ import ( "io" "log" "math/rand" - "net" "net/url" - "os" "time" "github.com/minio/minio-go/v7" @@ -33,6 +31,7 @@ import ( "github.com/coze-dev/coze-studio/backend/infra/contract/imagex" "github.com/coze-dev/coze-studio/backend/infra/contract/storage" + "github.com/coze-dev/coze-studio/backend/infra/impl/storage/proxy" "github.com/coze-dev/coze-studio/backend/pkg/ctxcache" "github.com/coze-dev/coze-studio/backend/pkg/errorx" "github.com/coze-dev/coze-studio/backend/types/consts" @@ -211,27 +210,9 @@ func (m *minioClient) GetObjectUrl(ctx context.Context, objectKey string, opts . } // logs.CtxDebugf(ctx, "[GetObjectUrl] origin presignedURL.String = %s", presignedURL.String()) - - proxyPort := os.Getenv(consts.MinIOProxyEndpoint) // :8889 - if len(proxyPort) > 0 { - currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx) - if !ok { - return presignedURL.String(), nil - } - - currentScheme, ok := ctxcache.Get[string](ctx, consts.RequestSchemeKeyInCtx) - if !ok { - return presignedURL.String(), nil - } - - host, _, err := net.SplitHostPort(currentHost) - if err != nil { - host = currentHost - } - minioProxyHost := host + proxyPort - presignedURL.Host = minioProxyHost - presignedURL.Scheme = currentScheme - // logs.CtxDebugf(ctx, "[GetObjectUrl] reset presignedURL.String = %s", presignedURL.String()) + ok, proxyURL := proxy.CheckIfNeedReplaceHost(ctx, presignedURL.String()) + if ok { + return proxyURL, nil } return presignedURL.String(), nil @@ -265,7 +246,6 @@ func (m *minioClient) GetUploadAuth(ctx context.Context, opt ...imagex.UploadAut } func (m *minioClient) GetResourceURL(ctx context.Context, uri string, opts ...imagex.GetResourceOpt) (*imagex.ResourceURL, error) { - url, err := m.GetObjectUrl(ctx, uri) if err != nil { return nil, err @@ -273,11 +253,12 @@ func (m *minioClient) GetResourceURL(ctx context.Context, uri string, opts ...im return &imagex.ResourceURL{ URL: url, }, nil - } + func (m *minioClient) Upload(ctx context.Context, data []byte, opts ...imagex.UploadAuthOpt) (*imagex.UploadResult, error) { return nil, nil } + func (m *minioClient) GetUploadAuthWithExpire(ctx context.Context, expire time.Duration, opt ...imagex.UploadAuthOpt) (*imagex.SecurityToken, error) { return nil, nil } diff --git a/backend/infra/impl/storage/proxy/proxy.go b/backend/infra/impl/storage/proxy/proxy.go new file mode 100644 index 00000000..a1368456 --- /dev/null +++ b/backend/infra/impl/storage/proxy/proxy.go @@ -0,0 +1,63 @@ +/* + * Copyright 2025 coze-dev Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package proxy + +import ( + "context" + "net" + "net/url" + "os" + + "github.com/coze-dev/coze-studio/backend/pkg/ctxcache" + "github.com/coze-dev/coze-studio/backend/pkg/logs" + "github.com/coze-dev/coze-studio/backend/types/consts" +) + +func CheckIfNeedReplaceHost(ctx context.Context, originURLStr string) (ok bool, proxyURL string) { + // url parse + originURL, err := url.Parse(originURLStr) + if err != nil { + logs.CtxWarnf(ctx, "[CheckIfNeedReplaceHost] url parse failed, err: %v", err) + return false, "" + } + + proxyPort := os.Getenv(consts.MinIOProxyEndpoint) // :8889 + if proxyPort == "" { + return false, "" + } + + currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx) + if !ok { + return false, "" + } + + currentScheme, ok := ctxcache.Get[string](ctx, consts.RequestSchemeKeyInCtx) + if !ok { + return false, "" + } + + host, _, err := net.SplitHostPort(currentHost) + if err != nil { + host = currentHost + } + + minioProxyHost := host + proxyPort + originURL.Host = minioProxyHost + originURL.Scheme = currentScheme + logs.CtxDebugf(ctx, "[CheckIfNeedReplaceHost] reset originURL.String = %s", originURL.String()) + return true, originURL.String() +} diff --git a/backend/infra/impl/storage/s3/s3.go b/backend/infra/impl/storage/s3/s3.go index 4eb2ff3f..ea3ec5b3 100644 --- a/backend/infra/impl/storage/s3/s3.go +++ b/backend/infra/impl/storage/s3/s3.go @@ -21,9 +21,6 @@ import ( "context" "fmt" "io" - "net" - "net/url" - "os" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -33,6 +30,7 @@ import ( "github.com/coze-dev/coze-studio/backend/infra/contract/imagex" "github.com/coze-dev/coze-studio/backend/infra/contract/storage" + "github.com/coze-dev/coze-studio/backend/infra/impl/storage/proxy" "github.com/coze-dev/coze-studio/backend/pkg/ctxcache" "github.com/coze-dev/coze-studio/backend/pkg/errorx" "github.com/coze-dev/coze-studio/backend/pkg/logs" @@ -219,34 +217,9 @@ func (t *s3Client) GetObjectUrl(ctx context.Context, objectKey string, opts ...s return "", fmt.Errorf("get object presigned url failed: %v", err) } - // url parse - url, err := url.Parse(req.URL) - if err != nil { - logs.CtxWarnf(ctx, "[GetObjectUrl] url parse failed, err: %v", err) - return req.URL, nil - } - - proxyPort := os.Getenv(consts.MinIOProxyEndpoint) // :8889 - if len(proxyPort) > 0 { - currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx) - if !ok { - return req.URL, nil - } - - currentScheme, ok := ctxcache.Get[string](ctx, consts.RequestSchemeKeyInCtx) - if !ok { - return req.URL, nil - } - - host, _, err := net.SplitHostPort(currentHost) - if err != nil { - host = currentHost - } - minioProxyHost := host + proxyPort - url.Host = minioProxyHost - url.Scheme = currentScheme - logs.CtxInfof(ctx, "[GetObjectUrl] reset ORG.URL = %s TOS.URL = %s", req.URL, url.String()) - return url.String(), nil + ok, proxyURL := proxy.CheckIfNeedReplaceHost(ctx, req.URL) + if ok { + return proxyURL, nil } return req.URL, nil @@ -258,7 +231,6 @@ func (i *s3Client) GetUploadHost(ctx context.Context) string { return "" } return currentHost + consts.ApplyUploadActionURI - } func (t *s3Client) GetServerID() string { diff --git a/backend/infra/impl/storage/tos/tos.go b/backend/infra/impl/storage/tos/tos.go index 5957ced6..19cdade5 100644 --- a/backend/infra/impl/storage/tos/tos.go +++ b/backend/infra/impl/storage/tos/tos.go @@ -21,10 +21,7 @@ import ( "context" "fmt" "io" - "net" "net/http" - "net/url" - "os" "time" "github.com/volcengine/ve-tos-golang-sdk/v2/tos" @@ -32,6 +29,7 @@ import ( "github.com/coze-dev/coze-studio/backend/infra/contract/imagex" "github.com/coze-dev/coze-studio/backend/infra/contract/storage" + "github.com/coze-dev/coze-studio/backend/infra/impl/storage/proxy" "github.com/coze-dev/coze-studio/backend/pkg/ctxcache" "github.com/coze-dev/coze-studio/backend/pkg/errorx" "github.com/coze-dev/coze-studio/backend/pkg/lang/conv" @@ -213,34 +211,9 @@ func (t *tosClient) GetObjectUrl(ctx context.Context, objectKey string, opts ... return "", err } - // url parse - url, err := url.Parse(output.SignedUrl) - if err != nil { - logs.CtxWarnf(ctx, "[GetObjectUrl] url parse failed, err: %v", err) - return output.SignedUrl, nil - } - - proxyPort := os.Getenv(consts.MinIOProxyEndpoint) // :8889 - if len(proxyPort) > 0 { - currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx) - if !ok { - return output.SignedUrl, nil - } - - currentScheme, ok := ctxcache.Get[string](ctx, consts.RequestSchemeKeyInCtx) - if !ok { - return output.SignedUrl, nil - } - - host, _, err := net.SplitHostPort(currentHost) - if err != nil { - host = currentHost - } - minioProxyHost := host + proxyPort - url.Host = minioProxyHost - url.Scheme = currentScheme - // logs.CtxDebugf(ctx, "[GetObjectUrl] reset \n ORG.URL = %s \n TOS.URL = %s", output.SignedUrl, url.String()) - return url.String(), nil + ok, proxyURL := proxy.CheckIfNeedReplaceHost(ctx, output.SignedUrl) + if ok { + return proxyURL, nil } return output.SignedUrl, nil diff --git a/backend/main.go b/backend/main.go index 6e69f328..1e6d0cdb 100755 --- a/backend/main.go +++ b/backend/main.go @@ -74,9 +74,10 @@ func startHttpServer() { server.WithMaxRequestBodySize(int(maxSize)), } - useSSL := getEnv("USE_SSL", "0") + useSSL := getEnv(consts.UseSSL, "0") if useSSL == "1" { - cert, err := tls.LoadX509KeyPair("cert.pem", "key.pem") + cert, err := tls.LoadX509KeyPair(getEnv(consts.SSLCertFile, ""), + getEnv(consts.SSLKeyFile, "")) if err != nil { fmt.Println(err.Error()) } @@ -193,10 +194,12 @@ func asyncStartMinioProxyServer(ctx context.Context) { originDirector(req) req.Host = req.URL.Host } - useSSL := getEnv("USE_SSL", "0") + useSSL := getEnv(consts.UseSSL, "0") if useSSL == "1" { logs.Infof("Minio proxy server is listening on %s with SSL", minioProxyEndpoint) - err := http.ListenAndServeTLS(minioProxyEndpoint, "cert.pem", "key.pem", proxy) + err := http.ListenAndServeTLS(minioProxyEndpoint, + getEnv(consts.SSLCertFile, ""), + getEnv(consts.SSLKeyFile, ""), proxy) if err != nil { log.Fatal(err) } diff --git a/backend/types/consts/consts.go b/backend/types/consts/consts.go index f1fb0403..06f5eb54 100644 --- a/backend/types/consts/consts.go +++ b/backend/types/consts/consts.go @@ -81,6 +81,10 @@ const ( CodeRunnerNodeModulesDir = "CODE_RUNNER_NODE_MODULES_DIR" CodeRunnerTimeoutSeconds = "CODE_RUNNER_TIMEOUT_SECONDS" CodeRunnerMemoryLimitMB = "CODE_RUNNER_MEMORY_LIMIT_MB" + + UseSSL = "USE_SSL" + SSLCertFile = "SSL_CERT_FILE" + SSLKeyFile = "SSL_KEY_FILE" ) const ( diff --git a/docker/.env.example b/docker/.env.example index 1f51c45e..8d12d4f6 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -4,6 +4,9 @@ export LOG_LEVEL="debug" export MAX_REQUEST_BODY_SIZE=1073741824 export SERVER_HOST="localhost${LISTEN_ADDR}" export MINIO_PROXY_ENDPOINT=":8889" +export USE_SSL="0" +export SSL_CERT_FILE="" +export SSL_KEY_FILE="" # MySQL export MYSQL_ROOT_PASSWORD=root