diff --git a/backend/go.mod b/backend/go.mod index 3c123eab..f701b6db 100755 --- a/backend/go.mod +++ b/backend/go.mod @@ -80,6 +80,10 @@ require ( cloud.google.com/go/auth v0.9.3 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.37 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.18 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.84.1 // indirect github.com/cloudwego/gopkg v0.1.4 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/extrame/ole2 v0.0.0-20160812065207-d69429661ad7 // indirect @@ -106,20 +110,20 @@ require ( require ( github.com/anthropics/anthropic-sdk-go v1.4.0 // indirect github.com/avast/retry-go v3.0.0+incompatible // indirect - github.com/aws/aws-sdk-go-v2 v1.33.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect + github.com/aws/aws-sdk-go-v2 v1.36.6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect github.com/aws/aws-sdk-go-v2/config v1.29.1 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.54 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.37 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.18 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect - github.com/aws/smithy-go v1.22.1 // indirect + github.com/aws/smithy-go v1.22.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/bytedance/gopkg v0.1.1 // indirect diff --git a/backend/go.sum b/backend/go.sum index d2dd214f..7c5fafc1 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -824,8 +824,12 @@ github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2 v1.33.0 h1:Evgm4DI9imD81V0WwD+TN4DCwjUMdc94TrduMLbgZJs= github.com/aws/aws-sdk-go-v2 v1.33.0/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2 v1.36.6 h1:zJqGjVbRdTPojeCGWn5IR5pbJwSQSBh5RWFTQcEQGdU= +github.com/aws/aws-sdk-go-v2 v1.36.6/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 h1:12SpdwU8Djs+YGklkinSSlcrPyj3H4VifVsKf78KbwA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11/go.mod h1:dd+Lkp6YmMryke+qxW/VnKyhMBDTYP41Q2Bb+6gNZgY= github.com/aws/aws-sdk-go-v2/config v1.29.1 h1:JZhGawAyZ/EuJeBtbQYnaoftczcb2drR2Iq36Wgz4sQ= github.com/aws/aws-sdk-go-v2/config v1.29.1/go.mod h1:7bR2YD5euaxBhzt2y/oDkt3uNRb6tjFp98GlTFueRwk= github.com/aws/aws-sdk-go-v2/credentials v1.17.54 h1:4UmqeOqJPvdvASZWrKlhzpRahAulBfyTJQUaYy4+hEI= @@ -834,15 +838,31 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 h1:5grmdTdMsovn9kPZPI23Hh github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24/go.mod h1:zqi7TVKTswH3Ozq28PkmBmgzG1tona7mo9G2IJg4Cis= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 h1:igORFSiH3bfq4lxKFkTSYDhJEUCYo6C8VKiWJjYwQuQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28/go.mod h1:3So8EA/aAYm36L7XIvCVwLa0s5N0P7o2b1oqnx/2R4g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.37 h1:osMWfm/sC/L4tvEdQ65Gri5ZZDCUpuYJZbTTDrsn4I0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.37/go.mod h1:ZV2/1fbjOPr4G4v38G3Ww5TBT4+hmsK45s/rxu1fGy0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 h1:1mOW9zAUMhTSrMDssEHS/ajx8JcAj/IcftzcmNlmVLI= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28/go.mod h1:kGlXVIWDfvt2Ox5zEaNglmq0hXPHgQFNMix33Tw22jA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.37 h1:v+X21AvTb2wZ+ycg1gx+orkB/9U6L7AOp93R7qYxsxM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.37/go.mod h1:G0uM1kyssELxmJ2VZEfG0q2npObR3BAkF3c1VsfVnfs= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.37 h1:XTZZ0I3SZUHAtBLBU6395ad+VOblE0DwQP6MuaNeics= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.37/go.mod h1:Pi6ksbniAWVwu2S8pEzcYPyhUkAcLaufxN7PfAUQjBk= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.5 h1:M5/B8JUaCI8+9QD+u3S/f4YHpvqE9RpSkV3rf0Iks2w= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.5/go.mod h1:Bktzci1bwdbpuLiu3AOksiNPMl/LLKmX1TWmqp2xbvs= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 h1:TQmKDyETFGiXVhZfQ/I0cCFziqqX58pi4tKJGYGFSz0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9/go.mod h1:HVLPK2iHQBUx7HfZeOQSEu3v2ubZaAY2YPbAm5/WUyY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.18 h1:vvbXsA2TVO80/KT7ZqCbx934dt6PY+vQ8hZpUZ/cpYg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.18/go.mod h1:m2JJHledjBGNMsLOF1g9gbAxprzq3KjC8e4lxtn+eWg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.18 h1:OS2e0SKqsU2LiJPqL8u9x41tKc6MMEHrWjLVLn3oysg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.18/go.mod h1:+Yrk+MDGzlNGxCXieljNeWpoZTCQUQVL+Jk9hGGJ8qM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.84.1 h1:RkHXU9jP0DptGy7qKI8CBGsUJruWz0v5IgwBa2DwWcU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.84.1/go.mod h1:3xAOf7tdKF+qbb+XpU+EPhNXAdun3Lu1RcDrj8KC24I= github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 h1:kuIyu4fTT38Kj7YCC7ouNbVZSSpqkZ+LzIfhCr6Dg+I= github.com/aws/aws-sdk-go-v2/service/sso v1.24.11/go.mod h1:Ro744S4fKiCCuZECXgOi760TiYylUM8ZBf6OGiZzJtY= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 h1:l+dgv/64iVlQ3WsBbnn+JSbkj01jIi+SM0wYsj3y/hY= @@ -852,6 +872,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.9/go.mod h1:f6vjfZER1M17Fokn0Izss github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw= +github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/bazelbuild/rules_go v0.49.0/go.mod h1:Dhcz716Kqg1RHNWos+N6MlXNkjNP2EwZQ0LukRKJfMs= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= diff --git a/backend/infra/impl/storage/s3/s3.go b/backend/infra/impl/storage/s3/s3.go new file mode 100644 index 00000000..4eb2ff3f --- /dev/null +++ b/backend/infra/impl/storage/s3/s3.go @@ -0,0 +1,299 @@ +/* + * Copyright 2025 coze-dev Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3 + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/url" + "os" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/coze-dev/coze-studio/backend/infra/contract/imagex" + "github.com/coze-dev/coze-studio/backend/infra/contract/storage" + "github.com/coze-dev/coze-studio/backend/pkg/ctxcache" + "github.com/coze-dev/coze-studio/backend/pkg/errorx" + "github.com/coze-dev/coze-studio/backend/pkg/logs" + "github.com/coze-dev/coze-studio/backend/types/consts" + "github.com/coze-dev/coze-studio/backend/types/errno" +) + +type s3Client struct { + client *s3.Client + bucketName string +} + +func NewStorageImagex(ctx context.Context, ak, sk, bucketName, endpoint, region string) (imagex.ImageX, error) { + t, err := getS3Client(ctx, ak, sk, bucketName, endpoint, region) + if err != nil { + return nil, err + } + return t, nil +} + +func getS3Client(ctx context.Context, ak, sk, bucketName, endpoint, region string) (*s3Client, error) { + creds := credentials.NewStaticCredentialsProvider(ak, sk, "") + customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + PartitionID: "aws", + URL: endpoint, + SigningRegion: region, + HostnameImmutable: true, + Source: aws.EndpointSourceCustom, + }, nil + }) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithCredentialsProvider(creds), + config.WithEndpointResolverWithOptions(customResolver), + config.WithRegion("auto"), + ) + if err != nil { + return nil, fmt.Errorf("init config failed, bucketName: %s, endpoint: %s, region: %s, err: %v", bucketName, endpoint, region, err) + } + + c := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = false // virtual-host mode + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + }) + + t := &s3Client{ + client: c, + bucketName: bucketName, + } + + err = t.CheckAndCreateBucket(ctx) + if err != nil { + return nil, err + } + + return t, nil +} + +func New(ctx context.Context, ak, sk, bucketName, endpoint, region string) (storage.Storage, error) { + t, err := getS3Client(ctx, ak, sk, bucketName, endpoint, region) + if err != nil { + return nil, err + } + return t, nil +} + +func (t *s3Client) test() { + // test upload + objectKey := fmt.Sprintf("test-%s.txt", time.Now().Format("20060102150405")) + err := t.PutObject(context.Background(), objectKey, []byte("hello world")) + if err != nil { + logs.CtxErrorf(context.Background(), "PutObject failed, objectKey: %s, err: %v", objectKey, err) + } + + // test download + content, err := t.GetObject(context.Background(), objectKey) + if err != nil { + logs.CtxErrorf(context.Background(), "GetObject failed, objectKey: %s, err: %v", objectKey, err) + } + + logs.CtxInfof(context.Background(), "GetObject content: %s", string(content)) + + // test get presigned url + url, err := t.GetObjectUrl(context.Background(), objectKey) + if err != nil { + logs.CtxErrorf(context.Background(), "GetObjectUrl failed, objectKey: %s, err: %v", objectKey, err) + } + + logs.CtxInfof(context.Background(), "GetObjectUrl url: %s", url) + + // test delete + err = t.DeleteObject(context.Background(), objectKey) + if err != nil { + logs.CtxErrorf(context.Background(), "DeleteObject failed, objectKey: %s, err: %v", objectKey, err) + } +} + +func (t *s3Client) CheckAndCreateBucket(ctx context.Context) error { + client := t.client + bucket := t.bucketName + + _, err := client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)}) + if err == nil { + return nil // already exist + } + + if err != nil { + // bucket not exist + if awsErr, ok := err.(interface{ ErrorCode() string }); ok && awsErr.ErrorCode() == "404" { + input := &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + } + // create bucket + _, err := client.CreateBucket(ctx, input) + return err + } + // other case + return err + } + + return nil +} + +func (t *s3Client) PutObject(ctx context.Context, objectKey string, content []byte, opts ...storage.PutOptFn) error { + client := t.client + body := bytes.NewReader(content) + bucket := t.bucketName + + // upload object + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + Body: body, + }) + return err +} + +func (t *s3Client) GetObject(ctx context.Context, objectKey string) ([]byte, error) { + client := t.client + bucket := t.bucketName + + result, err := client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + }) + if err != nil { + return nil, fmt.Errorf("get object failed : %v", err) + } + defer result.Body.Close() + + body, err := io.ReadAll(result.Body) + if err != nil { + return nil, err + } + + return body, nil +} + +func (t *s3Client) DeleteObject(ctx context.Context, objectKey string) error { + client := t.client + bucket := t.bucketName + + _, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + }) + + return err +} + +func (t *s3Client) GetObjectUrl(ctx context.Context, objectKey string, opts ...storage.GetOptFn) (string, error) { + client := t.client + bucket := t.bucketName + presignClient := s3.NewPresignClient(client) + + req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + }, func(options *s3.PresignOptions) { + options.Expires = time.Duration(60*60*24) * time.Second + }) + if err != nil { + return "", fmt.Errorf("get object presigned url failed: %v", err) + } + + // url parse + url, err := url.Parse(req.URL) + if err != nil { + logs.CtxWarnf(ctx, "[GetObjectUrl] url parse failed, err: %v", err) + return req.URL, nil + } + + proxyPort := os.Getenv(consts.MinIOProxyEndpoint) // :8889 + if len(proxyPort) > 0 { + currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx) + if !ok { + return req.URL, nil + } + + currentScheme, ok := ctxcache.Get[string](ctx, consts.RequestSchemeKeyInCtx) + if !ok { + return req.URL, nil + } + + host, _, err := net.SplitHostPort(currentHost) + if err != nil { + host = currentHost + } + minioProxyHost := host + proxyPort + url.Host = minioProxyHost + url.Scheme = currentScheme + logs.CtxInfof(ctx, "[GetObjectUrl] reset ORG.URL = %s TOS.URL = %s", req.URL, url.String()) + return url.String(), nil + } + + return req.URL, nil +} + +func (i *s3Client) GetUploadHost(ctx context.Context) string { + currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx) + if !ok { + return "" + } + return currentHost + consts.ApplyUploadActionURI + +} + +func (t *s3Client) GetServerID() string { + return "" +} + +func (t *s3Client) GetUploadAuth(ctx context.Context, opt ...imagex.UploadAuthOpt) (*imagex.SecurityToken, error) { + scheme, ok := ctxcache.Get[string](ctx, consts.RequestSchemeKeyInCtx) + if !ok { + return nil, errorx.New(errno.ErrUploadHostSchemaNotExistCode) + } + return &imagex.SecurityToken{ + AccessKeyID: "", + SecretAccessKey: "", + SessionToken: "", + ExpiredTime: time.Now().Add(time.Hour).Format("2006-01-02 15:04:05"), + CurrentTime: time.Now().Format("2006-01-02 15:04:05"), + HostScheme: scheme, + }, nil +} + +func (t *s3Client) GetResourceURL(ctx context.Context, uri string, opts ...imagex.GetResourceOpt) (*imagex.ResourceURL, error) { + url, err := t.GetObjectUrl(ctx, uri) + if err != nil { + return nil, err + } + return &imagex.ResourceURL{ + URL: url, + }, nil +} + +func (t *s3Client) Upload(ctx context.Context, data []byte, opts ...imagex.UploadAuthOpt) (*imagex.UploadResult, error) { + return nil, nil +} + +func (t *s3Client) GetUploadAuthWithExpire(ctx context.Context, expire time.Duration, opt ...imagex.UploadAuthOpt) (*imagex.SecurityToken, error) { + return nil, nil +} diff --git a/backend/infra/impl/storage/storage.go b/backend/infra/impl/storage/storage.go index ae2c7573..c16b16cf 100644 --- a/backend/infra/impl/storage/storage.go +++ b/backend/infra/impl/storage/storage.go @@ -24,6 +24,7 @@ import ( "github.com/coze-dev/coze-studio/backend/infra/contract/imagex" "github.com/coze-dev/coze-studio/backend/infra/contract/storage" "github.com/coze-dev/coze-studio/backend/infra/impl/storage/minio" + "github.com/coze-dev/coze-studio/backend/infra/impl/storage/s3" "github.com/coze-dev/coze-studio/backend/infra/impl/storage/tos" "github.com/coze-dev/coze-studio/backend/types/consts" ) @@ -51,6 +52,15 @@ func New(ctx context.Context) (Storage, error) { os.Getenv(consts.TOSEndpoint), os.Getenv(consts.TOSRegion), ) + case "s3": + return s3.New( + ctx, + os.Getenv(consts.S3AccessKey), + os.Getenv(consts.S3SecretKey), + os.Getenv(consts.StorageBucket), + os.Getenv(consts.S3Endpoint), + os.Getenv(consts.S3Region), + ) } return nil, fmt.Errorf("unknown storage type: %s", storageType) @@ -77,6 +87,15 @@ func NewImagex(ctx context.Context) (imagex.ImageX, error) { os.Getenv(consts.TOSEndpoint), os.Getenv(consts.TOSRegion), ) + case "s3": + return s3.NewStorageImagex( + ctx, + os.Getenv(consts.S3AccessKey), + os.Getenv(consts.S3SecretKey), + os.Getenv(consts.StorageBucket), + os.Getenv(consts.S3Endpoint), + os.Getenv(consts.S3Region), + ) } return nil, fmt.Errorf("unknown storage type: %s", storageType) } diff --git a/backend/main.go b/backend/main.go index 59bfe2e2..6e69f328 100755 --- a/backend/main.go +++ b/backend/main.go @@ -168,6 +168,10 @@ func asyncStartMinioProxyServer(ctx context.Context) { proxyURL = getEnv(consts.TOSBucketEndpoint, "https://opencoze.tos-cn-beijing.volces.com") } + if storageType == "s3" { + proxyURL = getEnv(consts.S3BucketEndpoint, "") + } + minioProxyEndpoint := getEnv(consts.MinIOProxyEndpoint, "") if len(minioProxyEndpoint) == 0 { return diff --git a/backend/types/consts/consts.go b/backend/types/consts/consts.go index 2b3cd4bb..b4a6597d 100644 --- a/backend/types/consts/consts.go +++ b/backend/types/consts/consts.go @@ -44,6 +44,11 @@ const ( TOSRegion = "TOS_REGION" TOSEndpoint = "TOS_ENDPOINT" TOSBucketEndpoint = "TOS_BUCKET_ENDPOINT" + S3AccessKey = "S3_ACCESS_KEY" + S3SecretKey = "S3_SECRET_KEY" + S3Region = "S3_REGION" + S3Endpoint = "S3_ENDPOINT" + S3BucketEndpoint = "S3_BUCKET_ENDPOINT" HostKeyInCtx = "HOST_KEY_IN_CTX" RequestSchemeKeyInCtx = "REQUEST_SCHEME_IN_CTX" diff --git a/docker/.env.example b/docker/.env.example index 4efd0798..ffa7599c 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -35,7 +35,7 @@ export VE_IMAGEX_TEMPLATE="" export VE_IMAGEX_UPLOAD_HOST="https://imagex.volcengineapi.com" # Storage component -export STORAGE_TYPE="minio" # minio / tos +export STORAGE_TYPE="minio" # minio / tos / s3 export STORAGE_BUCKET="opencoze" # MiniIO export MINIO_ROOT_USER=minioadmin @@ -53,6 +53,13 @@ export TOS_ENDPOINT=https://tos-cn-beijing.volces.com export TOS_BUCKET_ENDPOINT=https://opencoze.tos-cn-beijing.volces.com export TOS_REGION=cn-beijing +# S3 +export S3_ACCESS_KEY= +export S3_SECRET_KEY= +export S3_ENDPOINT= +export S3_BUCKET_ENDPOINT= +export S3_REGION= + # Elasticsearch export ES_ADDR="http://localhost:9200" export ES_VERSION="v8" @@ -190,4 +197,4 @@ export CODE_RUNNER_NODE_MODULES_DIR="" # Code execution timeout, default 60 seconds. e.g. "2.56" export CODE_RUNNER_TIMEOUT_SECONDS="" # Code execution memory limit, default 100MB. e.g. "256" -export CODE_RUNNER_MEMORY_LIMIT_MB="" \ No newline at end of file +export CODE_RUNNER_MEMORY_LIMIT_MB=""