ben/coze-studio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(infra): add file listing support (#1836)

Browse Source
This commit is contained in:
Ryo
2025-08-27 12:06:55 +08:00
committed by GitHub
parent 901d0252e8
commit 263a75b1c0
8 changed files with 512 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
backend/infra/impl/storage/minio/minio.go
View File

@@ -24,22 +24,17 @@ import (
"log"
"math/rand"
"net/url"
"os"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/coze-dev/coze-studio/backend/infra/contract/imagex"
"github.com/coze-dev/coze-studio/backend/infra/contract/storage"
"github.com/coze-dev/coze-studio/backend/infra/impl/storage/proxy"
"github.com/coze-dev/coze-studio/backend/pkg/ctxcache"
"github.com/coze-dev/coze-studio/backend/types/consts"
"github.com/coze-dev/coze-studio/backend/pkg/logs"
)
type minioClient struct {
host string
client *minio.Client
accessKeyID string
secretAccessKey string
@@ -47,11 +42,12 @@ type minioClient struct {
endpoint string
}
func NewStorageImagex(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (imagex.ImageX, error) {
func New(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (storage.Storage, error) {
m, err := getMinioClient(ctx, endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
if err != nil {
return nil, err
}
return m, nil
}
@@ -76,14 +72,8 @@ func getMinioClient(_ context.Context, endpoint, accessKeyID, secretAccessKey, b
if err != nil {
return nil, fmt.Errorf("init minio client failed %v", err)
}
return m, nil
}
func New(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (storage.Storage, error) {
m, err := getMinioClient(ctx, endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
if err != nil {
return nil, err
}
// m.test()
return m, nil
}
@@ -109,6 +99,8 @@ func (m *minioClient) test() {
ctx := context.Background()
objectName := fmt.Sprintf("test-file-%d.txt", rand.Int())
m.ListObjects(ctx, "")
err := m.PutObject(ctx, objectName, []byte("hello content"), storage.WithContentType("text/plain"))
if err != nil {
log.Fatalf("upload file failed: %v", err)
@@ -223,47 +215,48 @@ func (m *minioClient) GetObjectUrl(ctx context.Context, objectKey string, opts .
return presignedURL.String(), nil
}
func (m *minioClient) GetUploadHost(ctx context.Context) string {
currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx)
if !ok {
return ""
func (m *minioClient) ListObjectsPaginated(ctx context.Context, input *storage.ListObjectsPaginatedInput) (*storage.ListObjectsPaginatedOutput, error) {
if input == nil {
return nil, fmt.Errorf("input cannot be nil")
}
return currentHost + consts.ApplyUploadActionURI
}
func (m *minioClient) GetServerID() string {
return ""
}
func (m *minioClient) GetUploadAuth(ctx context.Context, opt ...imagex.UploadAuthOpt) (*imagex.SecurityToken, error) {
scheme := strings.ToLower(os.Getenv(consts.StorageUploadHTTPScheme))
if scheme == "" {
scheme = "http"
if input.PageSize <= 0 {
return nil, fmt.Errorf("page size must be positive")
}
return &imagex.SecurityToken{
AccessKeyID: "",
SecretAccessKey: "",
SessionToken: "",
ExpiredTime: time.Now().Add(time.Hour).Format("2006-01-02 15:04:05"),
CurrentTime: time.Now().Format("2006-01-02 15:04:05"),
HostScheme: scheme,
}, nil
}
func (m *minioClient) GetResourceURL(ctx context.Context, uri string, opts ...imagex.GetResourceOpt) (*imagex.ResourceURL, error) {
url, err := m.GetObjectUrl(ctx, uri)
files, err := m.ListObjects(ctx, input.Prefix)
if err != nil {
return nil, err
}
return &imagex.ResourceURL{
URL: url,
return &storage.ListObjectsPaginatedOutput{
Files: files,
IsTruncated: false,
Cursor: "",
}, nil
}
func (m *minioClient) Upload(ctx context.Context, data []byte, opts ...imagex.UploadAuthOpt) (*imagex.UploadResult, error) {
return nil, nil
}
func (m *minioClient) ListObjects(ctx context.Context, prefix string) ([]*storage.FileInfo, error) {
opts := minio.ListObjectsOptions{
Prefix: prefix,
Recursive: true,
}
func (m *minioClient) GetUploadAuthWithExpire(ctx context.Context, expire time.Duration, opt ...imagex.UploadAuthOpt) (*imagex.SecurityToken, error) {
return nil, nil
objectCh := m.client.ListObjects(ctx, m.bucketName, opts)
var files []*storage.FileInfo
for object := range objectCh {
if object.Err != nil {
return nil, object.Err
}
files = append(files, &storage.FileInfo{
Key: object.Key,
LastModified: object.LastModified,
ETag: object.ETag,
Size: object.Size,
})
logs.CtxDebugf(ctx, "key = %s, lastModified = %s, eTag = %s, size = %d", object.Key, object.LastModified, object.ETag, object.Size)
}
return files, nil
}

81
backend/infra/impl/storage/minio/minio_imagex.go Normal file
View File

@@ -0,0 +1,81 @@
/*
* Copyright 2025 coze-dev Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"context"
"os"
"strings"
"time"
"github.com/coze-dev/coze-studio/backend/infra/contract/imagex"
"github.com/coze-dev/coze-studio/backend/pkg/ctxcache"
"github.com/coze-dev/coze-studio/backend/types/consts"
)
func NewStorageImagex(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (imagex.ImageX, error) {
m, err := getMinioClient(ctx, endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
if err != nil {
return nil, err
}
return m, nil
}
func (m *minioClient) GetUploadHost(ctx context.Context) string {
currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx)
if !ok {
return ""
}
return currentHost + consts.ApplyUploadActionURI
}
func (m *minioClient) GetServerID() string {
return ""
}
func (m *minioClient) GetUploadAuth(ctx context.Context, opt ...imagex.UploadAuthOpt) (*imagex.SecurityToken, error) {
scheme := strings.ToLower(os.Getenv(consts.StorageUploadHTTPScheme))
if scheme == "" {
scheme = "http"
}
return &imagex.SecurityToken{
AccessKeyID: "",
SecretAccessKey: "",
SessionToken: "",
ExpiredTime: time.Now().Add(time.Hour).Format("2006-01-02 15:04:05"),
CurrentTime: time.Now().Format("2006-01-02 15:04:05"),
HostScheme: scheme,
}, nil
}
func (m *minioClient) GetResourceURL(ctx context.Context, uri string, opts ...imagex.GetResourceOpt) (*imagex.ResourceURL, error) {
url, err := m.GetObjectUrl(ctx, uri)
if err != nil {
return nil, err
}
return &imagex.ResourceURL{
URL: url,
}, nil
}
func (m *minioClient) Upload(ctx context.Context, data []byte, opts ...imagex.UploadAuthOpt) (*imagex.UploadResult, error) {
return nil, nil
}
func (m *minioClient) GetUploadAuthWithExpire(ctx context.Context, expire time.Duration, opt ...imagex.UploadAuthOpt) (*imagex.SecurityToken, error) {
return nil, nil
}