ben/coze-studio
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(infra): integrate PaddleOCR's PP-StructureV3 as a document parser backend (#714)

Browse Source
This commit is contained in:
Lin Manhui
2025-08-13 16:37:42 +08:00
committed by GitHub
parent 708a6ed0c0
commit 6b60c07c22
30 changed files with 657 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/infra/impl/document/parser/builtin/chunk_custom.go
View File

@@ -34,10 +34,10 @@ var (
emailRegex = regexp.MustCompile(`[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}`)
)
func chunkCustom(_ context.Context, text string, config *contract.Config, opts ...parser.Option) (docs []*schema.Document, err error) {
func ChunkCustom(_ context.Context, text string, config *contract.Config, opts ...parser.Option) (docs []*schema.Document, err error) {
cs := config.ChunkingStrategy
if cs.Overlap >= cs.ChunkSize {
return nil, fmt.Errorf("[chunkCustom] invalid param, overlap >= chunk_size")
return nil, fmt.Errorf("[ChunkCustom] invalid param, overlap >= chunk_size")
}
var (

2
backend/infra/impl/document/parser/builtin/chunk_custom_test.go
View File

@@ -39,7 +39,7 @@ func TestChunkCustom(t *testing.T) {
TrimURLAndEmail: true,
}
slices, err := chunkCustom(ctx, text, &parser.Config{ChunkingStrategy: cs})
slices, err := ChunkCustom(ctx, text, &parser.Config{ChunkingStrategy: cs})
assert.NoError(t, err)
assert.Len(t, slices, 10)

2
backend/infra/impl/document/parser/builtin/image.go
View File

@@ -24,7 +24,7 @@ import (
"github.com/coze-dev/coze-studio/backend/infra/contract/storage"
)
func putImageObject(ctx context.Context, st storage.Storage, imgExt string, uid int64, img []byte) (format string, err error) {
func PutImageObject(ctx context.Context, st storage.Storage, imgExt string, uid int64, img []byte) (format string, err error) {
secret := createSecret(uid, imgExt)
fileName := fmt.Sprintf("%d_%d_%s.%s", uid, time.Now().UnixNano(), secret, imgExt)
objectName := fmt.Sprintf("%s/%s", knowledgePrefix, fileName)

26
backend/infra/impl/document/parser/builtin/manager.go
View File

@@ -41,7 +41,7 @@ type manager struct {
}
func (m *manager) GetParser(config *parser.Config) (parser.Parser, error) {
var pFn parseFn
var pFn ParseFn
if config.ParsingStrategy.HeaderLine == 0 && config.ParsingStrategy.DataStartLine == 0 {
config.ParsingStrategy.DataStartLine = 1
@@ -52,26 +52,30 @@ func (m *manager) GetParser(config *parser.Config) (parser.Parser, error) {
switch config.FileExtension {
case parser.FileExtensionPDF:
pFn = parseByPython(config, m.storage, m.ocr, goutil.GetPython3Path(), goutil.GetPythonFilePath("parse_pdf.py"))
pFn = ParseByPython(config, m.storage, m.ocr, goutil.GetPython3Path(), goutil.GetPythonFilePath("parse_pdf.py"))
case parser.FileExtensionTXT:
pFn = parseText(config)
pFn = ParseText(config)
case parser.FileExtensionMarkdown:
pFn = parseMarkdown(config, m.storage, m.ocr)
pFn = ParseMarkdown(config, m.storage, m.ocr)
case parser.FileExtensionDocx:
pFn = parseByPython(config, m.storage, m.ocr, goutil.GetPython3Path(), goutil.GetPythonFilePath("parse_docx.py"))
pFn = ParseByPython(config, m.storage, m.ocr, goutil.GetPython3Path(), goutil.GetPythonFilePath("parse_docx.py"))
case parser.FileExtensionCSV:
pFn = parseCSV(config)
pFn = ParseCSV(config)
case parser.FileExtensionXLSX:
pFn = parseXLSX(config)
pFn = ParseXLSX(config)
case parser.FileExtensionJSON:
pFn = parseJSON(config)
pFn = ParseJSON(config)
case parser.FileExtensionJsonMaps:
pFn = parseJSONMaps(config)
pFn = ParseJSONMaps(config)
case parser.FileExtensionJPG, parser.FileExtensionJPEG, parser.FileExtensionPNG:
pFn = parseImage(config, m.model)
pFn = ParseImage(config, m.model)
default:
return nil, fmt.Errorf("[Parse] document type not support, type=%s", config.FileExtension)
}
return &p{parseFn: pFn}, nil
return &Parser{ParseFn: pFn}, nil
}
func (m *manager) IsAutoAnnotationSupported() bool {
return m.model != nil
}

2
backend/infra/impl/document/parser/builtin/parse_csv.go
View File

@@ -29,7 +29,7 @@ import (
contract "github.com/coze-dev/coze-studio/backend/infra/contract/document/parser"
)
func parseCSV(config *contract.Config) parseFn {
func ParseCSV(config *contract.Config) ParseFn {
return func(ctx context.Context, reader io.Reader, opts ...parser.Option) (docs []*schema.Document, err error) {
iter := &csvIterator{csv.NewReader(utfbom.SkipOnly(reader))}
return parseByRowIterator(iter, config, opts...)

8
backend/infra/impl/document/parser/builtin/parse_csv_test.go
View File

@@ -47,7 +47,7 @@ func TestParseCSV(t *testing.T) {
},
ChunkingStrategy: nil,
}
p1 := parseCSV(c1)
p1 := ParseCSV(c1)
docs, err := p1(ctx, r1, parser.WithExtraMeta(map[string]any{
"document_id": int64(123),
"knowledge_id": int64(456),
@@ -112,7 +112,7 @@ func TestParseCSV(t *testing.T) {
},
ChunkingStrategy: nil,
}
p2 := parseCSV(c2)
p2 := ParseCSV(c2)
docs, err = p2(ctx, r2, parser.WithExtraMeta(map[string]any{
"document_id": int64(123),
"knowledge_id": int64(456),
@@ -131,7 +131,7 @@ func TestParseCSVBadCases(t *testing.T) {
b, err := io.ReadAll(f)
assert.NoError(t, err)
pfn := parseCSV(&contract.Config{
pfn := ParseCSV(&contract.Config{
FileExtension: "csv",
ParsingStrategy: &contract.ParsingStrategy{
ExtractImage: true,
@@ -154,7 +154,7 @@ func TestParseCSVBadCases(t *testing.T) {
cols, err := document.GetDocumentColumns(resp[0])
assert.NoError(t, err)
cols[5].Nullable = false
npfn := parseCSV(&contract.Config{
npfn := ParseCSV(&contract.Config{
FileExtension: "csv",
ParsingStrategy: &contract.ParsingStrategy{
ExtractImage: true,

6
backend/infra/impl/document/parser/builtin/parse_image.go
View File

@@ -31,7 +31,7 @@ import (
"github.com/coze-dev/coze-studio/backend/types/errno"
)
func parseImage(config *contract.Config, model chatmodel.BaseChatModel) parseFn {
func ParseImage(config *contract.Config, model chatmodel.BaseChatModel) ParseFn {
return func(ctx context.Context, reader io.Reader, opts ...parser.Option) (docs []*schema.Document, err error) {
options := parser.GetCommonOptions(&parser.Options{}, opts...)
doc := &schema.Document{
@@ -76,14 +76,14 @@ func parseImage(config *contract.Config, model chatmodel.BaseChatModel) parseFn
output, err := model.Generate(ctx, []*schema.Message{input})
if err != nil {
return nil, fmt.Errorf("[parseImage] model generate failed: %w", err)
return nil, fmt.Errorf("[ParseImage] model generate failed: %w", err)
}
doc.Content = output.Content
case contract.ImageAnnotationTypeManual:
// do nothing
default:
return nil, fmt.Errorf("[parseImage] unknown image annotation type=%d", config.ParsingStrategy.ImageAnnotationType)
return nil, fmt.Errorf("[ParseImage] unknown image annotation type=%d", config.ParsingStrategy.ImageAnnotationType)
}
return []*schema.Document{doc}, nil

4
backend/infra/impl/document/parser/builtin/parse_json.go
View File

@@ -28,7 +28,7 @@ import (
contract "github.com/coze-dev/coze-studio/backend/infra/contract/document/parser"
)
func parseJSON(config *contract.Config) parseFn {
func ParseJSON(config *contract.Config) ParseFn {
return func(ctx context.Context, reader io.Reader, opts ...parser.Option) (docs []*schema.Document, err error) {
b, err := io.ReadAll(reader)
if err != nil {
@@ -41,7 +41,7 @@ func parseJSON(config *contract.Config) parseFn {
}
if len(rawSlices) == 0 {
return nil, fmt.Errorf("[parseJSON] json data is empty")
return nil, fmt.Errorf("[ParseJSON] json data is empty")
}
var header []string

2
backend/infra/impl/document/parser/builtin/parse_json_maps.go
View File

@@ -29,7 +29,7 @@ import (
contract "github.com/coze-dev/coze-studio/backend/infra/contract/document/parser"
)
func parseJSONMaps(config *contract.Config) parseFn {
func ParseJSONMaps(config *contract.Config) ParseFn {
return func(ctx context.Context, reader io.Reader, opts ...parser.Option) (docs []*schema.Document, err error) {
b, err := io.ReadAll(reader)
if err != nil {

2
backend/infra/impl/document/parser/builtin/parse_json_maps_test.go
View File

@@ -85,7 +85,7 @@ func TestParseTableCustomContent(t *testing.T) {
},
}
pfn := parseJSONMaps(config)
pfn := ParseJSONMaps(config)
docs, err := pfn(ctx, reader, parser.WithExtraMeta(map[string]any{
"document_id": int64(123),
"knowledge_id": int64(456),

4
backend/infra/impl/document/parser/builtin/parse_json_test.go
View File

@@ -55,7 +55,7 @@ func TestParseJSON(t *testing.T) {
},
ChunkingStrategy: nil,
}
pfn := parseJSON(config)
pfn := ParseJSON(config)
docs, err := pfn(context.Background(), reader, parser.WithExtraMeta(map[string]any{
"document_id": int64(123),
"knowledge_id": int64(456),
@@ -121,7 +121,7 @@ func TestParseJSONWithSchema(t *testing.T) {
},
},
}
pfn := parseJSON(config)
pfn := ParseJSON(config)
docs, err := pfn(context.Background(), reader, parser.WithExtraMeta(map[string]any{
"document_id": int64(123),
"knowledge_id": int64(456),

8
backend/infra/impl/document/parser/builtin/parse_markdown.go
View File

@@ -38,7 +38,7 @@ import (
"github.com/coze-dev/coze-studio/backend/pkg/logs"
)
func parseMarkdown(config *contract.Config, storage storage.Storage, ocr ocr.OCR) parseFn {
func ParseMarkdown(config *contract.Config, storage storage.Storage, ocr ocr.OCR) ParseFn {
return func(ctx context.Context, reader io.Reader, opts ...parser.Option) (docs []*schema.Document, err error) {
options := parser.GetCommonOptions(&parser.Options{}, opts...)
mdParser := goldmark.DefaultParser()
@@ -52,7 +52,7 @@ func parseMarkdown(config *contract.Config, storage storage.Storage, ocr ocr.OCR
ps := config.ParsingStrategy
if cs.ChunkType != contract.ChunkTypeCustom && cs.ChunkType != contract.ChunkTypeDefault {
return nil, fmt.Errorf("[parseMarkdown] chunk type not support, chunk type=%d", cs.ChunkType)
return nil, fmt.Errorf("[ParseMarkdown] chunk type not support, chunk type=%d", cs.ChunkType)
}
var (
@@ -173,7 +173,7 @@ func parseMarkdown(config *contract.Config, storage storage.Storage, ocr ocr.OCR
return ast.WalkStop, fmt.Errorf("failed to download image: %w", err)
}
imgSrc, err := putImageObject(ctx, storage, ext, getCreatorIDFromExtraMeta(options.ExtraMeta), img)
imgSrc, err := PutImageObject(ctx, storage, ext, GetCreatorIDFromExtraMeta(options.ExtraMeta), img)
if err != nil {
return ast.WalkStop, err
}
@@ -198,7 +198,7 @@ func parseMarkdown(config *contract.Config, storage storage.Storage, ocr ocr.OCR
pushSlice()
}
} else {
logs.CtxInfof(ctx, "[parseMarkdown] not a valid image url, skip, got=%s", imageURL)
logs.CtxInfof(ctx, "[ParseMarkdown] not a valid image url, skip, got=%s", imageURL)
}
}
}

2
backend/infra/impl/document/parser/builtin/parse_markdown_test.go
View File

@@ -37,7 +37,7 @@ func TestParseMarkdown(t *testing.T) {
mockStorage := ms.NewMockStorage(ctrl)
mockStorage.EXPECT().PutObject(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
pfn := parseMarkdown(&contract.Config{
pfn := ParseMarkdown(&contract.Config{
FileExtension: contract.FileExtensionMarkdown,
ParsingStrategy: &contract.ParsingStrategy{
ExtractImage: true,

6
backend/infra/impl/document/parser/builtin/parse_text.go
View File

@@ -27,7 +27,7 @@ import (
contract "github.com/coze-dev/coze-studio/backend/infra/contract/document/parser"
)
func parseText(config *contract.Config) parseFn {
func ParseText(config *contract.Config) ParseFn {
return func(ctx context.Context, reader io.Reader, opts ...parser.Option) (docs []*schema.Document, err error) {
content, err := io.ReadAll(reader)
if err != nil {
@@ -36,9 +36,9 @@ func parseText(config *contract.Config) parseFn {
switch config.ChunkingStrategy.ChunkType {
case contract.ChunkTypeCustom, contract.ChunkTypeDefault:
docs, err = chunkCustom(ctx, string(content), config, opts...)
docs, err = ChunkCustom(ctx, string(content), config, opts...)
default:
return nil, fmt.Errorf("[parseText] chunk type not support, type=%d", config.ChunkingStrategy.ChunkType)
return nil, fmt.Errorf("[ParseText] chunk type not support, type=%d", config.ChunkingStrategy.ChunkType)
}
if err != nil {
return nil, err

2
backend/infra/impl/document/parser/builtin/parse_xlsx.go
View File

@@ -27,7 +27,7 @@ import (
contract "github.com/coze-dev/coze-studio/backend/infra/contract/document/parser"
)
func parseXLSX(config *contract.Config) parseFn {
func ParseXLSX(config *contract.Config) ParseFn {
return func(ctx context.Context, reader io.Reader, opts ...parser.Option) (docs []*schema.Document, err error) {
f, err := excelize.OpenReader(reader)
if err != nil {

4
backend/infra/impl/document/parser/builtin/parse_xlsx_test.go
View File

@@ -88,7 +88,7 @@ func TestParseXLSX(t *testing.T) {
ChunkingStrategy: nil,
}
pfn := parseXLSX(config)
pfn := ParseXLSX(config)
docs, err := pfn(ctx, reader, parser.WithExtraMeta(map[string]any{
"document_id": int64(123),
"knowledge_id": int64(456),
@@ -159,7 +159,7 @@ func TestParseXLSXConvertColumnType(t *testing.T) {
ChunkingStrategy: nil,
}
pfn := parseXLSX(config)
pfn := ParseXLSX(config)
docs, err := pfn(ctx, reader, parser.WithExtraMeta(map[string]any{
"document_id": int64(123),
"knowledge_id": int64(456),

10
backend/infra/impl/document/parser/builtin/parser.go
View File

@@ -24,12 +24,12 @@ import (
"github.com/cloudwego/eino/schema"
)
type p struct {
parseFn
type Parser struct {
ParseFn
}
func (p p) Parse(ctx context.Context, reader io.Reader, opts ...parser.Option) ([]*schema.Document, error) {
return p.parseFn(ctx, reader, opts...)
func (p Parser) Parse(ctx context.Context, reader io.Reader, opts ...parser.Option) ([]*schema.Document, error) {
return p.ParseFn(ctx, reader, opts...)
}
type parseFn func(ctx context.Context, reader io.Reader, opts ...parser.Option) (docs []*schema.Document, err error)
type ParseFn func(ctx context.Context, reader io.Reader, opts ...parser.Option) (docs []*schema.Document, err error)

38
backend/infra/impl/document/parser/builtin/py_parser_protocol.go
View File

@@ -73,15 +73,15 @@ func (p *pyPDFTableIterator) NextRow() (row []string, end bool, err error) {
return row, false, nil
}
func parseByPython(config *contract.Config, storage storage.Storage, ocr ocr.OCR, pyPath, scriptPath string) parseFn {
func ParseByPython(config *contract.Config, storage storage.Storage, ocr ocr.OCR, pyPath, scriptPath string) ParseFn {
return func(ctx context.Context, reader io.Reader, opts ...parser.Option) (docs []*schema.Document, err error) {
pr, pw, err := os.Pipe()
if err != nil {
return nil, fmt.Errorf("[parseByPython] create rpipe failed, %w", err)
return nil, fmt.Errorf("[ParseByPython] create rpipe failed, %w", err)
}
r, w, err := os.Pipe()
if err != nil {
return nil, fmt.Errorf("[parseByPython] create pipe failed: %w", err)
return nil, fmt.Errorf("[ParseByPython] create pipe failed: %w", err)
}
options := parser.GetCommonOptions(&parser.Options{ExtraMeta: map[string]any{}}, opts...)
@@ -91,13 +91,13 @@ func parseByPython(config *contract.Config, storage storage.Storage, ocr ocr.OCR
FilterPages: config.ParsingStrategy.FilterPages,
})
if err != nil {
return nil, fmt.Errorf("[parseByPython] create parse request failed, %w", err)
return nil, fmt.Errorf("[ParseByPython] create parse request failed, %w", err)
}
if _, err = pw.Write(reqb); err != nil {
return nil, fmt.Errorf("[parseByPython] write parse request bytes failed, %w", err)
return nil, fmt.Errorf("[ParseByPython] write parse request bytes failed, %w", err)
}
if err = pw.Close(); err != nil {
return nil, fmt.Errorf("[parseByPython] close write request pipe failed, %w", err)
return nil, fmt.Errorf("[ParseByPython] close write request pipe failed, %w", err)
}
cmd := exec.Command(pyPath, scriptPath)
@@ -105,31 +105,31 @@ func parseByPython(config *contract.Config, storage storage.Storage, ocr ocr.OCR
cmd.Stdout = os.Stdout
cmd.ExtraFiles = []*os.File{w, pr}
if err = cmd.Start(); err != nil {
return nil, fmt.Errorf("[parseByPython] failed to start Python script: %w", err)
return nil, fmt.Errorf("[ParseByPython] failed to start Python script: %w", err)
}
if err = w.Close(); err != nil {
return nil, fmt.Errorf("[parseByPython] failed to close write pipe: %w", err)
return nil, fmt.Errorf("[ParseByPython] failed to close write pipe: %w", err)
}
result := &pyParseResult{}
if err = json.NewDecoder(r).Decode(result); err != nil {
return nil, fmt.Errorf("[parseByPython] failed to decode result: %w", err)
return nil, fmt.Errorf("[ParseByPython] failed to decode result: %w", err)
}
if err = cmd.Wait(); err != nil {
return nil, fmt.Errorf("[parseByPython] cmd wait err: %w", err)
return nil, fmt.Errorf("[ParseByPython] cmd wait err: %w", err)
}
if result.Error != "" {
return nil, fmt.Errorf("[parseByPython] python execution failed: %s", result.Error)
return nil, fmt.Errorf("[ParseByPython] python execution failed: %s", result.Error)
}
for i, item := range result.Content {
switch item.Type {
case contentTypeText:
partDocs, err := chunkCustom(ctx, item.Content, config, opts...)
partDocs, err := ChunkCustom(ctx, item.Content, config, opts...)
if err != nil {
return nil, fmt.Errorf("[parseByPython] chunk text failed, %w", err)
return nil, fmt.Errorf("[ParseByPython] chunk text failed, %w", err)
}
docs = append(docs, partDocs...)
case contentTypeImage:
@@ -138,9 +138,9 @@ func parseByPython(config *contract.Config, storage storage.Storage, ocr ocr.OCR
}
image, err := base64.StdEncoding.DecodeString(item.Content)
if err != nil {
return nil, fmt.Errorf("[parseByPython] decode image failed, %w", err)
return nil, fmt.Errorf("[ParseByPython] decode image failed, %w", err)
}
imgSrc, err := putImageObject(ctx, storage, "png", getCreatorIDFromExtraMeta(options.ExtraMeta), image)
imgSrc, err := PutImageObject(ctx, storage, "png", GetCreatorIDFromExtraMeta(options.ExtraMeta), image)
if err != nil {
return nil, err
}
@@ -148,7 +148,7 @@ func parseByPython(config *contract.Config, storage storage.Storage, ocr ocr.OCR
if config.ParsingStrategy.ImageOCR && ocr != nil {
texts, err := ocr.FromBase64(ctx, item.Content)
if err != nil {
return nil, fmt.Errorf("[parseByPython] FromBase64 failed, %w", err)
return nil, fmt.Errorf("[ParseByPython] FromBase64 failed, %w", err)
}
label += strings.Join(texts, "\n")
}
@@ -181,15 +181,15 @@ func parseByPython(config *contract.Config, storage storage.Storage, ocr ocr.OCR
ChunkingStrategy: config.ChunkingStrategy,
}, opts...)
if err != nil {
return nil, fmt.Errorf("[parseByPython] parse table failed, %w", err)
return nil, fmt.Errorf("[ParseByPython] parse table failed, %w", err)
}
fmtTableDocs, err := formatTablesInDocument(rawTableDocs)
if err != nil {
return nil, fmt.Errorf("[parseByPython] format table failed, %w", err)
return nil, fmt.Errorf("[ParseByPython] format table failed, %w", err)
}
docs = append(docs, fmtTableDocs...)
default:
return nil, fmt.Errorf("[parseByPython] invalid content type: %s", item.Type)
return nil, fmt.Errorf("[ParseByPython] invalid content type: %s", item.Type)
}
}

2
backend/infra/impl/document/parser/builtin/util.go
View File

@@ -61,7 +61,7 @@ func getExtension(uri string) string {
return ""
}
func getCreatorIDFromExtraMeta(extraMeta map[string]any) int64 {
func GetCreatorIDFromExtraMeta(extraMeta map[string]any) int64 {
if extraMeta == nil {
return 0
}