feat: manually mirror opencoze's code from bytedance

Change-Id: I09a73aadda978ad9511264a756b2ce51f5761adf

backend/infra/impl/document/nl2sql/builtin/nl2sql.go

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package builtin
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/cloudwego/eino/components/prompt"
+	"github.com/cloudwego/eino/compose"
+	"github.com/cloudwego/eino/schema"
+
+	"github.com/coze-dev/coze-studio/backend/infra/contract/chatmodel"
+	"github.com/coze-dev/coze-studio/backend/infra/contract/document"
+	"github.com/coze-dev/coze-studio/backend/infra/contract/document/nl2sql"
+	"github.com/coze-dev/coze-studio/backend/pkg/logs"
+)
+
+const (
+	defaultTableFmt  = "table name: %s.\ntable describe: %s.\n\n| field name | description | field type | is required |\n"
+	defaultColumnFmt = "| %s | %s | %s | %t |\n\n"
+)
+
+func NewNL2SQL(_ context.Context, cm chatmodel.BaseChatModel, tpl prompt.ChatTemplate) (nl2sql.NL2SQL, error) {
+	return &n2s{cm: cm, tpl: tpl}, nil
+}
+
+type n2s struct {
+	ch       *compose.Chain[*nl2sqlInput, string]
+	runnable compose.Runnable[*nl2sqlInput, string]
+
+	cm  chatmodel.BaseChatModel
+	tpl prompt.ChatTemplate
+}
+
+func (n *n2s) NL2SQL(ctx context.Context, messages []*schema.Message, tables []*document.TableSchema, opts ...nl2sql.Option) (sql string, err error) {
+	o := &nl2sql.Options{ChatModel: n.cm}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	if o.ChatModel == nil {
+		return "", fmt.Errorf("[NL2SQL] chat model not configured")
+	}
+
+	c := compose.NewChain[*nl2sqlInput, string]().
+		AppendLambda(compose.InvokableLambda(func(ctx context.Context, input *nl2sqlInput) (output map[string]any, err error) {
+			if len(input.tables) == 0 {
+				return nil, errors.New("table meta is empty")
+			}
+			tableDesc := strings.Builder{}
+			for _, table := range input.tables {
+				tableDesc.WriteString(fmt.Sprintf(defaultTableFmt, table.Name, table.Comment))
+				for _, column := range table.Columns {
+					tableDesc.WriteString(fmt.Sprintf(defaultColumnFmt, column.Name, column.Description, column.Type.String(), !column.Nullable))
+				}
+			}
+			//logs.CtxInfof(ctx, "table schema: %s", tableDesc.String())
+			return map[string]interface{}{
+				"messages":     input.messages,
+				"table_schema": tableDesc.String(),
+			}, nil
+		})).
+		AppendChatTemplate(n.tpl).
+		AppendChatModel(o.ChatModel).
+		AppendLambda(compose.InvokableLambda(func(ctx context.Context, msg *schema.Message) (sql string, err error) {
+			var promptResp *promptResponse
+			if err := json.Unmarshal([]byte(msg.Content), &promptResp); err != nil {
+				logs.CtxWarnf(ctx, "unmarshal failed: %v", err)
+				return "", err
+			}
+			if promptResp.SQL == "" {
+				logs.CtxInfof(ctx, "no sql generated, err_code: %v, err_msg: %v", promptResp.ErrCode, promptResp.ErrMsg)
+				return "", errors.New(promptResp.ErrMsg)
+			}
+			return promptResp.SQL, nil
+		}))
+
+	r, err := c.Compile(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	input := &nl2sqlInput{
+		messages: messages,
+		tables:   tables,
+	}
+
+	return r.Invoke(ctx, input)
+}
+
+type nl2sqlInput struct {
+	messages []*schema.Message
+	tables   []*document.TableSchema
+}
+
+type promptResponse struct {
+	SQL     string `json:"sql"`
+	ErrCode int    `json:"err_code"`
+	ErrMsg  string `json:"err_msg"`
+}

backend/infra/impl/document/nl2sql/builtin/nl2sql_test.go

@@ -0,0 +1,139 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package builtin
+
+import (
+	"context"
+	"testing"
+
+	"github.com/cloudwego/eino/components/model"
+	"github.com/cloudwego/eino/components/prompt"
+	"github.com/cloudwego/eino/schema"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/coze-dev/coze-studio/backend/infra/contract/document"
+)
+
+func TestNL2SQL(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("test table meta not provided", func(t *testing.T) {
+		impl, err := NewNL2SQL(ctx, &mockChatModel{"mock resp"}, prompt.FromMessages(schema.Jinja2,
+			schema.SystemMessage("system message 123"),
+			schema.UserMessage("{{messages}}, {{table_meta}}"),
+		))
+		assert.NoError(t, err)
+
+		sql, err := impl.NL2SQL(ctx, []*schema.Message{schema.UserMessage("hello")}, nil)
+		assert.Error(t, err)
+		assert.Equal(t, "", sql)
+	})
+
+	t.Run("test parse failed", func(t *testing.T) {
+		impl, err := NewNL2SQL(ctx, &mockChatModel{"mock resp"}, prompt.FromMessages(schema.Jinja2,
+			schema.SystemMessage("system message 123"),
+			schema.UserMessage("{{messages}}, {{table_meta}}"),
+		))
+		assert.NoError(t, err)
+
+		sql, err := impl.NL2SQL(ctx, []*schema.Message{schema.UserMessage("hello")}, []*document.TableSchema{
+			{
+				Name:    "mock_table_1",
+				Comment: "hello",
+				Columns: []*document.Column{
+					{
+						ID:          121,
+						Name:        "id",
+						Type:        document.TableColumnTypeInteger,
+						Description: "test",
+						Nullable:    false,
+						IsPrimary:   true,
+						Sequence:    0,
+					},
+					{
+						ID:          123,
+						Name:        "col_1",
+						Type:        document.TableColumnTypeString,
+						Description: "column_1",
+						Nullable:    true,
+						IsPrimary:   false,
+						Sequence:    1,
+					},
+				},
+			},
+		})
+		assert.Error(t, err)
+		assert.Equal(t, "", sql)
+	})
+
+	t.Run("test success", func(t *testing.T) {
+		impl, err := NewNL2SQL(ctx, &mockChatModel{`{"sql":"mock sql","err_code":0,"err_msg":""}`}, prompt.FromMessages(schema.Jinja2,
+			schema.SystemMessage("system message 123"),
+			schema.UserMessage("{{messages}}, {{table_meta}}"),
+		))
+		assert.NoError(t, err)
+
+		sql, err := impl.NL2SQL(ctx, []*schema.Message{schema.UserMessage("hello")}, []*document.TableSchema{
+			{
+				Name:    "mock_table_1",
+				Comment: "hello",
+				Columns: []*document.Column{
+					{
+						ID:          121,
+						Name:        "id",
+						Type:        document.TableColumnTypeInteger,
+						Description: "test",
+						Nullable:    false,
+						IsPrimary:   true,
+						Sequence:    0,
+					},
+					{
+						ID:          123,
+						Name:        "col_1",
+						Type:        document.TableColumnTypeString,
+						Description: "column_1",
+						Nullable:    true,
+						IsPrimary:   false,
+						Sequence:    1,
+					},
+				},
+			},
+		})
+		assert.NoError(t, err)
+		assert.Equal(t, "mock sql", sql)
+	})
+
+}
+
+type mockChatModel struct {
+	content string
+}
+
+func (m mockChatModel) Generate(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.Message, error) {
+	return schema.AssistantMessage(m.content, nil), nil
+}
+
+func (m mockChatModel) Stream(ctx context.Context, input []*schema.Message, opts ...model.Option) (*schema.StreamReader[*schema.Message], error) {
+	return nil, nil
+}
+
+func (m mockChatModel) BindTools(tools []*schema.ToolInfo) error {
+	return nil
+}
+
+const sys = "# Role: NL2SQL Consultant\n\n## Goals\nTranslate natural language statements into SQL queries in MySQL standard. Follow the Constraints and return only a JSON always.\n\n## Format\n- JSON format only. JSON contains field \"sql\" for generated SQL, filed \"err_code\" for reason type, field \"err_msg\" for detail reason (prefer more than 10 words)\n- Don't use \"```json\" markdown format\n\n## Skills\n- Good at Translate natural language statements into SQL queries in MySQL standard.\n\n## Define\n\"err_code\" Reason Type Define:\n- 0 means you generated a SQL\n- 3002 means you cannot generate a SQL because of timeout\n- 3003 means you cannot generate a SQL because of table schema missing\n- 3005 means you cannot generate a SQL because of some term is ambiguous\n\n## Example\nQ: Help me implement NL2SQL.\n​.table schema description: ​​CREATE TABLE `sales_records` (\\n  `sales_id` bigint(20) unsigned NOT NULL COMMENT 'id of sales person',\\n  `product_id` bigint(64) COMMENT 'id of product',\\n  `sale_date` datetime(3) COMMENT 'sold date and time',\\n  `quantity_sold` int(11) COMMENT 'sold amount',\\n  PRIMARY KEY (`sales_id`)\\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='销售记录表';\n​.natural language description of the SQL requirement:  ​​​​查询上月的销量总额第一名的销售员和他的销售总额\nA: {\n  \"sql\":\"SELECT sales_id, SUM(quantity_sold) AS total_sales FROM sales_records WHERE MONTH(sale_date) = MONTH(CURRENT_DATE - INTERVAL 1 MONTH) AND YEAR(sale_date) = YEAR(CURRENT_DATE - INTERVAL 1 MONTH) GROUP BY sales_id ORDER BY total_sales DESC LIMIT 1\",\n  \"err_code\":0,\n  \"err_msg\":\"SQL query generated successfully\"\n}"
+const usr = "help me implement NL2SQL.\ntable schema description:{{tableSchema}}\nnatural language description of the SQL requirement: {{chat_history}}."