feat: manually mirror opencoze's code from bytedance

Change-Id: I09a73aadda978ad9511264a756b2ce51f5761adf

backend/domain/workflow/internal/nodes/json/json_deserialization.go

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package json
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/coze-dev/coze-studio/backend/domain/workflow/entity/vo"
+	"github.com/coze-dev/coze-studio/backend/domain/workflow/internal/nodes"
+	"github.com/coze-dev/coze-studio/backend/pkg/ctxcache"
+	"github.com/coze-dev/coze-studio/backend/pkg/errorx"
+	"github.com/coze-dev/coze-studio/backend/pkg/sonic"
+	"github.com/coze-dev/coze-studio/backend/types/errno"
+)
+
+const (
+	InputKeyDeserialization  = "input"
+	OutputKeyDeserialization = "output"
+	warningsKey              = "deserialization_warnings"
+)
+
+type DeserializationConfig struct {
+	OutputFields map[string]*vo.TypeInfo `json:"outputFields,omitempty"`
+}
+
+type Deserializer struct {
+	config   *DeserializationConfig
+	typeInfo *vo.TypeInfo
+}
+
+func NewJsonDeserializer(_ context.Context, cfg *DeserializationConfig) (*Deserializer, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("config required")
+	}
+	if cfg.OutputFields == nil {
+		return nil, fmt.Errorf("OutputFields is required for deserialization")
+	}
+	typeInfo := cfg.OutputFields[OutputKeyDeserialization]
+	if typeInfo == nil {
+		return nil, fmt.Errorf("no output field specified in deserialization config")
+	}
+	return &Deserializer{
+		config:   cfg,
+		typeInfo: typeInfo,
+	}, nil
+}
+
+func (jd *Deserializer) Invoke(ctx context.Context, input map[string]any) (map[string]any, error) {
+	jsonStrValue := input[InputKeyDeserialization]
+
+	jsonStr, ok := jsonStrValue.(string)
+	if !ok {
+		return nil, fmt.Errorf("input is not a string, got %T", jsonStrValue)
+	}
+
+	typeInfo := jd.typeInfo
+
+	var rawValue any
+	var err error
+
+	// Unmarshal based on the root type
+	switch typeInfo.Type {
+	case vo.DataTypeString, vo.DataTypeInteger, vo.DataTypeNumber, vo.DataTypeBoolean, vo.DataTypeTime, vo.DataTypeFile:
+		// Scalar types - unmarshal to generic any
+		err = sonic.Unmarshal([]byte(jsonStr), &rawValue)
+	case vo.DataTypeArray:
+		// Array type - unmarshal to []any
+		var arr []any
+		err = sonic.Unmarshal([]byte(jsonStr), &arr)
+		rawValue = arr
+	case vo.DataTypeObject:
+		// Object type - unmarshal to map[string]any
+		var obj map[string]any
+		err = sonic.Unmarshal([]byte(jsonStr), &obj)
+		rawValue = obj
+	default:
+		return nil, fmt.Errorf("unsupported root data type: %s", typeInfo.Type)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("JSON unmarshaling failed: %w", err)
+	}
+
+	convertedValue, ws, err := nodes.Convert(ctx, rawValue, OutputKeyDeserialization, typeInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	if ws != nil && len(*ws) > 0 {
+		ctxcache.Store(ctx, warningsKey, *ws)
+	}
+
+	return map[string]any{OutputKeyDeserialization: convertedValue}, nil
+}
+
+func (jd *Deserializer) ToCallbackOutput(ctx context.Context, out map[string]any) (*nodes.StructuredCallbackOutput, error) {
+	var wfe vo.WorkflowError
+	if warnings, ok := ctxcache.Get[nodes.ConversionWarnings](ctx, warningsKey); ok {
+		wfe = vo.WrapWarn(errno.ErrNodeOutputParseFail, warnings, errorx.KV("warnings", warnings.Error()))
+	}
+	return &nodes.StructuredCallbackOutput{
+		Output:    out,
+		RawOutput: out,
+		Error:     wfe,
+	}, nil
+}

backend/domain/workflow/internal/nodes/json/json_deserialization_test.go

@@ -0,0 +1,360 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package json
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/coze-dev/coze-studio/backend/domain/workflow/entity/vo"
+	"github.com/coze-dev/coze-studio/backend/domain/workflow/internal/nodes"
+	"github.com/coze-dev/coze-studio/backend/pkg/ctxcache"
+	"github.com/coze-dev/coze-studio/backend/pkg/sonic"
+)
+
+func TestNewJsonDeserializer(t *testing.T) {
+	ctx := context.Background()
+
+	// Test with nil config
+	_, err := NewJsonDeserializer(ctx, nil)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "config required")
+
+	// Test with missing OutputFields config
+	_, err = NewJsonDeserializer(ctx, &DeserializationConfig{})
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "OutputFields is required")
+
+	// Test with missing output key in OutputFields
+	_, err = NewJsonDeserializer(ctx, &DeserializationConfig{
+		OutputFields: map[string]*vo.TypeInfo{
+			"testKey": {Type: vo.DataTypeString},
+		},
+	})
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "no output field specified in deserialization config")
+
+	// Test with valid config
+	validConfig := &DeserializationConfig{
+		OutputFields: map[string]*vo.TypeInfo{
+			OutputKeyDeserialization: {Type: vo.DataTypeString},
+		},
+	}
+	processor, err := NewJsonDeserializer(ctx, validConfig)
+	assert.NoError(t, err)
+	assert.NotNil(t, processor)
+}
+
+func TestJsonDeserializer_Invoke(t *testing.T) {
+	ctx := context.Background()
+
+	// Base type test config
+	baseTypeConfig := &DeserializationConfig{
+		OutputFields: map[string]*vo.TypeInfo{
+			"output": {Type: vo.DataTypeString},
+		},
+	}
+
+	// Object type test config
+	objectTypeConfig := &DeserializationConfig{
+		OutputFields: map[string]*vo.TypeInfo{
+			"output": {
+				Type: vo.DataTypeObject,
+				Properties: map[string]*vo.TypeInfo{
+					"name": {Type: vo.DataTypeString, Required: true},
+					"age":  {Type: vo.DataTypeInteger},
+				},
+			},
+		},
+	}
+
+	// Array type test config
+	arrayTypeConfig := &DeserializationConfig{
+		OutputFields: map[string]*vo.TypeInfo{
+			"output": {
+				Type:         vo.DataTypeArray,
+				ElemTypeInfo: &vo.TypeInfo{Type: vo.DataTypeInteger},
+			},
+		},
+	}
+
+	// Nested array object test config
+	nestedArrayConfig := &DeserializationConfig{
+		OutputFields: map[string]*vo.TypeInfo{
+			"output": {
+				Type: vo.DataTypeArray,
+				ElemTypeInfo: &vo.TypeInfo{
+					Type: vo.DataTypeObject,
+					Properties: map[string]*vo.TypeInfo{
+						"id":   {Type: vo.DataTypeInteger},
+						"name": {Type: vo.DataTypeString},
+					},
+				},
+			},
+		},
+	}
+
+	// Test cases
+	tests := []struct {
+		name           string
+		config         *DeserializationConfig
+		inputJSON      string
+		expectedOutput any
+		expectErr      bool
+		expectWarnings int
+	}{{
+		name:           "Test string deserialization",
+		config:         baseTypeConfig,
+		inputJSON:      `"test string"`,
+		expectedOutput: "test string",
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name: "Test integer deserialization",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {Type: vo.DataTypeInteger},
+			},
+		},
+		inputJSON:      `123`,
+		expectedOutput: 123,
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name: "Test boolean deserialization",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {Type: vo.DataTypeBoolean},
+			},
+		},
+		inputJSON:      `true`,
+		expectedOutput: true,
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name:           "Test object deserialization",
+		config:         objectTypeConfig,
+		inputJSON:      `{"name":"test","age":20}`,
+		expectedOutput: map[string]any{"name": "test", "age": 20.0},
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name:           "Test array deserialization",
+		config:         arrayTypeConfig,
+		inputJSON:      `[1,2,3]`,
+		expectedOutput: []any{1.0, 2.0, 3.0},
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name:      "Test nested array object deserialization",
+		config:    nestedArrayConfig,
+		inputJSON: `[{"id":1,"name":"a"},{"id":2,"name":"b"}]`,
+		expectedOutput: []any{
+			map[string]any{"id": 1.0, "name": "a"},
+			map[string]any{"id": 2.0, "name": "b"},
+		},
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name:           "Test invalid JSON format",
+		config:         baseTypeConfig,
+		inputJSON:      `{invalid json}`,
+		expectedOutput: nil,
+		expectErr:      true,
+		expectWarnings: 0,
+	}, {
+		name: "Test type mismatch warning",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {Type: vo.DataTypeInteger},
+			},
+		},
+		inputJSON:      `"not a number"`,
+		expectedOutput: nil,
+		expectErr:      false,
+		expectWarnings: 1,
+	}, {
+		name:           "Test null JSON input",
+		config:         baseTypeConfig,
+		inputJSON:      `null`,
+		expectedOutput: nil,
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name: "Test string to integer conversion",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {Type: vo.DataTypeInteger},
+			},
+		},
+		inputJSON:      `"123"`,
+		expectedOutput: 123,
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name: "Test float to integer conversion (integer part)",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {Type: vo.DataTypeInteger},
+			},
+		},
+		inputJSON:      `123.0`,
+		expectedOutput: 123,
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name: "Test float to integer conversion (non-integer part)",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {Type: vo.DataTypeInteger},
+			},
+		},
+		inputJSON:      `123.5`,
+		expectedOutput: 123,
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name: "Test boolean to integer conversion",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {Type: vo.DataTypeInteger},
+			},
+		},
+		inputJSON:      `true`,
+		expectedOutput: nil,
+		expectErr:      false,
+		expectWarnings: 1,
+	}, {
+		name: "Test string to boolean conversion",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {Type: vo.DataTypeBoolean},
+			},
+		},
+		inputJSON:      `"true"`,
+		expectedOutput: true,
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name: "Test string to integer conversion in nested object",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {
+					Type: vo.DataTypeObject,
+					Properties: map[string]*vo.TypeInfo{
+						"age": {Type: vo.DataTypeInteger},
+					},
+				},
+			},
+		},
+		inputJSON:      `{"age":"456"}`,
+		expectedOutput: map[string]any{"age": 456},
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name: "Test string to integer conversion for array elements",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {
+					Type:         vo.DataTypeArray,
+					ElemTypeInfo: &vo.TypeInfo{Type: vo.DataTypeInteger},
+				},
+			},
+		},
+		inputJSON:      `["1", "2", "3"]`,
+		expectedOutput: []any{1, 2, 3},
+		expectErr:      false,
+		expectWarnings: 0,
+	}, {
+		name: "Test string with non-numeric characters to integer conversion",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {Type: vo.DataTypeInteger},
+			},
+		},
+		inputJSON:      `"123abc"`,
+		expectedOutput: nil,
+		expectErr:      false,
+		expectWarnings: 1,
+	}, {
+		name: "Test type mismatch in nested object field",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {
+					Type: vo.DataTypeObject,
+					Properties: map[string]*vo.TypeInfo{
+						"score": {Type: vo.DataTypeInteger},
+					},
+				},
+			},
+		},
+		inputJSON:      `{"score":"invalid"}`,
+		expectedOutput: map[string]any{"score": nil},
+		expectErr:      false,
+		expectWarnings: 1,
+	}, {
+		name: "Test partial conversion failure in array elements",
+		config: &DeserializationConfig{
+			OutputFields: map[string]*vo.TypeInfo{
+				"output": {
+					Type:         vo.DataTypeArray,
+					ElemTypeInfo: &vo.TypeInfo{Type: vo.DataTypeInteger},
+				},
+			},
+		},
+		inputJSON:      `["1", "two", 3]`,
+		expectedOutput: []any{1, 3},
+		expectErr:      false,
+		expectWarnings: 1,
+	}}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			processor, err := NewJsonDeserializer(ctx, tt.config)
+			assert.NoError(t, err)
+
+			ctxWithCache := ctxcache.Init(ctx)
+			input := map[string]any{"input": tt.inputJSON}
+			result, err := processor.Invoke(ctxWithCache, input)
+
+			if tt.expectErr {
+				assert.Error(t, err)
+				return
+			}
+
+			assert.NoError(t, err)
+			assert.Contains(t, result, OutputKeyDeserialization)
+
+			// Verify the output
+			output := result[OutputKeyDeserialization]
+			if tt.expectedOutput == nil {
+				assert.Nil(t, output)
+			} else {
+				// Serialize expected and actual output to JSON for comparison, ignoring type differences (e.g., float64 vs. int)
+				actualJSON, _ := sonic.Marshal(output)
+				expectedJSON, _ := sonic.Marshal(tt.expectedOutput)
+				assert.JSONEq(t, string(expectedJSON), string(actualJSON))
+			}
+
+			// Verify the number of warnings
+			warnings, _ := ctxcache.Get[nodes.ConversionWarnings](ctxWithCache, warningsKey)
+			assert.Equal(t, tt.expectWarnings, len(warnings))
+		})
+	}
+}

backend/domain/workflow/internal/nodes/json/json_serialization.go

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package json
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/coze-dev/coze-studio/backend/domain/workflow/entity/vo"
+	"github.com/coze-dev/coze-studio/backend/pkg/sonic"
+)
+
+const (
+	InputKeySerialization  = "input"
+	OutputKeySerialization = "output"
+)
+
+type SerializationConfig struct {
+	InputTypes map[string]*vo.TypeInfo
+}
+
+type JsonSerializer struct {
+	config *SerializationConfig
+}
+
+func NewJsonSerializer(_ context.Context, cfg *SerializationConfig) (*JsonSerializer, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("config required")
+	}
+	if cfg.InputTypes == nil {
+		return nil, fmt.Errorf("InputTypes is required for serialization")
+	}
+
+	return &JsonSerializer{
+		config: cfg,
+	}, nil
+}
+
+func (js *JsonSerializer) Invoke(_ context.Context, input map[string]any) (map[string]any, error) {
+	// Directly use the input map for serialization
+	if input == nil {
+		return nil, fmt.Errorf("input data for serialization cannot be nil")
+	}
+
+	originData := input[InputKeySerialization]
+	serializedData, err := sonic.Marshal(originData) // Serialize the entire input map
+	if err != nil {
+		return nil, fmt.Errorf("serialization error: %w", err)
+	}
+	return map[string]any{OutputKeySerialization: string(serializedData)}, nil
+}

backend/domain/workflow/internal/nodes/json/json_serialization_test.go

@@ -0,0 +1,134 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package json
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/coze-dev/coze-studio/backend/domain/workflow/entity/vo"
+)
+
+func TestNewJsonSerialize(t *testing.T) {
+	ctx := context.Background()
+
+	// Test with nil config
+	_, err := NewJsonSerializer(ctx, nil)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "config required")
+
+	// Test with missing InputTypes config
+	_, err = NewJsonSerializer(ctx, &SerializationConfig{})
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "InputTypes is required")
+
+	// Test with valid config
+	validConfig := &SerializationConfig{
+		InputTypes: map[string]*vo.TypeInfo{
+			"testKey": {Type: "string"},
+		},
+	}
+	processor, err := NewJsonSerializer(ctx, validConfig)
+	assert.NoError(t, err)
+	assert.NotNil(t, processor)
+}
+
+func TestJsonSerialize_Invoke(t *testing.T) {
+	ctx := context.Background()
+	config := &SerializationConfig{
+		InputTypes: map[string]*vo.TypeInfo{
+			"stringKey": {Type: "string"},
+			"intKey":    {Type: "integer"},
+			"boolKey":   {Type: "boolean"},
+			"objKey":    {Type: "object"},
+		},
+	}
+
+	processor, err := NewJsonSerializer(ctx, config)
+	assert.NoError(t, err)
+
+	// Test cases
+	tests := []struct {
+		name      string
+		input     map[string]any
+		expected  string
+		expectErr bool
+	}{{
+		name: "Test string serialization",
+		input: map[string]any{
+			"input": "test",
+		},
+		expected:  `"test"`,
+		expectErr: false,
+	}, {
+		name: "Test integer serialization",
+		input: map[string]any{
+			"input": 123,
+		},
+		expected:  `123`,
+		expectErr: false,
+	}, {
+		name: "Test boolean serialization",
+		input: map[string]any{
+			"input": true,
+		},
+		expected:  `true`,
+		expectErr: false,
+	}, {
+		name: "Test object serialization",
+		input: map[string]any{
+			"input": map[string]any{
+				"nestedKey": "nestedValue",
+			},
+		},
+		expected:  `{"nestedKey":"nestedValue"}`,
+		expectErr: false,
+	}, {
+		name:      "Test nil input",
+		input:     nil,
+		expected:  "",
+		expectErr: true,
+	}, {
+		name: "Test special character handling",
+		input: map[string]any{
+			"input": "\"test\"\nwith\twhitespace",
+		},
+		expected:  `"\"test\"\nwith\twhitespace"`,
+		expectErr: false,
+	}}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := processor.Invoke(ctx, tt.input)
+
+			if tt.expectErr {
+				assert.Error(t, err)
+				return
+			}
+
+			assert.NoError(t, err)
+			assert.Contains(t, result, OutputKeySerialization)
+
+			jsonStr, ok := result[OutputKeySerialization].(string)
+			assert.True(t, ok, "The output should be of type string")
+
+			assert.JSONEq(t, tt.expected, jsonStr)
+		})
+	}
+}