feat: manually mirror opencoze's code from bytedance

Change-Id: I09a73aadda978ad9511264a756b2ce51f5761adf

backend/domain/workflow/internal/nodes/httprequester/http_requester.go

@@ -0,0 +1,684 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package httprequester
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"maps"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/coze-dev/coze-studio/backend/domain/workflow/internal/nodes"
+	"github.com/coze-dev/coze-studio/backend/pkg/lang/crypto"
+	"github.com/coze-dev/coze-studio/backend/pkg/lang/ptr"
+	"github.com/coze-dev/coze-studio/backend/pkg/sonic"
+)
+
+const defaultGetFileTimeout = 20       // second
+const maxSize int64 = 20 * 1024 * 1024 // 20MB
+
+const (
+	HeaderAuthorization = "Authorization"
+	HeaderBearerPrefix  = "Bearer "
+	HeaderContentType   = "Content-Type"
+)
+
+type AuthType uint
+
+const (
+	BearToken AuthType = 1
+	Custom    AuthType = 2
+)
+
+const (
+	ContentTypeJSON           = "application/json"
+	ContentTypePlainText      = "text/plain"
+	ContentTypeFormURLEncoded = "application/x-www-form-urlencoded"
+	ContentTypeBinary         = "application/octet-stream"
+)
+
+type Location uint8
+
+const (
+	Header     Location = 1
+	QueryParam Location = 2
+)
+
+type BodyType string
+
+const (
+	BodyTypeNone           BodyType = "EMPTY"
+	BodyTypeJSON           BodyType = "JSON"
+	BodyTypeRawText        BodyType = "RAW_TEXT"
+	BodyTypeFormData       BodyType = "FORM_DATA"
+	BodyTypeFormURLEncoded BodyType = "FORM_URLENCODED"
+	BodyTypeBinary         BodyType = "BINARY"
+)
+
+type URLConfig struct {
+	Tpl string `json:"tpl"`
+}
+
+type IgnoreExceptionSetting struct {
+	IgnoreException bool           `json:"ignore_exception"`
+	DefaultOutput   map[string]any `json:"default_output,omitempty"`
+}
+
+type BodyConfig struct {
+	BodyType        BodyType         `json:"body_type"`
+	FormDataConfig  *FormDataConfig  `json:"form_data_config,omitempty"`
+	TextPlainConfig *TextPlainConfig `json:"text_plain_config,omitempty"`
+	TextJsonConfig  *TextJsonConfig  `json:"text_json_config,omitempty"`
+}
+
+type FormDataConfig struct {
+	FileTypeMapping map[string]bool `json:"file_type_mapping"`
+}
+
+type TextPlainConfig struct {
+	Tpl string `json:"tpl"`
+}
+
+type TextJsonConfig struct {
+	Tpl string
+}
+
+type AuthenticationConfig struct {
+	Type     AuthType `json:"type"`
+	Location Location `json:"location"`
+}
+
+type Authentication struct {
+	Key   string
+	Value string
+	Token string
+}
+
+type Request struct {
+	URLVars            map[string]any
+	Headers            map[string]string
+	Params             map[string]string
+	Authentication     *Authentication
+	FormDataVars       map[string]string
+	FormURLEncodedVars map[string]string
+	JsonVars           map[string]any
+	TextPlainVars      map[string]any
+	FileURL            *string
+}
+
+var globalVariableReplaceRegexp = regexp.MustCompile(`global_variable_(\w+)\["(\w+)"\]`)
+
+type MD5FieldMapping struct {
+	HeaderMD5Mapping map[string]string `json:"header_md_5_mapping,omitempty"` // md5 vs key
+	ParamMD5Mapping  map[string]string `json:"param_md_5_mapping,omitempty"`
+	URLMD5Mapping    map[string]string `json:"url_md_5_mapping,omitempty"`
+	BodyMD5Mapping   map[string]string `json:"body_md_5_mapping,omitempty"`
+}
+
+func (fm *MD5FieldMapping) SetHeaderFields(fields ...string) {
+	if fm.HeaderMD5Mapping == nil && len(fields) > 0 {
+		fm.HeaderMD5Mapping = make(map[string]string)
+	}
+	for _, field := range fields {
+		fm.HeaderMD5Mapping[crypto.MD5HexValue(field)] = field
+	}
+
+}
+func (fm *MD5FieldMapping) SetParamFields(fields ...string) {
+	if fm.ParamMD5Mapping == nil && len(fields) > 0 {
+		fm.ParamMD5Mapping = make(map[string]string)
+	}
+
+	for _, field := range fields {
+		fm.ParamMD5Mapping[crypto.MD5HexValue(field)] = field
+	}
+
+}
+func (fm *MD5FieldMapping) SetURLFields(fields ...string) {
+	if fm.URLMD5Mapping == nil && len(fields) > 0 {
+		fm.URLMD5Mapping = make(map[string]string)
+	}
+	for _, field := range fields {
+		fm.URLMD5Mapping[crypto.MD5HexValue(field)] = field
+	}
+
+}
+func (fm *MD5FieldMapping) SetBodyFields(fields ...string) {
+	if fm.BodyMD5Mapping == nil && len(fields) > 0 {
+		fm.BodyMD5Mapping = make(map[string]string)
+	}
+	for _, field := range fields {
+		fm.BodyMD5Mapping[crypto.MD5HexValue(field)] = field
+	}
+
+}
+
+type Config struct {
+	URLConfig  URLConfig
+	AuthConfig *AuthenticationConfig
+	BodyConfig BodyConfig
+	Method     string
+	Timeout    time.Duration
+	RetryTimes uint64
+
+	IgnoreException bool
+	DefaultOutput   map[string]any
+
+	MD5FieldMapping
+}
+
+type HTTPRequester struct {
+	client *http.Client
+	config *Config
+}
+
+func NewHTTPRequester(_ context.Context, cfg *Config) (*HTTPRequester, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("config is requried")
+	}
+
+	if len(cfg.Method) == 0 {
+		return nil, fmt.Errorf("method is requried")
+	}
+
+	hg := &HTTPRequester{}
+	client := http.DefaultClient
+	if cfg.Timeout > 0 {
+		client.Timeout = cfg.Timeout
+	}
+
+	hg.client = client
+	hg.config = cfg
+
+	return hg, nil
+}
+
+func (hg *HTTPRequester) Invoke(ctx context.Context, input map[string]any) (output map[string]any, err error) {
+	var (
+		req         = &Request{}
+		method      = hg.config.Method
+		retryTimes  = hg.config.RetryTimes
+		body        io.ReadCloser
+		contentType string
+		response    *http.Response
+	)
+
+	req, err = hg.config.parserToRequest(input)
+	if err != nil {
+		return nil, err
+	}
+
+	httpRequest := &http.Request{
+		Method: method,
+		Header: http.Header{},
+	}
+
+	httpURL, err := nodes.TemplateRender(hg.config.URLConfig.Tpl, req.URLVars)
+	if err != nil {
+		return nil, err
+	}
+
+	for key, value := range req.Headers {
+		httpRequest.Header.Set(key, value)
+	}
+
+	u, err := url.Parse(httpURL)
+	if err != nil {
+		return nil, err
+	}
+
+	params := u.Query()
+	for key, value := range req.Params {
+		params.Set(key, value)
+	}
+
+	if hg.config.AuthConfig != nil {
+		httpRequest.Header, params, err = hg.config.AuthConfig.addAuthentication(ctx, req.Authentication, httpRequest.Header, params)
+		if err != nil {
+			return nil, err
+		}
+	}
+	u.RawQuery = params.Encode()
+	httpRequest.URL = u
+
+	body, contentType, err = hg.config.BodyConfig.getBodyAndContentType(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	if body != nil {
+		httpRequest.Body = body
+	}
+
+	if contentType != "" {
+		httpRequest.Header.Add(HeaderContentType, contentType)
+	}
+
+	for i := uint64(0); i <= retryTimes; i++ {
+		response, err = hg.client.Do(httpRequest)
+		if err == nil {
+			break
+		}
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	result := make(map[string]any)
+
+	headers := func() string {
+		// The structure of httpResp.Header is map[string][]string
+		// If there are multiple header values, the last one will be selected by default
+		hds := make(map[string]string, len(response.Header))
+		for key, values := range response.Header {
+			if len(values) == 0 {
+				hds[key] = ""
+			} else {
+				hds[key] = values[len(values)-1]
+			}
+		}
+		bs, _ := json.Marshal(hds)
+		return string(bs)
+	}()
+	result["headers"] = headers
+	var bodyBytes []byte
+
+	if response.Body != nil {
+		defer func() {
+			_ = response.Body.Close()
+		}()
+
+		bodyBytes, err = io.ReadAll(response.Body)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if response.StatusCode >= http.StatusBadRequest {
+		return nil, fmt.Errorf("request %v failed, response status code=%d, status=%v, headers=%v, body=%v",
+			httpURL, response.StatusCode, response.Status, headers, string(bodyBytes))
+	}
+
+	result["body"] = string(bodyBytes)
+	result["statusCode"] = int64(response.StatusCode)
+
+	return result, nil
+}
+
+// decodeUnicode parses the Unicode escape sequence in the string
+func decodeUnicode(s string) string {
+	var result strings.Builder
+	for i := 0; i < len(s); {
+		if i+1 < len(s) && s[i] == '\\' && s[i+1] == 'u' {
+			if i+6 <= len(s) {
+				hexStr := s[i+2 : i+6]
+				if code, err := strconv.ParseInt(hexStr, 16, 32); err == nil {
+					result.WriteRune(rune(code))
+					i += 6
+					continue
+				}
+			}
+		}
+		result.WriteByte(s[i])
+		i++
+	}
+	return result.String()
+}
+
+func (authCfg *AuthenticationConfig) addAuthentication(_ context.Context, auth *Authentication, header http.Header, params url.Values) (
+	http.Header, url.Values, error) {
+
+	if authCfg.Type == BearToken {
+		header.Set(HeaderAuthorization, HeaderBearerPrefix+auth.Token)
+		return header, params, nil
+	}
+	if authCfg.Type == Custom && authCfg.Location == Header {
+		header.Set(auth.Key, auth.Value)
+		return header, params, nil
+	}
+
+	if authCfg.Type == Custom && authCfg.Location == QueryParam {
+		params.Set(auth.Key, auth.Value)
+		return header, params, nil
+	}
+
+	return header, params, nil
+}
+
+func (b *BodyConfig) getBodyAndContentType(ctx context.Context, req *Request) (io.ReadCloser, string, error) {
+	var (
+		body        io.Reader
+		contentType string
+	)
+
+	// body none return body nil
+	if b.BodyType == BodyTypeNone {
+		return nil, "", nil
+	}
+
+	switch b.BodyType {
+	case BodyTypeJSON:
+		jsonString, err := nodes.TemplateRender(b.TextJsonConfig.Tpl, req.JsonVars)
+		if err != nil {
+			return nil, contentType, err
+		}
+		body = strings.NewReader(jsonString)
+		contentType = ContentTypeJSON
+	case BodyTypeFormURLEncoded:
+		form := url.Values{}
+		for key, value := range req.FormURLEncodedVars {
+			form.Add(key, value)
+		}
+
+		body = strings.NewReader(form.Encode())
+		contentType = ContentTypeFormURLEncoded
+	case BodyTypeRawText:
+		textString, err := nodes.TemplateRender(b.TextPlainConfig.Tpl, req.TextPlainVars)
+		if err != nil {
+			return nil, contentType, err
+		}
+
+		body = strings.NewReader(textString)
+		contentType = ContentTypePlainText
+	case BodyTypeBinary:
+		if req.FileURL == nil {
+			return nil, contentType, fmt.Errorf("file url is required")
+		}
+
+		fileURL := *req.FileURL
+		response, err := httpGet(ctx, fileURL)
+		if err != nil {
+			return nil, contentType, err
+		}
+
+		body = response.Body
+		contentType = ContentTypeBinary
+	case BodyTypeFormData:
+		var buffer = &bytes.Buffer{}
+		formDataConfig := b.FormDataConfig
+		writer := multipart.NewWriter(buffer)
+
+		total := int64(0)
+		for key, value := range req.FormDataVars {
+			if ok := formDataConfig.FileTypeMapping[key]; ok {
+				fileWrite, err := writer.CreateFormFile(key, key)
+				if err != nil {
+					return nil, contentType, err
+				}
+
+				response, err := httpGet(ctx, value)
+				if err != nil {
+					return nil, contentType, err
+				}
+
+				if response.StatusCode != http.StatusOK {
+					return nil, contentType, fmt.Errorf("failed to download file: %s, status code %v", value, response.StatusCode)
+				}
+
+				size, err := io.Copy(fileWrite, response.Body)
+				if err != nil {
+					return nil, contentType, err
+				}
+
+				total += size
+				if total > maxSize {
+					return nil, contentType, fmt.Errorf("too large body, total size: %d", total)
+				}
+			} else {
+				err := writer.WriteField(key, value)
+				if err != nil {
+					return nil, contentType, err
+				}
+			}
+		}
+
+		_ = writer.Close()
+		contentType = writer.FormDataContentType()
+		body = buffer
+	default:
+		return nil, contentType, fmt.Errorf("unknown content type %s", b.BodyType)
+	}
+
+	if _, ok := body.(io.ReadCloser); ok {
+		return body.(io.ReadCloser), contentType, nil
+	}
+
+	return io.NopCloser(body), contentType, nil
+}
+
+func httpGet(ctx context.Context, url string) (*http.Response, error) {
+	request, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	http.DefaultClient.Timeout = time.Second * defaultGetFileTimeout
+	return http.DefaultClient.Do(request)
+}
+
+func (hg *HTTPRequester) ToCallbackInput(_ context.Context, input map[string]any) (map[string]any, error) {
+	var (
+		request = &Request{}
+		config  = hg.config
+	)
+	request, err := hg.config.parserToRequest(input)
+	if err != nil {
+		return nil, err
+	}
+	result := make(map[string]any)
+	result["method"] = config.Method
+
+	u, err := nodes.TemplateRender(config.URLConfig.Tpl, request.URLVars)
+	if err != nil {
+		return nil, err
+	}
+	result["url"] = u
+
+	params := make(map[string]any, len(request.Params))
+	for k, v := range request.Params {
+		params[k] = v
+	}
+	result["param"] = params
+
+	headers := make(map[string]any, len(request.Headers))
+	for k, v := range request.Headers {
+		headers[k] = v
+	}
+	result["header"] = headers
+	result["auth"] = nil
+	if config.AuthConfig != nil {
+		if config.AuthConfig.Type == Custom {
+			result["auth"] = map[string]interface{}{
+				"Key":   request.Authentication.Key,
+				"Value": request.Authentication.Value,
+			}
+		} else if config.AuthConfig.Type == BearToken {
+			result["auth"] = map[string]interface{}{
+				"token": request.Authentication.Token,
+			}
+		}
+	}
+
+	result["body"] = nil
+	switch config.BodyConfig.BodyType {
+	case BodyTypeJSON:
+		js, err := nodes.TemplateRender(config.BodyConfig.TextJsonConfig.Tpl, request.JsonVars)
+		if err != nil {
+			return nil, err
+		}
+		ret := make(map[string]any)
+		err = sonic.Unmarshal([]byte(js), &ret)
+		if err != nil {
+			return nil, err
+		}
+		result["body"] = ret
+	case BodyTypeRawText:
+		tx, err := nodes.TemplateRender(config.BodyConfig.TextPlainConfig.Tpl, request.TextPlainVars)
+		if err != nil {
+
+			return nil, err
+		}
+		result["body"] = tx
+	case BodyTypeFormData:
+		result["body"] = request.FormDataVars
+	case BodyTypeFormURLEncoded:
+		result["body"] = request.FormURLEncodedVars
+	case BodyTypeBinary:
+		result["body"] = request.FileURL
+
+	}
+	return result, nil
+}
+
+const (
+	apiInfoURLPrefix = "__apiInfo_url_"
+	headersPrefix    = "__headers_"
+	paramsPrefix     = "__params_"
+
+	authDataPrefix            = "__auth_authData_"
+	authBearerTokenDataPrefix = "bearerTokenData_token"
+	authCustomDataPrefix      = "customData_data"
+
+	bodyDataPrefix           = "__body_bodyData_"
+	bodyJsonPrefix           = "json_"
+	bodyFormDataPrefix       = "formData_"
+	bodyFormURLEncodedPrefix = "formURLEncoded_"
+	bodyRawTextPrefix        = "rawText_"
+	bodyBinaryFileURLPrefix  = "binary_fileURL"
+)
+
+func (cfg *Config) parserToRequest(input map[string]any) (*Request, error) {
+	request := &Request{
+		URLVars:            make(map[string]any),
+		Headers:            make(map[string]string),
+		Params:             make(map[string]string),
+		Authentication:     &Authentication{},
+		FormURLEncodedVars: make(map[string]string),
+		JsonVars:           make(map[string]any),
+		TextPlainVars:      make(map[string]any),
+		FormDataVars:       map[string]string{},
+	}
+	for key, value := range input {
+		if strings.HasPrefix(key, apiInfoURLPrefix) {
+			urlMD5 := strings.TrimPrefix(key, apiInfoURLPrefix)
+			if urlKey, ok := cfg.URLMD5Mapping[urlMD5]; ok {
+				if strings.HasPrefix(urlKey, "global_variable_") {
+					urlKey = globalVariableReplaceRegexp.ReplaceAllString(urlKey, "global_variable_$1.$2")
+				}
+				nodes.SetMapValue(request.URLVars, strings.Split(urlKey, "."), value.(string))
+			}
+		}
+		if strings.HasPrefix(key, headersPrefix) {
+			headerKeyMD5 := strings.TrimPrefix(key, headersPrefix)
+			if headerKey, ok := cfg.HeaderMD5Mapping[headerKeyMD5]; ok {
+				request.Headers[headerKey] = value.(string)
+			}
+		}
+		if strings.HasPrefix(key, paramsPrefix) {
+			paramKeyMD5 := strings.TrimPrefix(key, paramsPrefix)
+			if paramKey, ok := cfg.ParamMD5Mapping[paramKeyMD5]; ok {
+				request.Params[paramKey] = value.(string)
+			}
+		}
+
+		if strings.HasPrefix(key, authDataPrefix) {
+			authKey := strings.TrimPrefix(key, authDataPrefix)
+			if strings.HasPrefix(authKey, authBearerTokenDataPrefix) {
+				request.Authentication.Token = value.(string) // bear
+			}
+			if strings.HasPrefix(authKey, authCustomDataPrefix) {
+				if key == "__auth_authData_customData_data_Key" {
+					request.Authentication.Key = value.(string)
+				}
+				if key == "__auth_authData_customData_data_Value" {
+					request.Authentication.Value = value.(string)
+				}
+			}
+		}
+
+		if strings.HasPrefix(key, bodyDataPrefix) {
+			bodyKey := strings.TrimPrefix(key, bodyDataPrefix)
+			if strings.HasPrefix(bodyKey, bodyJsonPrefix) {
+				jsonMd5Key := strings.TrimPrefix(bodyKey, bodyJsonPrefix)
+				if jsonKey, ok := cfg.BodyMD5Mapping[jsonMd5Key]; ok {
+					if strings.HasPrefix(jsonKey, "global_variable_") {
+						jsonKey = globalVariableReplaceRegexp.ReplaceAllString(jsonKey, "global_variable_$1.$2")
+					}
+					nodes.SetMapValue(request.JsonVars, strings.Split(jsonKey, "."), value)
+				}
+
+			}
+			if strings.HasPrefix(bodyKey, bodyFormDataPrefix) {
+				formDataMd5Key := strings.TrimPrefix(bodyKey, bodyFormDataPrefix)
+				if formDataKey, ok := cfg.BodyMD5Mapping[formDataMd5Key]; ok {
+					request.FormDataVars[formDataKey] = value.(string)
+				}
+
+			}
+
+			if strings.HasPrefix(bodyKey, bodyFormURLEncodedPrefix) {
+				formURLEncodeMd5Key := strings.TrimPrefix(bodyKey, bodyFormURLEncodedPrefix)
+				if formURLEncodeKey, ok := cfg.BodyMD5Mapping[formURLEncodeMd5Key]; ok {
+					request.FormURLEncodedVars[formURLEncodeKey] = value.(string)
+				}
+			}
+
+			if strings.HasPrefix(bodyKey, bodyRawTextPrefix) {
+				rawTextMd5Key := strings.TrimPrefix(bodyKey, bodyRawTextPrefix)
+				if rawTextKey, ok := cfg.BodyMD5Mapping[rawTextMd5Key]; ok {
+					if strings.HasPrefix(rawTextKey, "global_variable_") {
+						rawTextKey = globalVariableReplaceRegexp.ReplaceAllString(rawTextKey, "global_variable_$1.$2")
+					}
+					nodes.SetMapValue(request.TextPlainVars, strings.Split(rawTextKey, "."), value)
+				}
+			}
+
+			if strings.HasPrefix(bodyKey, bodyBinaryFileURLPrefix) {
+				request.FileURL = ptr.Of(value.(string))
+			}
+
+		}
+
+	}
+
+	return request, nil
+}
+
+func (hg *HTTPRequester) ToCallbackOutput(_ context.Context, out map[string]any) (*nodes.StructuredCallbackOutput, error) {
+	if body, ok := out["body"]; !ok {
+		return &nodes.StructuredCallbackOutput{
+			RawOutput: out,
+			Output:    out,
+		}, nil
+	} else {
+		output := maps.Clone(out)
+		output["body"] = decodeUnicode(body.(string))
+		return &nodes.StructuredCallbackOutput{
+			RawOutput: out,
+			Output:    output,
+		}, nil
+	}
+
+}