coze-studio/backend/domain/workflow/internal/nodes/httprequester/http_requester.go

/*
 * Copyright 2025 coze-dev Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package httprequester

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"maps"
	"mime/multipart"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/coze-dev/coze-studio/backend/domain/workflow/internal/nodes"
	"github.com/coze-dev/coze-studio/backend/pkg/lang/crypto"
	"github.com/coze-dev/coze-studio/backend/pkg/lang/ptr"
	"github.com/coze-dev/coze-studio/backend/pkg/sonic"
)

const defaultGetFileTimeout = 20       // second
const maxSize int64 = 20 * 1024 * 1024 // 20MB

const (
	HeaderAuthorization = "Authorization"
	HeaderBearerPrefix  = "Bearer "
	HeaderContentType   = "Content-Type"
)

type AuthType uint

const (
	BearToken AuthType = 1
	Custom    AuthType = 2
)

const (
	ContentTypeJSON           = "application/json"
	ContentTypePlainText      = "text/plain"
	ContentTypeFormURLEncoded = "application/x-www-form-urlencoded"
	ContentTypeBinary         = "application/octet-stream"
)

type Location uint8

const (
	Header     Location = 1
	QueryParam Location = 2
)

type BodyType string

const (
	BodyTypeNone           BodyType = "EMPTY"
	BodyTypeJSON           BodyType = "JSON"
	BodyTypeRawText        BodyType = "RAW_TEXT"
	BodyTypeFormData       BodyType = "FORM_DATA"
	BodyTypeFormURLEncoded BodyType = "FORM_URLENCODED"
	BodyTypeBinary         BodyType = "BINARY"
)

type URLConfig struct {
	Tpl string `json:"tpl"`
}

type IgnoreExceptionSetting struct {
	IgnoreException bool           `json:"ignore_exception"`
	DefaultOutput   map[string]any `json:"default_output,omitempty"`
}

type BodyConfig struct {
	BodyType        BodyType         `json:"body_type"`
	FormDataConfig  *FormDataConfig  `json:"form_data_config,omitempty"`
	TextPlainConfig *TextPlainConfig `json:"text_plain_config,omitempty"`
	TextJsonConfig  *TextJsonConfig  `json:"text_json_config,omitempty"`
}

type FormDataConfig struct {
	FileTypeMapping map[string]bool `json:"file_type_mapping"`
}

type TextPlainConfig struct {
	Tpl string `json:"tpl"`
}

type TextJsonConfig struct {
	Tpl string
}

type AuthenticationConfig struct {
	Type     AuthType `json:"type"`
	Location Location `json:"location"`
}

type Authentication struct {
	Key   string
	Value string
	Token string
}

type Request struct {
	URLVars            map[string]any
	Headers            map[string]string
	Params             map[string]string
	Authentication     *Authentication
	FormDataVars       map[string]string
	FormURLEncodedVars map[string]string
	JsonVars           map[string]any
	TextPlainVars      map[string]any
	FileURL            *string
}

var globalVariableReplaceRegexp = regexp.MustCompile(`global_variable_(\w+)\["(\w+)"\]`)

type MD5FieldMapping struct {
	HeaderMD5Mapping map[string]string `json:"header_md_5_mapping,omitempty"` // md5 vs key
	ParamMD5Mapping  map[string]string `json:"param_md_5_mapping,omitempty"`
	URLMD5Mapping    map[string]string `json:"url_md_5_mapping,omitempty"`
	BodyMD5Mapping   map[string]string `json:"body_md_5_mapping,omitempty"`
}

func (fm *MD5FieldMapping) SetHeaderFields(fields ...string) {
	if fm.HeaderMD5Mapping == nil && len(fields) > 0 {
		fm.HeaderMD5Mapping = make(map[string]string)
	}
	for _, field := range fields {
		fm.HeaderMD5Mapping[crypto.MD5HexValue(field)] = field
	}

}
func (fm *MD5FieldMapping) SetParamFields(fields ...string) {
	if fm.ParamMD5Mapping == nil && len(fields) > 0 {
		fm.ParamMD5Mapping = make(map[string]string)
	}

	for _, field := range fields {
		fm.ParamMD5Mapping[crypto.MD5HexValue(field)] = field
	}

}
func (fm *MD5FieldMapping) SetURLFields(fields ...string) {
	if fm.URLMD5Mapping == nil && len(fields) > 0 {
		fm.URLMD5Mapping = make(map[string]string)
	}
	for _, field := range fields {
		fm.URLMD5Mapping[crypto.MD5HexValue(field)] = field
	}

}
func (fm *MD5FieldMapping) SetBodyFields(fields ...string) {
	if fm.BodyMD5Mapping == nil && len(fields) > 0 {
		fm.BodyMD5Mapping = make(map[string]string)
	}
	for _, field := range fields {
		fm.BodyMD5Mapping[crypto.MD5HexValue(field)] = field
	}

}

type Config struct {
	URLConfig  URLConfig
	AuthConfig *AuthenticationConfig
	BodyConfig BodyConfig
	Method     string
	Timeout    time.Duration
	RetryTimes uint64

	IgnoreException bool
	DefaultOutput   map[string]any

	MD5FieldMapping
}

type HTTPRequester struct {
	client *http.Client
	config *Config
}

func NewHTTPRequester(_ context.Context, cfg *Config) (*HTTPRequester, error) {
	if cfg == nil {
		return nil, fmt.Errorf("config is requried")
	}

	if len(cfg.Method) == 0 {
		return nil, fmt.Errorf("method is requried")
	}

	hg := &HTTPRequester{}
	client := http.DefaultClient
	if cfg.Timeout > 0 {
		client.Timeout = cfg.Timeout
	}

	hg.client = client
	hg.config = cfg

	return hg, nil
}

func (hg *HTTPRequester) Invoke(ctx context.Context, input map[string]any) (output map[string]any, err error) {
	var (
		req         = &Request{}
		method      = hg.config.Method
		retryTimes  = hg.config.RetryTimes
		body        io.ReadCloser
		contentType string
		response    *http.Response
	)

	req, err = hg.config.parserToRequest(input)
	if err != nil {
		return nil, err
	}

	httpRequest := &http.Request{
		Method: method,
		Header: http.Header{},
	}

	httpURL, err := nodes.TemplateRender(hg.config.URLConfig.Tpl, req.URLVars)
	if err != nil {
		return nil, err
	}

	for key, value := range req.Headers {
		httpRequest.Header.Set(key, value)
	}

	u, err := url.Parse(httpURL)
	if err != nil {
		return nil, err
	}

	params := u.Query()
	for key, value := range req.Params {
		params.Set(key, value)
	}

	if hg.config.AuthConfig != nil {
		httpRequest.Header, params, err = hg.config.AuthConfig.addAuthentication(ctx, req.Authentication, httpRequest.Header, params)
		if err != nil {
			return nil, err
		}
	}
	u.RawQuery = params.Encode()
	httpRequest.URL = u

	body, contentType, err = hg.config.BodyConfig.getBodyAndContentType(ctx, req)
	if err != nil {
		return nil, err
	}
	if body != nil {
		httpRequest.Body = body
	}

	if contentType != "" {
		httpRequest.Header.Add(HeaderContentType, contentType)
	}

	for i := uint64(0); i <= retryTimes; i++ {
		response, err = hg.client.Do(httpRequest)
		if err == nil {
			break
		}
	}

	if err != nil {
		return nil, err
	}
	result := make(map[string]any)

	headers := func() string {
		// The structure of httpResp.Header is map[string][]string
		// If there are multiple header values, the last one will be selected by default
		hds := make(map[string]string, len(response.Header))
		for key, values := range response.Header {
			if len(values) == 0 {
				hds[key] = ""
			} else {
				hds[key] = values[len(values)-1]
			}
		}
		bs, _ := json.Marshal(hds)
		return string(bs)
	}()
	result["headers"] = headers
	var bodyBytes []byte

	if response.Body != nil {
		defer func() {
			_ = response.Body.Close()
		}()

		bodyBytes, err = io.ReadAll(response.Body)
		if err != nil {
			return nil, err
		}
	}

	if response.StatusCode >= http.StatusBadRequest {
		return nil, fmt.Errorf("request %v failed, response status code=%d, status=%v, headers=%v, body=%v",
			httpURL, response.StatusCode, response.Status, headers, string(bodyBytes))
	}

	result["body"] = string(bodyBytes)
	result["statusCode"] = int64(response.StatusCode)

	return result, nil
}

// decodeUnicode parses the Unicode escape sequence in the string
func decodeUnicode(s string) string {
	var result strings.Builder
	for i := 0; i < len(s); {
		if i+1 < len(s) && s[i] == '\\' && s[i+1] == 'u' {
			if i+6 <= len(s) {
				hexStr := s[i+2 : i+6]
				if code, err := strconv.ParseInt(hexStr, 16, 32); err == nil {
					result.WriteRune(rune(code))
					i += 6
					continue
				}
			}
		}
		result.WriteByte(s[i])
		i++
	}
	return result.String()
}

func (authCfg *AuthenticationConfig) addAuthentication(_ context.Context, auth *Authentication, header http.Header, params url.Values) (
	http.Header, url.Values, error) {

	if authCfg.Type == BearToken {
		header.Set(HeaderAuthorization, HeaderBearerPrefix+auth.Token)
		return header, params, nil
	}
	if authCfg.Type == Custom && authCfg.Location == Header {
		header.Set(auth.Key, auth.Value)
		return header, params, nil
	}

	if authCfg.Type == Custom && authCfg.Location == QueryParam {
		params.Set(auth.Key, auth.Value)
		return header, params, nil
	}

	return header, params, nil
}

func (b *BodyConfig) getBodyAndContentType(ctx context.Context, req *Request) (io.ReadCloser, string, error) {
	var (
		body        io.Reader
		contentType string
	)

	// body none return body nil
	if b.BodyType == BodyTypeNone {
		return nil, "", nil
	}

	switch b.BodyType {
	case BodyTypeJSON:
		jsonString, err := nodes.TemplateRender(b.TextJsonConfig.Tpl, req.JsonVars)
		if err != nil {
			return nil, contentType, err
		}
		body = strings.NewReader(jsonString)
		contentType = ContentTypeJSON
	case BodyTypeFormURLEncoded:
		form := url.Values{}
		for key, value := range req.FormURLEncodedVars {
			form.Add(key, value)
		}

		body = strings.NewReader(form.Encode())
		contentType = ContentTypeFormURLEncoded
	case BodyTypeRawText:
		textString, err := nodes.TemplateRender(b.TextPlainConfig.Tpl, req.TextPlainVars)
		if err != nil {
			return nil, contentType, err
		}

		body = strings.NewReader(textString)
		contentType = ContentTypePlainText
	case BodyTypeBinary:
		if req.FileURL == nil {
			return nil, contentType, fmt.Errorf("file url is required")
		}

		fileURL := *req.FileURL
		response, err := httpGet(ctx, fileURL)
		if err != nil {
			return nil, contentType, err
		}

		body = response.Body
		contentType = ContentTypeBinary
	case BodyTypeFormData:
		var buffer = &bytes.Buffer{}
		formDataConfig := b.FormDataConfig
		writer := multipart.NewWriter(buffer)

		total := int64(0)
		for key, value := range req.FormDataVars {
			if ok := formDataConfig.FileTypeMapping[key]; ok {
				fileWrite, err := writer.CreateFormFile(key, key)
				if err != nil {
					return nil, contentType, err
				}

				response, err := httpGet(ctx, value)
				if err != nil {
					return nil, contentType, err
				}

				if response.StatusCode != http.StatusOK {
					return nil, contentType, fmt.Errorf("failed to download file: %s, status code %v", value, response.StatusCode)
				}

				size, err := io.Copy(fileWrite, response.Body)
				if err != nil {
					return nil, contentType, err
				}

				total += size
				if total > maxSize {
					return nil, contentType, fmt.Errorf("too large body, total size: %d", total)
				}
			} else {
				err := writer.WriteField(key, value)
				if err != nil {
					return nil, contentType, err
				}
			}
		}

		_ = writer.Close()
		contentType = writer.FormDataContentType()
		body = buffer
	default:
		return nil, contentType, fmt.Errorf("unknown content type %s", b.BodyType)
	}

	if _, ok := body.(io.ReadCloser); ok {
		return body.(io.ReadCloser), contentType, nil
	}

	return io.NopCloser(body), contentType, nil
}

func httpGet(ctx context.Context, url string) (*http.Response, error) {
	request, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		return nil, err
	}

	http.DefaultClient.Timeout = time.Second * defaultGetFileTimeout
	return http.DefaultClient.Do(request)
}

func (hg *HTTPRequester) ToCallbackInput(_ context.Context, input map[string]any) (map[string]any, error) {
	var (
		request = &Request{}
		config  = hg.config
	)
	request, err := hg.config.parserToRequest(input)
	if err != nil {
		return nil, err
	}
	result := make(map[string]any)
	result["method"] = config.Method

	u, err := nodes.TemplateRender(config.URLConfig.Tpl, request.URLVars)
	if err != nil {
		return nil, err
	}
	result["url"] = u

	params := make(map[string]any, len(request.Params))
	for k, v := range request.Params {
		params[k] = v
	}
	result["param"] = params

	headers := make(map[string]any, len(request.Headers))
	for k, v := range request.Headers {
		headers[k] = v
	}
	result["header"] = headers
	result["auth"] = nil
	if config.AuthConfig != nil {
		if config.AuthConfig.Type == Custom {
			result["auth"] = map[string]interface{}{
				"Key":   request.Authentication.Key,
				"Value": request.Authentication.Value,
			}
		} else if config.AuthConfig.Type == BearToken {
			result["auth"] = map[string]interface{}{
				"token": request.Authentication.Token,
			}
		}
	}

	result["body"] = nil
	switch config.BodyConfig.BodyType {
	case BodyTypeJSON:
		js, err := nodes.TemplateRender(config.BodyConfig.TextJsonConfig.Tpl, request.JsonVars)
		if err != nil {
			return nil, err
		}
		ret := make(map[string]any)
		err = sonic.Unmarshal([]byte(js), &ret)
		if err != nil {
			return nil, err
		}
		result["body"] = ret
	case BodyTypeRawText:
		tx, err := nodes.TemplateRender(config.BodyConfig.TextPlainConfig.Tpl, request.TextPlainVars)
		if err != nil {

			return nil, err
		}
		result["body"] = tx
	case BodyTypeFormData:
		result["body"] = request.FormDataVars
	case BodyTypeFormURLEncoded:
		result["body"] = request.FormURLEncodedVars
	case BodyTypeBinary:
		result["body"] = request.FileURL

	}
	return result, nil
}

const (
	apiInfoURLPrefix = "__apiInfo_url_"
	headersPrefix    = "__headers_"
	paramsPrefix     = "__params_"

	authDataPrefix            = "__auth_authData_"
	authBearerTokenDataPrefix = "bearerTokenData_token"
	authCustomDataPrefix      = "customData_data"

	bodyDataPrefix           = "__body_bodyData_"
	bodyJsonPrefix           = "json_"
	bodyFormDataPrefix       = "formData_"
	bodyFormURLEncodedPrefix = "formURLEncoded_"
	bodyRawTextPrefix        = "rawText_"
	bodyBinaryFileURLPrefix  = "binary_fileURL"
)

func (cfg *Config) parserToRequest(input map[string]any) (*Request, error) {
	request := &Request{
		URLVars:            make(map[string]any),
		Headers:            make(map[string]string),
		Params:             make(map[string]string),
		Authentication:     &Authentication{},
		FormURLEncodedVars: make(map[string]string),
		JsonVars:           make(map[string]any),
		TextPlainVars:      make(map[string]any),
		FormDataVars:       map[string]string{},
	}
	for key, value := range input {
		if strings.HasPrefix(key, apiInfoURLPrefix) {
			urlMD5 := strings.TrimPrefix(key, apiInfoURLPrefix)
			if urlKey, ok := cfg.URLMD5Mapping[urlMD5]; ok {
				if strings.HasPrefix(urlKey, "global_variable_") {
					urlKey = globalVariableReplaceRegexp.ReplaceAllString(urlKey, "global_variable_$1.$2")
				}
				nodes.SetMapValue(request.URLVars, strings.Split(urlKey, "."), value.(string))
			}
		}
		if strings.HasPrefix(key, headersPrefix) {
			headerKeyMD5 := strings.TrimPrefix(key, headersPrefix)
			if headerKey, ok := cfg.HeaderMD5Mapping[headerKeyMD5]; ok {
				request.Headers[headerKey] = value.(string)
			}
		}
		if strings.HasPrefix(key, paramsPrefix) {
			paramKeyMD5 := strings.TrimPrefix(key, paramsPrefix)
			if paramKey, ok := cfg.ParamMD5Mapping[paramKeyMD5]; ok {
				request.Params[paramKey] = value.(string)
			}
		}

		if strings.HasPrefix(key, authDataPrefix) {
			authKey := strings.TrimPrefix(key, authDataPrefix)
			if strings.HasPrefix(authKey, authBearerTokenDataPrefix) {
				request.Authentication.Token = value.(string) // bear
			}
			if strings.HasPrefix(authKey, authCustomDataPrefix) {
				if key == "__auth_authData_customData_data_Key" {
					request.Authentication.Key = value.(string)
				}
				if key == "__auth_authData_customData_data_Value" {
					request.Authentication.Value = value.(string)
				}
			}
		}

		if strings.HasPrefix(key, bodyDataPrefix) {
			bodyKey := strings.TrimPrefix(key, bodyDataPrefix)
			if strings.HasPrefix(bodyKey, bodyJsonPrefix) {
				jsonMd5Key := strings.TrimPrefix(bodyKey, bodyJsonPrefix)
				if jsonKey, ok := cfg.BodyMD5Mapping[jsonMd5Key]; ok {
					if strings.HasPrefix(jsonKey, "global_variable_") {
						jsonKey = globalVariableReplaceRegexp.ReplaceAllString(jsonKey, "global_variable_$1.$2")
					}
					nodes.SetMapValue(request.JsonVars, strings.Split(jsonKey, "."), value)
				}

			}
			if strings.HasPrefix(bodyKey, bodyFormDataPrefix) {
				formDataMd5Key := strings.TrimPrefix(bodyKey, bodyFormDataPrefix)
				if formDataKey, ok := cfg.BodyMD5Mapping[formDataMd5Key]; ok {
					request.FormDataVars[formDataKey] = value.(string)
				}

			}

			if strings.HasPrefix(bodyKey, bodyFormURLEncodedPrefix) {
				formURLEncodeMd5Key := strings.TrimPrefix(bodyKey, bodyFormURLEncodedPrefix)
				if formURLEncodeKey, ok := cfg.BodyMD5Mapping[formURLEncodeMd5Key]; ok {
					request.FormURLEncodedVars[formURLEncodeKey] = value.(string)
				}
			}

			if strings.HasPrefix(bodyKey, bodyRawTextPrefix) {
				rawTextMd5Key := strings.TrimPrefix(bodyKey, bodyRawTextPrefix)
				if rawTextKey, ok := cfg.BodyMD5Mapping[rawTextMd5Key]; ok {
					if strings.HasPrefix(rawTextKey, "global_variable_") {
						rawTextKey = globalVariableReplaceRegexp.ReplaceAllString(rawTextKey, "global_variable_$1.$2")
					}
					nodes.SetMapValue(request.TextPlainVars, strings.Split(rawTextKey, "."), value)
				}
			}

			if strings.HasPrefix(bodyKey, bodyBinaryFileURLPrefix) {
				request.FileURL = ptr.Of(value.(string))
			}

		}

	}

	return request, nil
}

func (hg *HTTPRequester) ToCallbackOutput(_ context.Context, out map[string]any) (*nodes.StructuredCallbackOutput, error) {
	if body, ok := out["body"]; !ok {
		return &nodes.StructuredCallbackOutput{
			RawOutput: out,
			Output:    out,
		}, nil
	} else {
		output := maps.Clone(out)
		output["body"] = decodeUnicode(body.(string))
		return &nodes.StructuredCallbackOutput{
			RawOutput: out,
			Output:    output,
		}, nil
	}

}