feat: manually mirror opencoze's code from bytedance

Change-Id: I09a73aadda978ad9511264a756b2ce51f5761adf

frontend/packages/agent-ide/chat-background-shared/src/hooks/use-background-content.ts

@@ -0,0 +1,164 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+import { useShallow } from 'zustand/react/shallow';
+import { useRequest } from 'ahooks';
+import { GenerateType } from '@coze-studio/components';
+import { useBotInfoStore } from '@coze-studio/bot-detail-store/bot-info';
+import {
+  DotStatus,
+  useGenerateImageStore,
+} from '@coze-studio/bot-detail-store';
+import { PicType } from '@coze-arch/bot-api/playground_api';
+import { type BackgroundImageInfo } from '@coze-arch/bot-api/developer_api';
+import { PlaygroundApi } from '@coze-arch/bot-api';
+
+export interface UseBackgroundContentProps {
+  openConfig?: () => void;
+  setBackgroundImageInfoList?: (value: BackgroundImageInfo[]) => void;
+}
+
+const getShowDot = (imageDotStatus: DotStatus, gifDotStatus: DotStatus) =>
+  imageDotStatus !== DotStatus.None || gifDotStatus !== DotStatus.None;
+
+const getGeneratingType = (
+  imageDotStatus: DotStatus,
+  gifDotStatus: DotStatus,
+) =>
+  imageDotStatus === DotStatus.Generating
+    ? PicType.BackgroundStatic
+    : gifDotStatus === DotStatus.Generating
+    ? PicType.BackgroundGif
+    : undefined;
+
+export const useBackgroundContent = (props?: UseBackgroundContentProps) => {
+  const { openConfig, setBackgroundImageInfoList } = props ?? {};
+  const {
+    messageList,
+    imageDotStatus,
+    gifDotStatus,
+    setGenerateBackgroundModalByImmer,
+  } = useGenerateImageStore(
+    useShallow(state => ({
+      messageList: state.imageList,
+      imageDotStatus: state.generateBackGroundModal.image.dotStatus,
+      gifDotStatus: state.generateBackGroundModal.gif.dotStatus,
+      setGenerateBackgroundModalByImmer:
+        state.setGenerateBackgroundModalByImmer,
+      selectedImage: state.generateBackGroundModal.selectedImage,
+    })),
+  );
+
+  const botId = useBotInfoStore(s => s.botId);
+  //最新的 静图与动图 未读 - 生成中/成功/失败，展示原点状态
+  const showDot = getShowDot(imageDotStatus, gifDotStatus);
+
+  const hasDotType =
+    imageDotStatus !== DotStatus.None
+      ? PicType.BackgroundStatic
+      : PicType.BackgroundGif;
+
+  const generatingType = getGeneratingType(imageDotStatus, gifDotStatus);
+
+  const { runAsync: markReadNotice } = useRequest(
+    async () =>
+      await PlaygroundApi.MarkReadNotice({
+        pic_type: hasDotType,
+        bot_id: botId,
+      }),
+    {
+      manual: true,
+    },
+  );
+  const imageReadExpression = (status: DotStatus) =>
+    status !== DotStatus.None && status !== DotStatus.Generating;
+
+  const markRead = async () => {
+    if (showDot) {
+      setGenerateBackgroundModalByImmer(state => {
+        // 设置当前tab
+        state.activeKey =
+          imageDotStatus !== DotStatus.None
+            ? GenerateType.Static
+            : GenerateType.Gif;
+        // 设置已读状态：失败/成功需要设置已读，进行中/无状态 不需要
+        if (
+          imageReadExpression(imageDotStatus) &&
+          hasDotType === PicType.BackgroundStatic
+        ) {
+          state.image.dotStatus = DotStatus.None;
+        }
+        if (
+          imageReadExpression(gifDotStatus) &&
+          hasDotType === PicType.BackgroundGif
+        ) {
+          state.gif.dotStatus = DotStatus.None;
+        }
+      });
+
+      if (
+        imageReadExpression(imageDotStatus) ||
+        imageReadExpression(gifDotStatus)
+      ) {
+        await markReadNotice();
+      }
+    }
+  };
+
+  const handleEdit = () => {
+    // 打开编辑弹窗
+    openConfig?.();
+  };
+
+  const handleRemove = async () => {
+    // 存在进行中的任务时
+    if (generatingType) {
+      // 取消继续生成
+      const generatingTaskId = messageList.find(
+        item => item.type === generatingType,
+      )?.id;
+      await PlaygroundApi.CancelGenerateGif({
+        task_id: generatingTaskId,
+      });
+      setGenerateBackgroundModalByImmer(state => {
+        if (generatingType === PicType.BackgroundGif) {
+          state.gif.loading = false;
+          state.gif.dotStatus = DotStatus.None;
+        }
+        if (generatingType === PicType.BackgroundStatic) {
+          state.image.loading = false;
+          state.image.dotStatus = DotStatus.None;
+        }
+        state.generatingTaskId = '';
+      });
+    }
+    // 有状态的标记已读
+    await markRead();
+    // 清空当前渲染的背景图
+    setBackgroundImageInfoList?.([]);
+  };
+
+  const showDotStatus =
+    imageDotStatus !== DotStatus.None ? imageDotStatus : gifDotStatus;
+
+  return {
+    handleEdit,
+    showDot,
+    showDotStatus,
+    handleRemove,
+    markRead,
+  };
+};

frontend/packages/agent-ide/chat-background-shared/src/hooks/use-crop-image.ts

@@ -0,0 +1,174 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+import { type ReactCropperElement } from 'react-cropper';
+import { type RefObject, useState, useEffect, useRef } from 'react';
+
+import { ceil, floor } from 'lodash-es';
+import { MODE_CONFIG } from '@coze-common/chat-uikit';
+import {
+  type BackgroundImageDetail,
+  type GradientPosition,
+} from '@coze-arch/bot-api/developer_api';
+
+import { computePosition, getImageThemeColor } from '../utils';
+
+export const useCropperImg = ({
+  cropperRef,
+  url,
+  mode,
+  setLoading,
+  backgroundInfo,
+}: {
+  cropperRef: RefObject<ReactCropperElement>;
+  url: string;
+  mode: 'pc' | 'mobile';
+  setLoading: (loading: boolean) => void;
+  backgroundInfo?: BackgroundImageDetail;
+}) => {
+  const [gradientPosition, setGradientPosition] = useState<GradientPosition>({
+    left: 0,
+    right: 0,
+  });
+  const [themeColor, setThemeColor] = useState(
+    backgroundInfo?.theme_color ?? '#fff',
+  );
+  const currentUrl = useRef(url);
+  const { size, centerWidth } = MODE_CONFIG[mode];
+  useEffect(() => {
+    currentUrl.current = url;
+    if (!url) {
+      setThemeColor('#fff');
+    }
+    handleGradientPosition();
+  }, [url]);
+
+  // 设置最大缩放比例
+  const onZoom = () => {
+    const {
+      width = 0,
+      height = 0,
+      naturalWidth = 0,
+      naturalHeight = 0,
+    } = cropperRef.current?.cropper?.getCanvasData() ?? {};
+
+    if (naturalWidth > naturalHeight) {
+      if (height >= size.height * 2) {
+        cropperRef.current?.cropper.setCanvasData({
+          height: size.width * 2,
+        });
+      }
+    } else {
+      if (width > size.width * 2) {
+        cropperRef.current?.cropper.setCanvasData({
+          width: size.width * 2,
+        });
+      }
+    }
+    // TODO:因没有缩放end事件，缩放实时获取主题色大图卡顿严重，故此场景临时先不获取主题色，修改交互or尝试webworker解决此问题
+    // await handleThemeColor();
+  };
+
+  const handleGradientPosition = () => {
+    const position = computePosition(mode, cropperRef);
+    setGradientPosition(position);
+  };
+
+  const handleDragLimit = (y: number) => {
+    const cropperObj = cropperRef?.current?.cropper;
+
+    if (!cropperObj) {
+      return;
+    }
+    const canvasData = cropperObj?.getCanvasData();
+    const imgData = cropperObj?.getImageData();
+    // 图片下边缘 距离 裁剪区下边缘的偏移距离
+    const scaleTop = imgData.height + canvasData.top - size.height;
+    // 图片左边缘 距离 裁剪区左 边缘的偏移距离
+    const scaleLeft = ceil(imgData.left + canvasData?.left, 2);
+
+    // 图片上下拖拽不能有超出图片容器外
+    if (y < 0) {
+      cropperRef.current?.cropper.setCanvasData({
+        top: 0,
+      });
+    }
+    if (scaleTop < 0) {
+      cropperRef.current?.cropper.setCanvasData({
+        top: size.height - imgData.height,
+      });
+    }
+
+    // 产品需求： 左右拖动不能超过 固定“对话气泡容器” left or right 80%
+    const maxRightOffset = floor(
+      (size.width - centerWidth) / 2 + centerWidth * 0.4,
+      2,
+    );
+    const maxLeftOffset = floor(size.width - imgData.width - maxRightOffset, 2);
+    if (scaleLeft > maxRightOffset || scaleLeft < maxLeftOffset) {
+      cropperRef.current?.cropper.setCanvasData({
+        left: scaleLeft > maxRightOffset ? maxRightOffset : maxLeftOffset,
+      });
+    }
+  };
+
+  const handleCrop = (detail: Cropper.CropEvent) => {
+    handleGradientPosition();
+    handleDragLimit(detail.detail.y);
+  };
+
+  const cropEnd = async () => {
+    await handleThemeColor();
+    handleGradientPosition();
+  };
+
+  const handleThemeColor = async () => {
+    const cropperObj = cropperRef.current?.cropper;
+    // 大图move卡顿优化方案：move停止时获取到主题色前 禁止移动
+    cropperObj?.disable();
+    // 为了加载快一些，设置图片质量中等
+    const corp = cropperObj?.getCroppedCanvas()?.toDataURL('image/webp', 0.7);
+    if (corp) {
+      const color = await getImageThemeColor(corp);
+      setThemeColor(color);
+      cropperObj?.enable();
+    }
+  };
+
+  const handleReady = async () => {
+    const cropperObj = cropperRef.current?.cropper;
+    if (
+      backgroundInfo?.canvas_position &&
+      currentUrl.current === backgroundInfo.origin_image_url
+    ) {
+      cropperObj?.setCanvasData(backgroundInfo?.canvas_position);
+    }
+    await handleThemeColor();
+
+    setLoading(false);
+  };
+
+  return {
+    gradientPosition,
+    handleReady,
+    handleThemeColor,
+    handleCrop,
+    cropEnd,
+    onZoom,
+    themeColor,
+    size,
+  };
+};

frontend/packages/agent-ide/chat-background-shared/src/hooks/use-drag-image.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+import type React from 'react';
+import { type DragEventHandler, useRef, useState } from 'react';
+
+const checkHasFileOnDrag = (e: React.DragEvent<HTMLDivElement>) =>
+  Boolean(e.dataTransfer?.types.includes('Files'));
+
+export const useDragImage = () => {
+  const timer = useRef<ReturnType<typeof setTimeout> | null>(null);
+
+  const [isDragIn, setIsDragIn] = useState(false);
+
+  const clearTimer = () => {
+    if (!timer.current) {
+      return;
+    }
+    clearTimeout(timer.current);
+    timer.current = null;
+  };
+  const onDragEnter: DragEventHandler<HTMLDivElement> = e => {
+    clearTimer();
+    if (!checkHasFileOnDrag(e)) {
+      return;
+    }
+    setIsDragIn(true);
+  };
+
+  const onDragEnd = () => {
+    clearTimer();
+    timer.current = setTimeout(() => {
+      setIsDragIn(false);
+    }, 100);
+  };
+
+  const onDragOver: DragEventHandler<HTMLDivElement> = e => {
+    e.preventDefault();
+    clearTimer();
+    if (!checkHasFileOnDrag(e)) {
+      return;
+    }
+    setIsDragIn(true);
+  };
+
+  return {
+    isDragIn,
+    setIsDragIn,
+    onDragEnter,
+    onDragEnd,
+    onDragOver,
+  };
+};

frontend/packages/agent-ide/chat-background-shared/src/hooks/use-submit-cropped-image.ts

@@ -0,0 +1,162 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+import { type ReactCropperElement } from 'react-cropper';
+import { useRef, type RefObject } from 'react';
+
+import { logger } from '@coze-arch/logger';
+import { type FileItem } from '@coze-arch/bot-semi/Upload';
+import {
+  type BackgroundImageDetail,
+  type BackgroundImageInfo,
+} from '@coze-arch/bot-api/developer_api';
+import { useBotInfoAuditor } from '@coze-studio/bot-audit-adapter';
+
+import { canvasPosition, computePosition, computeThemeColor } from '../utils';
+import { useUploadImage } from './use-upload-img';
+
+export interface SubmitCroppedImageParams {
+  cropperWebRef: RefObject<ReactCropperElement>;
+  cropperMobileRef: RefObject<ReactCropperElement>;
+  setLoading: (loading: boolean) => void;
+  getUserId: () => {
+    userId: string;
+  };
+  onSuccess: (value: BackgroundImageInfo[]) => void;
+  currentOriginImage: Partial<FileItem>;
+  handleCancel: () => void;
+  onAuditCheck: (notPass: boolean) => void;
+}
+
+export interface FileValue {
+  uri: string;
+  url: string;
+}
+
+interface GetBackgroundInfoItemParams {
+  mode: 'pc' | 'mobile';
+  originImageInfo: {
+    origin_image_uri: string;
+    origin_image_url: string;
+  };
+  themeColorList: string[];
+}
+export const useSubmitCroppedImage = ({
+  cropperWebRef,
+  cropperMobileRef,
+  setLoading,
+  getUserId,
+  onSuccess,
+  currentOriginImage,
+  handleCancel,
+  onAuditCheck,
+}: SubmitCroppedImageParams) => {
+  const fileList = useRef<File[]>([]);
+  const { check } = useBotInfoAuditor();
+
+  const getBackgroundInfoItem = ({
+    mode,
+    originImageInfo,
+    themeColorList,
+  }: GetBackgroundInfoItemParams): BackgroundImageDetail => {
+    const cropperRef = mode === 'pc' ? cropperWebRef : cropperMobileRef;
+    return {
+      ...originImageInfo,
+      theme_color: mode === 'pc' ? themeColorList[0] : themeColorList[1],
+      gradient_position: computePosition(mode, cropperRef),
+      canvas_position: canvasPosition(cropperRef),
+    };
+  };
+
+  const handleUploadAllSuccess = async (croppedImageList?: FileValue[]) => {
+    setLoading(false);
+
+    const themeColorList = await computeThemeColor([
+      cropperWebRef,
+      cropperMobileRef,
+    ]);
+
+    if (!currentOriginImage?.url) {
+      return;
+    }
+
+    const originImageInfo = {
+      origin_image_uri: croppedImageList?.[0]?.uri || currentOriginImage.uri,
+      origin_image_url: croppedImageList?.[0]?.url || currentOriginImage.url,
+    };
+    const info = {
+      themeColorList,
+      originImageInfo,
+    };
+    const backgroundImageList = [
+      {
+        web_background_image: getBackgroundInfoItem({
+          mode: 'pc',
+          ...info,
+        }),
+        mobile_background_image: getBackgroundInfoItem({
+          mode: 'mobile',
+          ...info,
+        }),
+      },
+    ];
+    fileList.current = [];
+
+    const res = await check({
+      background_images_struct: backgroundImageList?.[0],
+    });
+    const notPass = Boolean(res?.check_not_pass);
+    onAuditCheck(notPass);
+    if (notPass) {
+      // 机审未通过，就不执行下面回调
+      return;
+    }
+
+    onSuccess(backgroundImageList);
+    handleCancel();
+  };
+  const { uploadFileList } = useUploadImage({
+    onUploadAllSuccess: handleUploadAllSuccess,
+    getUserId: () => getUserId().userId,
+    onUploadError: () => {
+      setLoading(false);
+    },
+    onAuditError: () => onAuditCheck(true),
+  });
+
+  const handleSubmit = () => {
+    setLoading(true);
+
+    try {
+      if (
+        currentOriginImage?.fileInstance &&
+        currentOriginImage?.fileInstance instanceof File
+      ) {
+        fileList.current = [currentOriginImage.fileInstance];
+        uploadFileList(fileList.current);
+      } else {
+        // 回填文件时 不需要存原图
+        handleUploadAllSuccess();
+      }
+    } catch (error) {
+      if (error instanceof Error) {
+        logger.error({ error });
+      }
+    }
+  };
+
+  return { handleSubmit };
+};

frontend/packages/agent-ide/chat-background-shared/src/hooks/use-upload-img.ts

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+import { useEffect, useRef } from 'react';
+
+import { nanoid } from 'nanoid';
+import { withSlardarIdButton } from '@coze-studio/bot-utils';
+import { I18n } from '@coze-arch/i18n';
+import { Toast } from '@coze-arch/bot-semi';
+
+// import { type UploadState } from '../type';
+
+import { UploadController } from '../service/upload-controller';
+
+export const useUploadImage = ({
+  getUserId,
+  onUploadError,
+  onUploadAllSuccess,
+  onAuditError,
+}: {
+  getUserId: () => string;
+  onUploadAllSuccess: (param: { url: string; uri: string }[]) => void;
+  onUploadError: () => void;
+  onAuditError?: () => void;
+}) => {
+  const uploadControllerMap = useRef<Record<string, UploadController>>({});
+
+  const deleteUploadControllerById = (id: string) => {
+    delete uploadControllerMap.current[id];
+  };
+  const cancelUploadById = (id: string) => {
+    const controller = uploadControllerMap.current[id];
+    if (!controller) {
+      return;
+    }
+    controller.cancel();
+    deleteUploadControllerById(id);
+  };
+
+  const handleError = (_e: unknown, controllerId: string) => {
+    cancelUploadById(controllerId);
+    onUploadError();
+    Toast.error({
+      content: withSlardarIdButton(I18n.t('Upload_failed')),
+      showClose: false,
+    });
+  };
+
+  const onAuditFailed = () => {
+    if (onAuditError) {
+      onAuditError();
+    } else {
+      Toast.error({
+        content: I18n.t('inappropriate_contents'),
+        showClose: false,
+      });
+    }
+    onUploadError();
+  };
+
+  const uploadFileList = (fileList: File[]) => {
+    const controllerId = nanoid();
+
+    if (!fileList.length) {
+      return;
+    }
+
+    uploadControllerMap.current[controllerId] = new UploadController({
+      fileList,
+      controllerId,
+      userId: getUserId(),
+      onComplete: event => {
+        onUploadAllSuccess(event);
+      },
+      onUploadError: handleError,
+      onGetTokenError: handleError,
+      onGetUploadInstanceError: handleError,
+      onAuditFailed,
+    });
+  };
+
+  const clearAllSideEffect = () => {
+    Object.entries(uploadControllerMap.current).forEach(([, controller]) =>
+      controller.cancel(),
+    );
+    uploadControllerMap.current = {};
+  };
+
+  useEffect(() => clearAllSideEffect, []);
+
+  return {
+    uploadFileList,
+  };
+};