feat: manually mirror opencoze's code from bytedance

Change-Id: I09a73aadda978ad9511264a756b2ce51f5761adf

frontend/scripts/block-unresolved-conflict.sh

@@ -0,0 +1,55 @@
+#!/usr/bash
+
+block_unresolved_conflict() {
+  set -e
+  [ "$CI" = "true" ] && set -x
+  # git冲突标记符，一般为7个字符
+  local match="<<<<<<<|=======|>>>>>>>"
+  local diff_params="$1 --name-status -G $match"
+  local count=0
+
+  if [[ $1 == *..* ]]; then
+    # 检查分支是否存在，可以解决 merge 之后 feature 分支被 removed ，导致 git 报错的问题。
+    sourceBranch=${1%%..*}
+    targetBranch=${1#*..}
+    if ! git branch -a | grep -qE "$sourceBranch"; then
+      echo "branch do not exist: $sourceBranch"
+      return 0
+    fi
+    if ! git branch -a | grep -qE "$targetBranch"; then
+      echo "branch do not exist: $targetBranch"
+      return 0
+    fi
+  fi
+
+  # Specify the pattern you want to exclude
+  EXCLUDE_PATTERNS=(
+    'frontend/scripts/block-unresolved-conflict.sh'
+    'frontend/packages/arch/bot-api/src/auto-generate/**'
+    'frontend/packages/arch/idl/src/**'
+    'common/git-hooks/**'
+  )
+
+  for pattern in "${EXCLUDE_PATTERNS[@]}"; do
+    exclude_string+=":(exclude)$pattern "
+  done
+
+  diff_params+=" $exclude_string"
+
+  # 只检测修改的文件
+  conflicts=$(git diff $diff_params | grep '^M' | cut -f 2-)
+
+  if [[ -n "$conflicts" ]]; then
+    for conflict in $conflicts; do
+      if grep -Eq $match $conflict; then
+        echo $conflict
+        ((count++))
+      fi
+    done
+    if [[ $count -ne 0 ]]; then
+      echo "Unresolved merge conflicts in these files, please check"
+      exit 1
+    fi
+  fi
+  return 0
+}

frontend/scripts/card_builder_static_build.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# Switch cwd to the project folder
+cd $(dirname "$0")
+
+# Import the utilities functions
+source ./scm_base.sh
+
+# Clean up the build directory
+rm -rf dist output output_resource ${ROOT}/output ${ROOT}/output_resource
+
+# Prepare
+prepare_environment
+
+# Install the dependencies
+install_package_deps @flowpd/card-lynx-runtime
+
+SKIP_POST_INSTALL=true rush build -t @flowpd/card-lynx-runtime
+
+# Create artifact
+OUTPUT_DIR="${ROOT_DIR}/output"
+OUTPUT_RESOURCE_DIR="${ROOT_DIR}/output_resource"
+
+# copy lynx runtime sdk
+cp -r "${ROOT_DIR}"/packages/lynx-runtime-sdk/dist "${OUTPUT_DIR}"/lynx_sdk/"${CUSTOM_LYNX_VERSION}"
+cp -r "${ROOT_DIR}"/packages/lynx-runtime-sdk/dist "${OUTPUT_RESOURCE_DIR}"/lynx_sdk/"${CUSTOM_LYNX_VERSION}"
+
+# if [[ $CUSTOM_VERSION = 'release' ]]; then
+#   TARGETS=("sg" "va")
+#   SOURCES=("sg_release" "va_release")
+# else
+#   TARGETS=("boe" "cn" "sg")
+#   SOURCES=("cn_boe" "cn_inhouse" "sg_inhouse")
+# fi
+
+
+# # Loop through targets and sources
+# for ((i=0; i<${#TARGETS[@]}; i++)); do
+#   # Create output directories
+#   mkdir -p "${OUTPUT_DIR}/${TARGETS[i]}"
+#   mkdir -p "${OUTPUT_RESOURCE_DIR}/${TARGETS[i]}"
+
+#   # Copy index.html
+#   cp "./output/${SOURCES[i]}/"*.html "${OUTPUT_DIR}/${TARGETS[i]}"
+
+#   # Copy static files
+#   cp -r "./output/${SOURCES[i]}/"* "${OUTPUT_RESOURCE_DIR}/${TARGETS[i]}"
+# done

frontend/scripts/del-branch.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+git fetch
+
+# 要删除的远程分支完整名称列表，包含origin前缀
+declare -a branches_to_delete_full=(
+  origin/feat/analysis-tyy
+  origin/feat/query_classify
+)
+
+# 远程仓库的名称，默认为origin
+remote_name="origin"
+
+# 函数，用于删除远程分支
+delete_branch() {
+  local branch_name_with_origin=$1
+  # 去除origin前缀
+  local branch_name=$(echo "$branch_name_with_origin" | sed 's/^'"$remote_name"'\///')
+
+  if git show-ref --verify --quiet "refs/remotes/$branch_name_with_origin"; then
+    echo "正在删除远程分支: $branch_name"
+    git push "$remote_name" --delete "$branch_name" --no-verify
+    if [ $? -eq 0 ]; then
+      echo "远程分支 $branch_name 已被删除。"
+    else
+      echo "删除远程分支 $branch_name 失败，请检查分支名称或权限。"
+    fi
+  else
+    echo "分支 $branch_name 不存在，无需删除。"
+  fi
+}
+
+# 遍历分支列表并调用删除函数
+for full_branch_name in "${branches_to_delete_full[@]}"; do
+  delete_branch "$full_branch_name"
+done
+
+echo "批量删除操作完成。"

frontend/scripts/find-empty-files.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# 需要搜索的目录
+SEARCH_DIR=${1:-.}  # 默认目录为当前目录
+
+# 检查输入参数
+if [ -z "$SEARCH_DIR" ]; then
+  echo "Usage: $0 <search_directory>"
+  exit 1
+fi
+
+# 获取所有被 Git 跟踪的 .tsx 和 .less 文件
+git ls-files --others --ignored --exclude-standard -o -c -- "$SEARCH_DIR" ':!*.tsx' ':!*.less' | while read -r FILE; do
+  if [[ "$FILE" == *.tsx || "$FILE" == *.less ]]; then
+    # 获取文件行数
+    LINE_COUNT=$(wc -l < "$FILE")
+    # 如果文件行数为空，删除文件并输出文件路径
+    if [ "$LINE_COUNT" -eq 0 ]; then
+      echo "Deleting empty file: $FILE"
+      rm "$FILE"
+    fi
+  fi
+done

frontend/scripts/post-rush-install.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+ROOT_DIR=$(realpath "$SCRIPT_DIR/..")
+
+if [ "$CUSTOM_SKIP_POST_INSTALL" == "true" ]; then
+  exit 0
+fi
+
+# pushd $ROOT_DIR/packages/arch/i18n && npm run pull-i18n && popd || exit
+node $ROOT_DIR/common/scripts/install-run-rush.js pull-idl -a install || exit
+if [ "$NO_STARLING" != true ]; then
+  # 更新文案
+  pushd $ROOT_DIR/ee/infra/sync-scripts && npm run sync:starling && popd || exit
+  pushd $ROOT_DIR/ee/infra/sync-scripts && npm run sync:starling-cozeloop && popd || exit
+fi
+
+if [ "$CI" != "true" ]; then
+  node $ROOT_DIR/common/scripts/install-run-rush.js pre-build -o tag:phase-prebuild -v
+fi
+
+# if [ -z "$BUILD_TYPE" ]; then
+#   # 更新 icon
+#   pushd $ROOT_DIR/ee/infra/sync-scripts && npm run sync:icon && popd || exit
+#   pushd $ROOT_DIR/ee/infra/sync-scripts && npm run sync:illustration && popd || exit
+# fi

frontend/scripts/pre-push-hook.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# 设置红色颜色的 ANSI 转义码
+RED='\033[0;31m'
+# 重置颜色的 ANSI 转义码
+NC='\033[0m'
+
+CURRENT_USER=$(git config user.email)
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+CURRENT_ORIGIN_BRANCH=$(git rev-parse --abbrev-ref @{u})
+
+# if [[ -n "$CURRENT_ORIGIN_BRANCH" ]]; then
+#   block_unresolved_conflict "$CURRENT_BRANCH..$CURRENT_ORIGIN_BRANCH"
+# fi
+
+if [ "$CURRENT_BRANCH" = "master" ] && [ "$CURRENT_USER" != "ci_flow@bytedance.com" ]; then
+  echo "${RED}Do not push to master branch manually!!!${NC}"
+  exit 1
+fi
+
+if git status --porcelain | grep -q "pnpm-lock.yaml"; then
+  echo -e "${RED}Error: pnpm-lock.yaml is included in the changes. Please commit it before push!${NC}"
+  exit 1
+fi
+

frontend/scripts/publish-pro.sh

@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+# 该脚本用与在 scm 内执行 包发布逻辑
+
+source ./scripts/scm_base.sh
+
+# Prepare
+prepare_environment
+
+# ignore install
+export PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=true
+export CYPRESS_INSTALL_BINARY=0
+export TAIKO_SKIP_CHROMIUM_DOWNLOAD=0
+export RE2_DOWNLOAD_SKIP_PATH=1
+export RE2_DOWNLOAD_MIRROR=​https://bnpm.bytedance.net/mirrors
+
+# custom_env
+export NPM_AUTH_TOKEN=$CUSTOM_NPM_AUTH_TOKEN
+export GITLAB_TOKEN=$CUSTOM_GITLAB_TOKEN
+export CODEBASE_JWT=$CUSTOM_CODEBASE_JWT
+export COZE_HUB_APP_ID=$CUSTOM_COZE_HUB_APP_ID
+export COZE_HUB_KEY_ID=$CUSTOM_COZE_HUB_KEY_ID
+export COZE_HUB_PRIVATE_KEY=$CUSTOM_COZE_HUB_PRIVATE_KEY
+
+export DEBUG=$CUSTOM_DEBUG
+# resolve 400 error https://bytedance.feishu.cn/wiki/wikcnOUP2D6rGipIaN8q7UdzO0e
+export CI_NAME=allow_same_version_$BUILD_VERSION
+export DEFAULT_BRANCH=$BUILD_REPO_BRANCH
+export WEBHOOK_URL=$CUSTOM_WEBHOOK_URL
+# publish type example: beta or alpha
+PUBLISH_TYPE=$CUSTOM_PUBLISH_TYPE
+# use for npm tag
+PUBLISH_TAG=$CUSTOM_PUBLISH_TAG
+# only beta use for now
+PATCH_TAG=$CUSTOM_PATCH_TAG
+BY_DIFF=$CUSTOM_BY_DIFF
+
+# to packages
+TO_PACKAGES=$CUSTOM_TO_PACKAGES
+# from tag
+FROM_TAG=$CUSTOM_FROM_TAG
+# independent
+INDEPENDENT=$CUSTOM_INDEPENDENT
+
+# scm env
+BUILD_USER=$BUILD_USER
+# version or publiish or tag
+MODE=$CUSTOM_MODE
+# publish sha
+PUBLISH_SHA=$CUSTOM_PUBLISH_SHA
+# just list and detch not version
+LIST_ONLY=$CUSTOM_LIST_ONLY
+
+initialization() {
+  # npm
+  npm config set registry https://bnpm.byted.org
+  npm config set //bnpm.byted.org/:_authToken $NPM_AUTH_TOKEN
+  npm whoami
+
+  # git
+  git config user.name ci_flow
+  git config user.email ci_flow@bytedance.com
+  git remote set-url origin https://bot-studio-monorepo:$GITLAB_TOKEN@code.byted.org/obric/bot-studio-monorepo.git
+}
+
+fetch() {
+  git fetch --filter=blob:none --unshallow -q
+}
+
+initialization
+
+echo "to packages"
+echo $TO_PACKAGES
+
+echo "from tag"
+echo $FROM_TAG
+
+if [ -n "${PUBLISH_TAG}" ]; then
+  PUBLISH_TAG="--tag ${PUBLISH_TAG}"
+fi
+
+if [ -n "${PUBLISH_TYPE}" ]; then
+  PUBLISH_TYPE="--pre ${PUBLISH_TYPE}"
+fi
+
+if [ -n "${WEBHOOK_URL}" ]; then
+  WEBHOOK_URL="-w ${WEBHOOK_URL}"
+fi
+
+if [ -n "${BUILD_USER}" ]; then
+  BUILD_USER="-a ${BUILD_USER}"
+fi
+
+echo "安装依赖"
+node infra/rush-increment-install/lib/index.js
+node common/scripts/install-run-rush.js build -t @coze/cli -v
+
+if [ "$MODE" == "version" -o "$MODE" == "multiple" ]; then
+  echo "获取仓库 tags 列表"
+  fetch
+
+  echo "计算版本更新列表"
+  # Copy output to root dir
+  mkdir -p $ROOT_DIR/output
+  OUTPUT_COMMAND="node ee/infra/rush-x/bin/run version --list $PUBLISH_TYPE $PUBLISH_TAG $PATCH_TAG $TO_PACKAGES $FROM_TAG $BY_DIFF $INDEPENDENT -b $DEFAULT_BRANCH"
+  $OUTPUT_COMMAND >>$ROOT_DIR/output/output.txt
+
+  if [ "$LIST_ONLY" != "true" ]; then
+
+    echo "版本更新列表写入 output.txt"
+
+    echo "执行 version"
+    if [ "$MODE" == "multiple" ]; then
+      # multiple 模式下只输出 publish 步骤的结果即可
+      node ee/infra/rush-x/bin/run version $PUBLISH_TYPE $PUBLISH_TAG $PATCH_TAG $TO_PACKAGES $FROM_TAG $BY_DIFF $INDEPENDENT -b $DEFAULT_BRANCH
+    else
+      node ee/infra/rush-x/bin/run version $PUBLISH_TYPE $PUBLISH_TAG $PATCH_TAG $TO_PACKAGES $FROM_TAG $BY_DIFF $INDEPENDENT $WEBHOOK_URL $BUILD_USER -b $DEFAULT_BRANCH
+    fi
+    echo "version 完成"
+  fi
+fi
+
+if [ "$MODE" == "multiple" ]; then
+  PUBLISH_SHA=$(git rev-parse HEAD)
+fi
+
+if [ "$MODE" == "publish" -o "$MODE" == "multiple" ]; then
+  mkdir -p $ROOT_DIR/output
+  echo "Publish sha $PUBLISH_SHA"
+  echo "Publish sha $PUBLISH_SHA" >>$ROOT_DIR/output/output.txt
+
+  echo "执行 publish"
+  node ee/infra/rush-x/bin/run publish --sha $PUBLISH_SHA $PUBLISH_TAG $WEBHOOK_URL $BUILD_USER
+  echo "publish 完成"
+fi
+
+# 更新 distTag
+if [ "$MODE" == "tag" ]; then
+  fetch
+  mkdir -p $ROOT_DIR/output
+  echo "Publish sha $PUBLISH_SHA"
+  echo "Publish sha $PUBLISH_SHA" >>$ROOT_DIR/output/output.txt
+
+  echo "执行 dist tag"
+  # 不同模式下点 WEBHOOK_URL 基本不同，需要再传参时注意
+  node ee/infra/rush-x/bin/run publish --sha $PUBLISH_SHA $PUBLISH_TAG --dist-tag $WEBHOOK_URL $BUILD_USER
+  echo "dist-tag 完成"
+fi

frontend/scripts/retry.sh

@@ -0,0 +1,21 @@
+retry() {
+  local retries=$1   # 重试次数
+  local wait_time=$2 # 等待时间
+  shift 2
+  local count=0
+
+  cd $(pwd)
+
+  until "$@"; do
+    exit_code=$?
+    count=$((count + 1))
+    if [ $count -lt $retries ]; then
+      echo "Attempt $count/$retries failed with exit code $exit_code. Retrying in $wait_time seconds..."
+      sleep $wait_time
+    else
+      echo "Attempt $count/$retries failed with exit code $exit_code. No more retries left."
+      return $exit_code
+    fi
+  done
+  return 0
+}

frontend/scripts/scan-outdate-branch.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Define the cut-off date, 3 months from now
+cutoff=$(date -v-2m +%s)
+branches_to_delete=()
+outdated_branches_with_change=()
+
+echo "| 分支名 | commits 数量 | 作者 | 最后更新时间 | log |"
+echo "| ----- | ----- | ----- | ----- | ----- |"
+
+# Get all remote branches
+
+for branch in $(git for-each-ref refs/remotes/origin --format '%(refname:short)'); do
+  # Skip branches with 'release/' in their name
+  if [[ $branch == *'release/'* ]]; then
+    continue
+  fi
+
+  # Get the date of the last commit on the branch
+  last_commit=$(git log -1 --pretty=format:%ct $branch)
+
+  # If the last commit is older than the cutoff date, delete the branch
+  if [ $last_commit -lt $cutoff ]; then
+    # Check if there are commits that are not merged into master
+    if [ "$(git log origin/master..$branch)" != "" ]; then
+      commits=$(git log --pretty=format:'%an' origin/master..$branch)
+      echo "| $branch | $(echo "$commits" | wc -l) | $(echo "$(git show -s --format="%an | %ci | %s" origin/master..$branch )" | head -n 1) |"
+      outdated_branches_with_change+=("$branch")
+    else
+      branches_to_delete+=("$branch")
+    fi
+  fi
+done
+
+echo "these branches should be deleted:\n"
+
+for branch in "${branches_to_delete[@]}"
+do
+  echo "Deleting $branch"
+  git push origin --delete ${branch#origin/} --no-verify
+done
+
+echo "----------------------------------------------"
+
+# echo "these branches outdated but with not commit changes:\n"
+
+# for branch in "${outdated_branches_with_change[@]}"
+# do
+#   echo "$branch"
+# done
+

frontend/scripts/scm_base.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+ROOT_DIR=$(realpath "$SCRIPT_DIR/..")
+echo $ROOT_DIR
+RUSH_FILE="$ROOT_DIR/common/scripts/install-run-rush.js"
+
+source "$ROOT_DIR/scripts/setup-env.sh"
+
+# Fix https://code.byted.org/apaas/monorepo/issues/45
+# Use the latest git version from leafboat
+export PATH=/tmp/leafboat/bin:$PATH
+
+die() {
+  echo $1
+  exit 1
+}
+
+rushx() {
+  node $RUSH_FILE "$@"
+}
+
+rushtool () {
+  RUSH_X_PATH=$ROOT_DIR/ee/infra/rush-x/bin/run
+  node $RUSH_X_PATH "$@"
+}
+
+# Install multiple package dependencies at the same time.
+#
+# Usage:
+# install_package_deps @kunlun/typings @apaas/vite.app @apaas/react.app
+install_package_deps()
+{
+  if [ $# -eq 0 ]; then
+    die "install_package_deps: missing arguments"
+  fi
+
+  args=""
+  for pkg in "$@";
+  do
+    args="-t ${pkg} -t tag:rush-tools ${args}"
+  done
+  rushx --debug install ${args}
+}
+
+# Prepare the rush project environment.
+prepare_environment()
+{
+  npm i -g pnpm@8.15.8
+}
+
+
+# Install current project dependencies.
+# It will abort with errors if package.json is not found in the current directory.
+#
+# Usage:
+# install_project_deps
+install_project_deps()
+{
+  if [ ! -f package.json ]; then
+    die "install_project_deps: package.json not found"
+  fi
+  install_package_deps .
+}
+
+
+# Build current project.
+# It will abort with errors if package.json is not found in the current directory.
+#
+# Usage:
+# build_project
+build_project()
+{
+  if [ ! -f package.json ]; then
+    die "build_project: package.json not found"
+  fi
+  rushx build -t .
+}
+
+# Build current project with dependencies.
+# It will abort with errors if package.json is not found in the current directory.
+#
+# Usage:
+# build_project_deps
+build_project_deps()
+{
+  if [ ! -f package.json ]; then
+    die "build_project: package.json not found"
+  fi
+  rushx build -T .
+}
+
+# Force Build current project.
+# It will abort with errors if package.json is not found in the current directory.
+#
+# Usage:
+# build_project_ignore_cache
+build_project_ignore_cache()
+{
+  if [ ! -f package.json ]; then
+    die "build_project: package.json not found"
+  fi
+  rushx build -T . -v
+  rushx rebuild -o . -v
+}

frontend/scripts/setup-env.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# Setup common env for CI & SCM
+# 1. 忽略不影响构建的 install
+export PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=true
+export CYPRESS_INSTALL_BINARY=0
+export TAIKO_SKIP_CHROMIUM_DOWNLOAD=0
+export RE2_DOWNLOAD_SKIP_PATH=1
+export RE2_DOWNLOAD_MIRROR="​https://bnpm.bytedance.net/mirrors"
+
+# 2. 在 CI 环境生效：
+echo ::set-env name=PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD::$PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD
+echo ::set-env name=CYPRESS_INSTALL_BINARY::$CYPRESS_INSTALL_BINARY
+echo ::set-env name=TAIKO_SKIP_CHROMIUM_DOWNLOAD::$TAIKO_SKIP_CHROMIUM_DOWNLOAD
+echo ::set-env name=RE2_DOWNLOAD_SKIP_PATH::$RE2_DOWNLOAD_SKIP_PATH
+echo ::set-env name=RE2_DOWNLOAD_MIRROR::$RE2_DOWNLOAD_MIRROR

frontend/scripts/subdir-cloc.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# usage:
+# cd $path/to/monorepo
+# sh scripts/subdir-cloc.sh $relative_path/to/some/dir
+#
+# example: sh scripts/subdir-cloc.sh apps/bot/src/routes
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 directory"
+    exit 1
+fi
+
+directory=$1
+
+#首先创建或清空已有的结果文件
+echo "Directory, files,language,blank,comment,code" > cloc_results2.csv
+
+#使用find命令来查找foo目录下的所有子目录
+#如果你只想遍历直接子目录，可以去掉-maxdepth和-mindepth选项
+find $directory -type d -mindepth 1 -maxdepth 1 | while read subdir
+do
+    #使用cloc工具计算每个子目录的代码行数
+    #然后使用awk工具来提取需要的数据：目录名、文件数、代码行数
+    cloc_result=$(cloc $subdir --csv --quiet | tail -n 1)
+    lines=$(echo "$cloc_result" | awk -F "\"*,\"*" 'NR==3 {print $4}')
+    echo "$subdir, $cloc_result" >> cloc_results2.csv
+done