feat: manually mirror opencoze's code from bytedance

Change-Id: I09a73aadda978ad9511264a756b2ce51f5761adf

.codebase/pipelines/backend.yml

@@ -0,0 +1,73 @@
+name: "ci@backend pipeline"
+
+trigger:
+  change:
+    paths:
+      - "backend/**"
+
+jobs:
+  unit_test:
+    image: hub.byted.org/codebase/ci_go_1_24:latest
+    name: Unit Test
+    steps:
+      - id: mysql_scripts
+        name: mysql scripts
+        uses: actions/mysql-scripts@v1
+        inputs:
+          db_name: opencoze
+          host: mysql
+          path: "docker/atlas/migrations/*.sql"
+          user: root
+          password: root
+      - name: Go Test
+        uses: actions/codecov
+        inputs:
+          driver: go
+          commands:
+            - modules=`find . -name "go.mod" -exec dirname {} \;`
+            - echo $modules
+            - list=""
+            - coverpkg=""
+            - if [[ ! -f "go.work" ]];then go work init;fi
+            - for module in $modules; do go work use $module; list=$module"/... "$list; coverpkg=$module"/...,"$coverpkg; done
+            - go work sync
+            - go test -coverprofile=coverage.out -gcflags="all=-l -N" -coverpkg=$coverpkg $list
+          config:
+            status:
+              project:
+                opencoze:
+                  minimum_coverage: 0%
+                  paths:
+                    - "!tests"
+                    - "!examples/**"
+                    - "!*/examples/**"
+                    - "!*/mock/**"
+                    - "!*/model/**"
+              diff:
+                eino:
+                  line_limit: 10 # 增量行数少于多少行时，默认置成功
+                  minimum_coverage: 0%
+                  paths:
+                    - "!tests"
+                    - "!examples/**"
+                    - "!*/examples/**"
+                    - "!*/mock/**"
+                    - "!*/model/**"
+      - name: Sonar
+        uses: actions/sonar@v1
+        inputs:
+          coverage_path: /tmp/coverage.out
+          coverage_type: go
+          disable_quality_gates: true
+          exclusions_list:
+            - "!tests"
+            - "examples/**"
+            - "*/mock/**"
+            - "*/model/**"
+          language: go
+    services:
+      - id: mysql
+        image: hub.byted.org/ee/mysql:8.0
+        envs:
+          MYSQL_ROOT_PASSWORD: root
+          MYSQL_DATABASE: opencoze

.codebase/pipelines/ci@fe-master.yaml

@@ -0,0 +1,180 @@
+name: CI@fe-master
+trigger:
+  cron:
+  push:
+    branches: []
+    paths:
+      - "frontend/**"
+    types: [branch]
+    notification:
+      when: failure
+      # 搜索群名：Bot Studio Monorepo CI 报警群
+      to: ['7350862409475031044']
+  manual:
+    notification:
+      when: failure
+
+jobs:
+  lint: &base_job
+    runs-on:
+      env: online
+    name: Lint
+    image: hub.byted.org/base/bot_monorepo_ci_env:ae543e9bbc6d8155cffbd8f5ed27fb73
+    # 从最近的 CI 执行记录看，install 步骤有概率会等待超时
+    # 目前已 oncall，这里先设置一个超时时间，避免阻塞
+    timeout: 30
+    envs:
+      RUSH_BUILD_CACHE_WRITE_ALLOWED: ${{RUSH_BUILD_CACHE_WRITE_ALLOWED}}
+      RUSH_BUILD_CACHE_ENABLED: ${{RUSH_BUILD_CACHE_ENABLED}}
+      RUSH_BUILD_CACHE_CREDENTIAL: ${{RUSH_BUILD_CACHE_CREDENTIAL}}
+      ACCESS_TOKEN: ${{CI_BOT_ACCESS_TOKEN}}
+      CI: 'true'
+      RUN_BYTEST_COV: 'false' # disable Jupiter default behavior of invoke bytestcov
+      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 'true'
+      CYPRESS_INSTALL_BINARY: '0'
+      TAIKO_SKIP_CHROMIUM_DOWNLOAD: '0'
+      RE2_DOWNLOAD_SKIP_PATH: '1'
+      RE2_DOWNLOAD_MIRROR: https://bnpm.bytedance.net/mirrors
+      # https://typescript-eslint.io/packages/parser/#allowautomaticsingleruninference
+      TSESTREE_SINGLE_RUN: 'true'
+    # 使用云盘缓存：https://bytedance.larkoffice.com/wiki/A60lwgolSinteMkOz71cFPvFneg
+    # TODO:不同job的缓存key暂时使用同一个，后续可以考虑区分
+    caches:
+      - backend: ebs
+        key: bot-studio-monorepo-master-v2-lint
+        size: 256
+    steps:
+      - &checkout_step
+        id: Checkout
+        uses: actions/checkout
+        inputs:
+          depth: 1
+      - &proxy_step
+        name: SetupProxy
+        commands:
+          - bash .codebase/scripts/env.sh
+      - &init_env_step
+        name: Initialization
+        commands:
+          - printenv
+          - git config user.name ci_flow
+          - git config user.email ci_flow@bytedance.com
+      - &install_deps_step
+        name: Install dependencies
+        commands:
+          - node common/scripts/install-run-rush.js install
+      # 暂时不启用，因为eslintignore文件移除会导致解析报错，后期接入可考虑动态读取eslint ignore配置
+      # - name: PreLint
+      #   parallel-with-next-step: true
+      #   commands:
+      #     - npx oxlint@latest -A all .
+      - name: Lint
+        commands:
+          - NODE_OPTIONS="--max-old-space-size=4096" node common/scripts/install-run-rush.js lint
+
+  ts_check:
+    <<: *base_job
+    name: Check TS Type
+    timeout: 30
+    caches:
+      - backend: ebs
+        key: bot-studio-monorepo-master-v2-ts-check
+        size: 256
+    steps:
+      - *checkout_step
+      - *proxy_step
+      - *init_env_step
+      - *install_deps_step
+      - name: Check TS Type
+        commands:
+          - node common/scripts/install-run-rush.js ts-check -v
+
+  build:
+    <<: *base_job
+    name: Build
+    caches:
+      - backend: ebs
+        key: bot-studio-monorepo-master-v2-build
+        size: 256
+    steps:
+      - *checkout_step
+      - *proxy_step
+      - *init_env_step
+      - *install_deps_step
+      - name: Build
+        commands:
+          - CUSTOM_VERSION=release REGION=sg BUILD_BRANCH=test node common/scripts/install-run-rush.js build -v
+
+  test:
+    <<: *base_job
+    caches:
+      - backend: ebs
+        key: bot-studio-monorepo-master-v2-test
+        size: 256
+    name: Test
+    services:
+      # 下面启动一个mysql和mongo服务，主要用于coze.hub.core相关服务的集成测试
+      # id需要保持跟代码中连接的host一致
+      - id: hub_mysql
+        image: hub.byted.org/ee/mysql:5.7
+        envs:
+          MYSQL_ROOT_PASSWORD: test
+          MYSQL_DATABASE: coze_hub
+          MYSQL_USER: test
+          MYSQL_PASSWORD: test
+          MYSQL_TCP_PORT: 3306
+      - id: hub_mongo
+        image: hub.byted.org/ee/mongo:4.0.24
+        commands:
+          - mongod --replSet rs0
+      - id: mongo-init
+        image: hub.byted.org/ee/mongo:4.0.24
+        commands:
+          - sleep 10s
+          - mongo mongodb://127.0.0.1:27017 --eval 'rs.initiate()'
+    steps:
+      - *checkout_step
+      - *proxy_step
+      - *init_env_step
+      - *install_deps_step
+      - name: wait mongo ready
+        uses: actions/mongodb-scripts
+        inputs:
+          host: 127.0.0.1
+          port: 27017
+      - name: wait mysql ready
+        uses: actions/mysql-scripts
+        inputs:
+          host: hub_mysql
+          user: test
+          password: test
+        commands:
+          - mysql -u test -ptest -h hub_mysql -P 3306 -e 'show databases;'
+      # 服务发现
+      - id: consul
+        uses: actions/setup-consul
+      - name: Test Coverage
+        id: test
+        commands:
+          - node common/scripts/install-run-rush.js test:cov -v
+      # https://bytedance.feishu.cn/wiki/wikcn9Z4azEdfJalVPKcOs9vxsR
+
+  security_scan:
+    <<: *base_job
+    name: Security Scan
+    caches:
+      - backend: ebs
+        key: bot-studio-monorepo-master-v2-scan
+        size: 256
+    steps:
+      - *checkout_step
+      - *proxy_step
+      - *init_env_step
+      - *install_deps_step
+      - name: build
+        commands:
+          - CUSTOM_VERSION=release REGION=sg node common/scripts/install-run-rush.js build -o app-botstudio-main -v
+      - id: Argus
+        commands:
+          - npm i @ies/argus-scan@0.30.37 -g
+          - bash .codebase/scripts/argus-scan.sh

.codebase/pipelines/ci@fe-tsc.yaml

@@ -0,0 +1,89 @@
+name: CI@fe-tsc
+trigger:
+  manual:
+  change:
+    # 临时支持 project-ide 项目需求，临时修改为如下格式
+    # 后续需调整回 integration/**
+    types: [create, push, restore]
+    paths:
+      - "frontend/**"
+    notification:
+      when: failure
+
+jobs:
+  ts_check: &base_job
+    runs-on:
+      env: online
+    image: hub.byted.org/base/bot_monorepo_ci_env:ae543e9bbc6d8155cffbd8f5ed27fb73
+    # 从最近的 CI 执行记录看，install 步骤有概率会等待超时
+    # 目前已 oncall，这里先设置一个超时时间，避免阻塞
+    timeout: 30
+    if: ${{ !Event.Change.IsPreSubmit }}
+    envs:
+      targetBranch: ${{Event.Change.Target.Branch}}
+      RUSH_BUILD_CACHE_WRITE_ALLOWED: ${{RUSH_BUILD_CACHE_WRITE_ALLOWED}}
+      RUSH_BUILD_CACHE_ENABLED: ${{RUSH_BUILD_CACHE_ENABLED}}
+      RUSH_BUILD_CACHE_CREDENTIAL: ${{RUSH_BUILD_CACHE_CREDENTIAL}}
+      ACCESS_TOKEN: ${{CI_BOT_ACCESS_TOKEN}}
+      CI: 'true'
+      RUN_BYTEST_COV: 'false'
+      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 'true'
+      CYPRESS_INSTALL_BINARY: '0'
+      TAIKO_SKIP_CHROMIUM_DOWNLOAD: '0'
+      RE2_DOWNLOAD_SKIP_PATH: '1'
+      RE2_DOWNLOAD_MIRROR: https://bnpm.bytedance.net/mirrors
+      MERGE_REQUEST_TITLE: ${{Event.Change.Title}}
+      TSESTREE_SINGLE_RUN: 'true'
+    steps:
+      - &checkout_step
+        name: Checkout
+        uses: actions/checkout
+        inputs:
+          submodules: true
+          depth: 1
+
+      - &proxy_step
+        name: SetupProxy
+        commands:
+          - bash .codebase/scripts/env.sh
+
+      - &remote_info_step
+        id: RemoteInfo
+        uses: 'actions/mr-remote-info'
+        inputs:
+          info_list: ['behind_commits', 'changed_files', 'changed_files_path']
+
+      - &cache_step
+        id: Cache
+        uses: actions/cache
+        inputs:
+          key: flow-monorepo-${{Event.Change.Source.Branch}}
+          paths:
+            - common/temp/pnpm-store
+          restore_keys:
+            - flow-monorepo-${{Event.Change.Source.Branch}}
+            - flow-monorepo-store-master
+
+      - &init_env_step
+        id: InitEnv
+        name: Initialization
+        commands:
+          - printenv
+          - git config user.name ci_flow
+          - git config user.email ci_flow@bytedance.com
+          - npm config set registry=https://registry.npmjs.org
+          - pnpm config set network-concurrency 32
+          - echo "$(<${{Steps.RemoteInfo.Outputs.changed_files_path}})"
+          - echo "::set-output name=ShouldRunBuild::${{ Event.Change.IsPreSubmit || int(Steps.RemoteInfo.Outputs.behind_commits) <= 6 }}"
+
+      - &install_deps_step
+        name: Install dependencies
+        commands:
+          - npx why-is-node-running@v2.x common/scripts/install-run-rush.js increment --action install -p ${{Steps.RemoteInfo.Outputs.changed_files_path}}
+
+      - name: Prepare basic packages
+        commands:
+          - node common/scripts/install-run-rush.js pre-build -v
+      - name: Check TS Type
+        commands:
+          - node common/scripts/install-run-rush.js increment --action ts-check -p ${{Steps.RemoteInfo.Outputs.changed_files_path}}

.codebase/pipelines/ci@fe.yml

@@ -0,0 +1,184 @@
+name: CI@fe
+trigger:
+  manual:
+  change:
+    source-branches: ['!release/**', '!task/**']
+    types: [create, push, restore]
+    paths:
+      - "frontend/**"
+    notification:
+      when: failure
+
+jobs:
+  lint: &base_job
+    runs-on:
+      env: online
+    image: hub.byted.org/base/bot_monorepo_ci_env:ae543e9bbc6d8155cffbd8f5ed27fb73
+    # 从最近的 CI 执行记录看，install 步骤有概率会等待超时
+    # 目前已 oncall，这里先设置一个超时时间，避免阻塞
+    timeout: 30
+    if: ${{ !Event.Change.IsPreSubmit }}
+    envs:
+      targetBranch: ${{Event.Change.Target.Branch}}
+      RUSH_BUILD_CACHE_WRITE_ALLOWED: ${{RUSH_BUILD_CACHE_WRITE_ALLOWED}}
+      RUSH_BUILD_CACHE_ENABLED: ${{RUSH_BUILD_CACHE_ENABLED}}
+      RUSH_BUILD_CACHE_CREDENTIAL: ${{RUSH_BUILD_CACHE_CREDENTIAL}}
+      ACCESS_TOKEN: ${{CI_BOT_ACCESS_TOKEN}}
+      CI: 'true'
+      RUN_BYTEST_COV: 'false'
+      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 'true'
+      CYPRESS_INSTALL_BINARY: '0'
+      TAIKO_SKIP_CHROMIUM_DOWNLOAD: '0'
+      BUILD_BRANCH: ${{Event.Change.Source.Branch}}
+      RE2_DOWNLOAD_SKIP_PATH: '1'
+      MERGE_REQUEST_TITLE: ${{Event.Change.Title}}
+      TSESTREE_SINGLE_RUN: 'true'
+    caches:
+      - backend: ebs
+        key: bot-studio-monorepo-ci-v2-lint
+        size: 256
+    steps:
+      - &checkout_step
+        name: Checkout
+        uses: actions/checkout
+        inputs:
+          submodules: true
+          depth: 1
+
+      - &proxy_step
+        name: SetupProxy
+        commands:
+          - bash .codebase/scripts/env.sh
+
+      - &remote_info_step
+        id: RemoteInfo
+        uses: 'actions/mr-remote-info'
+        inputs:
+          info_list: ['behind_commits', 'changed_files', 'changed_files_path']
+
+      - &cache_step
+        id: Cache
+        uses: actions/cache
+        inputs:
+          key: flow-monorepo-${{Event.Change.Source.Branch}}
+          paths:
+            - common/temp/pnpm-store
+            - packages/arch/idl/src/auto-generated
+          restore_keys:
+            - flow-monorepo-${{Event.Change.Source.Branch}}
+            - flow-monorepo-store-master
+
+      - &init_env_step
+        id: InitEnv
+        name: Initialization
+        commands:
+          - printenv
+          - git config user.name ci_flow
+          - git config user.email ci_flow@bytedance.com
+          - npm config set registry=https://registry.npmjs.org
+          - pnpm config set network-concurrency 32
+          - echo "$(<${{Steps.RemoteInfo.Outputs.changed_files_path}})"
+          - echo "::set-output name=ShouldRunBuild::${{ Event.Change.IsPreSubmit || int(Steps.RemoteInfo.Outputs.behind_commits) <= 6 }}"
+
+      - &install_deps_step
+        name: Install dependencies
+        commands:
+          - npx why-is-node-running@v2.x common/scripts/install-run-rush.js increment --action install -p ${{Steps.RemoteInfo.Outputs.changed_files_path}}
+
+      - name: Check Lint
+        commands:
+          - node common/scripts/install-run-rush.js increment --action lint -p ${{Steps.RemoteInfo.Outputs.changed_files_path}}
+
+  style_check:
+    <<: *base_job
+    caches:
+      - backend: ebs
+        key: bot-studio-monorepo-ci-v2-style-check
+        size: 256
+    name: Check Stylelint
+    steps:
+      - *checkout_step
+      - *proxy_step
+      - *remote_info_step
+      - *cache_step
+      - *init_env_step
+      - *install_deps_step
+      - name: Check Stylelint
+        commands:
+          - node common/scripts/install-run-rush.js increment --action style -p ${{Steps.RemoteInfo.Outputs.changed_files_path}}
+
+  build:
+    <<: *base_job
+    caches:
+      - backend: ebs
+        key: bot-studio-monorepo-ci-v2-build
+        size: 256
+    name: Build
+    timeout: 20
+    steps:
+      - *checkout_step
+      - *proxy_step
+      - *remote_info_step
+      - *cache_step
+      - *init_env_step
+      - *install_deps_step
+      - name: Build
+        id: build
+        if: ${{ Steps.InitEnv.Outputs.ShouldRunBuild }}
+        commands:
+          - node common/scripts/install-run-rush.js increment --action build -p ${{Steps.RemoteInfo.Outputs.changed_files_path}}
+
+  test:
+    <<: *base_job
+    caches:
+      - backend: ebs
+        key: bot-studio-monorepo-ci-v2-test
+        size: 256
+    name: Test Coverage
+    services:
+      # 下面启动一个mysql和mongo服务，主要用于coze.hub.core相关服务的集成测试
+      # id需要保持跟代码中连接的host一致
+      - id: hub_mysql
+        image: hub.byted.org/ee/mysql:5.7
+        envs:
+          MYSQL_ROOT_PASSWORD: test
+          MYSQL_DATABASE: coze_hub
+          MYSQL_USER: test
+          MYSQL_PASSWORD: test
+          MYSQL_TCP_PORT: 3306
+      - id: hub_mongo
+        image: hub.byted.org/ee/mongo:4.0.24
+        commands:
+          - mongod --replSet rs0
+      - id: mongo-init
+        image: hub.byted.org/ee/mongo:4.0.24
+        commands:
+          - sleep 10s
+          - mongo mongodb://127.0.0.1:27017 --eval 'rs.initiate()'
+    steps:
+      - *checkout_step
+      - *proxy_step
+      - *remote_info_step
+      - *cache_step
+      - *init_env_step
+      - *install_deps_step
+      - name: wait mongo ready
+        uses: actions/mongodb-scripts
+        inputs:
+          host: 127.0.0.1
+          port: 27017
+      - name: wait mysql ready
+        uses: actions/mysql-scripts
+        inputs:
+          host: hub_mysql
+          user: test
+          password: test
+        commands:
+          - mysql -u test -ptest -h hub_mysql -P 3306 -e 'show databases;'
+      # 服务发现
+      - id: consul
+        uses: actions/setup-consul
+      - name: Test Coverage
+        commands:
+          - git fetch --filter=blob:none --unshallow -q
+          - node common/scripts/install-run-rush.js increment --action test:cov -p ${{Steps.RemoteInfo.Outputs.changed_files_path}}