From f0c339d2311849caffff02efa5a3f45f5238f86a Mon Sep 17 00:00:00 2001 From: Ryo Date: Mon, 28 Jul 2025 12:26:58 +0800 Subject: [PATCH] =?UTF-8?q?chore(ci):=20Optimize=20Elasticsearch=20index?= =?UTF-8?q?=20init=20script,=20remove=20Docker=20im=E2=80=A6=20(#106)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 9 ++- backend/Dockerfile | 7 +- backend/script/bootstrap.sh | 4 + docker/docker-compose.yml | 25 ------- docker/volumes/elasticsearch/setup_es.sh | 93 ++++++++++++++++++------ 5 files changed, 88 insertions(+), 50 deletions(-) diff --git a/Makefile b/Makefile index b89f8257..018abd79 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,8 @@ MYSQL_SCHEMA := ./docker/volumes/mysql/schema.sql MYSQL_INIT_SQL := ./docker/volumes/mysql/sql_init.sql ENV_FILE := ./docker/.env STATIC_DIR := ./bin/resources/static +ES_INDEX_SCHEMA := ./docker/volumes/elasticsearch/es_index_schema +ES_SETUP_SCRIPT := ./docker/volumes/elasticsearch/setup_es.sh debug: env middleware python server @@ -26,7 +28,7 @@ fe: @echo "Building frontend..." @bash $(BUILD_FE_SCRIPT) -server: env +server: env setup_es_index @if [ ! -d "$(STATIC_DIR)" ]; then \ echo "Static directory '$(STATIC_DIR)' not found, building frontend..."; \ $(MAKE) fe; \ @@ -84,6 +86,10 @@ atlas-hash: @echo "Rehash atlas migration files..." @(cd ./docker/atlas && atlas migrate hash) +setup_es_index: + @echo "Setting up Elasticsearch index..." + @bash $(ES_SETUP_SCRIPT) --index-dir $(ES_INDEX_SCHEMA) --docker-host false + help: @echo "Usage: make [target]" @echo "" @@ -103,4 +109,5 @@ help: @echo " clean - Stop the docker containers and clean volumes." @echo " python - Setup python environment." @echo " atlas-hash - Rehash atlas migration files." + @echo " setup_es_index - Setup elasticsearch index." @echo " help - Show this help message." diff --git a/backend/Dockerfile b/backend/Dockerfile index 322e166d..26d2f798 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -31,7 +31,7 @@ WORKDIR /app # Install runtime dependencies for Go app and base for Python # pax-utils for scanelf, python3 for running Python, python3-dev for headers/shared libs # bind-tools for nslookup etc., file for debugging file types -RUN apk add --no-cache pax-utils python3 python3-dev bind-tools file deno +RUN apk add --no-cache pax-utils python3 python3-dev bind-tools file deno curl # Install Python build dependencies, create venv, install packages, then remove build deps RUN apk add --no-cache --virtual .python-build-deps build-base py3-pip git && \ @@ -81,6 +81,9 @@ EXPOSE 8888 # Use a script to start both applications COPY backend/script/bootstrap.sh /app/bootstrap.sh -RUN chmod +x /app/bootstrap.sh +COPY docker/volumes/elasticsearch/setup_es.sh /app/setup_es.sh +COPY docker/volumes/elasticsearch/es_index_schema /app/es_index_schemas + +RUN chmod +x /app/bootstrap.sh /app/setup_es.sh CMD ["/app/bootstrap.sh"] diff --git a/backend/script/bootstrap.sh b/backend/script/bootstrap.sh index 30090352..4bf47d07 100755 --- a/backend/script/bootstrap.sh +++ b/backend/script/bootstrap.sh @@ -1,5 +1,9 @@ #!/bin/sh +# Set up Elasticsearch +echo "Setting up Elasticsearch..." +/app/setup_es.sh --index-dir /app/es_index_schemas + # Start the proxy application in the background echo "Starting proxy application..." /app/proxy >/tmp/proxy.log 2>&1 & diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2e66105f..f74e91ae 100755 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -65,8 +65,6 @@ services: depends_on: minio-setup: condition: service_completed_successfully - elasticsearch-setup: - condition: service_completed_successfully mysql-setup-schema: condition: service_completed_successfully mysql-setup-init-sql: @@ -409,27 +407,6 @@ services: networks: - coze-network - elasticsearch-setup: - image: alpine/curl:8.12.1 - container_name: coze-elasticsearch-setup - profiles: ['middleware', 'volcano-setup'] - env_file: *env_file - depends_on: - elasticsearch: - condition: service_healthy - volumes: - - ./volumes/elasticsearch/setup_es.sh:/setup_es.sh - - ./volumes/elasticsearch/es_index_schema:/es_index_schema - command: - - /bin/sh - - -c - - | - set -ex - /setup_es.sh - echo 'Elasticsearch setup complete.' - networks: - - coze-network - restart: 'no' minio-setup: image: minio/mc:RELEASE.2025-05-21T01-59-54Z-cpuv1 container_name: coze-minio-setup @@ -562,8 +539,6 @@ services: condition: service_healthy minio-setup: condition: service_completed_successfully - elasticsearch-setup: - condition: service_completed_successfully mysql-setup-init-sql: condition: service_completed_successfully command: ['/app/bootstrap.sh'] diff --git a/docker/volumes/elasticsearch/setup_es.sh b/docker/volumes/elasticsearch/setup_es.sh index e2ecfd36..e1255a8f 100755 --- a/docker/volumes/elasticsearch/setup_es.sh +++ b/docker/volumes/elasticsearch/setup_es.sh @@ -1,14 +1,61 @@ #!/bin/sh set -e -if [[ "$ES_ADDR" == *"localhost"* || "$ES_ADDR" == *"127.0.0.1"* ]]; then - echo "ES_ADDR is localhost, using docker address: http://elasticsearch:9200" - ES_ADDR="http://elasticsearch:9200" +# Parse command-line arguments +while [ $# -gt 0 ]; do + case "$1" in + --es-address) + case "$2" in + "" | -*) echo "Error: Missing value for $1" >&2; exit 1;; + esac + ES_ADDR_ARG="$2" + shift 2 + ;; + --index-dir) + case "$2" in + "" | -*) echo "Error: Missing value for $1" >&2; exit 1;; + esac + INDEX_DIR_ARG="$2" + shift 2 + ;; + --docker-host) + case "$2" in + "true" | "false") REPLACE_ES_ADDR_WITH_DOCKER_HOST_ARG="$2" ;; + *) echo "Error: Invalid value for $1. Must be 'true' or 'false'" >&2; exit 1;; + esac + shift 2 + ;; + *) + # unknown option + shift + ;; + esac +done + +# If ES_ADDR_ARG argument is provided, use it. Otherwise, use environment variable or default logic. +if [ -n "$ES_ADDR_ARG" ]; then + ES_ADDR=$ES_ADDR_ARG +fi + +REPLACE_ES_ADDR=${REPLACE_ES_ADDR_WITH_DOCKER_HOST_ARG:-true} +if [ "$REPLACE_ES_ADDR" = "true" ]; then + case "$ES_ADDR" in + *localhost*|*"127.0.0.1"*) + echo "ES_ADDR is localhost, using docker address: http://elasticsearch:9200" + ES_ADDR="http://elasticsearch:9200" + ;; + esac +fi + +# If INDEX_DIR_ARG argument is provided, use it. Otherwise, use the default. +if [ -n "$INDEX_DIR_ARG" ]; then + INDEX_DIR=$INDEX_DIR_ARG +else + INDEX_DIR=/es_index_schema fi -# ES_ADDR=http://localhost:31160 -INDEX_DIR=/es_index_schema echo "ES_ADDR: $ES_ADDR" +echo "INDEX_DIR: $INDEX_DIR" AUTH_PARAM="" if [ -n "$ES_USERNAME" ]; then @@ -38,30 +85,32 @@ if [ -z "$ES_TEMPLATES" ]; then exit 1 else # Add index creation logic - echo -e "🔄 Creating Elasticsearch indexes..." for template_file in $ES_TEMPLATES; do - template_name=$(basename "$template_file" | sed 's/\.index-template\.json$//') - echo -e "➡ī¸ Registering template: $template_name" - - # Attempt to register index template - response=$(curl -s $AUTH_PARAM -X PUT "${ES_ADDR}/_index_template/$template_name" \ - -H "Content-Type: application/json" \ - -d @"$template_file" 2>&1) - - # Check if successful - if echo "$response" | grep -q '"acknowledged":true'; then - echo -e "✅ Template $template_name registered successfully" + # Check if index template exists + if curl -s -f $AUTH_PARAM -I "${ES_ADDR}/_index_template/$template_name" >/dev/null; then + echo -e "ℹī¸ Index template $template_name already exists" else - echo -e "❌ Failed to register template $template_name. Response: $response" - exit 1 + echo -e "➡ī¸ Registering template: $template_name" + + # Attempt to register index template + response=$(curl -s $AUTH_PARAM -X PUT "${ES_ADDR}/_index_template/$template_name" \ + -H "Content-Type: application/json" \ + -d @"$template_file" 2>&1) + + # Check if successful + if echo "$response" | grep -q '"acknowledged":true'; then + echo -e "✅ Template $template_name registered successfully" + else + echo -e "❌ Failed to register template $template_name. Response: $response" + exit 1 + fi fi index_name=$(basename "$template_file" | sed 's/\.index-template\.json$//') - echo -e "➡ī¸ Creating index: $index_name" - # Check if index exists if ! curl -s -f $AUTH_PARAM "${ES_ADDR}/_cat/indices/$index_name" >/dev/null; then + echo -e "➡ī¸ Creating index: $index_name" # Create index (matching template's index_patterns) curl $AUTH_PARAM -X PUT "${ES_ADDR}/$index_name" -H "Content-Type: application/json" echo "" @@ -90,4 +139,4 @@ else fi echo "Elasticsearch setup completed." -exit 0 +exit 0 \ No newline at end of file