feat: manually mirror opencoze's code from bytedance
Change-Id: I09a73aadda978ad9511264a756b2ce51f5761adf
This commit is contained in:
fanlv
BIN
docker/volumes/elasticsearch/analysis-smartcn.zip
Normal file
BIN
docker/volumes/elasticsearch/analysis-smartcn.zip
Normal file
Binary file not shown.
14
docker/volumes/elasticsearch/elasticsearch.yml
Executable file
14
docker/volumes/elasticsearch/elasticsearch.yml
Executable file
@@ -0,0 +1,14 @@
|
||||
cluster:
|
||||
name: "docker-cluster"
|
||||
routing:
|
||||
allocation:
|
||||
disk:
|
||||
watermark:
|
||||
low: "99%"
|
||||
high: "99%"
|
||||
flood_stage: "99%"
|
||||
discovery:
|
||||
type: single-node
|
||||
xpack:
|
||||
security:
|
||||
enabled: false
|
||||
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"index_patterns": ["coze_resource*"],
|
||||
"template": {
|
||||
"settings": {
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0,
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"text_analyzer": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "stop", "snowball"]
|
||||
},
|
||||
"smartcn": {
|
||||
"type": "smartcn"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"dynamic": false,
|
||||
"properties": {
|
||||
"res_type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"app_id": {
|
||||
"type": "keyword",
|
||||
"null_value": "NULL"
|
||||
},
|
||||
"res_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"res_sub_type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"name": {
|
||||
"type": "text",
|
||||
"analyzer": "smartcn",
|
||||
"search_analyzer": "smartcn",
|
||||
"fields": {
|
||||
"raw": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"owner_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"space_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"biz_status": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"publish_status": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"create_time": {
|
||||
"type": "long"
|
||||
},
|
||||
"update_time": {
|
||||
"type": "long"
|
||||
},
|
||||
"publish_time": {
|
||||
"type": "long"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"priority": 200,
|
||||
"_meta": {
|
||||
"description": "resource library's index template"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"index_patterns": ["project_draft*"],
|
||||
"template": {
|
||||
"settings": {
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0,
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"text_analyzer": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "stop", "snowball"]
|
||||
},
|
||||
"smartcn": {
|
||||
"type": "smartcn"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"dynamic": false,
|
||||
"properties": {
|
||||
"create_time": {
|
||||
"type": "long"
|
||||
},
|
||||
"has_published": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"name": {
|
||||
"type": "text",
|
||||
"analyzer": "smartcn",
|
||||
"search_analyzer": "smartcn",
|
||||
"fields": {
|
||||
"raw": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
},
|
||||
"owner_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"publish_time": {
|
||||
"type": "long"
|
||||
},
|
||||
"space_id": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"status": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"type": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"update_time": {
|
||||
"type": "long"
|
||||
},
|
||||
"fav_time": {
|
||||
"type": "long"
|
||||
},
|
||||
"recently_open_time": {
|
||||
"type": "long"
|
||||
},
|
||||
"is_fav": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"is_recently_open": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"priority": 200,
|
||||
"_meta": {
|
||||
"description": "Project draft index template"
|
||||
}
|
||||
}
|
||||
93
docker/volumes/elasticsearch/setup_es.sh
Executable file
93
docker/volumes/elasticsearch/setup_es.sh
Executable file
@@ -0,0 +1,93 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
if [[ "$ES_ADDR" == *"localhost"* || "$ES_ADDR" == *"127.0.0.1"* ]]; then
|
||||
echo "ES_ADDR is localhost, using docker address: http://elasticsearch:9200"
|
||||
ES_ADDR="http://elasticsearch:9200"
|
||||
fi
|
||||
# ES_ADDR=http://localhost:31160
|
||||
INDEX_DIR=/es_index_schema
|
||||
|
||||
echo "ES_ADDR: $ES_ADDR"
|
||||
|
||||
AUTH_PARAM=""
|
||||
if [ -n "$ES_USERNAME" ]; then
|
||||
AUTH_PARAM="-k -u $ES_USERNAME:$ES_PASSWORD"
|
||||
fi
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
echo "Checking Elasticsearch availability... (attempt $i)"
|
||||
if curl -s -f $AUTH_PARAM "${ES_ADDR}/_cat/health"; then
|
||||
echo "Elasticsearch is up and running!"
|
||||
break
|
||||
fi
|
||||
echo "Elasticsearch not available, retrying in 1 seconds..."
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo -e "🔍 Checking smartcn plugin status..."
|
||||
if ! curl -s $AUTH_PARAM "${ES_ADDR}/_cat/plugins" | grep -q "analysis-smartcn"; then
|
||||
echo -e "❌ smartcn plugin not loaded correctly, please ensure the plugin is installed and Elasticsearch is restarted"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "🔍 Initializing Elasticsearch index templates..."
|
||||
ES_TEMPLATES=$(find "$INDEX_DIR" -type f -name "*.index-template.json" | sort)
|
||||
if [ -z "$ES_TEMPLATES" ]; then
|
||||
echo -e "ℹ️ No Elasticsearch index templates found in $INDEX_DIR"
|
||||
exit 1
|
||||
else
|
||||
# Add index creation logic
|
||||
echo -e "🔄 Creating Elasticsearch indexes..."
|
||||
for template_file in $ES_TEMPLATES; do
|
||||
|
||||
template_name=$(basename "$template_file" | sed 's/\.index-template\.json$//')
|
||||
echo -e "➡️ Registering template: $template_name"
|
||||
|
||||
# Attempt to register index template
|
||||
response=$(curl -s $AUTH_PARAM -X PUT "${ES_ADDR}/_index_template/$template_name" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d @"$template_file" 2>&1)
|
||||
|
||||
# Check if successful
|
||||
if echo "$response" | grep -q '"acknowledged":true'; then
|
||||
echo -e "✅ Template $template_name registered successfully"
|
||||
else
|
||||
echo -e "❌ Failed to register template $template_name. Response: $response"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
index_name=$(basename "$template_file" | sed 's/\.index-template\.json$//')
|
||||
echo -e "➡️ Creating index: $index_name"
|
||||
|
||||
# Check if index exists
|
||||
if ! curl -s -f $AUTH_PARAM "${ES_ADDR}/_cat/indices/$index_name" >/dev/null; then
|
||||
# Create index (matching template's index_patterns)
|
||||
curl $AUTH_PARAM -X PUT "${ES_ADDR}/$index_name" -H "Content-Type: application/json"
|
||||
echo ""
|
||||
|
||||
# Set refresh interval if index was just created
|
||||
echo -e "🔄 Setting refresh_interval for index: $index_name..."
|
||||
CURL_OUTPUT=$(curl -s $AUTH_PARAM -w "\nHTTP_STATUS_CODE:%{http_code}" -X PUT "${ES_ADDR}/${index_name}/_settings" -H 'Content-Type: application/json' -d'
|
||||
{
|
||||
"index": {
|
||||
"refresh_interval": "10ms"
|
||||
}
|
||||
}')
|
||||
echo -e "📄 Curl command output for $index_name:\n$CURL_OUTPUT"
|
||||
# Extract the JSON body from the output, excluding the HTTP_STATUS_CODE line
|
||||
JSON_BODY=$(echo "$CURL_OUTPUT" | sed '$d')
|
||||
if ! echo "$JSON_BODY" | grep -q '"acknowledged":true'; then
|
||||
echo -e "⚠️ Warning: Failed to set refresh interval for $index_name index. Response Body: $JSON_BODY. Please check and set manually."
|
||||
exit 1
|
||||
else
|
||||
echo -e "✅ Successfully set refresh_interval for $index_name."
|
||||
fi
|
||||
else
|
||||
echo -e "ℹ️ Index $index_name already exists"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
echo "Elasticsearch setup completed."
|
||||
exit 0
|
||||
Reference in New Issue
Block a user