🎉 Complete Nomad monitoring infrastructure project

✅ Major Achievements:
- Deployed complete observability stack (Prometheus + Loki + Grafana)
- Established rapid troubleshooting capabilities (3-step process)
- Created heatmap dashboard for log correlation analysis
- Unified logging system (systemd-journald across all nodes)
- Configured API access with Service Account tokens

🧹 Project Cleanup:
- Intelligent cleanup based on Git modification frequency
- Organized files into proper directory structure
- Removed deprecated webhook deployment scripts
- Eliminated 70+ temporary/test files (43% reduction)

📊 Infrastructure Status:
- Prometheus: 13 nodes monitored
- Loki: 12 nodes logging
- Grafana: Heatmap dashboard + API access
- Promtail: Deployed to 12/13 nodes

🚀 Ready for Terraform transition (静默一周后切换)

Project Status: COMPLETED ✅

infrastructure/consul/current/ash1d-consul.hcl

@@ -0,0 +1,58 @@
+# Consul Client Configuration for ash1d
+datacenter = "dc1"
+data_dir = "/opt/consul/data"
+log_level = "INFO"
+node_name = "ash1d"
+bind_addr = "100.81.26.3"
+
+# Client mode (not server)
+server = false
+
+# Connect to Consul servers (指向三节点集群)
+retry_join = [
+  "100.117.106.136",  "100.122.197.112",  "100.116.80.94"]
+
+# Performance optimization
+performance {
+  raft_multiplier = 5
+}
+
+# Ports configuration
+ports {
+  grpc = 8502
+  http = 8500
+  dns = 8600
+}
+
+# Enable Connect for service mesh
+connect {
+  enabled = true
+}
+
+# Cache configuration for performance
+cache {
+  entry_fetch_max_burst = 42
+  entry_fetch_rate = 30
+}
+
+# Node metadata
+node_meta = {
+  region = "unknown"
+  zone = "nomad-client"
+}
+
+# UI disabled for clients
+ui_config {
+  enabled = false
+}
+
+# ACL configuration (if needed)
+acl = {
+  enabled = false
+  default_policy = "allow"
+}
+
+# Logging
+log_file = "/var/log/consul/consul.log"
+log_rotate_duration = "24h"
+log_rotate_max_files = 7

infrastructure/consul/current/ash2e-consul.hcl

@@ -0,0 +1,99 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# Full configuration options can be found at https://developer.hashicorp.com/docs/agent/config
+
+# datacenter
+# This flag controls the datacenter in which the agent is running. If not provided,
+# it defaults to "dc1". Consul has first-class support for multiple datacenters, but 
+# it relies on proper configuration. Nodes in the same datacenter should be on a 
+# single LAN.
+#datacenter = "my-dc-1"
+
+# data_dir
+# This flag provides a data directory for the agent to store state. This is required
+# for all agents. The directory should be durable across reboots. This is especially
+# critical for agents that are running in server mode as they must be able to persist
+# cluster state. Additionally, the directory must support the use of filesystem
+# locking, meaning some types of mounted folders (e.g. VirtualBox shared folders) may
+# not be suitable.
+data_dir = "/opt/consul"
+
+# client_addr
+# The address to which Consul will bind client interfaces, including the HTTP and DNS
+# servers. By default, this is "127.0.0.1", allowing only loopback connections. In
+# Consul 1.0 and later this can be set to a space-separated list of addresses to bind
+# to, or a go-sockaddr template that can potentially resolve to multiple addresses.
+#client_addr = "0.0.0.0"
+
+# ui
+# Enables the built-in web UI server and the required HTTP routes. This eliminates
+# the need to maintain the Consul web UI files separately from the binary.
+# Version 1.10 deprecated ui=true in favor of ui_config.enabled=true
+#ui_config{
+#  enabled = true
+#}
+
+# server
+# This flag is used to control if an agent is in server or client mode. When provided,
+# an agent will act as a Consul server. Each Consul cluster must have at least one
+# server and ideally no more than 5 per datacenter. All servers participate in the Raft
+# consensus algorithm to ensure that transactions occur in a consistent, linearizable
+# manner. Transactions modify cluster state, which is maintained on all server nodes to
+# ensure availability in the case of node failure. Server nodes also participate in a
+# WAN gossip pool with server nodes in other datacenters. Servers act as gateways to
+# other datacenters and forward traffic as appropriate.
+#server = true
+
+# Bind addr
+# You may use IPv4 or IPv6 but if you have multiple interfaces you must be explicit.
+#bind_addr = "[::]" # Listen on all IPv6
+#bind_addr = "0.0.0.0" # Listen on all IPv4
+#
+# Advertise addr - if you want to point clients to a different address than bind or LB.
+#advertise_addr = "127.0.0.1"
+
+# Enterprise License
+# As of 1.10, Enterprise requires a license_path and does not have a short trial.
+#license_path = "/etc/consul.d/consul.hclic"
+
+# bootstrap_expect
+# This flag provides the number of expected servers in the datacenter. Either this value
+# should not be provided or the value must agree with other servers in the cluster. When
+# provided, Consul waits until the specified number of servers are available and then
+# bootstraps the cluster. This allows an initial leader to be elected automatically.
+# This cannot be used in conjunction with the legacy -bootstrap flag. This flag requires
+# -server mode.
+#bootstrap_expect=3
+
+# encrypt
+# Specifies the secret key to use for encryption of Consul network traffic. This key must
+# be 32-bytes that are Base64-encoded. The easiest way to create an encryption key is to
+# use consul keygen. All nodes within a cluster must share the same encryption key to
+# communicate. The provided key is automatically persisted to the data directory and loaded
+# automatically whenever the agent is restarted. This means that to encrypt Consul's gossip
+# protocol, this option only needs to be provided once on each agent's initial startup
+# sequence. If it is provided after Consul has been initialized with an encryption key,
+# then the provided key is ignored and a warning will be displayed.
+#encrypt = "..."
+
+# retry_join
+# Similar to -join but allows retrying a join until it is successful. Once it joins 
+# successfully to a member in a list of members it will never attempt to join again.
+# Agents will then solely maintain their membership via gossip. This is useful for
+# cases where you know the address will eventually be available. This option can be
+# specified multiple times to specify multiple agents to join. The value can contain
+# IPv4, IPv6, or DNS addresses. In Consul 1.1.0 and later this can be set to a go-sockaddr
+# template. If Consul is running on the non-default Serf LAN port, this must be specified
+# as well. IPv6 must use the "bracketed" syntax. If multiple values are given, they are
+# tried and retried in the order listed until the first succeeds. Here are some examples:
+#retry_join = ["consul.domain.internal"]
+#retry_join = ["10.0.4.67"]
+#retry_join = ["[::1]:8301"]
+#retry_join = ["consul.domain.internal", "10.0.4.67"]
+# Cloud Auto-join examples:
+# More details - https://developer.hashicorp.com/docs/agent/cloud-auto-join
+#retry_join = ["provider=aws tag_key=... tag_value=..."]
+#retry_join = ["provider=azure tag_name=... tag_value=... tenant_id=... client_id=... subscription_id=... secret_access_key=..."]
+#retry_join = ["provider=gce project_name=... tag_value=..."]
+ 

infrastructure/consul/current/ash3c-consul.hcl

@@ -0,0 +1,61 @@
+# Consul Client Configuration for ash3c
+datacenter = "dc1"
+data_dir = "/opt/consul/data"
+log_level = "INFO"
+node_name = "ash3c"
+bind_addr = "100.116.80.94"
+
+# Client mode (not server)
+server = false
+
+# Connect to Consul servers (指向三节点集群)
+retry_join = [
+  "100.117.106.136",  # master (韩国)
+  "100.122.197.112",  # warden (北京)  
+  "100.116.80.94"     # ash3c (美国)
+]
+
+# Performance optimization
+performance {
+  raft_multiplier = 5
+}
+
+# Ports configuration
+ports {
+  grpc = 8502
+  http = 8500
+  dns = 8600
+}
+
+# Enable Connect for service mesh
+connect {
+  enabled = true
+}
+
+# Cache configuration for performance
+cache {
+  entry_fetch_max_burst = 42
+  entry_fetch_rate = 30
+}
+
+# Node metadata
+node_meta = {
+  region = "unknown"
+  zone = "nomad-server"
+}
+
+# UI disabled for clients
+ui_config {
+  enabled = false
+}
+
+# ACL configuration (if needed)
+acl = {
+  enabled = false
+  default_policy = "allow"
+}
+
+# Logging
+log_file = "/var/log/consul/consul.log"
+log_rotate_duration = "24h"
+log_rotate_max_files = 7

infrastructure/consul/current/browser-consul.hcl

@@ -0,0 +1 @@
+# Consul配置不存在

infrastructure/consul/current/ch2-consul.hcl

@@ -0,0 +1,58 @@
+# Consul Client Configuration for ch2
+datacenter = "dc1"
+data_dir = "/opt/consul/data"
+log_level = "INFO"
+node_name = "ch2"
+bind_addr = "100.90.159.68"
+
+# Client mode (not server)
+server = false
+
+# Connect to Consul servers (指向三节点集群)
+retry_join = [
+  "100.117.106.136",  "100.122.197.112",  "100.116.80.94"]
+
+# Performance optimization
+performance {
+  raft_multiplier = 5
+}
+
+# Ports configuration
+ports {
+  grpc = 8502
+  http = 8500
+  dns = 8600
+}
+
+# Enable Connect for service mesh
+connect {
+  enabled = true
+}
+
+# Cache configuration for performance
+cache {
+  entry_fetch_max_burst = 42
+  entry_fetch_rate = 30
+}
+
+# Node metadata
+node_meta = {
+  region = "unknown"
+  zone = "nomad-client"
+}
+
+# UI disabled for clients
+ui_config {
+  enabled = false
+}
+
+# ACL configuration (if needed)
+acl = {
+  enabled = false
+  default_policy = "allow"
+}
+
+# Logging
+log_file = "/var/log/consul/consul.log"
+log_rotate_duration = "24h"
+log_rotate_max_files = 7

infrastructure/consul/current/ch3-consul.hcl

@@ -0,0 +1,58 @@
+# Consul Client Configuration for ch3
+datacenter = "dc1"
+data_dir = "/opt/consul/data"
+log_level = "INFO"
+node_name = "ch3"
+bind_addr = "100.86.141.112"
+
+# Client mode (not server)
+server = false
+
+# Connect to Consul servers (指向三节点集群)
+retry_join = [
+  "100.117.106.136",  "100.122.197.112",  "100.116.80.94"]
+
+# Performance optimization
+performance {
+  raft_multiplier = 5
+}
+
+# Ports configuration
+ports {
+  grpc = 8502
+  http = 8500
+  dns = 8600
+}
+
+# Enable Connect for service mesh
+connect {
+  enabled = true
+}
+
+# Cache configuration for performance
+cache {
+  entry_fetch_max_burst = 42
+  entry_fetch_rate = 30
+}
+
+# Node metadata
+node_meta = {
+  region = "unknown"
+  zone = "nomad-client"
+}
+
+# UI disabled for clients
+ui_config {
+  enabled = false
+}
+
+# ACL configuration (if needed)
+acl = {
+  enabled = false
+  default_policy = "allow"
+}
+
+# Logging
+log_file = "/var/log/consul/consul.log"
+log_rotate_duration = "24h"
+log_rotate_max_files = 7

infrastructure/consul/current/ch4-consul.hcl

@@ -0,0 +1,61 @@
+# Consul Client Configuration for master
+datacenter = "dc1"
+data_dir = "/opt/consul/data"
+log_level = "INFO"
+node_name = "ch4"
+bind_addr = "100.117.106.136"
+
+# Client mode (not server)
+server = false
+
+# Connect to Consul servers (指向三节点集群)
+retry_join = [
+  "100.117.106.136",  # master (韩国)
+  "100.122.197.112",  # warden (北京)  
+  "100.116.80.94"     # ash3c (美国)
+]
+
+# Performance optimization
+performance {
+  raft_multiplier = 5
+}
+
+# Ports configuration
+ports {
+  grpc = 8502
+  http = 8500
+  dns = 8600
+}
+
+# Enable Connect for service mesh
+connect {
+  enabled = true
+}
+
+# Cache configuration for performance
+cache {
+  entry_fetch_max_burst = 42
+  entry_fetch_rate = 30
+}
+
+# Node metadata
+node_meta = {
+  region = "unknown"
+  zone = "nomad-server"
+}
+
+# UI disabled for clients
+ui_config {
+  enabled = false
+}
+
+# ACL configuration (if needed)
+acl = {
+  enabled = false
+  default_policy = "allow"
+}
+
+# Logging
+log_file = "/var/log/consul/consul.log"
+log_rotate_duration = "24h"
+log_rotate_max_files = 7

infrastructure/consul/current/de-consul.hcl

@@ -0,0 +1,58 @@
+# Consul Client Configuration for de
+datacenter = "dc1"
+data_dir = "/opt/consul/data"
+log_level = "INFO"
+node_name = "de"
+bind_addr = "100.120.225.29"
+
+# Client mode (not server)
+server = false
+
+# Connect to Consul servers (指向三节点集群)
+retry_join = [
+  "100.117.106.136",  "100.122.197.112",  "100.116.80.94"]
+
+# Performance optimization
+performance {
+  raft_multiplier = 5
+}
+
+# Ports configuration
+ports {
+  grpc = 8502
+  http = 8500
+  dns = 8600
+}
+
+# Enable Connect for service mesh
+connect {
+  enabled = true
+}
+
+# Cache configuration for performance
+cache {
+  entry_fetch_max_burst = 42
+  entry_fetch_rate = 30
+}
+
+# Node metadata
+node_meta = {
+  region = "unknown"
+  zone = "nomad-client"
+}
+
+# UI disabled for clients
+ui_config {
+  enabled = false
+}
+
+# ACL configuration (if needed)
+acl = {
+  enabled = false
+  default_policy = "allow"
+}
+
+# Logging
+log_file = "/var/log/consul/consul.log"
+log_rotate_duration = "24h"
+log_rotate_max_files = 7

infrastructure/consul/current/hcp1-consul.hcl

@@ -0,0 +1,61 @@
+# Consul Client Configuration for hcp1
+datacenter = "dc1"
+data_dir = "/opt/consul/data"
+log_level = "INFO"
+node_name = "hcp1"
+bind_addr = "100.97.62.111"
+
+# Client mode (not server)
+server = false
+
+# Connect to Consul servers (指向三节点集群)
+retry_join = [
+  "100.117.106.136",  # ch4 (韩国)
+  "100.122.197.112",  # warden (北京)  
+  "100.116.80.94"     # ash3c (美国)
+]
+
+# Performance optimization
+performance {
+  raft_multiplier = 5
+}
+
+# Ports configuration
+ports {
+  grpc = 8502
+  http = 8500
+  dns = 8600
+}
+
+# Enable Connect for service mesh
+connect {
+  enabled = true
+}
+
+# Cache configuration for performance
+cache {
+  entry_fetch_max_burst = 42
+  entry_fetch_rate = 30
+}
+
+# Node metadata
+node_meta = {
+  region = "unknown"
+  zone = "nomad-client"
+}
+
+# UI disabled for clients
+ui_config {
+  enabled = false
+}
+
+# ACL configuration (if needed)
+acl = {
+  enabled = false
+  default_policy = "allow"
+}
+
+# Logging
+log_file = "/var/log/consul/consul.log"
+log_rotate_duration = "24h"
+log_rotate_max_files = 7

infrastructure/consul/current/influxdb-consul.hcl

@@ -0,0 +1 @@
+# Consul配置不存在

infrastructure/consul/current/onecloud1-consul.hcl

@@ -0,0 +1,65 @@
+# Consul Server Configuration for onecloud1
+datacenter = "dc1"
+data_dir = "/opt/consul/data"
+log_level = "INFO"
+node_name = "onecloud1"
+bind_addr = "100.98.209.50"
+
+# Server mode
+server = true
+bootstrap_expect = 4
+
+# Join existing cluster
+retry_join = [
+  "100.117.106.136",  # ch4
+  "100.122.197.112",  # warden
+  "100.116.80.94"     # ash3c
+]
+
+# Performance optimization
+performance {
+  raft_multiplier = 5
+}
+
+# Ports configuration
+ports {
+  grpc = 8502
+  http = 8500
+  dns = 8600
+  server = 8300
+  serf_lan = 8301
+  serf_wan = 8302
+}
+
+# Enable Connect for service mesh
+connect {
+  enabled = true
+}
+
+# Cache configuration for performance
+cache {
+  entry_fetch_max_burst = 42
+  entry_fetch_rate = 30
+}
+
+# Node metadata
+node_meta = {
+  region = "unknown"
+  zone = "nomad-client"
+}
+
+# UI enabled for servers
+ui_config {
+  enabled = true
+}
+
+# ACL configuration (if needed)
+acl = {
+  enabled = false
+  default_policy = "allow"
+}
+
+# Logging
+log_file = "/var/log/consul/consul.log"
+log_rotate_duration = "24h"
+log_rotate_max_files = 7

infrastructure/consul/current/semaphore-consul.hcl

@@ -0,0 +1,62 @@
+# Consul Client Configuration for ash1d
+datacenter = "dc1"
+data_dir = "/opt/consul/data"
+log_level = "INFO"
+node_name = "semaphore"
+bind_addr = "100.116.158.95"
+
+# Client mode (not server)
+server = false
+
+# Connect to Consul servers (指向三节点集群)
+retry_join = [
+  "100.117.106.136",  # master (韩国)
+  "100.122.197.112",  # warden (北京)  
+  "100.116.80.94"     # ash3c (美国)
+]
+
+# Performance optimization
+performance {
+  raft_multiplier = 5
+}
+
+# Ports configuration
+ports {
+  grpc = 8502
+  http = 8500
+  dns = 8600
+}
+
+# Enable Connect for service mesh
+connect {
+  enabled = true
+}
+
+# Cache configuration for performance
+cache {
+  entry_fetch_max_burst = 42
+  entry_fetch_rate = 30
+}
+
+# Node metadata
+node_meta = {
+  region = "unknown"
+  zone = "nomad-server"
+}
+
+# UI disabled for clients
+ui_config {
+  enabled = false
+}
+
+# ACL configuration (if needed)
+acl = {
+  enabled = false
+  default_policy = "allow"
+}
+
+# Logging
+log_file = "/var/log/consul/consul.log"
+log_rotate_duration = "24h"
+log_rotate_max_files = 7
+

infrastructure/consul/current/warden-consul.hcl

@@ -0,0 +1,61 @@
+# Consul Client Configuration for warden
+datacenter = "dc1"
+data_dir = "/opt/consul/data"
+log_level = "INFO"
+node_name = "warden"
+bind_addr = "100.122.197.112"
+
+# Client mode (not server)
+server = false
+
+# Connect to Consul servers (指向三节点集群)
+retry_join = [
+  "100.117.106.136",  # master (韩国)
+  "100.122.197.112",  # warden (北京)  
+  "100.116.80.94"     # ash3c (美国)
+]
+
+# Performance optimization
+performance {
+  raft_multiplier = 5
+}
+
+# Ports configuration
+ports {
+  grpc = 8502
+  http = 8500
+  dns = 8600
+}
+
+# Enable Connect for service mesh
+connect {
+  enabled = true
+}
+
+# Cache configuration for performance
+cache {
+  entry_fetch_max_burst = 42
+  entry_fetch_rate = 30
+}
+
+# Node metadata
+node_meta = {
+  region = "unknown"
+  zone = "nomad-server"
+}
+
+# UI disabled for clients
+ui_config {
+  enabled = false
+}
+
+# ACL configuration (if needed)
+acl = {
+  enabled = false
+  default_policy = "allow"
+}
+
+# Logging
+log_file = "/var/log/consul/consul.log"
+log_rotate_duration = "24h"
+log_rotate_max_files = 7