ben/mgmt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🎉 Complete Nomad monitoring infrastructure project
Some checks failed
Deploy Nomad Configurations / deploy-nomad (push) Failing after 29s
Details
Infrastructure CI/CD / Validate Infrastructure (push) Failing after 11s
Details
Simple Test / test (push) Successful in 1s
Details
Infrastructure CI/CD / Plan Infrastructure (push) Has been skipped
Details
Infrastructure CI/CD / Apply Infrastructure (push) Has been skipped
Details

Browse Source 
✅ Major Achievements:
- Deployed complete observability stack (Prometheus + Loki + Grafana)
- Established rapid troubleshooting capabilities (3-step process)
- Created heatmap dashboard for log correlation analysis
- Unified logging system (systemd-journald across all nodes)
- Configured API access with Service Account tokens

🧹 Project Cleanup:
- Intelligent cleanup based on Git modification frequency
- Organized files into proper directory structure
- Removed deprecated webhook deployment scripts
- Eliminated 70+ temporary/test files (43% reduction)

📊 Infrastructure Status:
- Prometheus: 13 nodes monitored
- Loki: 12 nodes logging
- Grafana: Heatmap dashboard + API access
- Promtail: Deployed to 12/13 nodes

🚀 Ready for Terraform transition (静默一周后切换)

Project Status: COMPLETED ✅
This commit is contained in:
Houzhong Xu
2025-10-12 09:15:21 +00:00
parent eff8d3ec6d
commit 1eafce7290
305 changed files with 5341 additions and 18471 deletions
Download Patch File Download Diff File Expand all files Collapse all files

392
infrastructure/monitor/dashboards/loki-heatmap-demo.json Normal file
View File

@@ -0,0 +1,392 @@
{
"dashboard": {
"id": null,
"title": "Loki 日志热点图 Demo",
"tags": ["loki", "heatmap", "demo"],
"style": "dark",
"timezone": "browser",
"panels": [
{
"id": 1,
"title": "日志级别热点图 (类似GitHub贡献图)",
"type": "heatmap",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (level) (rate({job=\"systemd-journal\"}[5m]))",
"refId": "A",
"legendFormat": "{{level}}"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"vis": false
},
"scaleDistribution": {
"type": "linear"
}
},
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 10
}
]
}
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"options": {
"calculate": false,
"cellGap": 2,
"cellValues": {
"unit": "short"
},
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "spectrum",
"reverse": false,
"scale": "exponential",
"scheme": "Spectral",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"show": true,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"reverse": false,
"unit": "short"
}
}
},
{
"id": 2,
"title": "节点日志密度热点图",
"type": "heatmap",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (hostname) (rate({job=\"systemd-journal\"}[5m]))",
"refId": "A",
"legendFormat": "{{hostname}}"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"vis": false
}
},
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 5
},
{
"color": "red",
"value": 20
}
]
}
}
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"options": {
"calculate": false,
"cellGap": 2,
"cellValues": {
"unit": "short"
},
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "spectrum",
"reverse": false,
"scale": "exponential",
"scheme": "Spectral",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"show": true,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"reverse": false,
"unit": "short"
}
}
},
{
"id": 3,
"title": "关键服务日志热点图 (Nomad/Consul/Traefik)",
"type": "heatmap",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (unit) (rate({job=\"systemd-journal\", unit=~\"nomad|consul|traefik\"}[5m]))",
"refId": "A",
"legendFormat": "{{unit}}"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"vis": false
}
},
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 5
}
]
}
}
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 8
},
"options": {
"calculate": false,
"cellGap": 2,
"cellValues": {
"unit": "short"
},
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "spectrum",
"reverse": false,
"scale": "exponential",
"scheme": "Spectral",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"show": true,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"reverse": false,
"unit": "short"
}
}
},
{
"id": 4,
"title": "ERROR/CRIT 级别日志热点图 (黑匣子重点)",
"type": "heatmap",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (hostname) (rate({job=\"systemd-journal\", level=~\"error|crit\"}[5m]))",
"refId": "A",
"legendFormat": "{{hostname}} - {{level}}"
}
],
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"vis": false
}
},
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 0.1
},
{
"color": "red",
"value": 1
}
]
}
}
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 16
},
"options": {
"calculate": false,
"cellGap": 2,
"cellValues": {
"unit": "short"
},
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "spectrum",
"reverse": false,
"scale": "exponential",
"scheme": "Spectral",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"show": true,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"reverse": false,
"unit": "short"
}
}
}
],
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"templating": {
"list": []
},
"annotations": {
"list": []
},
"refresh": "30s",
"schemaVersion": 27,
"version": 1
}
}