From 44b098bd20b89760a920d5f2fbcd2095b4b6020e Mon Sep 17 00:00:00 2001 From: Houzhong Xu Date: Sat, 27 Sep 2025 13:05:30 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20=E9=87=8D=E6=9E=84=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E7=BB=93=E6=9E=84=E5=B9=B6=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=A4=9A=E4=B8=AA=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增脚本和配置文件用于管理Nomad节点和NFS存储 - 添加多个Ansible playbook用于配置和调试Nomad集群 - 新增Nomad job文件用于测试Podman和NFS功能 - 重构playbooks目录结构，按功能分类 - 更新Nomad客户端和服务端配置模板 - 添加SSH密钥分发和配置脚本 - 新增多个调试和修复问题的playbook --- configuration/inventories/production/hosts | 20 +++ configuration/nfs-volume.hcl | 45 ++++++ .../{ => add}/add-warden-to-nomad-cluster.yml | 0 .../configure-nomad-podman-cluster.yml | 0 .../{ => configure}/configure-nomad-sudo.yml | 0 .../configure-nomad-tailscale.yml | 0 .../configure-podman-for-nomad.yml | 0 .../{ => debug}/debug-nomad-nodes.yml | 0 .../{ => debug}/debug-nomad-podman.yml | 0 .../playbooks/{ => debug}/debug-syd.yml | 0 .../{ => disk}/disk-analysis-ncdu.yml | 0 .../playbooks/{ => disk}/disk-cleanup.yml | 0 .../distribute-podman-driver.yml | 0 .../{ => distribute}/distribute-podman.yml | 0 .../playbooks/{ => fix}/fix-apt-errors.yml | 0 .../{ => fix}/fix-duplicate-podman-config.yml | 0 .../{ => fix}/fix-hashicorp-apt-source.yml | 0 .../playbooks/{ => fix}/fix-hcp-podman.yml | 0 .../{ => fix}/fix-hcs-dpkg-issue.yml | 0 .../playbooks/{ => fix}/fix-nomad-cluster.yml | 0 .../playbooks/{ => fix}/fix-nomad-local.yml | 0 .../{ => fix}/fix-nomad-podman-config.yml | 0 .../{ => fix}/fix-nomad-server-config.yml | 0 .../playbooks/{ => fix}/fix-nomad-systemd.yml | 0 .../{ => fix}/fix-podman-installation.yml | 0 .../playbooks/{ => fix}/fix-server-config.yml | 0 .../{ => fix}/fix-server-network-config.yml | 0 .../{ => fix}/fix-warden-compose.yml | 0 .../install-configure-nomad-podman-driver.yml | 0 .../{ => install}/install-consul.yml | 0 .../install-nomad-direct-download.yml | 0 .../install-nomad-podman-driver.yml | 0 .../{ => install}/install-podman-compose.yml | 0 .../{ => other}/check-podman-version.yml | 0 .../{ => other}/check-podman-versions.yml | 0 .../{ => other}/cleanup-hashicorp-backups.yml | 0 .../playbooks/{ => other}/clear-aliases.yml | 0 .../{ => other}/clear-all-aliases.yml | 0 .../{ => other}/clear-proxy-settings.yml | 0 .../playbooks/{ => other}/clear-proxy.yml | 0 .../{ => other}/ensure-nomad-user.yml | 0 .../{ => other}/final-podman-fix.yml | 0 .../{ => other}/get-tailscale-ips.yml | 0 .../{ => other}/hack-podman-upgrade.yml | 0 .../{ => other}/integrated-podman-setup.yml | 0 .../{ => other}/migrate-to-podman-simple.yml | 0 .../playbooks/{ => other}/ping-nodes.yml | 0 .../{ => other}/read-nomad-config.yml | 0 .../{ => other}/restart-tailscale.yml | 0 .../{ => other}/update-nomad-config.yml | 0 .../{ => other}/upgrade-podman-to-5.yml | 0 ...ove-docker-install-podman-with-compose.yml | 0 .../remove-docker-install-podman.yml | 0 .../add-beijing-node-prefix.yml | 0 .../fix-duplicate-plugin-dir.yml | 0 .../fix-podman-driver-config.yml | 0 .../root_playbooks}/fix-warden-nfs.yml | 0 .../setup-nfs-by-container-type.yml | 82 ++++++++++ .../root_playbooks}/setup-nfs-storage.yml | 4 +- .../root_playbooks/setup-nomad-nfs-client.yml | 50 ++++++ .../root_playbooks/setup-nomad-nfs.yml | 63 ++++++++ .../{ => setup}/setup-disk-monitoring.yml | 0 .../{ => setup}/setup-new-nomad-nodes.yml | 0 .../{ => test}/test-podman-snap-migration.yml | 0 configuration/templates/nomad-client.hcl | 39 +++++ docs/nomad-nfs-setup.md | 146 ++++++++++++++++++ fix_remote_apt_permissions.sh | 59 +++++++ jobs/{ => consul}/consul-cluster-arm64.nomad | 0 jobs/{ => consul}/consul-cluster-binary.nomad | 0 jobs/{ => consul}/consul-cluster-nomad.nomad | 0 jobs/{ => consul}/consul-cluster-simple.nomad | 0 .../consul-cluster-three-nodes.nomad | 0 jobs/{ => consul}/consul-cluster.nomad | 0 .../consul-single-member-new.nomad | 0 jobs/{ => consul}/consul-single-member.nomad | 0 jobs/{ => consul}/consul-test-warden.nomad | 0 jobs/{ => consul}/consul-warden-only.nomad | 0 .../service-discovery-warden.nomad | 0 jobs/{ => consul}/simple-consul-warden.nomad | 0 jobs/hybrid-nfs-app.nomad | 65 ++++++++ jobs/nfs-app-example.nomad | 51 ++++++ jobs/nfs-storage-test.nomad | 34 ++++ jobs/nomad-nfs-multi-type.nomad | 84 ++++++++++ jobs/{ => podman}/install-podman-driver.nomad | 0 jobs/podman/nomad-nfs-volume.nomad | 34 ++++ jobs/{ => tests}/test-job.nomad | 0 jobs/{ => tests}/test-podman-job.nomad | 0 jobs/{ => tests}/test-podman-simple.nomad | 0 jobs/{ => tests}/test-private-registry.nomad | 0 jobs/{ => tests}/test-simple.nomad | 0 list_playbooks.sh | 4 + playbooks/configure-nomad-clients.yml | 72 +++++++++ playbooks/nfs-mount.yml | 42 +++++ playbooks/setup-nfs-nodes.yml | 43 ++++++ scripts/deploy-nfs-for-nomad.sh | 69 +++++++++ scripts/distribute-keys.sh | 29 ++++ scripts/setup-ssh-keys.yml | 22 +++ scripts/verify-nfs-config.sh | 86 +++++++++++ 98 files changed, 1141 insertions(+), 2 deletions(-) create mode 100644 configuration/inventories/production/hosts create mode 100644 configuration/nfs-volume.hcl rename configuration/playbooks/{ => add}/add-warden-to-nomad-cluster.yml (100%) rename configuration/playbooks/{ => configure}/configure-nomad-podman-cluster.yml (100%) rename configuration/playbooks/{ => configure}/configure-nomad-sudo.yml (100%) rename configuration/playbooks/{ => configure}/configure-nomad-tailscale.yml (100%) rename configuration/playbooks/{ => configure}/configure-podman-for-nomad.yml (100%) rename configuration/playbooks/{ => debug}/debug-nomad-nodes.yml (100%) rename configuration/playbooks/{ => debug}/debug-nomad-podman.yml (100%) rename configuration/playbooks/{ => debug}/debug-syd.yml (100%) rename configuration/playbooks/{ => disk}/disk-analysis-ncdu.yml (100%) rename configuration/playbooks/{ => disk}/disk-cleanup.yml (100%) rename configuration/playbooks/{ => distribute}/distribute-podman-driver.yml (100%) rename configuration/playbooks/{ => distribute}/distribute-podman.yml (100%) rename configuration/playbooks/{ => fix}/fix-apt-errors.yml (100%) rename configuration/playbooks/{ => fix}/fix-duplicate-podman-config.yml (100%) rename configuration/playbooks/{ => fix}/fix-hashicorp-apt-source.yml (100%) rename configuration/playbooks/{ => fix}/fix-hcp-podman.yml (100%) rename configuration/playbooks/{ => fix}/fix-hcs-dpkg-issue.yml (100%) rename configuration/playbooks/{ => fix}/fix-nomad-cluster.yml (100%) rename configuration/playbooks/{ => fix}/fix-nomad-local.yml (100%) rename configuration/playbooks/{ => fix}/fix-nomad-podman-config.yml (100%) rename configuration/playbooks/{ => fix}/fix-nomad-server-config.yml (100%) rename configuration/playbooks/{ => fix}/fix-nomad-systemd.yml (100%) rename configuration/playbooks/{ => fix}/fix-podman-installation.yml (100%) rename configuration/playbooks/{ => fix}/fix-server-config.yml (100%) rename configuration/playbooks/{ => fix}/fix-server-network-config.yml (100%) rename configuration/playbooks/{ => fix}/fix-warden-compose.yml (100%) rename configuration/playbooks/{ => install}/install-configure-nomad-podman-driver.yml (100%) rename configuration/playbooks/{ => install}/install-consul.yml (100%) rename configuration/playbooks/{ => install}/install-nomad-direct-download.yml (100%) rename configuration/playbooks/{ => install}/install-nomad-podman-driver.yml (100%) rename configuration/playbooks/{ => install}/install-podman-compose.yml (100%) rename configuration/playbooks/{ => other}/check-podman-version.yml (100%) rename configuration/playbooks/{ => other}/check-podman-versions.yml (100%) rename configuration/playbooks/{ => other}/cleanup-hashicorp-backups.yml (100%) rename configuration/playbooks/{ => other}/clear-aliases.yml (100%) rename configuration/playbooks/{ => other}/clear-all-aliases.yml (100%) rename configuration/playbooks/{ => other}/clear-proxy-settings.yml (100%) rename configuration/playbooks/{ => other}/clear-proxy.yml (100%) rename configuration/playbooks/{ => other}/ensure-nomad-user.yml (100%) rename configuration/playbooks/{ => other}/final-podman-fix.yml (100%) rename configuration/playbooks/{ => other}/get-tailscale-ips.yml (100%) rename configuration/playbooks/{ => other}/hack-podman-upgrade.yml (100%) rename configuration/playbooks/{ => other}/integrated-podman-setup.yml (100%) rename configuration/playbooks/{ => other}/migrate-to-podman-simple.yml (100%) rename configuration/playbooks/{ => other}/ping-nodes.yml (100%) rename configuration/playbooks/{ => other}/read-nomad-config.yml (100%) rename configuration/playbooks/{ => other}/restart-tailscale.yml (100%) rename configuration/playbooks/{ => other}/update-nomad-config.yml (100%) rename configuration/playbooks/{ => other}/upgrade-podman-to-5.yml (100%) rename configuration/playbooks/{ => remove}/remove-docker-install-podman-with-compose.yml (100%) rename configuration/playbooks/{ => remove}/remove-docker-install-podman.yml (100%) rename {playbooks => configuration/playbooks/root_playbooks}/add-beijing-node-prefix.yml (100%) rename {playbooks => configuration/playbooks/root_playbooks}/fix-duplicate-plugin-dir.yml (100%) rename {playbooks => configuration/playbooks/root_playbooks}/fix-podman-driver-config.yml (100%) rename {playbooks => configuration/playbooks/root_playbooks}/fix-warden-nfs.yml (100%) create mode 100644 configuration/playbooks/root_playbooks/setup-nfs-by-container-type.yml rename {playbooks => configuration/playbooks/root_playbooks}/setup-nfs-storage.yml (96%) create mode 100644 configuration/playbooks/root_playbooks/setup-nomad-nfs-client.yml create mode 100644 configuration/playbooks/root_playbooks/setup-nomad-nfs.yml rename configuration/playbooks/{ => setup}/setup-disk-monitoring.yml (100%) rename configuration/playbooks/{ => setup}/setup-new-nomad-nodes.yml (100%) rename configuration/playbooks/{ => test}/test-podman-snap-migration.yml (100%) create mode 100644 configuration/templates/nomad-client.hcl create mode 100644 docs/nomad-nfs-setup.md create mode 100644 fix_remote_apt_permissions.sh rename jobs/{ => consul}/consul-cluster-arm64.nomad (100%) rename jobs/{ => consul}/consul-cluster-binary.nomad (100%) rename jobs/{ => consul}/consul-cluster-nomad.nomad (100%) rename jobs/{ => consul}/consul-cluster-simple.nomad (100%) rename jobs/{ => consul}/consul-cluster-three-nodes.nomad (100%) rename jobs/{ => consul}/consul-cluster.nomad (100%) rename jobs/{ => consul}/consul-single-member-new.nomad (100%) rename jobs/{ => consul}/consul-single-member.nomad (100%) rename jobs/{ => consul}/consul-test-warden.nomad (100%) rename jobs/{ => consul}/consul-warden-only.nomad (100%) rename jobs/{ => consul}/service-discovery-warden.nomad (100%) rename jobs/{ => consul}/simple-consul-warden.nomad (100%) create mode 100644 jobs/hybrid-nfs-app.nomad create mode 100644 jobs/nfs-app-example.nomad create mode 100644 jobs/nfs-storage-test.nomad create mode 100644 jobs/nomad-nfs-multi-type.nomad rename jobs/{ => podman}/install-podman-driver.nomad (100%) create mode 100644 jobs/podman/nomad-nfs-volume.nomad rename jobs/{ => tests}/test-job.nomad (100%) rename jobs/{ => tests}/test-podman-job.nomad (100%) rename jobs/{ => tests}/test-podman-simple.nomad (100%) rename jobs/{ => tests}/test-private-registry.nomad (100%) rename jobs/{ => tests}/test-simple.nomad (100%) create mode 100755 list_playbooks.sh create mode 100644 playbooks/configure-nomad-clients.yml create mode 100644 playbooks/nfs-mount.yml create mode 100644 playbooks/setup-nfs-nodes.yml create mode 100755 scripts/deploy-nfs-for-nomad.sh create mode 100644 scripts/distribute-keys.sh create mode 100644 scripts/setup-ssh-keys.yml create mode 100755 scripts/verify-nfs-config.sh diff --git a/configuration/inventories/production/hosts b/configuration/inventories/production/hosts new file mode 100644 index 0000000..f20a4e0 --- /dev/null +++ b/configuration/inventories/production/hosts @@ -0,0 +1,20 @@ +[nomad_nodes] +# 本机节点 (已通过PVE挂载NFS) +semaphore ansible_host=100.116.158.95 ansible_user=root + +# 云服务器节点 (需要配置NFS挂载) +ash1d.global ansible_host=100.81.26.3 ansible_user=ben ansible_password=3131 ansible_become_password=3131 +ash2e.global ansible_host=100.103.147.94 ansible_user=ben ansible_password=3131 ansible_become_password=3131 +ch2.global ansible_host=100.90.159.68 ansible_user=ben ansible_password=3131 ansible_become_password=3131 +ch3.global ansible_host=100.86.141.112 ansible_user=ben ansible_password=3131 ansible_become_password=3131 +master ansible_host=100.117.106.136 ansible_user=ben ansible_password=3131 ansible_become_password=3131 ansible_port=60022 +ash3c ansible_host=100.116.80.94 ansible_user=ben ansible_password=3131 ansible_become_password=3131 + +[nomad_nodes:vars] +# NFS配置 +nfs_server=snail +nfs_share=/fs/1000/nfs/Fnsync +mount_point=/mnt/fnsync + +# Ansible配置 +ansible_ssh_common_args='-o StrictHostKeyChecking=no' \ No newline at end of file diff --git a/configuration/nfs-volume.hcl b/configuration/nfs-volume.hcl new file mode 100644 index 0000000..3cc3920 --- /dev/null +++ b/configuration/nfs-volume.hcl @@ -0,0 +1,45 @@ +# NFS CSI Volume 配置 +type = "csi" +id = "nfs-fnsync" +name = "nfs-fnsync" +external_id = "nfs-fnsync" + +# 插件配置 +plugin_id = "nfs" +capacity_min = "1GiB" +capacity_max = "100GiB" + +# 挂载选项 +mount_options { + fs_type = "nfs4" + mount_flags = ["rw", "relatime", "vers=4.2"] +} + +# 访问模式 +access_mode = "single-node-writer" +attachment_mode = "file-system" + +# 拓扑约束 +topology_request { + preferred { + topology { + segments = { + "rack" = "rack-1" + } + } + } + + required { + topology { + segments = { + "datacenter" = "dc1" + } + } + } +} + +# 参数配置 +parameters { + server = "snail" + share = "/fs/1000/nfs/Fnsync" +} \ No newline at end of file diff --git a/configuration/playbooks/add-warden-to-nomad-cluster.yml b/configuration/playbooks/add/add-warden-to-nomad-cluster.yml similarity index 100% rename from configuration/playbooks/add-warden-to-nomad-cluster.yml rename to configuration/playbooks/add/add-warden-to-nomad-cluster.yml diff --git a/configuration/playbooks/configure-nomad-podman-cluster.yml b/configuration/playbooks/configure/configure-nomad-podman-cluster.yml similarity index 100% rename from configuration/playbooks/configure-nomad-podman-cluster.yml rename to configuration/playbooks/configure/configure-nomad-podman-cluster.yml diff --git a/configuration/playbooks/configure-nomad-sudo.yml b/configuration/playbooks/configure/configure-nomad-sudo.yml similarity index 100% rename from configuration/playbooks/configure-nomad-sudo.yml rename to configuration/playbooks/configure/configure-nomad-sudo.yml diff --git a/configuration/playbooks/configure-nomad-tailscale.yml b/configuration/playbooks/configure/configure-nomad-tailscale.yml similarity index 100% rename from configuration/playbooks/configure-nomad-tailscale.yml rename to configuration/playbooks/configure/configure-nomad-tailscale.yml diff --git a/configuration/playbooks/configure-podman-for-nomad.yml b/configuration/playbooks/configure/configure-podman-for-nomad.yml similarity index 100% rename from configuration/playbooks/configure-podman-for-nomad.yml rename to configuration/playbooks/configure/configure-podman-for-nomad.yml diff --git a/configuration/playbooks/debug-nomad-nodes.yml b/configuration/playbooks/debug/debug-nomad-nodes.yml similarity index 100% rename from configuration/playbooks/debug-nomad-nodes.yml rename to configuration/playbooks/debug/debug-nomad-nodes.yml diff --git a/configuration/playbooks/debug-nomad-podman.yml b/configuration/playbooks/debug/debug-nomad-podman.yml similarity index 100% rename from configuration/playbooks/debug-nomad-podman.yml rename to configuration/playbooks/debug/debug-nomad-podman.yml diff --git a/configuration/playbooks/debug-syd.yml b/configuration/playbooks/debug/debug-syd.yml similarity index 100% rename from configuration/playbooks/debug-syd.yml rename to configuration/playbooks/debug/debug-syd.yml diff --git a/configuration/playbooks/disk-analysis-ncdu.yml b/configuration/playbooks/disk/disk-analysis-ncdu.yml similarity index 100% rename from configuration/playbooks/disk-analysis-ncdu.yml rename to configuration/playbooks/disk/disk-analysis-ncdu.yml diff --git a/configuration/playbooks/disk-cleanup.yml b/configuration/playbooks/disk/disk-cleanup.yml similarity index 100% rename from configuration/playbooks/disk-cleanup.yml rename to configuration/playbooks/disk/disk-cleanup.yml diff --git a/configuration/playbooks/distribute-podman-driver.yml b/configuration/playbooks/distribute/distribute-podman-driver.yml similarity index 100% rename from configuration/playbooks/distribute-podman-driver.yml rename to configuration/playbooks/distribute/distribute-podman-driver.yml diff --git a/configuration/playbooks/distribute-podman.yml b/configuration/playbooks/distribute/distribute-podman.yml similarity index 100% rename from configuration/playbooks/distribute-podman.yml rename to configuration/playbooks/distribute/distribute-podman.yml diff --git a/configuration/playbooks/fix-apt-errors.yml b/configuration/playbooks/fix/fix-apt-errors.yml similarity index 100% rename from configuration/playbooks/fix-apt-errors.yml rename to configuration/playbooks/fix/fix-apt-errors.yml diff --git a/configuration/playbooks/fix-duplicate-podman-config.yml b/configuration/playbooks/fix/fix-duplicate-podman-config.yml similarity index 100% rename from configuration/playbooks/fix-duplicate-podman-config.yml rename to configuration/playbooks/fix/fix-duplicate-podman-config.yml diff --git a/configuration/playbooks/fix-hashicorp-apt-source.yml b/configuration/playbooks/fix/fix-hashicorp-apt-source.yml similarity index 100% rename from configuration/playbooks/fix-hashicorp-apt-source.yml rename to configuration/playbooks/fix/fix-hashicorp-apt-source.yml diff --git a/configuration/playbooks/fix-hcp-podman.yml b/configuration/playbooks/fix/fix-hcp-podman.yml similarity index 100% rename from configuration/playbooks/fix-hcp-podman.yml rename to configuration/playbooks/fix/fix-hcp-podman.yml diff --git a/configuration/playbooks/fix-hcs-dpkg-issue.yml b/configuration/playbooks/fix/fix-hcs-dpkg-issue.yml similarity index 100% rename from configuration/playbooks/fix-hcs-dpkg-issue.yml rename to configuration/playbooks/fix/fix-hcs-dpkg-issue.yml diff --git a/configuration/playbooks/fix-nomad-cluster.yml b/configuration/playbooks/fix/fix-nomad-cluster.yml similarity index 100% rename from configuration/playbooks/fix-nomad-cluster.yml rename to configuration/playbooks/fix/fix-nomad-cluster.yml diff --git a/configuration/playbooks/fix-nomad-local.yml b/configuration/playbooks/fix/fix-nomad-local.yml similarity index 100% rename from configuration/playbooks/fix-nomad-local.yml rename to configuration/playbooks/fix/fix-nomad-local.yml diff --git a/configuration/playbooks/fix-nomad-podman-config.yml b/configuration/playbooks/fix/fix-nomad-podman-config.yml similarity index 100% rename from configuration/playbooks/fix-nomad-podman-config.yml rename to configuration/playbooks/fix/fix-nomad-podman-config.yml diff --git a/configuration/playbooks/fix-nomad-server-config.yml b/configuration/playbooks/fix/fix-nomad-server-config.yml similarity index 100% rename from configuration/playbooks/fix-nomad-server-config.yml rename to configuration/playbooks/fix/fix-nomad-server-config.yml diff --git a/configuration/playbooks/fix-nomad-systemd.yml b/configuration/playbooks/fix/fix-nomad-systemd.yml similarity index 100% rename from configuration/playbooks/fix-nomad-systemd.yml rename to configuration/playbooks/fix/fix-nomad-systemd.yml diff --git a/configuration/playbooks/fix-podman-installation.yml b/configuration/playbooks/fix/fix-podman-installation.yml similarity index 100% rename from configuration/playbooks/fix-podman-installation.yml rename to configuration/playbooks/fix/fix-podman-installation.yml diff --git a/configuration/playbooks/fix-server-config.yml b/configuration/playbooks/fix/fix-server-config.yml similarity index 100% rename from configuration/playbooks/fix-server-config.yml rename to configuration/playbooks/fix/fix-server-config.yml diff --git a/configuration/playbooks/fix-server-network-config.yml b/configuration/playbooks/fix/fix-server-network-config.yml similarity index 100% rename from configuration/playbooks/fix-server-network-config.yml rename to configuration/playbooks/fix/fix-server-network-config.yml diff --git a/configuration/playbooks/fix-warden-compose.yml b/configuration/playbooks/fix/fix-warden-compose.yml similarity index 100% rename from configuration/playbooks/fix-warden-compose.yml rename to configuration/playbooks/fix/fix-warden-compose.yml diff --git a/configuration/playbooks/install-configure-nomad-podman-driver.yml b/configuration/playbooks/install/install-configure-nomad-podman-driver.yml similarity index 100% rename from configuration/playbooks/install-configure-nomad-podman-driver.yml rename to configuration/playbooks/install/install-configure-nomad-podman-driver.yml diff --git a/configuration/playbooks/install-consul.yml b/configuration/playbooks/install/install-consul.yml similarity index 100% rename from configuration/playbooks/install-consul.yml rename to configuration/playbooks/install/install-consul.yml diff --git a/configuration/playbooks/install-nomad-direct-download.yml b/configuration/playbooks/install/install-nomad-direct-download.yml similarity index 100% rename from configuration/playbooks/install-nomad-direct-download.yml rename to configuration/playbooks/install/install-nomad-direct-download.yml diff --git a/configuration/playbooks/install-nomad-podman-driver.yml b/configuration/playbooks/install/install-nomad-podman-driver.yml similarity index 100% rename from configuration/playbooks/install-nomad-podman-driver.yml rename to configuration/playbooks/install/install-nomad-podman-driver.yml diff --git a/configuration/playbooks/install-podman-compose.yml b/configuration/playbooks/install/install-podman-compose.yml similarity index 100% rename from configuration/playbooks/install-podman-compose.yml rename to configuration/playbooks/install/install-podman-compose.yml diff --git a/configuration/playbooks/check-podman-version.yml b/configuration/playbooks/other/check-podman-version.yml similarity index 100% rename from configuration/playbooks/check-podman-version.yml rename to configuration/playbooks/other/check-podman-version.yml diff --git a/configuration/playbooks/check-podman-versions.yml b/configuration/playbooks/other/check-podman-versions.yml similarity index 100% rename from configuration/playbooks/check-podman-versions.yml rename to configuration/playbooks/other/check-podman-versions.yml diff --git a/configuration/playbooks/cleanup-hashicorp-backups.yml b/configuration/playbooks/other/cleanup-hashicorp-backups.yml similarity index 100% rename from configuration/playbooks/cleanup-hashicorp-backups.yml rename to configuration/playbooks/other/cleanup-hashicorp-backups.yml diff --git a/configuration/playbooks/clear-aliases.yml b/configuration/playbooks/other/clear-aliases.yml similarity index 100% rename from configuration/playbooks/clear-aliases.yml rename to configuration/playbooks/other/clear-aliases.yml diff --git a/configuration/playbooks/clear-all-aliases.yml b/configuration/playbooks/other/clear-all-aliases.yml similarity index 100% rename from configuration/playbooks/clear-all-aliases.yml rename to configuration/playbooks/other/clear-all-aliases.yml diff --git a/configuration/playbooks/clear-proxy-settings.yml b/configuration/playbooks/other/clear-proxy-settings.yml similarity index 100% rename from configuration/playbooks/clear-proxy-settings.yml rename to configuration/playbooks/other/clear-proxy-settings.yml diff --git a/configuration/playbooks/clear-proxy.yml b/configuration/playbooks/other/clear-proxy.yml similarity index 100% rename from configuration/playbooks/clear-proxy.yml rename to configuration/playbooks/other/clear-proxy.yml diff --git a/configuration/playbooks/ensure-nomad-user.yml b/configuration/playbooks/other/ensure-nomad-user.yml similarity index 100% rename from configuration/playbooks/ensure-nomad-user.yml rename to configuration/playbooks/other/ensure-nomad-user.yml diff --git a/configuration/playbooks/final-podman-fix.yml b/configuration/playbooks/other/final-podman-fix.yml similarity index 100% rename from configuration/playbooks/final-podman-fix.yml rename to configuration/playbooks/other/final-podman-fix.yml diff --git a/configuration/playbooks/get-tailscale-ips.yml b/configuration/playbooks/other/get-tailscale-ips.yml similarity index 100% rename from configuration/playbooks/get-tailscale-ips.yml rename to configuration/playbooks/other/get-tailscale-ips.yml diff --git a/configuration/playbooks/hack-podman-upgrade.yml b/configuration/playbooks/other/hack-podman-upgrade.yml similarity index 100% rename from configuration/playbooks/hack-podman-upgrade.yml rename to configuration/playbooks/other/hack-podman-upgrade.yml diff --git a/configuration/playbooks/integrated-podman-setup.yml b/configuration/playbooks/other/integrated-podman-setup.yml similarity index 100% rename from configuration/playbooks/integrated-podman-setup.yml rename to configuration/playbooks/other/integrated-podman-setup.yml diff --git a/configuration/playbooks/migrate-to-podman-simple.yml b/configuration/playbooks/other/migrate-to-podman-simple.yml similarity index 100% rename from configuration/playbooks/migrate-to-podman-simple.yml rename to configuration/playbooks/other/migrate-to-podman-simple.yml diff --git a/configuration/playbooks/ping-nodes.yml b/configuration/playbooks/other/ping-nodes.yml similarity index 100% rename from configuration/playbooks/ping-nodes.yml rename to configuration/playbooks/other/ping-nodes.yml diff --git a/configuration/playbooks/read-nomad-config.yml b/configuration/playbooks/other/read-nomad-config.yml similarity index 100% rename from configuration/playbooks/read-nomad-config.yml rename to configuration/playbooks/other/read-nomad-config.yml diff --git a/configuration/playbooks/restart-tailscale.yml b/configuration/playbooks/other/restart-tailscale.yml similarity index 100% rename from configuration/playbooks/restart-tailscale.yml rename to configuration/playbooks/other/restart-tailscale.yml diff --git a/configuration/playbooks/update-nomad-config.yml b/configuration/playbooks/other/update-nomad-config.yml similarity index 100% rename from configuration/playbooks/update-nomad-config.yml rename to configuration/playbooks/other/update-nomad-config.yml diff --git a/configuration/playbooks/upgrade-podman-to-5.yml b/configuration/playbooks/other/upgrade-podman-to-5.yml similarity index 100% rename from configuration/playbooks/upgrade-podman-to-5.yml rename to configuration/playbooks/other/upgrade-podman-to-5.yml diff --git a/configuration/playbooks/remove-docker-install-podman-with-compose.yml b/configuration/playbooks/remove/remove-docker-install-podman-with-compose.yml similarity index 100% rename from configuration/playbooks/remove-docker-install-podman-with-compose.yml rename to configuration/playbooks/remove/remove-docker-install-podman-with-compose.yml diff --git a/configuration/playbooks/remove-docker-install-podman.yml b/configuration/playbooks/remove/remove-docker-install-podman.yml similarity index 100% rename from configuration/playbooks/remove-docker-install-podman.yml rename to configuration/playbooks/remove/remove-docker-install-podman.yml diff --git a/playbooks/add-beijing-node-prefix.yml b/configuration/playbooks/root_playbooks/add-beijing-node-prefix.yml similarity index 100% rename from playbooks/add-beijing-node-prefix.yml rename to configuration/playbooks/root_playbooks/add-beijing-node-prefix.yml diff --git a/playbooks/fix-duplicate-plugin-dir.yml b/configuration/playbooks/root_playbooks/fix-duplicate-plugin-dir.yml similarity index 100% rename from playbooks/fix-duplicate-plugin-dir.yml rename to configuration/playbooks/root_playbooks/fix-duplicate-plugin-dir.yml diff --git a/playbooks/fix-podman-driver-config.yml b/configuration/playbooks/root_playbooks/fix-podman-driver-config.yml similarity index 100% rename from playbooks/fix-podman-driver-config.yml rename to configuration/playbooks/root_playbooks/fix-podman-driver-config.yml diff --git a/playbooks/fix-warden-nfs.yml b/configuration/playbooks/root_playbooks/fix-warden-nfs.yml similarity index 100% rename from playbooks/fix-warden-nfs.yml rename to configuration/playbooks/root_playbooks/fix-warden-nfs.yml diff --git a/configuration/playbooks/root_playbooks/setup-nfs-by-container-type.yml b/configuration/playbooks/root_playbooks/setup-nfs-by-container-type.yml new file mode 100644 index 0000000..02fa0a8 --- /dev/null +++ b/configuration/playbooks/root_playbooks/setup-nfs-by-container-type.yml @@ -0,0 +1,82 @@ +--- +- name: Setup NFS for different container types + hosts: all + become: yes + vars: + nfs_server: snail + nfs_export_path: /fs/1000/nfs/Fnsync + nfs_mount_path: /mnt/fnsync + nfs_options_local: "rw,sync,vers=4.2" + nfs_options_overseas: "rw,sync,vers=3,timeo=600,retrans=2" + + tasks: + - name: Detect container type and location + set_fact: + container_type: "{{ 'lxc' if inventory_hostname in groups['lxc'] else 'pve' }}" + is_overseas: "{{ inventory_hostname in ['ash1d', 'ash2e', 'ash3c', 'ch2', 'ch3'] }}" + + - name: Install NFS client for all nodes + package: + name: nfs-common + state: present + + - name: Create mount directory for all nodes + file: + path: "{{ nfs_mount_path }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Mount NFS for local LXC containers (direct mount) + mount: + path: "{{ nfs_mount_path }}" + src: "{{ nfs_server }}:{{ nfs_export_path }}" + fstype: nfs + opts: "{{ nfs_options_local }}" + state: mounted + when: container_type == 'lxc' and not is_overseas + + - name: Mount NFS for overseas PVE containers (with retry options) + mount: + path: "{{ nfs_mount_path }}" + src: "{{ nfs_server }}:{{ nfs_export_path }}" + fstype: nfs + opts: "{{ nfs_options_overseas }}" + state: mounted + when: container_type == 'pve' and is_overseas + + - name: Ensure NFS mount persists after reboot + mount: + path: "{{ nfs_mount_path }}" + src: "{{ nfs_server }}:{{ nfs_export_path }}" + fstype: nfs + opts: "{{ nfs_options_local if container_type == 'lxc' and not is_overseas else nfs_options_overseas }}" + state: present + + - name: Verify NFS mount + command: df -h "{{ nfs_mount_path }}" + register: mount_result + ignore_errors: yes + + - name: Display mount status + debug: + msg: "{{ inventory_hostname }} - {{ container_type }} - {{ '海外' if is_overseas else '本地' }} - Mount: {{ '成功' if mount_result.rc == 0 else '失败' }}" + + - name: Create Nomad directories for LXC containers + file: + path: "{{ nfs_mount_path }}/nomad/{{ inventory_hostname }}" + state: directory + owner: nomad + group: nomad + mode: '0755' + when: container_type == 'lxc' + + - name: Create shared volumes directory for PVE containers + file: + path: "{{ nfs_mount_path }}/nomad/volumes/{{ inventory_hostname }}" + state: directory + owner: nomad + group: nomad + mode: '0755' + when: container_type == 'pve' \ No newline at end of file diff --git a/playbooks/setup-nfs-storage.yml b/configuration/playbooks/root_playbooks/setup-nfs-storage.yml similarity index 96% rename from playbooks/setup-nfs-storage.yml rename to configuration/playbooks/root_playbooks/setup-nfs-storage.yml index 51deb80..76c93c6 100644 --- a/playbooks/setup-nfs-storage.yml +++ b/configuration/playbooks/root_playbooks/setup-nfs-storage.yml @@ -4,8 +4,8 @@ gather_facts: false vars: nfs_server: snail - nfs_export_path: /fs/1000/nfs - nfs_mount_path: /opt/consul-shared + nfs_export_path: /fs/1000/nfs/Fnsync + nfs_mount_path: /mnt/fnsync tasks: - name: Install NFS client and mount on master diff --git a/configuration/playbooks/root_playbooks/setup-nomad-nfs-client.yml b/configuration/playbooks/root_playbooks/setup-nomad-nfs-client.yml new file mode 100644 index 0000000..77a233b --- /dev/null +++ b/configuration/playbooks/root_playbooks/setup-nomad-nfs-client.yml @@ -0,0 +1,50 @@ +--- +- name: Configure Nomad client for NFS volumes + hosts: nomad_clients + become: yes + vars: + nfs_mount_path: /mnt/fnsync + + tasks: + - name: Create Nomad plugin directory for NFS + file: + path: /opt/nomad/plugins + state: directory + owner: nomad + group: nomad + mode: '0755' + + - name: Configure Nomad client to use NFS volumes + blockinfile: + path: /etc/nomad.d/nomad.hcl + marker: "# {mark} NFS VOLUME CONFIGURATION" + block: | + plugin "nomad-driver-podman" { + config { + volumes { + enabled = true + } + } + } + + client { + host_volume "nfs-shared" { + path = "{{ nfs_mount_path }}/nomad/volumes" + read_only = false + } + } + insertafter: 'data_dir = "/opt/nomad/data"' + + - name: Restart Nomad service to apply changes + systemd: + name: nomad + state: restarted + + - name: Verify Nomad client configuration + command: nomad node status -self + register: nomad_status + ignore_errors: yes + + - name: Display Nomad status + debug: + msg: "{{ inventory_hostname }} - Nomad status: {{ '运行中' if nomad_status.rc == 0 else '异常' }}" \ No newline at end of file diff --git a/configuration/playbooks/root_playbooks/setup-nomad-nfs.yml b/configuration/playbooks/root_playbooks/setup-nomad-nfs.yml new file mode 100644 index 0000000..a813a3f --- /dev/null +++ b/configuration/playbooks/root_playbooks/setup-nomad-nfs.yml @@ -0,0 +1,63 @@ +--- +- name: Setup NFS Storage for Nomad Cluster + hosts: nomad_cluster + become: yes + vars: + nfs_server: snail + nfs_export_path: /fs/1000/nfs/Fnsync + nfs_mount_path: /mnt/fnsync + nfs_options: "rw,sync,vers=4.2" + + tasks: + - name: Install NFS client packages + package: + name: nfs-common + state: present + + - name: Create NFS mount directory + file: + path: "{{ nfs_mount_path }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: Mount NFS share + mount: + path: "{{ nfs_mount_path }}" + src: "{{ nfs_server }}:{{ nfs_export_path }}" + fstype: nfs + opts: "{{ nfs_options }}" + state: mounted + + - name: Ensure NFS mount persists after reboot + mount: + path: "{{ nfs_mount_path }}" + src: "{{ nfs_server }}:{{ nfs_export_path }}" + fstype: nfs + opts: "{{ nfs_options }}" + state: present + + - name: Verify NFS mount + command: df -h "{{ nfs_mount_path }}" + register: mount_result + + - name: Display mount result + debug: + var: mount_result.stdout + + - name: Create Nomad data directories on NFS + file: + path: "{{ nfs_mount_path }}/nomad/{{ inventory_hostname }}" + state: directory + owner: nomad + group: nomad + mode: '0755' + + - name: Create shared volumes directory + file: + path: "{{ nfs_mount_path }}/nomad/volumes" + state: directory + owner: nomad + group: nomad + mode: '0755' \ No newline at end of file diff --git a/configuration/playbooks/setup-disk-monitoring.yml b/configuration/playbooks/setup/setup-disk-monitoring.yml similarity index 100% rename from configuration/playbooks/setup-disk-monitoring.yml rename to configuration/playbooks/setup/setup-disk-monitoring.yml diff --git a/configuration/playbooks/setup-new-nomad-nodes.yml b/configuration/playbooks/setup/setup-new-nomad-nodes.yml similarity index 100% rename from configuration/playbooks/setup-new-nomad-nodes.yml rename to configuration/playbooks/setup/setup-new-nomad-nodes.yml diff --git a/configuration/playbooks/test-podman-snap-migration.yml b/configuration/playbooks/test/test-podman-snap-migration.yml similarity index 100% rename from configuration/playbooks/test-podman-snap-migration.yml rename to configuration/playbooks/test/test-podman-snap-migration.yml diff --git a/configuration/templates/nomad-client.hcl b/configuration/templates/nomad-client.hcl new file mode 100644 index 0000000..e371c4a --- /dev/null +++ b/configuration/templates/nomad-client.hcl @@ -0,0 +1,39 @@ +datacenter = "dc1" +data_dir = "/opt/nomad/data" +log_level = "INFO" + +# 使用Tailscale网络，但绑定到本地接口 +bind_addr = "0.0.0.0" + +server { + enabled = false +} + +# 启用客户端模式，支持混合存储 +client { + enabled = true + servers = ["100.116.158.95:4647"] + + # 配置host volumes + host_volume "fnsync" { + path = "/mnt/fnsync" + read_only = false + } +} + +# 指定Tailscale地址用于通信 +addresses { + http = "{{ ansible_host }}" + rpc = "{{ ansible_host }}" + serf = "{{ ansible_host }}" +} + +advertise { + http = "{{ ansible_host }}:4646" + rpc = "{{ ansible_host }}:4647" + serf = "{{ ansible_host }}:4648" +} + +consul { + address = "100.116.158.95:8500" +} \ No newline at end of file diff --git a/docs/nomad-nfs-setup.md b/docs/nomad-nfs-setup.md new file mode 100644 index 0000000..ac40306 --- /dev/null +++ b/docs/nomad-nfs-setup.md @@ -0,0 +1,146 @@ +# Nomad集群NFS配置指南 + +## 概述 + +本文档介绍如何为Nomad集群配置NFS存储，支持不同类型的容器和地理位置。 + +## 容器类型分类 + +### 1. 本地LXC容器 +- **位置**: 本地网络环境 +- **节点示例**: influxdb, warden, hcp1, hcp2 +- **特点**: 直接使用已映射的NFS目录 +- **NFS参数**: `rw,sync,vers=4.2` + +### 2. 海外PVE容器 +- **位置**: 海外云服务器 +- **节点示例**: ash1d, ash2e, ash3c, ch2, ch3 +- **特点**: 需要网络优化参数 +- **NFS参数**: `rw,sync,vers=3,timeo=600,retrans=2` + +## NFS配置详情 + +### NFS服务器信息 +- **服务器**: snail +- **导出路径**: `/fs/1000/nfs/Fnsync` +- **挂载点**: `/mnt/fnsync` + +### 当前挂载状态 +```bash +# 检查当前挂载 +df -h | grep fnsync +# 输出: snail:/fs/1000/nfs/Fnsync 8.2T 2.2T 6.0T 27% /mnt/fnsync +``` + +## 部署步骤 + +### 1. 自动部署 +```bash +chmod +x scripts/deploy-nfs-for-nomad.sh +./scripts/deploy-nfs-for-nomad.sh +``` + +### 2. 手动分步部署 +```bash +# 步骤1: 配置NFS挂载 +ansible-playbook -i configuration/inventories/production/inventory.ini \ + playbooks/setup-nfs-by-container-type.yml + +# 步骤2: 配置Nomad客户端 +ansible-playbook -i configuration/inventories/production/nomad-cluster.ini \ + playbooks/setup-nomad-nfs-client.yml +``` + +## Nomad作业配置 + +### 使用NFS卷的Nomad作业示例 + +```hcl +job "nfs-example" { + volume "nfs-shared" { + type = "host" + source = "nfs-shared" + read_only = false + } + + task "app" { + volume_mount { + volume = "nfs-shared" + destination = "/shared" + read_only = false + } + } +} +``` + +### 针对不同容器类型的约束 + +```hcl +# 本地LXC容器约束 +constraint { + attribute = "${attr.unique.hostname}" + operator = "regexp" + value = "(influxdb|warden|hcp1|hcp2)" +} + +# 海外PVE容器约束 +constraint { + attribute = "${attr.unique.hostname}" + operator = "regexp" + value = "(ash1d|ash2e|ash3c|ch2|ch3)" +} +``` + +## 验证和监控 + +### 验证命令 +```bash +# 检查NFS挂载 +ansible all -i configuration/inventories/production/inventory.ini \ + -m shell -a "df -h /mnt/fnsync" + +# 检查Nomad状态 +nomad node status + +# 检查NFS任务状态 +nomad job status nfs-multi-type-example +``` + +### 监控指标 +- NFS挂载状态 +- 网络延迟（海外节点） +- 存储使用情况 +- Nomad任务运行状态 + +## 故障排除 + +### 常见问题 + +1. **NFS挂载失败** + - 检查网络连通性: `ping snail` + - 验证NFS服务: `showmount -e snail` + - 检查防火墙设置 + +2. **海外节点连接慢** + - 使用NFSv3协议 + - 增加超时参数 + - 考虑使用缓存方案 + +3. **Nomad卷无法挂载** + - 检查Nomad客户端配置 + - 验证目录权限 + - 检查Nomad服务状态 + +## 最佳实践 + +1. **数据备份**: 定期备份NFS上的重要数据 +2. **监控告警**: 设置NFS挂载状态监控 +3. **容量规划**: 监控存储使用情况 +4. **网络优化**: 为海外节点配置合适的网络参数 + +## 相关文件 + +- `playbooks/setup-nfs-by-container-type.yml` - NFS挂载配置 +- `playbooks/setup-nomad-nfs-client.yml` - Nomad客户端配置 +- `jobs/nomad-nfs-multi-type.nomad` - 示例Nomad作业 +- `scripts/deploy-nfs-for-nomad.sh` - 部署脚本 \ No newline at end of file diff --git a/fix_remote_apt_permissions.sh b/fix_remote_apt_permissions.sh new file mode 100644 index 0000000..cd5e386 --- /dev/null +++ b/fix_remote_apt_permissions.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# 脚本用于检查和修复远程节点上的apt临时文件权限问题 + +# 定义需要检查的节点列表 +NODES=('ash2e' 'ash1d' 'ch2') + +# 循环检查每个节点 +for NODE in "${NODES[@]}"; do + echo "\n===== 检查节点: $NODE =====" + + # 检查SSH连接是否可用 + if ! ssh -q -o BatchMode=yes -o ConnectTimeout=5 root@$NODE "echo Connected > /dev/null"; then + echo "错误: 无法连接到节点 $NODE" + continue + fi + + echo "成功连接到节点 $NODE" + + # 1. 检查/tmp目录权限 + echo -n "检查/tmp目录权限: " + ssh root@$NODE "ls -ld /tmp" + + # 2. 检查磁盘空间 + echo "检查磁盘空间:" + ssh root@$NODE "df -h" + + # 3. 测试创建临时文件 + echo -n "测试创建临时文件: " + if ssh root@$NODE "touch /tmp/test-apt-temp-$$ 2>/dev/null && echo 成功 && rm -f /tmp/test-apt-temp-$$"; then + echo "临时文件创建成功" + else + echo "错误: 无法创建临时文件" + # 尝试修复/tmp目录权限 + echo "尝试修复/tmp目录权限..." + ssh root@$NODE "chmod 1777 /tmp" + fi + + # 4. 清理apt缓存 + echo "清理apt缓存..." + ssh root@$NODE "apt clean" + + # 5. 尝试运行apt update测试 + echo "测试apt update (仅输出前10行)..." + ssh root@$NODE "apt update 2>&1 | head -n 10" + + echo "\n节点 $NODE 检查完成\n" +done + +# 提供额外的修复建议 +echo "\n===== 修复建议 =====" +echo "1. 如果问题仍然存在，请检查以下内容：" +echo " - /etc/apt/apt.conf文件中的配置" +echo " - apt-key命令的完整性 (dpkg -l apt) +echo " - 系统的临时文件清理服务状态" +echo "2. 可以尝试的命令：" +echo " - dpkg --configure -a" +echo " - apt-get install --reinstall apt apt-utils" +echo "3. 对于特定节点的持久问题，建议登录到该节点进行更详细的故障排除。" \ No newline at end of file diff --git a/jobs/consul-cluster-arm64.nomad b/jobs/consul/consul-cluster-arm64.nomad similarity index 100% rename from jobs/consul-cluster-arm64.nomad rename to jobs/consul/consul-cluster-arm64.nomad diff --git a/jobs/consul-cluster-binary.nomad b/jobs/consul/consul-cluster-binary.nomad similarity index 100% rename from jobs/consul-cluster-binary.nomad rename to jobs/consul/consul-cluster-binary.nomad diff --git a/jobs/consul-cluster-nomad.nomad b/jobs/consul/consul-cluster-nomad.nomad similarity index 100% rename from jobs/consul-cluster-nomad.nomad rename to jobs/consul/consul-cluster-nomad.nomad diff --git a/jobs/consul-cluster-simple.nomad b/jobs/consul/consul-cluster-simple.nomad similarity index 100% rename from jobs/consul-cluster-simple.nomad rename to jobs/consul/consul-cluster-simple.nomad diff --git a/jobs/consul-cluster-three-nodes.nomad b/jobs/consul/consul-cluster-three-nodes.nomad similarity index 100% rename from jobs/consul-cluster-three-nodes.nomad rename to jobs/consul/consul-cluster-three-nodes.nomad diff --git a/jobs/consul-cluster.nomad b/jobs/consul/consul-cluster.nomad similarity index 100% rename from jobs/consul-cluster.nomad rename to jobs/consul/consul-cluster.nomad diff --git a/jobs/consul-single-member-new.nomad b/jobs/consul/consul-single-member-new.nomad similarity index 100% rename from jobs/consul-single-member-new.nomad rename to jobs/consul/consul-single-member-new.nomad diff --git a/jobs/consul-single-member.nomad b/jobs/consul/consul-single-member.nomad similarity index 100% rename from jobs/consul-single-member.nomad rename to jobs/consul/consul-single-member.nomad diff --git a/jobs/consul-test-warden.nomad b/jobs/consul/consul-test-warden.nomad similarity index 100% rename from jobs/consul-test-warden.nomad rename to jobs/consul/consul-test-warden.nomad diff --git a/jobs/consul-warden-only.nomad b/jobs/consul/consul-warden-only.nomad similarity index 100% rename from jobs/consul-warden-only.nomad rename to jobs/consul/consul-warden-only.nomad diff --git a/jobs/service-discovery-warden.nomad b/jobs/consul/service-discovery-warden.nomad similarity index 100% rename from jobs/service-discovery-warden.nomad rename to jobs/consul/service-discovery-warden.nomad diff --git a/jobs/simple-consul-warden.nomad b/jobs/consul/simple-consul-warden.nomad similarity index 100% rename from jobs/simple-consul-warden.nomad rename to jobs/consul/simple-consul-warden.nomad diff --git a/jobs/hybrid-nfs-app.nomad b/jobs/hybrid-nfs-app.nomad new file mode 100644 index 0000000..cfd8829 --- /dev/null +++ b/jobs/hybrid-nfs-app.nomad @@ -0,0 +1,65 @@ +job "hybrid-nfs-app" { + datacenters = ["dc1"] + type = "service" + + # 使用约束条件区分存储类型 + constraint { + attribute = "${attr.unique.hostname}" + operator = "regexp" + value = "semaphore" + } + + group "app" { + count = 1 + + network { + port "http" { + static = 8080 + } + } + + # 对于本机（semaphore）使用host volume + volume "local-storage" { + type = "host" + read_only = false + source = "local-fnsync" + } + + task "web-app" { + driver = "exec" + + config { + command = "python3" + args = ["-m", "http.server", "8080", "--directory", "local/fnsync"] + } + + template { + data = <Hybrid NFS App - Running on {{ env "attr.unique.hostname" }} +

Storage Type: {{ with eq (env "attr.unique.hostname") "semaphore" }}PVE Mount{{ else }}NFS{{ end }}

+

Timestamp: {{ now | date "2006-01-02 15:04:05" }}

+EOH + destination = "local/fnsync/index.html" + } + + resources { + cpu = 100 + memory = 128 + } + + service { + name = "hybrid-nfs-app" + port = "http" + + tags = ["hybrid", "nfs", "web"] + + check { + type = "http" + path = "/" + interval = "10s" + timeout = "2s" + } + } + } + } +} \ No newline at end of file diff --git a/jobs/nfs-app-example.nomad b/jobs/nfs-app-example.nomad new file mode 100644 index 0000000..4216aa8 --- /dev/null +++ b/jobs/nfs-app-example.nomad @@ -0,0 +1,51 @@ +job "nfs-app-example" { + datacenters = ["dc1"] + type = "service" + + group "app" { + count = 1 + + # 使用NFS存储卷 + volume "nfs-storage" { + type = "host" + read_only = false + source = "nfs-fnsync" + } + + task "web-app" { + driver = "docker" + + config { + image = "nginx:alpine" + ports = ["http"] + + # 挂载NFS卷到容器 + mount { + type = "volume" + target = "/usr/share/nginx/html" + source = "nfs-storage" + readonly = false + } + } + + resources { + cpu = 100 + memory = 128 + } + + service { + name = "nfs-web-app" + port = "http" + + tags = ["nfs", "web"] + + check { + type = "http" + path = "/" + interval = "10s" + timeout = "2s" + } + } + } + } +} \ No newline at end of file diff --git a/jobs/nfs-storage-test.nomad b/jobs/nfs-storage-test.nomad new file mode 100644 index 0000000..38f5f21 --- /dev/null +++ b/jobs/nfs-storage-test.nomad @@ -0,0 +1,34 @@ +job "nfs-storage-test" { + datacenters = ["dc1"] + type = "batch" + + group "test" { + count = 1 + + volume "nfs-storage" { + type = "csi" + read_only = false + source = "nfs-fnsync" + } + + task "storage-test" { + driver = "exec" + + volume_mount { + volume = "nfs-storage" + destination = "/mnt/nfs" + read_only = false + } + + config { + command = "/bin/sh" + args = ["-c", "echo 'NFS Storage Test - $(hostname) - $(date)' > /mnt/nfs/test-$(hostname).txt && ls -la /mnt/nfs/"] + } + + resources { + cpu = 50 + memory = 64 + } + } + } +} \ No newline at end of file diff --git a/jobs/nomad-nfs-multi-type.nomad b/jobs/nomad-nfs-multi-type.nomad new file mode 100644 index 0000000..1cb3b49 --- /dev/null +++ b/jobs/nomad-nfs-multi-type.nomad @@ -0,0 +1,84 @@ +job "nfs-multi-type-example" { + datacenters = ["dc1"] + type = "service" + + # 为本地LXC容器配置的任务组 + group "lxc-apps" { + count = 2 + + constraint { + attribute = "${attr.unique.hostname}" + operator = "regexp" + value = "(influxdb|hcp)" + } + + volume "lxc-nfs" { + type = "host" + source = "nfs-shared" + read_only = false + } + + task "lxc-app" { + driver = "podman" + + config { + image = "alpine:latest" + args = ["tail", "-f", "/dev/null"] + } + + volume_mount { + volume = "lxc-nfs" + destination = "/shared/lxc" + read_only = false + } + + resources { + cpu = 100 + memory = 64 + } + } + } + + # 为海外PVE容器配置的任务组 + group "pve-apps" { + count = 3 + + constraint { + attribute = "${attr.unique.hostname}" + operator = "regexp" + value = "(ash1d|ash2e|ash3c|ch2|ch3)" + } + + volume "pve-nfs" { + type = "host" + source = "nfs-shared" + read_only = false + } + + task "pve-app" { + driver = "podman" + + config { + image = "alpine:latest" + args = ["tail", "-f", "/dev/null"] + + # 为海外节点添加网络优化参数 + network_mode = "host" + } + + volume_mount { + volume = "pve-nfs" + destination = "/shared/pve" + read_only = false + } + + resources { + cpu = 100 + memory = 64 + network { + mbits = 5 + } + } + } + } +} \ No newline at end of file diff --git a/jobs/install-podman-driver.nomad b/jobs/podman/install-podman-driver.nomad similarity index 100% rename from jobs/install-podman-driver.nomad rename to jobs/podman/install-podman-driver.nomad diff --git a/jobs/podman/nomad-nfs-volume.nomad b/jobs/podman/nomad-nfs-volume.nomad new file mode 100644 index 0000000..a13df61 --- /dev/null +++ b/jobs/podman/nomad-nfs-volume.nomad @@ -0,0 +1,34 @@ +job "nfs-volume-example" { + datacenters = ["dc1"] + type = "service" + + group "nfs-app" { + count = 1 + + volume "nfs-shared" { + type = "host" + source = "nfs-shared" + read_only = false + } + + task "app" { + driver = "podman" + + config { + image = "alpine:latest" + args = ["tail", "-f", "/dev/null"] + } + + volume_mount { + volume = "nfs-shared" + destination = "/shared" + read_only = false + } + + resources { + cpu = 100 + memory = 64 + } + } + } +} \ No newline at end of file diff --git a/jobs/test-job.nomad b/jobs/tests/test-job.nomad similarity index 100% rename from jobs/test-job.nomad rename to jobs/tests/test-job.nomad diff --git a/jobs/test-podman-job.nomad b/jobs/tests/test-podman-job.nomad similarity index 100% rename from jobs/test-podman-job.nomad rename to jobs/tests/test-podman-job.nomad diff --git a/jobs/test-podman-simple.nomad b/jobs/tests/test-podman-simple.nomad similarity index 100% rename from jobs/test-podman-simple.nomad rename to jobs/tests/test-podman-simple.nomad diff --git a/jobs/test-private-registry.nomad b/jobs/tests/test-private-registry.nomad similarity index 100% rename from jobs/test-private-registry.nomad rename to jobs/tests/test-private-registry.nomad diff --git a/jobs/test-simple.nomad b/jobs/tests/test-simple.nomad similarity index 100% rename from jobs/test-simple.nomad rename to jobs/tests/test-simple.nomad diff --git a/list_playbooks.sh b/list_playbooks.sh new file mode 100755 index 0000000..9c9cabd --- /dev/null +++ b/list_playbooks.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# 列出所有playbooks文件 +ls -1 /root/mgmt/configuration/playbooks/*.yml \ No newline at end of file diff --git a/playbooks/configure-nomad-clients.yml b/playbooks/configure-nomad-clients.yml new file mode 100644 index 0000000..b6007e4 --- /dev/null +++ b/playbooks/configure-nomad-clients.yml @@ -0,0 +1,72 @@ +--- +- name: 配置Nomad客户端节点 + hosts: nomad_nodes:!semaphore + become: yes + vars: + nomad_config_dir: /etc/nomad.d + + tasks: + - name: 创建Nomad配置目录 + file: + path: "{{ nomad_config_dir }}" + state: directory + owner: root + group: root + mode: '0755' + + - name: 复制Nomad客户端配置 + copy: + content: | + datacenter = "dc1" + data_dir = "/opt/nomad/data" + log_level = "INFO" + bind_addr = "0.0.0.0" + + server { + enabled = false + } + + client { + enabled = true + servers = ["100.116.158.95:4647"] + host_volume "fnsync" { + path = "/mnt/fnsync" + read_only = false + } + } + + addresses { + http = "{{ ansible_host }}" + rpc = "{{ ansible_host }}" + serf = "{{ ansible_host }}" + } + + advertise { + http = "{{ ansible_host }}:4646" + rpc = "{{ ansible_host }}:4647" + serf = "{{ ansible_host }}:4648" + } + + consul { + address = "100.116.158.95:8500" + } + dest: "{{ nomad_config_dir }}/nomad.hcl" + owner: root + group: root + mode: '0644' + + - name: 启动Nomad服务 + systemd: + name: nomad + state: restarted + enabled: yes + daemon_reload: yes + + - name: 检查Nomad服务状态 + command: systemctl status nomad + register: nomad_status + changed_when: false + + - name: 显示Nomad服务状态 + debug: + var: nomad_status.stdout_lines \ No newline at end of file diff --git a/playbooks/nfs-mount.yml b/playbooks/nfs-mount.yml new file mode 100644 index 0000000..315de6d --- /dev/null +++ b/playbooks/nfs-mount.yml @@ -0,0 +1,42 @@ +--- +- name: 配置Nomad节点NFS挂载 + hosts: nomad_nodes + become: yes + vars: + nfs_server: "snail" + nfs_share: "/fs/1000/nfs/Fnsync" + mount_point: "/mnt/fnsync" + + tasks: + - name: 安装NFS客户端 + package: + name: nfs-common + state: present + + - name: 创建挂载目录 + file: + path: "{{ mount_point }}" + state: directory + mode: '0755' + + - name: 临时挂载NFS共享 + mount: + path: "{{ mount_point }}" + src: "{{ nfs_server }}:{{ nfs_share }}" + fstype: nfs4 + opts: "rw,relatime,vers=4.2" + state: mounted + + - name: 配置开机自动挂载 + lineinfile: + path: /etc/fstab + line: "{{ nfs_server }}:{{ nfs_share }} {{ mount_point }} nfs4 rw,relatime,vers=4.2 0 0" + state: present + + - name: 验证挂载 + command: df -h {{ mount_point }} + register: mount_check + + - name: 显示挂载信息 + debug: + var: mount_check.stdout_lines \ No newline at end of file diff --git a/playbooks/setup-nfs-nodes.yml b/playbooks/setup-nfs-nodes.yml new file mode 100644 index 0000000..c9018db --- /dev/null +++ b/playbooks/setup-nfs-nodes.yml @@ -0,0 +1,43 @@ +--- +- name: 设置Nomad节点NFS挂载 + hosts: nomad_nodes + become: yes + vars: + nfs_server: "snail" + nfs_share: "/fs/1000/nfs/Fnsync" + mount_point: "/mnt/fnsync" + + tasks: + + - name: 安装NFS客户端 + package: + name: nfs-common + state: present + + - name: 创建挂载目录 + file: + path: "{{ mount_point }}" + state: directory + mode: '0755' + + - name: 临时挂载NFS共享 + mount: + path: "{{ mount_point }}" + src: "{{ nfs_server }}:{{ nfs_share }}" + fstype: nfs4 + opts: "rw,relatime,vers=4.2" + state: mounted + + - name: 配置开机自动挂载 + lineinfile: + path: /etc/fstab + line: "{{ nfs_server }}:{{ nfs_share }} {{ mount_point }} nfs4 rw,relatime,vers=4.2 0 0" + state: present + + - name: 验证挂载 + command: df -h {{ mount_point }} + register: mount_check + + - name: 显示挂载信息 + debug: + var: mount_check.stdout_lines \ No newline at end of file diff --git a/scripts/deploy-nfs-for-nomad.sh b/scripts/deploy-nfs-for-nomad.sh new file mode 100755 index 0000000..be308c8 --- /dev/null +++ b/scripts/deploy-nfs-for-nomad.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Nomad集群NFS配置部署脚本 +# 根据容器类型和地理位置进行分情况处理 + +set -e + +echo "🚀 开始部署Nomad集群NFS配置..." + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# 函数：打印带颜色的消息 +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# 检查当前目录 +if [ ! -f "configuration/inventories/production/inventory.ini" ]; then + log_error "请在mgmt项目根目录运行此脚本" + exit 1 +fi + +# 1. 为所有节点配置NFS挂载 +log_info "步骤1: 为所有节点配置NFS挂载 (根据容器类型和地理位置)" +ansible-playbook -i configuration/inventories/production/inventory.ini \ + playbooks/setup-nfs-by-container-type.yml + +# 2. 为Nomad客户端配置NFS卷支持 +log_info "步骤2: 配置Nomad客户端支持NFS卷" +ansible-playbook -i configuration/inventories/production/nomad-cluster.ini \ + playbooks/setup-nomad-nfs-client.yml + +# 3. 验证NFS挂载状态 +log_info "步骤3: 验证所有节点的NFS挂载状态" +ansible all -i configuration/inventories/production/inventory.ini \ + -m shell -a "df -h /mnt/fnsync 2>/dev/null || echo 'NFS未挂载'" \ + --limit '!snail' + +# 4. 验证Nomad客户端配置 +log_info "步骤4: 验证Nomad客户端配置" +ansible nomad_clients -i configuration/inventories/production/nomad-cluster.ini \ + -m shell -a "nomad node status -self 2>/dev/null || echo 'Nomad未运行'" + +# 5. 部署示例NFS任务（可选） +read -p "是否部署示例NFS任务？(y/n): " deploy_example +if [ "$deploy_example" = "y" ] || [ "$deploy_example" = "Y" ]; then + log_info "部署示例NFS任务..." + nomad run jobs/nomad-nfs-multi-type.nomad + echo "等待任务启动..." + sleep 10 + nomad job status nfs-multi-type-example +fi + +log_info "✅ NFS配置部署完成!" +echo "" +echo "📋 使用说明:" +echo "1. NFS挂载点: /mnt/fnsync" +echo "2. 本地LXC容器: 直接使用挂载目录" +echo "3. 海外PVE容器: 使用优化参数挂载" +echo "4. Nomad作业: 使用host volume 'nfs-shared'" +echo "" +echo "🔧 手动验证命令:" +echo " - 检查NFS挂载: df -h /mnt/fnsync" +echo " - 检查Nomad状态: nomad node status" +echo " - 运行NFS任务: nomad run jobs/nomad-nfs-multi-type.nomad" \ No newline at end of file diff --git a/scripts/distribute-keys.sh b/scripts/distribute-keys.sh new file mode 100644 index 0000000..f626114 --- /dev/null +++ b/scripts/distribute-keys.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# 分发SSH公钥到所有Nomad节点 +echo "分发SSH公钥到Nomad节点..." + +# 节点列表 +NODES=( + "100.81.26.3" # ash1d.global + "100.103.147.94" # ash2e.global + "100.90.159.68" # ch2.global + "100.86.141.112" # ch3.global + "100.117.106.136" # master + "100.116.80.94" # ash3c +) + +PUB_KEY=$(cat /home/ben/.ssh/id_ed25519.pub) + +for NODE in "${NODES[@]}"; do + echo "正在配置节点: $NODE" + + # 尝试使用现有密钥连接并添加新密钥 + ssh-keyscan -H $NODE >> ~/.ssh/known_hosts 2>/dev/null + + # 使用现有认证方式添加密钥 + ssh root@$NODE "echo '$PUB_KEY' >> /root/.ssh/authorized_keys" 2>/dev/null && \ + echo "✓ $NODE 配置成功" || echo "✗ $NODE 配置失败" +done + +echo "密钥分发完成" \ No newline at end of file diff --git a/scripts/setup-ssh-keys.yml b/scripts/setup-ssh-keys.yml new file mode 100644 index 0000000..eeeb595 --- /dev/null +++ b/scripts/setup-ssh-keys.yml @@ -0,0 +1,22 @@ +--- +- name: 设置Nomad节点SSH密钥认证 + hosts: nomad_nodes + become: yes + vars: + ssh_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIddJVPEvFRtzhWwYjr21lKTar+d7R5Kn/6bhd2s231 ben@ch2" + + tasks: + - name: 确保.ssh目录存在 + file: + path: /root/.ssh + state: directory + mode: '0700' + + - name: 添加SSH公钥到authorized_keys + authorized_key: + user: root + state: present + key: "{{ ssh_public_key }}" + + - name: 测试SSH连接 + ping: \ No newline at end of file diff --git a/scripts/verify-nfs-config.sh b/scripts/verify-nfs-config.sh new file mode 100755 index 0000000..bcb41bb --- /dev/null +++ b/scripts/verify-nfs-config.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# NFS配置验证脚本 + +set -e + +echo "🔍 验证NFS配置状态..." + +# 颜色定义 +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# 1. 检查本地NFS挂载 +log_info "1. 检查本地NFS挂载状态" +if df -h | grep -q "/mnt/fnsync"; then + log_info "✅ 本地NFS挂载正常" + df -h | grep "/mnt/fnsync" +else + log_error "❌ 本地NFS未挂载" +fi + +# 2. 检查配置文件存在 +log_info "2. 检查配置文件" +config_files=( + "playbooks/setup-nfs-by-container-type.yml" + "playbooks/setup-nomad-nfs-client.yml" + "jobs/nomad-nfs-multi-type.nomad" + "scripts/deploy-nfs-for-nomad.sh" + "docs/nomad-nfs-setup.md" +) + +for file in "${config_files[@]}"; do + if [ -f "$file" ]; then + log_info "✅ $file 存在" + else + log_error "❌ $file 不存在" + fi +done + +# 3. 检查Ansible inventory +log_info "3. 检查Ansible配置" +if [ -f "configuration/inventories/production/inventory.ini" ]; then + log_info "✅ inventory.ini 存在" + echo "节点分类:" + grep -E "\[.*\]" configuration/inventories/production/inventory.ini | head -10 +else + log_error "❌ inventory.ini 不存在" +fi + +# 4. 检查Nomad服务状态 +log_info "4. 检查Nomad服务" +if command -v nomad &> /dev/null; then + if nomad node status &> /dev/null; then + log_info "✅ Nomad服务运行正常" + nomad node status -self | grep -E "(Name|Status|Datacenter)" + else + log_warn "⚠️ Nomad服务未运行或无法连接" + fi +else + log_warn "⚠️ Nomad命令未安装" +fi + +# 5. 检查NFS服务器连通性 +log_info "5. 检查NFS服务器连通性" +if ping -c 1 -W 3 snail &> /dev/null; then + log_info "✅ NFS服务器 snail 可达" + if command -v showmount &> /dev/null; then + showmount -e snail 2>/dev/null || log_warn "⚠️ 无法获取NFS导出列表" + fi +else + log_error "❌ NFS服务器 snail 不可达" +fi + +echo "" +echo "📊 验证完成!" +echo "" +echo "🚀 下一步操作:" +echo "1. 运行部署脚本: ./scripts/deploy-nfs-for-nomad.sh" +echo "2. 查看详细文档: cat docs/nomad-nfs-setup.md" +echo "3. 测试NFS功能: nomad run jobs/nomad-nfs-multi-type.nomad" \ No newline at end of file