diff --git a/.kilocode/mcp.json.backup b/.kilocode/mcp.json.backup deleted file mode 100644 index 8bafd83..0000000 --- a/.kilocode/mcp.json.backup +++ /dev/null @@ -1,90 +0,0 @@ -{ - "mcpServers": { - "context7": { - "command": "npx", - "args": [ - "-y", - "@upstash/context7-mcp" - ], - "env": { - "DEFAULT_MINIMUM_TOKENS": "" - } - }, - "filesystem": { - "command": "npx", - "args": [ - "-y", - "@modelcontextprotocol/server-filesystem", - "./" - ], - "disabled": false, - "alwaysAllow": [] - }, - "sequentialthinking": { - "command": "npx", - "args": [ - "-y", - "@modelcontextprotocol/server-sequential-thinking" - ], - "alwaysAllow": [ - "sequentialthinking" - ] - }, - "git": { - "command": "uvx", - "args": [ - "mcp-server-git", - "--repository", - "./" - ], - "alwaysAllow": [ - "git_status", - "git_diff_unstaged", - "git_diff", - "git_diff_staged", - "git_commit", - "git_add", - "git_reset", - "git_log", - "git_create_branch", - "git_checkout", - "git_show", - "git_branch" - ] - }, - "time": { - "command": "uvx", - "args": [ - "mcp-server-time" - ], - "alwaysAllow": [ - "get_current_time", - "convert_time" - ] - }, - "memory": { - "command": "npx", - "args": [ - "-y", - "@modelcontextprotocol/server-memory" - ], - "alwaysAllow": [ - "create_entities", - "create_relations", - "add_observations", - "delete_entities", - "delete_observations" - ] - }, - "tavily": { - "command": "npx", - "args": [ - "-y", - "tavily-mcp@0.2.3" - ], - "env": { - "TAVILY_API_KEY": "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" - } - } - } -} diff --git a/.kilocode/mcp.json.backup b/.kilocode/mcp.json.backup new file mode 120000 index 0000000..413e870 --- /dev/null +++ b/.kilocode/mcp.json.backup @@ -0,0 +1 @@ +/mnt/fnsync/mcp/mcp_shared_config.json \ No newline at end of file diff --git a/Makefile b/Makefile index 9e0dbf3..459a78f 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,22 @@ test: ## 运行测试 @echo "🧪 运行测试..." @bash scripts/utilities/run-tests.sh +test-mcp: ## 运行MCP服务器测试 + @echo "🧪 运行MCP服务器测试..." + @./run_tests.sh + +test-kali: ## 运行Kali Linux快速健康检查 + @echo "🧪 运行Kali Linux快速健康检查..." + @cd configuration && ansible-playbook -i inventories/production/inventory.ini playbooks/test/kali-health-check.yml + +test-kali-security: ## 运行Kali Linux安全工具测试 + @echo "🧪 运行Kali Linux安全工具测试..." + @cd configuration && ansible-playbook -i inventories/production/inventory.ini playbooks/test/kali-security-tools.yml + +test-kali-full: ## 运行Kali Linux完整测试套件 + @echo "🧪 运行Kali Linux完整测试套件..." + @cd configuration && ansible-playbook playbooks/test/kali-full-test-suite.yml + lint: ## 代码检查 @echo "🔍 代码检查..." @bash scripts/utilities/lint.sh diff --git a/README.md b/README.md index b28938f..d2c9de1 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,10 @@ mgmt/ ├── security/ # 安全配置 │ ├── certificates/ # 证书文件 │ └── policies/ # 安全策略 +├── tests/ # 测试脚本和报告 +│ ├── mcp_servers/ # MCP服务器测试脚本 +│ ├── mcp_server_test_report.md # MCP服务器测试报告 +│ └── legacy/ # 旧的测试脚本 ├── tools/ # 工具和实用程序 ├── playbooks/ # 核心Ansible剧本 └── Makefile # 项目管理命令 @@ -195,6 +199,10 @@ nomad node status -address=http://${LEADER} | `nomad node status` | 查看 Nomad 节点状态 | | `podman ps` | 查看运行中的容器 | | `ansible-playbook playbooks/configure-nomad-clients.yml` | 配置 Nomad 客户端 | +| `./run_tests.sh` 或 `make test-mcp` | 运行所有MCP服务器测试 | +| `make test-kali` | 运行Kali Linux快速健康检查 | +| `make test-kali-security` | 运行Kali Linux安全工具测试 | +| `make test-kali-full` | 运行Kali Linux完整测试套件 | ## 🌩️ 支持的云服务商 @@ -283,6 +291,45 @@ nomad node status -address=http://${LEADER} - **集成测试**: 服务间集成测试 - **端到端测试**: 完整流程测试 +### MCP服务器测试 +项目包含完整的MCP（Model Context Protocol）服务器测试套件，位于 `tests/mcp_servers/` 目录： + +- **context7服务器测试**: 验证初始化、工具列表和搜索功能 +- **qdrant服务器测试**: 测试文档添加、搜索和删除功能 +- **qdrant-ollama服务器测试**: 验证向量数据库与LLM集成功能 + +测试脚本包括Shell脚本和Python脚本，支持通过JSON-RPC协议直接测试MCP服务器功能。详细的测试结果和问题修复记录请参考 `tests/mcp_server_test_report.md`。 + +运行测试： +```bash +# 运行单个测试脚本 +cd tests/mcp_servers +./test_local_mcp_servers.sh + +# 或运行Python测试 +python test_mcp_servers_simple.py +``` + +### Kali Linux系统测试 +项目包含完整的Kali Linux系统测试套件，位于 `configuration/playbooks/test/` 目录。测试包括： + +1. **快速健康检查** (`kali-health-check.yml`): 基本系统状态检查 +2. **安全工具测试** (`kali-security-tools.yml`): 测试各种安全工具的安装和功能 +3. **完整系统测试** (`test-kali.yml`): 全面的系统测试和报告生成 +4. **完整测试套件** (`kali-full-test-suite.yml`): 按顺序执行所有测试 + +运行测试： +```bash +# Kali Linux快速健康检查 +make test-kali + +# Kali Linux安全工具测试 +make test-kali-security + +# Kali Linux完整测试套件 +make test-kali-full +``` + ## 📚 文档 - [Consul集群故障排除](docs/consul-cluster-troubleshooting.md) @@ -312,6 +359,37 @@ nomad node status -address=http://${LEADER} 2. 搜索 [Issues](../../issues) 3. 创建新的 [Issue](../../issues/new) +## ⚠️ 重要经验教训 + +### Consul 和 Nomad 访问问题 +**问题**：尝试访问 Consul 服务时，使用 `http://localhost:8500` 或 `http://127.0.0.1:8500` 无法连接。 + +**根本原因**：本项目中的 Consul 和 Nomad 服务通过 Nomad + Podman 在集群中运行，并通过 Tailscale 网络进行访问。这些服务不在本地运行，因此无法通过 localhost 访问。 + +**解决方案**： +1. **使用 Tailscale IP**：必须使用 Tailscale 分配的 IP 地址访问服务 + ```bash + # 查看当前节点的 Tailscale IP + tailscale ip -4 + + # 查看所有 Tailscale 网络中的节点 + tailscale status + + # 访问 Consul (使用实际的 Tailscale IP) + curl http://100.x.x.x:8500/v1/status/leader + + # 访问 Nomad (使用实际的 Tailscale IP) + curl http://100.x.x.x:4646/v1/status/leader + ``` + +2. **服务发现**：Consul 集群由 3 个节点组成，Nomad 集群由十多个节点组成，需要正确识别服务运行的节点 + +3. **集群架构**： + - Consul 集群：3 个节点 (kr-master, us-ash3c, bj-warden) + - Nomad 集群：十多个节点，包括服务器节点和客户端节点 + +**重要提醒**：在开发和调试过程中，始终记住使用 Tailscale IP 而不是 localhost 访问集群服务。这是本项目架构的基本要求，必须严格遵守。 + ## 🎉 致谢 感谢所有为这个项目做出贡献的开发者和社区成员！ \ No newline at end of file diff --git a/jobs/consul/consul-cluster-simple.nomad b/components/consul/jobs/consul-cluster-simple.nomad similarity index 100% rename from jobs/consul/consul-cluster-simple.nomad rename to components/consul/jobs/consul-cluster-simple.nomad diff --git a/jobs/consul/consul-cluster.nomad b/components/consul/jobs/consul-cluster.nomad similarity index 100% rename from jobs/consul/consul-cluster.nomad rename to components/consul/jobs/consul-cluster.nomad diff --git a/jobs/podman/install-podman-driver.nomad b/components/nomad/jobs/install-podman-driver.nomad similarity index 100% rename from jobs/podman/install-podman-driver.nomad rename to components/nomad/jobs/install-podman-driver.nomad diff --git a/jobs/podman/nomad-nfs-volume.nomad b/components/nomad/jobs/nomad-nfs-volume.nomad similarity index 100% rename from jobs/podman/nomad-nfs-volume.nomad rename to components/nomad/jobs/nomad-nfs-volume.nomad diff --git a/components/vault/jobs/vault-cluster-exec.nomad b/components/vault/jobs/vault-cluster-exec.nomad new file mode 100644 index 0000000..2620dde --- /dev/null +++ b/components/vault/jobs/vault-cluster-exec.nomad @@ -0,0 +1,211 @@ +job "vault-cluster-exec" { + datacenters = ["dc1"] + type = "service" + + group "vault-master" { + count = 1 + + constraint { + attribute = "${node.unique.name}" + value = "kr-master" + } + + network { + port "api" { + static = 8200 + } + port "cluster" { + static = 8201 + } + } + + task "vault" { + driver = "exec" + + config { + command = "vault" + args = [ + "server", + "-config=/opt/nomad/data/vault/config/vault.hcl" + ] + } + + template { + data = < /home/ben/.vnc/passwd + echo "{{ vnc_password }}" | vncpasswd -f > /home/ben/.vnc/passwd2 + become_user: ben + + - name: 设置VNC密码文件权限 + file: + path: /home/ben/.vnc/passwd + owner: ben + group: ben + mode: '0600' + + - name: 设置VNC密码文件2权限 + file: + path: /home/ben/.vnc/passwd2 + owner: ben + group: ben + mode: '0600' + + - name: 创建VNC启动脚本 + copy: + dest: /home/ben/.vnc/xstartup + content: | + #!/bin/bash + unset SESSION_MANAGER + unset DBUS_SESSION_BUS_ADDRESS + exec startxfce4 + owner: ben + group: ben + mode: '0755' + + - name: 创建VNC服务文件 + copy: + dest: /etc/systemd/system/vncserver@.service + content: | + [Unit] + Description=Start TigerVNC server at startup + After=syslog.target network.target + + [Service] + Type=forking + User=ben + Group=ben + WorkingDirectory=/home/ben + + PIDFile=/home/ben/.vnc/%H:%i.pid + ExecStartPre=-/usr/bin/vncserver -kill :%i > /dev/null 2>&1 + ExecStart=/usr/bin/vncserver -depth {{ vnc_depth }} -geometry {{ vnc_geometry }} :%i + ExecStop=/usr/bin/vncserver -kill :%i + + [Install] + WantedBy=multi-user.target + + - name: 重新加载systemd配置 + systemd: + daemon_reload: yes + + - name: 启用并启动VNC服务 + systemd: + name: vncserver@1.service + enabled: yes + state: started + + - name: 检查VNC服务状态 + command: systemctl status vncserver@1.service + register: vnc_status + ignore_errors: yes + + - name: 显示VNC服务状态 + debug: + msg: "{{ vnc_status.stdout_lines }}" + + - name: 显示VNC连接信息 + debug: + msg: | + VNC服务器已成功配置！ + 连接信息： + - 地址: {{ ansible_host }} + - 端口: {{ vnc_port }} + - 密码: {{ vnc_password }} + - 连接命令: vnc://{{ ansible_host }}:{{ vnc_port }} + - 使用macOS屏幕共享应用连接到上述地址 \ No newline at end of file diff --git a/configuration/playbooks/security/setup-browser-ssh-auth.yml b/configuration/playbooks/security/setup-browser-ssh-auth.yml new file mode 100644 index 0000000..d3c5944 --- /dev/null +++ b/configuration/playbooks/security/setup-browser-ssh-auth.yml @@ -0,0 +1,81 @@ +--- +- name: Setup complete SSH key authentication for browser host + hosts: browser + become: yes + vars: + target_user: ben + ssh_key_comment: "ansible-generated-key-for-{{ inventory_hostname }}" + + tasks: + - name: Copy existing Ed25519 SSH public key to target user + copy: + src: /root/.ssh/id_ed25519.pub + dest: /home/{{ target_user }}/.ssh/id_ed25519.pub + owner: "{{ target_user }}" + group: "{{ target_user }}" + mode: '0644' + + - name: Copy existing Ed25519 SSH private key to target user + copy: + src: /root/.ssh/id_ed25519 + dest: /home/{{ target_user }}/.ssh/id_ed25519 + owner: "{{ target_user }}" + group: "{{ target_user }}" + mode: '0600' + + - name: Get SSH public key content + command: cat /home/{{ target_user }}/.ssh/id_ed25519.pub + register: ssh_public_key + become_user: "{{ target_user }}" + changed_when: false + + - name: Ensure .ssh directory exists for user + file: + path: /home/{{ target_user }}/.ssh + state: directory + owner: "{{ target_user }}" + group: "{{ target_user }}" + mode: '0700' + + - name: Add public key to authorized_keys + authorized_key: + user: "{{ target_user }}" + state: present + key: "{{ ssh_public_key.stdout }}" + become_user: "{{ target_user }}" + + - name: Configure SSH to prefer key authentication + lineinfile: + path: /etc/ssh/sshd_config + regexp: '^PasswordAuthentication' + line: 'PasswordAuthentication yes' + backup: yes + notify: restart sshd + when: ansible_connection != 'local' + + - name: Configure SSH to allow key authentication + lineinfile: + path: /etc/ssh/sshd_config + regexp: '^PubkeyAuthentication' + line: 'PubkeyAuthentication yes' + backup: yes + notify: restart sshd + when: ansible_connection != 'local' + + - name: Configure SSH authorized keys file permissions + file: + path: /home/{{ target_user }}/.ssh/authorized_keys + owner: "{{ target_user }}" + group: "{{ target_user }}" + mode: '0600' + + - name: Display success message + debug: + msg: "SSH key authentication has been configured for user {{ target_user }} on {{ inventory_hostname }}" + + handlers: + - name: restart sshd + systemd: + name: sshd + state: restarted + when: ansible_connection != 'local' \ No newline at end of file diff --git a/configuration/playbooks/security/setup-ssh-keys.yml b/configuration/playbooks/security/setup-ssh-keys.yml new file mode 100644 index 0000000..28708f1 --- /dev/null +++ b/configuration/playbooks/security/setup-ssh-keys.yml @@ -0,0 +1,62 @@ +--- +- name: Setup SSH key authentication for browser host + hosts: browser + become: yes + vars: + target_user: ben + ssh_key_comment: "ansible-generated-key" + tasks: + - name: Generate SSH key pair if it doesn't exist + user: + name: "{{ target_user }}" + generate_ssh_key: yes + ssh_key_bits: 4096 + ssh_key_comment: "{{ ssh_key_comment }}" + become_user: "{{ target_user }}" + + - name: Get SSH public key content + command: cat /home/{{ target_user }}/.ssh/id_rsa.pub + register: ssh_public_key + become_user: "{{ target_user }}" + changed_when: false + + - name: Display SSH public key for manual configuration + debug: + msg: | + SSH Public Key for {{ inventory_hostname }}: + {{ ssh_public_key.stdout }} + + To complete key-based authentication setup: + 1. Copy the above public key to the target system's authorized_keys + 2. Or use ssh-copy-id command from this system: + ssh-copy-id -i /home/{{ target_user }}/.ssh/id_rsa.pub {{ target_user }}@{{ inventory_hostname }} + + - name: Ensure .ssh directory exists for user + file: + path: /home/{{ target_user }}/.ssh + state: directory + owner: "{{ target_user }}" + group: "{{ target_user }}" + mode: '0700' + + - name: Configure SSH to prefer key authentication + lineinfile: + path: /etc/ssh/sshd_config + regexp: '^PasswordAuthentication' + line: 'PasswordAuthentication yes' + backup: yes + notify: restart sshd + + - name: Configure SSH to allow key authentication + lineinfile: + path: /etc/ssh/sshd_config + regexp: '^PubkeyAuthentication' + line: 'PubkeyAuthentication yes' + backup: yes + notify: restart sshd + + handlers: + - name: restart sshd + systemd: + name: sshd + state: restarted \ No newline at end of file diff --git a/configuration/playbooks/setup/setup-xfce-chrome-dev.yml b/configuration/playbooks/setup/setup-xfce-chrome-dev.yml new file mode 100644 index 0000000..fa7ba74 --- /dev/null +++ b/configuration/playbooks/setup/setup-xfce-chrome-dev.yml @@ -0,0 +1,114 @@ +--- +- name: Setup Xfce desktop environment and Chrome Dev for browser automation + hosts: browser + become: yes + vars: + target_user: ben + + tasks: + - name: Update package lists + apt: + update_cache: yes + cache_valid_time: 3600 + + - name: Install Xfce desktop environment + apt: + name: + - xfce4 + - xfce4-goodies + - lightdm + - xorg + - dbus-x11 + state: present + + - name: Install additional useful packages for desktop environment + apt: + name: + - firefox-esr + - geany + - thunar-archive-plugin + - xfce4-terminal + - gvfs + - fonts-noto + - fonts-noto-cjk + state: present + + - name: Download Google Chrome Dev .deb package + get_url: + url: https://dl.google.com/linux/direct/google-chrome-unstable_current_amd64.deb + dest: /tmp/google-chrome-unstable_current_amd64.deb + mode: '0644' + + - name: Install Google Chrome Dev + apt: + deb: /tmp/google-chrome-unstable_current_amd64.deb + + - name: Clean up downloaded .deb package + file: + path: /tmp/google-chrome-unstable_current_amd64.deb + state: absent + + - name: Install Chrome automation dependencies + apt: + name: + - python3-pip + - python3-venv + - python3-dev + - build-essential + - libssl-dev + - libffi-dev + state: present + + - name: Install Python packages for browser automation + pip: + name: + - selenium + - webdriver-manager + - pyvirtualdisplay + executable: pip3 + + - name: Set up Xfce as default desktop environment + copy: + dest: /etc/lightdm/lightdm.conf + content: | + [Seat:*] + autologin-user={{ target_user }} + autologin-user-timeout=0 + autologin-session=xfce + user-session=xfce + + - name: Ensure user is in necessary groups + user: + name: "{{ target_user }}" + groups: + - audio + - video + - input + - netdev + append: yes + + - name: Create .xprofile for user + copy: + dest: /home/{{ target_user }}/.xprofile + content: | + # Start Xfce on login + startxfce4 + owner: "{{ target_user }}" + group: "{{ target_user }}" + mode: '0644' + + - name: Enable and start lightdm service + systemd: + name: lightdm + enabled: yes + state: started + + - name: Display success message + debug: + msg: "Xfce desktop environment and Chrome Dev have been configured for user {{ target_user }} on {{ inventory_hostname }}" + + handlers: + - name: restart lightdm + systemd: + name: lightdm + state: restarted \ No newline at end of file diff --git a/configuration/playbooks/test/README.md b/configuration/playbooks/test/README.md new file mode 100644 index 0000000..eaac977 --- /dev/null +++ b/configuration/playbooks/test/README.md @@ -0,0 +1,110 @@ +# Kali Linux Ansible 测试套件 + +本目录包含用于测试Kali Linux系统的Ansible playbook集合。 + +## 测试Playbook列表 + +### 1. kali-health-check.yml +**用途**: Kali Linux快速健康检查 +**描述**: 执行基本的系统状态检查，包括系统信息、更新状态、磁盘空间、关键工具安装状态、网络连接、系统负载和SSH服务状态。 + +**运行方式**: +```bash +cd /root/mgmt/configuration +ansible-playbook -i inventories/production/inventory.ini playbooks/test/kali-health-check.yml +``` + +### 2. kali-security-tools.yml +**用途**: Kali Linux安全工具测试 +**描述**: 专门测试各种Kali Linux安全工具的安装和基本功能，包括： +- Nmap +- Metasploit Framework +- Wireshark +- John the Ripper +- Hydra +- SQLMap +- Aircrack-ng +- Burp Suite +- Netcat +- Curl + +**运行方式**: +```bash +cd /root/mgmt/configuration +ansible-playbook -i inventories/production/inventory.ini playbooks/test/kali-security-tools.yml +``` + +### 3. test-kali.yml +**用途**: Kali Linux完整系统测试 +**描述**: 执行全面的系统测试，包括： +- 系统基本信息收集 +- 网络连接测试 +- 包管理器测试 +- Kali工具检查 +- 系统安全性检查 +- 系统性能测试 +- 网络工具测试 +- 生成详细测试报告 + +**运行方式**: +```bash +cd /root/mgmt/configuration +ansible-playbook -i inventories/production/inventory.ini playbooks/test/test-kali.yml +``` + +### 4. kali-full-test-suite.yml +**用途**: Kali Linux完整测试套件 +**描述**: 按顺序执行所有上述测试，提供全面的系统测试覆盖。 + +**运行方式**: +```bash +cd /root/mgmt/configuration +ansible-playbook playbooks/test/kali-full-test-suite.yml +``` + +## 测试结果 + +### 健康检查 +- 直接在终端显示测试结果 +- 无额外文件生成 + +### 安全工具测试 +- 终端显示测试结果摘要 +- 在Kali系统上生成 `/tmp/kali_security_tools_report.md` 报告文件 + +### 完整系统测试 +- 终端显示测试进度 +- 在Kali系统上生成 `/tmp/kali_test_results/` 目录，包含： + - `system_info.txt`: 系统基本信息 + - `tool_check.txt`: Kali工具检查结果 + - `security_check.txt`: 系统安全检查 + - `performance.txt`: 系统性能信息 + - `network_tools.txt`: 网络工具测试 + - `kali_test.log`: 完整测试日志 + - `README.md`: 测试报告摘要 + +## 前提条件 + +1. 确保Kali系统在inventory中正确配置 +2. 确保Ansible可以连接到Kali系统 +3. 确保有足够的权限在Kali系统上执行测试 + +## 注意事项 + +1. 某些测试可能需要网络连接 +2. 完整系统测试可能需要较长时间 +3. 测试结果文件会保存在Kali系统的临时目录中 +4. 建议定期清理测试结果文件以节省磁盘空间 + +## 故障排除 + +如果测试失败，请检查： +1. 网络连接是否正常 +2. Ansible inventory配置是否正确 +3. SSH连接是否正常 +4. Kali系统是否正常运行 +5. 是否有足够的权限执行测试 + +## 自定义测试 + +您可以根据需要修改playbook中的测试内容，或添加新的测试任务。所有playbook都使用模块化设计，便于扩展和维护。 \ No newline at end of file diff --git a/configuration/playbooks/test/kali-full-test-suite.yml b/configuration/playbooks/test/kali-full-test-suite.yml new file mode 100644 index 0000000..37addb0 --- /dev/null +++ b/configuration/playbooks/test/kali-full-test-suite.yml @@ -0,0 +1,50 @@ +--- +- name: Kali Linux 完整测试套件 + hosts: localhost + gather_facts: no + tasks: + - name: 显示测试开始信息 + debug: + msg: "开始执行 Kali Linux 完整测试套件" + + - name: 执行Kali快速健康检查 + command: "ansible-playbook -i ../inventories/production/inventory.ini kali-health-check.yml" + args: + chdir: "/root/mgmt/configuration/playbooks/test" + register: health_check_result + + - name: 显示健康检查结果 + debug: + msg: "健康检查完成，退出码: {{ health_check_result.rc }}" + + - name: 执行Kali安全工具测试 + command: "ansible-playbook -i ../inventories/production/inventory.ini kali-security-tools.yml" + args: + chdir: "/root/mgmt/configuration/playbooks/test" + register: security_tools_result + + - name: 显示安全工具测试结果 + debug: + msg: "安全工具测试完成，退出码: {{ security_tools_result.rc }}" + + - name: 执行Kali完整系统测试 + command: "ansible-playbook -i ../inventories/production/inventory.ini test-kali.yml" + args: + chdir: "/root/mgmt/configuration/playbooks/test" + register: full_test_result + + - name: 显示完整测试结果 + debug: + msg: "完整系统测试完成，退出码: {{ full_test_result.rc }}" + + - name: 显示测试完成信息 + debug: + msg: | + Kali Linux 完整测试套件执行完成！ + + 测试结果摘要: + - 健康检查: {{ '成功' if health_check_result.rc == 0 else '失败' }} + - 安全工具测试: {{ '成功' if security_tools_result.rc == 0 else '失败' }} + - 完整系统测试: {{ '成功' if full_test_result.rc == 0 else '失败' }} + + 详细测试结果请查看各测试生成的报告文件。 \ No newline at end of file diff --git a/configuration/playbooks/test/kali-health-check.yml b/configuration/playbooks/test/kali-health-check.yml new file mode 100644 index 0000000..61a0cd2 --- /dev/null +++ b/configuration/playbooks/test/kali-health-check.yml @@ -0,0 +1,86 @@ +--- +- name: Kali Linux 快速健康检查 + hosts: kali + become: yes + gather_facts: yes + + tasks: + - name: 显示系统基本信息 + debug: + msg: | + === Kali Linux 系统信息 === + 主机名: {{ ansible_hostname }} + 操作系统: {{ ansible_distribution }} {{ ansible_distribution_version }} + 内核版本: {{ ansible_kernel }} + 架构: {{ ansible_architecture }} + CPU核心数: {{ ansible_processor_vcpus }} + 内存总量: {{ ansible_memtotal_mb }} MB + + - name: 修复损坏的依赖关系 + command: apt --fix-broken install -y + when: ansible_os_family == "Debian" + ignore_errors: yes + + - name: 检查系统更新状态 + apt: + update_cache: yes + upgrade: dist + check_mode: yes + register: update_check + changed_when: false + ignore_errors: yes + + - name: 显示系统更新状态 + debug: + msg: "{% if update_check.changed %}系统有可用更新{% else %}系统已是最新{% endif %}" + + - name: 检查磁盘空间 + command: "df -h /" + register: disk_space + + - name: 显示根分区磁盘空间 + debug: + msg: "根分区使用情况: {{ disk_space.stdout_lines[1] }}" + + - name: 检查关键Kali工具 + command: "which {{ item }}" + loop: + - nmap + - metasploit-framework + - wireshark + register: tool_check + ignore_errors: yes + changed_when: false + + - name: 显示工具检查结果 + debug: + msg: "{% for result in tool_check.results %}{{ result.item }}: {% if result.rc == 0 %}已安装{% else %}未安装{% endif %}{% endfor %}" + + - name: 检查网络连接 + uri: + url: https://httpbin.org/get + method: GET + timeout: 5 + register: network_test + ignore_errors: yes + + - name: 显示网络连接状态 + debug: + msg: "{% if network_test.failed %}网络连接测试失败{% else %}网络连接正常{% endif %}" + + - name: 检查系统负载 + command: "uptime" + register: uptime + + - name: 显示系统负载 + debug: + msg: "系统负载: {{ uptime.stdout }}" + + - name: 检查SSH服务状态 + systemd: + name: ssh + register: ssh_service + + - name: 显示SSH服务状态 + debug: + msg: "SSH服务状态: {{ ssh_service.status.ActiveState }}" \ No newline at end of file diff --git a/configuration/playbooks/test/kali-security-tools.yml b/configuration/playbooks/test/kali-security-tools.yml new file mode 100644 index 0000000..ebb3e7f --- /dev/null +++ b/configuration/playbooks/test/kali-security-tools.yml @@ -0,0 +1,228 @@ +--- +- name: Kali Linux 安全工具测试 + hosts: kali + become: yes + gather_facts: yes + + vars: + test_results: [] + + tasks: + - name: 初始化测试结果 + set_fact: + test_results: [] + + - name: 测试Nmap + block: + - name: 检查Nmap是否安装 + command: "which nmap" + register: nmap_check + ignore_errors: yes + changed_when: false + + - name: 测试Nmap基本功能 + command: "nmap -sn 127.0.0.1" + register: nmap_test + when: nmap_check.rc == 0 + ignore_errors: yes + changed_when: false + + - name: 记录Nmap测试结果 + set_fact: + test_results: "{{ test_results + ['Nmap: ' + ('✓ 正常工作' if nmap_check.rc == 0 and nmap_test.rc == 0 else '✗ 未安装或异常')] }}" + + - name: 测试Metasploit Framework + block: + - name: 检查Metasploit是否安装 + command: "which msfconsole" + register: msf_check + ignore_errors: yes + changed_when: false + + - name: 测试Metasploit版本 + command: "msfconsole --version" + register: msf_version + when: msf_check.rc == 0 + ignore_errors: yes + changed_when: false + + - name: 记录Metasploit测试结果 + set_fact: + test_results: "{{ test_results + ['Metasploit: ' + ('✓ 正常工作' if msf_check.rc == 0 else '✗ 未安装')] }}" + + - name: 测试Wireshark + block: + - name: 检查Wireshark是否安装 + command: "which wireshark" + register: wireshark_check + ignore_errors: yes + changed_when: false + + - name: 检查tshark是否可用 + command: "which tshark" + register: tshark_check + when: wireshark_check.rc == 0 + ignore_errors: yes + changed_when: false + + - name: 记录Wireshark测试结果 + set_fact: + test_results: "{{ test_results + ['Wireshark: ' + ('✓ 正常工作' if wireshark_check.rc == 0 else '✗ 未安装')] }}" + + - name: 测试John the Ripper + block: + - name: 检查John是否安装 + command: "which john" + register: john_check + ignore_errors: yes + changed_when: false + + - name: 测试John版本 + command: "john --version" + register: john_version + when: john_check.rc == 0 + ignore_errors: yes + changed_when: false + + - name: 记录John测试结果 + set_fact: + test_results: "{{ test_results + ['John the Ripper: ' + ('✓ 正常工作' if john_check.rc == 0 else '✗ 未安装')] }}" + + - name: 测试Hydra + block: + - name: 检查Hydra是否安装 + command: "which hydra" + register: hydra_check + ignore_errors: yes + changed_when: false + + - name: 测试Hydra帮助 + command: "hydra -h" + register: hydra_help + when: hydra_check.rc == 0 + ignore_errors: yes + changed_when: false + + - name: 记录Hydra测试结果 + set_fact: + test_results: "{{ test_results + ['Hydra: ' + ('✓ 正常工作' if hydra_check.rc == 0 else '✗ 未安装')] }}" + + - name: 测试SQLMap + block: + - name: 检查SQLMap是否安装 + command: "which sqlmap" + register: sqlmap_check + ignore_errors: yes + changed_when: false + + - name: 测试SQLMap版本 + command: "sqlmap --version" + register: sqlmap_version + when: sqlmap_check.rc == 0 + ignore_errors: yes + changed_when: false + + - name: 记录SQLMap测试结果 + set_fact: + test_results: "{{ test_results + ['SQLMap: ' + ('✓ 正常工作' if sqlmap_check.rc == 0 else '✗ 未安装')] }}" + + - name: 测试Aircrack-ng + block: + - name: 检查Aircrack-ng是否安装 + command: "which airmon-ng" + register: aircrack_check + ignore_errors: yes + changed_when: false + + - name: 测试Aircrack-ng版本 + command: "airmon-ng --version" + register: aircrack_version + when: aircrack_check.rc == 0 + ignore_errors: yes + changed_when: false + + - name: 记录Aircrack-ng测试结果 + set_fact: + test_results: "{{ test_results + ['Aircrack-ng: ' + ('✓ 正常工作' if aircrack_check.rc == 0 else '✗ 未安装')] }}" + + - name: 测试Burp Suite + block: + - name: 检查Burp Suite是否安装 + command: "which burpsuite" + register: burp_check + ignore_errors: yes + changed_when: false + + - name: 记录Burp Suite测试结果 + set_fact: + test_results: "{{ test_results + ['Burp Suite: ' + ('✓ 正常工作' if burp_check.rc == 0 else '✗ 未安装')] }}" + + - name: 测试Netcat + block: + - name: 检查Netcat是否安装 + command: "which nc" + register: nc_check + ignore_errors: yes + changed_when: false + + - name: 测试Netcat基本功能 + command: "nc -z 127.0.0.1 22" + register: nc_test + when: nc_check.rc == 0 + ignore_errors: yes + changed_when: false + + - name: 记录Netcat测试结果 + set_fact: + test_results: "{{ test_results + ['Netcat: ' + ('✓ 正常工作' if nc_check.rc == 0 else '✗ 未安装')] }}" + + - name: 测试Curl + block: + - name: 检查Curl是否安装 + command: "which curl" + register: curl_check + ignore_errors: yes + changed_when: false + + - name: 测试Curl基本功能 + command: "curl -s -o /dev/null -w '%{http_code}' https://httpbin.org/get" + register: curl_test + when: curl_check.rc == 0 + ignore_errors: yes + changed_when: false + + - name: 记录Curl测试结果 + set_fact: + test_results: "{{ test_results + ['Curl: ' + ('✓ 正常工作' if curl_check.rc == 0 else '✗ 未安装')] }}" + + - name: 显示所有测试结果 + debug: + msg: | + === Kali Linux 安全工具测试结果 === + {% for result in test_results %} + {{ result }} + {% endfor %} + + - name: 生成测试报告 + copy: + content: | + # Kali Linux 安全工具测试报告 + + **测试时间**: {{ ansible_date_time.iso8601 }} + **测试主机**: {{ ansible_hostname }} + + ## 测试结果 + + {% for result in test_results %} + {{ result }} + {% endfor %} + + ## 建议 + + {% for result in test_results %} + {% if '✗' in result %} + - {{ result.split(':')[0] }} 未安装，可以使用以下命令安装: `sudo apt install {{ result.split(':')[0].lower().replace(' ', '-') }}` + {% endif %} + {% endfor %} + + dest: "/tmp/kali_security_tools_report.md" \ No newline at end of file diff --git a/configuration/playbooks/test/test-kali.yml b/configuration/playbooks/test/test-kali.yml new file mode 100644 index 0000000..a31a81f --- /dev/null +++ b/configuration/playbooks/test/test-kali.yml @@ -0,0 +1,260 @@ +--- +- name: Kali Linux 系统测试 + hosts: kali + become: yes + gather_facts: yes + + vars: + test_results_dir: "/tmp/kali_test_results" + test_log_file: "{{ test_results_dir }}/kali_test.log" + + tasks: + - name: 创建测试结果目录 + file: + path: "{{ test_results_dir }}" + state: directory + mode: '0755' + + - name: 初始化测试日志 + copy: + content: "Kali Linux 系统测试日志 - {{ ansible_date_time.iso8601 }}\n\n" + dest: "{{ test_log_file }}" + + - name: 记录系统基本信息 + block: + - name: 获取系统信息 + setup: + register: system_info + + - name: 记录系统信息到日志 + copy: + content: | + === 系统基本信息 === + 主机名: {{ ansible_hostname }} + 操作系统: {{ ansible_distribution }} {{ ansible_distribution_version }} + 内核版本: {{ ansible_kernel }} + 架构: {{ ansible_architecture }} + CPU核心数: {{ ansible_processor_vcpus }} + 内存总量: {{ ansible_memtotal_mb }} MB + 磁盘空间: {{ ansible_mounts | map(attribute='size_total') | sum | human_readable }} + + dest: "{{ test_results_dir }}/system_info.txt" + + - name: 记录到主日志 + lineinfile: + path: "{{ test_log_file }}" + line: "[✓] 系统基本信息收集完成" + + - name: 测试网络连接 + block: + - name: 测试网络连通性 + uri: + url: https://www.google.com + method: GET + timeout: 10 + register: network_test + ignore_errors: yes + + - name: 记录网络测试结果 + lineinfile: + path: "{{ test_log_file }}" + line: "{% if network_test.failed %}[✗] 网络连接测试失败{% else %}[✓] 网络连接测试成功{% endif %}" + + - name: 测试包管理器 + block: + - name: 更新包列表 + apt: + update_cache: yes + changed_when: false + + - name: 记录包管理器测试结果 + lineinfile: + path: "{{ test_log_file }}" + line: "[✓] APT包管理器工作正常" + + - name: 检查Kali工具 + block: + - name: 检查常见Kali工具是否安装 + command: "which {{ item }}" + loop: + - nmap + - metasploit-framework + - wireshark + - john + - hydra + - sqlmap + - burpsuite + - aircrack-ng + register: tool_check + ignore_errors: yes + changed_when: false + + - name: 记录工具检查结果 + copy: + content: | + === Kali工具检查结果 === + {% for result in tool_check.results %} + {{ result.item }}: {% if result.rc == 0 %}已安装{% else %}未安装{% endif %} + {% endfor %} + + dest: "{{ test_results_dir }}/tool_check.txt" + + - name: 记录到主日志 + lineinfile: + path: "{{ test_log_file }}" + line: "[✓] Kali工具检查完成" + + - name: 测试系统安全性 + block: + - name: 检查防火墙状态 + command: "ufw status" + register: firewall_status + ignore_errors: yes + changed_when: false + + - name: 检查SSH配置 + command: "grep -E '^PermitRootLogin|^PasswordAuthentication' /etc/ssh/sshd_config" + register: ssh_config + ignore_errors: yes + changed_when: false + + - name: 记录安全检查结果 + copy: + content: | + === 系统安全检查 === + 防火墙状态: + {{ firewall_status.stdout }} + + SSH配置: + {{ ssh_config.stdout }} + + dest: "{{ test_results_dir }}/security_check.txt" + + - name: 记录到主日志 + lineinfile: + path: "{{ test_log_file }}" + line: "[✓] 系统安全检查完成" + + - name: 测试系统性能 + block: + - name: 获取CPU使用率 + command: "top -bn1 | grep 'Cpu(s)'" + register: cpu_usage + changed_when: false + + - name: 获取内存使用情况 + command: "free -h" + register: memory_usage + changed_when: false + + - name: 获取磁盘使用情况 + command: "df -h" + register: disk_usage + changed_when: false + + - name: 记录性能测试结果 + copy: + content: | + === 系统性能信息 === + CPU使用率: + {{ cpu_usage.stdout }} + + 内存使用情况: + {{ memory_usage.stdout }} + + 磁盘使用情况: + {{ disk_usage.stdout }} + + dest: "{{ test_results_dir }}/performance.txt" + + - name: 记录到主日志 + lineinfile: + path: "{{ test_log_file }}" + line: "[✓] 系统性能测试完成" + + - name: 测试网络工具 + block: + - name: 测试ping命令 + command: "ping -c 4 8.8.8.8" + register: ping_test + ignore_errors: yes + changed_when: false + + - name: 测试nslookup命令 + command: "nslookup google.com" + register: nslookup_test + ignore_errors: yes + changed_when: false + + - name: 记录网络工具测试结果 + copy: + content: | + === 网络工具测试 === + Ping测试结果: + {{ ping_test.stdout }} + + NSlookup测试结果: + {{ nslookup_test.stdout }} + + dest: "{{ test_results_dir }}/network_tools.txt" + + - name: 记录到主日志 + lineinfile: + path: "{{ test_log_file }}" + line: "[✓] 网络工具测试完成" + + - name: 生成测试报告 + block: + - name: 创建测试报告 + copy: + content: | + # Kali Linux 系统测试报告 + + **测试时间**: {{ ansible_date_time.iso8601 }} + **测试主机**: {{ ansible_hostname }} + + ## 测试结果摘要 + + {% if network_test.failed %}- [✗] 网络连接测试失败{% else %}- [✓] 网络连接测试成功{% endif %} + - [✓] APT包管理器工作正常 + - [✓] Kali工具检查完成 + - [✓] 系统安全检查完成 + - [✓] 系统性能测试完成 + - [✓] 网络工具测试完成 + + ## 详细结果 + + 请查看以下文件获取详细测试结果： + - system_info.txt: 系统基本信息 + - tool_check.txt: Kali工具检查结果 + - security_check.txt: 系统安全检查 + - performance.txt: 系统性能信息 + - network_tools.txt: 网络工具测试 + - kali_test.log: 完整测试日志 + + ## 建议 + + {% for result in tool_check.results %} + {% if result.rc != 0 %} + - 建议安装 {{ result.item }} 工具: `sudo apt install {{ result.item }}` + {% endif %} + {% endfor %} + + dest: "{{ test_results_dir }}/README.md" + + - name: 记录到主日志 + lineinfile: + path: "{{ test_log_file }}" + line: "[✓] 测试报告生成完成" + + - name: 显示测试结果位置 + debug: + msg: "Kali Linux 系统测试完成！测试结果保存在 {{ test_results_dir }} 目录中" + + - name: 显示测试日志最后几行 + command: "tail -10 {{ test_log_file }}" + register: log_tail + + - name: 输出测试日志摘要 + debug: + msg: "{{ log_tail.stdout_lines }}" \ No newline at end of file diff --git a/docs/vault/vault_deployment_guide.md b/docs/vault/vault_deployment_guide.md new file mode 100644 index 0000000..330a634 --- /dev/null +++ b/docs/vault/vault_deployment_guide.md @@ -0,0 +1,117 @@ +# Vault 通过 Nomad 部署指南 + +本文档提供了使用 Nomad 的 exec 驱动部署 HashiCorp Vault 的详细步骤，类似于 Consul 的部署方式。 + +## 部署架构 + +- **驱动方式**：使用 Nomad 的 `exec` 驱动 +- **节点分布**：在三个节点上部署（kr-master、us-ash3c、bj-warden） +- **存储后端**：使用本地 Consul 作为存储后端 +- **网络设置**：API 端口为 8200，集群通信端口为 8201 + +## 自动部署方法 + +我们提供了一个自动化脚本来简化部署过程。该脚本会： + +1. 使用 Ansible 在所有节点上安装 Vault +2. 通过 Nomad 部署 Vault 服务 +3. 初始化和解封 Vault（如果需要） + +### 使用自动部署脚本 + +```bash +# 确保脚本有执行权限 +chmod +x scripts/deploy_vault.sh + +# 运行部署脚本 +./scripts/deploy_vault.sh +``` + +脚本执行完成后，Vault 将在主节点上初始化并解封。您需要在其他节点上手动执行解封操作。 + +## 手动部署步骤 + +如果您想手动部署，请按照以下步骤操作： + +### 1. 安装 Vault + +使用 Ansible 在所有节点上安装 Vault： + +```bash +ansible-playbook -i configuration/inventories/production/vault.ini configuration/playbooks/install/install_vault.yml +``` + +### 2. 部署 Vault 服务 + +使用 Nomad 部署 Vault 服务： + +```bash +nomad job run jobs/vault-cluster-exec.nomad +``` + +### 3. 初始化 Vault + +在一个节点上初始化 Vault： + +```bash +export VAULT_ADDR='http://127.0.0.1:8200' +vault operator init -key-shares=5 -key-threshold=3 +``` + +请安全保存生成的解封密钥和根令牌！ + +### 4. 解封 Vault + +在每个节点上解封 Vault： + +```bash +export VAULT_ADDR='http://127.0.0.1:8200' +vault operator unseal <解封密钥1> +vault operator unseal <解封密钥2> +vault operator unseal <解封密钥3> +``` + +## 验证部署 + +验证 Vault 状态： + +```bash +export VAULT_ADDR='http://127.0.0.1:8200' +vault status +``` + +## 配置文件说明 + +### Nomad 作业文件 + +`jobs/vault-cluster-exec.nomad` 定义了 Vault 服务的 Nomad 作业配置，使用 exec 驱动在三个节点上部署 Vault。 + +### Ansible Playbook + +`configuration/playbooks/install/install_vault.yml` 负责在目标节点上安装 Vault 软件包和创建必要的目录结构。 + +## 故障排除 + +### Vault 无法启动 + +- 检查 Nomad 作业状态：`nomad job status vault-cluster-exec` +- 检查 Nomad 分配日志：`nomad alloc logs ` +- 确保 Consul 正在运行：`consul members` + +### Vault 无法解封 + +- 确保使用正确的解封密钥 +- 检查 Vault 状态：`vault status` +- 检查 Consul 中的 Vault 数据：`consul kv get -recurse vault/` + +## 后续步骤 + +成功部署 Vault 后，您可能需要： + +1. 配置访问策略 +2. 启用密钥引擎 +3. 与 Nomad 集成 +4. 配置审计日志 +5. 设置自动解封机制（生产环境） + +请参考 `docs/vault/vault_setup_guide.md` 获取更多信息。 \ No newline at end of file diff --git a/jobs/consul/jobs b/jobs/consul/jobs new file mode 120000 index 0000000..33e07e9 --- /dev/null +++ b/jobs/consul/jobs @@ -0,0 +1 @@ +components/consul/jobs/ \ No newline at end of file diff --git a/jobs/nomad b/jobs/nomad new file mode 120000 index 0000000..ed21927 --- /dev/null +++ b/jobs/nomad @@ -0,0 +1 @@ +components/nomad/jobs/ \ No newline at end of file diff --git a/jobs/vault b/jobs/vault new file mode 120000 index 0000000..13397ba --- /dev/null +++ b/jobs/vault @@ -0,0 +1 @@ +components/vault/jobs/ \ No newline at end of file diff --git a/lxc_chrome_automation_config.md b/lxc_chrome_automation_config.md new file mode 100644 index 0000000..7eb9fd0 --- /dev/null +++ b/lxc_chrome_automation_config.md @@ -0,0 +1,294 @@ +# LXC 容器浏览器自动化环境配置方案 + +## 1. LXC 容器基础配置 + +```bash +# 创建 Ubuntu 22.04 基础容器 +lxc launch ubuntu:22.04 chrome-automation + +# 配置容器资源限制 +lxc config set chrome-automation limits.cpu 2 +lxc config set chrome-automation limits.memory 4GB + +# 映射端口（如果需要外部访问） +lxc config device add chrome-automation proxy-port8080 proxy listen=tcp:0.0.0.0:8080 connect=tcp:127.0.0.1:8080 +``` + +## 2. 容器内环境配置 + +### 2.1 基础系统包安装 +```bash +# 进入容器 +lxc exec chrome-automation -- bash + +# 更新系统 +apt update && apt upgrade -y + +# 安装基础开发工具和图形支持 +apt install -y \ + curl \ + wget \ + unzip \ + git \ + build-essential \ + xvfb \ + x11-utils \ + x11-xserver-utils \ + xdg-utils \ + libnss3 \ + libatk-bridge2.0-0 \ + libdrm2 \ + libxkbcommon0 \ + libxcomposite1 \ + libxdamage1 \ + libxrandr2 \ + libgbm1 \ + libxss1 \ + libasound2 \ + fonts-liberation \ + libappindicator3-1 \ + xdg-utils \ + libsecret-1-dev \ + libgconf-2-4 +``` + +### 2.2 安装 Chrome 浏览器 +```bash +# 下载并安装 Google Chrome +wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | apt-key add - +echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list +apt update +apt install -y google-chrome-stable +``` + +### 2.3 安装浏览器自动化工具 +```bash +# 安装 Node.js 和 npm +curl -fsSL https://deb.nodesource.com/setup_18.x | bash - +apt install -y nodejs + +# 安装 Python 和相关工具 +apt install -y python3 python3-pip python3-venv + +# 安装 Selenium 和浏览器驱动 +pip3 install selenium webdriver-manager + +# 下载 ChromeDriver +npm install -g chromedriver +``` + +### 2.4 配置无头模式运行环境 +```bash +# 创建自动化脚本目录 +mkdir -p /opt/browser-automation +cd /opt/browser-automation + +# 创建 Chrome 无头模式启动脚本 +cat > chrome-headless.sh << 'EOF' +#!/bin/bash +export DISPLAY=:99 +Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & +sleep 2 +google-chrome --headless --no-sandbox --disable-dev-shm-usage --disable-gpu --remote-debugging-port=9222 --disable-extensions --disable-plugins --disable-images & +sleep 3 +EOF + +chmod +x chrome-headless.sh +``` + +## 3. 自动化工具配置 + +### 3.1 Python Selenium 配置示例 +```python +# selenium_automation.py +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.chrome.service import Service +from webdriver_manager.chrome import ChromeDriverManager + +def create_chrome_driver(): + chrome_options = Options() + chrome_options.add_argument("--headless") + chrome_options.add_argument("--no-sandbox") + chrome_options.add_argument("--disable-dev-shm-usage") + chrome_options.add_argument("--disable-gpu") + chrome_options.add_argument("--remote-debugging-port=9222") + chrome_options.add_argument("--disable-extensions") + chrome_options.add_argument("--disable-plugins") + chrome_options.add_argument("--window-size=1920,1080") + + service = Service(ChromeDriverManager().install()) + driver = webdriver.Chrome(service=service, options=chrome_options) + return driver + +# 使用示例 +driver = create_chrome_driver() +driver.get("https://www.example.com") +print(driver.title) +driver.quit() +``` + +### 3.2 Node.js Puppeteer 配置示例 +```javascript +// puppeteer_automation.js +const puppeteer = require('puppeteer'); + +async function runAutomation() { + const browser = await puppeteer.launch({ + headless: true, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-gpu', + '--window-size=1920,1080' + ] + }); + + const page = await browser.newPage(); + await page.goto('https://www.example.com'); + const title = await page.title(); + console.log(title); + + await browser.close(); +} + +runAutomation(); +``` + +## 4. 容器启动配置 + +### 4.1 启动脚本 +```bash +cat > /opt/browser-automation/start.sh << 'EOF' +#!/bin/bash + +# 启动 Xvfb 虚拟显示 +export DISPLAY=:99 +Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & +sleep 2 + +# 启动 Chrome 浏览器 +google-chrome --headless --no-sandbox --disable-dev-shm-usage --disable-gpu --remote-debugging-port=9222 --disable-extensions --disable-plugins --disable-images & +sleep 3 + +# 可选：启动自动化服务 +# python3 /opt/browser-automation/service.py + +echo "Browser automation environment ready!" +EOF + +chmod +x /opt/browser-automation/start.sh +``` + +### 4.2 系统服务配置 +```bash +cat > /etc/systemd/system/browser-automation.service << 'EOF' +[Unit] +Description=Browser Automation Service +After=network.target + +[Service] +Type=forking +ExecStart=/opt/browser-automation/start.sh +Restart=always +User=root +Environment=DISPLAY=:99 + +[Install] +WantedBy=multi-user.target +EOF + +systemctl enable browser-automation.service +``` + +## 5. 安全配置 + +### 5.1 非 root 用户配置 +```bash +# 创建专用用户 +useradd -m -s /bin/bash browser-user +usermod -a -G sudo browser-user + +# 设置 Chrome 以非 root 用户运行 +echo 'chrome --no-sandbox --user-data-dir=/home/browser-user/.config/google-chrome' > /home/browser-user/run-chrome.sh +chown browser-user:browser-user /home/browser-user/run-chrome.sh +``` + +### 5.2 网络安全 +```bash +# 配置防火墙（如果需要） +ufw allow 22/tcp +# 仅在需要外部访问时开放特定端口 +# ufw allow 8080/tcp +``` + +## 6. 监控和日志 + +### 6.1 日志配置 +```bash +# 创建日志目录 +mkdir -p /var/log/browser-automation + +# 配置日志轮转 +cat > /etc/logrotate.d/browser-automation << 'EOF' +/var/log/browser-automation/*.log { + daily + missingok + rotate 30 + compress + delaycompress + notifempty + create 644 root root +} +EOF +``` + +## 7. 备份和恢复 + +### 7.1 创建容器快照 +```bash +# 创建快照 +lxc snapshot chrome-automation initial-setup + +# 列出快照 +lxc info chrome-automation --snapshots + +# 恢复快照 +lxc restore chrome-automation initial-setup +``` + +### 7.2 配置文件备份 +```bash +# 备份重要配置 +lxc file pull chrome-automation/etc/systemd/system/browser-automation.service ./ +lxc file pull chrome-automation/opt/browser-automation/start.sh ./ +``` + +## 8. 性能优化 + +### 8.1 Chrome 启动参数优化 +```bash +CHROME_OPTS="--headless \ +--no-sandbox \ +--disable-dev-shm-usage \ +--disable-gpu \ +--remote-debugging-port=9222 \ +--disable-extensions \ +--disable-plugins \ +--disable-images \ +--disable-javascript \ +--memory-pressure-off \ +--max_old_space_size=4096 \ +--js-flags=--max-old-space-size=2048" +``` + +### 8.2 容器资源优化 +```bash +# 在容器配置中设置资源限制 +lxc config set chrome-automation limits.cpu 2 +lxc config set chrome-automation limits.memory 4GB +lxc config set chrome-automation limits.memory.swap false +``` + +这个配置方案提供了完整的LXC容器环境，专门用于浏览器自动化任务，具有良好的性能、安全性和可维护性。 \ No newline at end of file diff --git a/playbooks/install/install_vault.yml b/playbooks/install/install_vault.yml new file mode 100644 index 0000000..f2ea382 --- /dev/null +++ b/playbooks/install/install_vault.yml @@ -0,0 +1,36 @@ +--- +# install_vault.yml +- name: Install HashiCorp Vault + hosts: vault_servers + become: yes + tasks: + - name: Check if Vault is already installed + command: which vault + register: vault_check + ignore_errors: yes + changed_when: false + + - name: Install Vault using apt + apt: + name: vault + state: present + update_cache: yes + when: vault_check.rc != 0 + + - name: Create Vault data directory + file: + path: "{{ vault_data_dir | default('/opt/nomad/data/vault/config') }}" + state: directory + owner: root + group: root + mode: '0755' + recurse: yes + + - name: Verify Vault installation + command: vault --version + register: vault_version + changed_when: false + + - name: Display Vault version + debug: + var: vault_version.stdout \ No newline at end of file diff --git a/qdrant_mcp_server.py b/qdrant_mcp_server.py new file mode 100644 index 0000000..3a2644a --- /dev/null +++ b/qdrant_mcp_server.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +""" +Qdrant MCP 服务器 +此脚本实现了一个 MCP 服务器，与 Qdrant 向量数据库集成 +""" + +import asyncio +import json +import os +import sys +from typing import Any, Dict, List, Optional +import logging + +from qdrant_client import QdrantClient +from qdrant_client.models import Distance, VectorParams, PointStruct, Filter + +# 设置日志 +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +class QdrantMCPServer: + def __init__(self): + # 从环境变量获取配置 + self.qdrant_url = os.getenv("QDRANT_URL", "http://localhost:6333") + self.qdrant_api_key = os.getenv("QDRANT_API_KEY", "") + self.collection_name = os.getenv("COLLECTION_NAME", "mcp") + self.embedding_model = os.getenv("EMBEDDING_MODEL", "bge-m3") + + # 初始化 Qdrant 客户端 + self.client = QdrantClient( + url=self.qdrant_url, + api_key=self.qdrant_api_key if self.qdrant_api_key else None + ) + + # 确保集合存在 + self._ensure_collection_exists() + + logger.info(f"Qdrant MCP 服务器已初始化") + logger.info(f"Qdrant URL: {self.qdrant_url}") + logger.info(f"集合名称: {self.collection_name}") + logger.info(f"嵌入模型: {self.embedding_model}") + + def _ensure_collection_exists(self): + """确保集合存在，如果不存在则创建""" + try: + collections = self.client.get_collections().collections + collection_names = [collection.name for collection in collections] + + if self.collection_name not in collection_names: + # 创建新集合 + self.client.create_collection( + collection_name=self.collection_name, + vectors_config=VectorParams(size=1024, distance=Distance.COSINE) + ) + logger.info(f"已创建新集合: {self.collection_name}") + else: + logger.info(f"集合已存在: {self.collection_name}") + except Exception as e: + logger.error(f"确保集合存在时出错: {e}") + raise + + async def handle_request(self, request: Dict[str, Any]) -> Dict[str, Any]: + """处理 MCP 请求""" + method = request.get("method") + params = request.get("params", {}) + request_id = request.get("id") + + logger.info(f"收到请求: {method}") + + try: + if method == "initialize": + result = await self.initialize(params) + elif method == "tools/list": + result = await self.list_tools(params) + elif method == "tools/call": + result = await self.call_tool(params) + elif method == "resources/list": + result = await self.list_resources(params) + elif method == "resources/read": + result = await self.read_resource(params) + else: + result = { + "error": { + "code": -32601, + "message": f"未知方法: {method}" + } + } + except Exception as e: + logger.error(f"处理请求时出错: {e}") + result = { + "error": { + "code": -32603, + "message": f"内部错误: {str(e)}" + } + } + + response = { + "jsonrpc": "2.0", + "id": request_id, + **result + } + + return response + + async def initialize(self, params: Dict[str, Any]) -> Dict[str, Any]: + """初始化 MCP 服务器""" + logger.info("初始化 Qdrant MCP 服务器") + + return { + "result": { + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": { + "listChanged": False + }, + "resources": { + "subscribe": False, + "listChanged": False + } + }, + "serverInfo": { + "name": "qdrant-mcp-server", + "version": "1.0.0" + } + } + } + + async def list_tools(self, params: Dict[str, Any]) -> Dict[str, Any]: + """列出可用工具""" + return { + "result": { + "tools": [ + { + "name": "qdrant_search", + "description": "在 Qdrant 中搜索相似向量", + "inputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "搜索查询文本" + }, + "limit": { + "type": "integer", + "default": 5, + "description": "返回结果数量限制" + } + }, + "required": ["query"] + } + }, + { + "name": "qdrant_add", + "description": "向 Qdrant 添加向量", + "inputSchema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "要添加的文本内容" + }, + "metadata": { + "type": "object", + "description": "与文本关联的元数据" + } + }, + "required": ["text"] + } + }, + { + "name": "qdrant_delete", + "description": "从 Qdrant 删除向量", + "inputSchema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "要删除的向量ID" + } + }, + "required": ["id"] + } + } + ] + } + } + + async def call_tool(self, params: Dict[str, Any]) -> Dict[str, Any]: + """调用工具""" + name = params.get("name") + arguments = params.get("arguments", {}) + + if name == "qdrant_search": + return await self._search_vectors(arguments) + elif name == "qdrant_add": + return await self._add_vector(arguments) + elif name == "qdrant_delete": + return await self._delete_vector(arguments) + else: + return { + "error": { + "code": -32601, + "message": f"未知工具: {name}" + } + } + + async def _search_vectors(self, params: Dict[str, Any]) -> Dict[str, Any]: + """搜索相似向量""" + query = params.get("query", "") + limit = params.get("limit", 5) + + # 这里应该使用嵌入模型将查询转换为向量 + # 由于我们没有实际的嵌入模型，这里使用一个简单的模拟 + query_vector = [0.1] * 1024 # 模拟向量 + + try: + search_result = self.client.search( + collection_name=self.collection_name, + query_vector=query_vector, + limit=limit + ) + + results = [] + for hit in search_result: + results.append({ + "id": hit.id, + "score": hit.score, + "payload": hit.payload + }) + + return { + "result": { + "content": [ + { + "type": "text", + "text": f"搜索结果: {json.dumps(results, ensure_ascii=False)}" + } + ] + } + } + except Exception as e: + logger.error(f"搜索向量时出错: {e}") + return { + "error": { + "code": -32603, + "message": f"搜索向量时出错: {str(e)}" + } + } + + async def _add_vector(self, params: Dict[str, Any]) -> Dict[str, Any]: + """添加向量""" + text = params.get("text", "") + metadata = params.get("metadata", {}) + + # 生成一个简单的ID + import hashlib + vector_id = hashlib.md5(text.encode()).hexdigest() + + # 这里应该使用嵌入模型将文本转换为向量 + # 由于我们没有实际的嵌入模型，这里使用一个简单的模拟 + vector = [0.1] * 1024 # 模拟向量 + + try: + self.client.upsert( + collection_name=self.collection_name, + points=[ + PointStruct( + id=vector_id, + vector=vector, + payload={ + "text": text, + **metadata + } + ) + ] + ) + + return { + "result": { + "content": [ + { + "type": "text", + "text": f"已添加向量，ID: {vector_id}" + } + ] + } + } + except Exception as e: + logger.error(f"添加向量时出错: {e}") + return { + "error": { + "code": -32603, + "message": f"添加向量时出错: {str(e)}" + } + } + + async def _delete_vector(self, params: Dict[str, Any]) -> Dict[str, Any]: + """删除向量""" + vector_id = params.get("id", "") + + try: + self.client.delete( + collection_name=self.collection_name, + points_selector=[vector_id] + ) + + return { + "result": { + "content": [ + { + "type": "text", + "text": f"已删除向量，ID: {vector_id}" + } + ] + } + } + except Exception as e: + logger.error(f"删除向量时出错: {e}") + return { + "error": { + "code": -32603, + "message": f"删除向量时出错: {str(e)}" + } + } + + async def list_resources(self, params: Dict[str, Any]) -> Dict[str, Any]: + """列出资源""" + return { + "result": { + "resources": [] + } + } + + async def read_resource(self, params: Dict[str, Any]) -> Dict[str, Any]: + """读取资源""" + return { + "error": { + "code": -32601, + "message": "不支持读取资源" + } + } + +async def main(): + """主函数""" + server = QdrantMCPServer() + + # 从标准输入读取请求 + for line in sys.stdin: + try: + request = json.loads(line) + response = await server.handle_request(request) + print(json.dumps(response, ensure_ascii=False)) + sys.stdout.flush() + except json.JSONDecodeError as e: + logger.error(f"解析 JSON 时出错: {e}") + error_response = { + "jsonrpc": "2.0", + "id": None, + "error": { + "code": -32700, + "message": f"解析 JSON 时出错: {str(e)}" + } + } + print(json.dumps(error_response, ensure_ascii=False)) + sys.stdout.flush() + except Exception as e: + logger.error(f"处理请求时出错: {e}") + error_response = { + "jsonrpc": "2.0", + "id": None, + "error": { + "code": -32603, + "message": f"内部错误: {str(e)}" + } + } + print(json.dumps(error_response, ensure_ascii=False)) + sys.stdout.flush() + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/qdrant_ollama_integration.py b/qdrant_ollama_integration.py new file mode 100644 index 0000000..f2af4f3 --- /dev/null +++ b/qdrant_ollama_integration.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +Qdrant 与 Ollama 嵌入模型集成示例 +此脚本演示如何使用 Ollama 作为嵌入模型提供者与 Qdrant 向量数据库集成 +""" + +from langchain_ollama import OllamaEmbeddings +from qdrant_client import QdrantClient +from qdrant_client.models import Distance, VectorParams, PointStruct +import os + +def main(): + # 1. 初始化 Ollama 嵌入模型 + # 使用 nomic-embed-text 模型，这是 Ollama 推荐的嵌入模型 + print("初始化 Ollama 嵌入模型...") + embeddings = OllamaEmbeddings( + model="nomic-embed-text", + base_url="http://localhost:11434" # Ollama 默认地址 + ) + + # 2. 初始化 Qdrant 客户端 + print("连接到 Qdrant 数据库...") + client = QdrantClient( + url="http://localhost:6333", # Qdrant 默认地址 + api_key="313131" # 从之前查看的配置中获取的 API 密钥 + ) + + # 3. 创建集合（如果不存在） + collection_name = "ollama_integration_test" + print(f"创建或检查集合: {collection_name}") + + # 首先检查集合是否已存在 + collections = client.get_collections().collections + collection_exists = any(collection.name == collection_name for collection in collections) + + if not collection_exists: + # 创建新集合 + # 首先获取嵌入模型的维度 + sample_embedding = embeddings.embed_query("sample text") + vector_size = len(sample_embedding) + + client.create_collection( + collection_name=collection_name, + vectors_config=VectorParams( + size=vector_size, + distance=Distance.COSINE + ) + ) + print(f"已创建新集合，向量维度: {vector_size}") + else: + print("集合已存在") + + # 4. 准备示例数据 + documents = [ + "Qdrant 是一个高性能的向量搜索引擎", + "Ollama 是一个本地运行大语言模型的工具", + "向量数据库用于存储和检索高维向量", + "嵌入模型将文本转换为数值向量表示" + ] + + metadata = [ + {"source": "qdrant_docs", "category": "database"}, + {"source": "ollama_docs", "category": "llm"}, + {"source": "vector_db_docs", "category": "database"}, + {"source": "embedding_docs", "category": "ml"} + ] + + # 5. 使用 Ollama 生成嵌入并存储到 Qdrant + print("生成嵌入并存储到 Qdrant...") + points = [] + + for idx, (doc, meta) in enumerate(zip(documents, metadata)): + # 使用 Ollama 生成嵌入 + embedding = embeddings.embed_query(doc) + + # 创建 Qdrant 点 + point = PointStruct( + id=idx, + vector=embedding, + payload={ + "text": doc, + "metadata": meta + } + ) + points.append(point) + + # 上传点到 Qdrant + client.upsert( + collection_name=collection_name, + points=points + ) + print(f"已上传 {len(points)} 个文档到 Qdrant") + + # 6. 执行相似性搜索 + query = "什么是向量数据库？" + print(f"\n执行搜索查询: '{query}'") + + # 使用 Ollama 生成查询嵌入 + query_embedding = embeddings.embed_query(query) + + # 在 Qdrant 中搜索 + search_result = client.search( + collection_name=collection_name, + query_vector=query_embedding, + limit=2 + ) + + # 7. 显示搜索结果 + print("\n搜索结果:") + for i, hit in enumerate(search_result, 1): + print(f"{i}. {hit.payload['text']} (得分: {hit.score:.4f})") + print(f" 元数据: {hit.payload['metadata']}") + + print("\n集成测试完成！") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/qdrant_ollama_mcp_server.py b/qdrant_ollama_mcp_server.py new file mode 100644 index 0000000..77bb129 --- /dev/null +++ b/qdrant_ollama_mcp_server.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +""" +Qdrant 与 Ollama 嵌入模型集成的 MCP 服务器 +此脚本实现了一个 MCP 服务器，使用 Ollama 作为嵌入模型提供者与 Qdrant 向量数据库集成 +""" + +import asyncio +import json +import os +import sys +from typing import Any, Dict, List, Optional +import logging + +from langchain_ollama import OllamaEmbeddings +from qdrant_client import QdrantClient +from qdrant_client.models import Distance, VectorParams, PointStruct, Filter + +# 设置日志 +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +class QdrantOllamaMCPServer: + def __init__(self): + # 在初始化之前打印环境变量 + print(f"环境变量:") + print(f"QDRANT_URL: {os.getenv('QDRANT_URL', '未设置')}") + print(f"QDRANT_API_KEY: {os.getenv('QDRANT_API_KEY', '未设置')}") + print(f"OLLAMA_URL: {os.getenv('OLLAMA_URL', '未设置')}") + print(f"OLLAMA_MODEL: {os.getenv('OLLAMA_MODEL', '未设置')}") + print(f"COLLECTION_NAME: {os.getenv('COLLECTION_NAME', '未设置')}") + + # 从环境变量获取配置 + self.qdrant_url = os.getenv("QDRANT_URL", "http://dev1:6333") # dev1服务器上的Qdrant地址 + self.qdrant_api_key = os.getenv("QDRANT_API_KEY", "313131") + self.collection_name = os.getenv("COLLECTION_NAME", "ollama_mcp") + self.ollama_model = os.getenv("OLLAMA_MODEL", "nomic-embed-text") + self.ollama_url = os.getenv("OLLAMA_URL", "http://dev1:11434") # dev1服务器上的Ollama地址 + + # 初始化客户端 + self.embeddings = OllamaEmbeddings( + model=self.ollama_model, + base_url=self.ollama_url + ) + + self.client = QdrantClient( + url=self.qdrant_url, + api_key=self.qdrant_api_key + ) + + # 确保集合存在 + self._ensure_collection_exists() + + logger.info(f"初始化完成，使用集合: {self.collection_name}") + + def _ensure_collection_exists(self): + """确保集合存在，如果不存在则创建""" + collections = self.client.get_collections().collections + collection_exists = any(collection.name == self.collection_name for collection in collections) + + if not collection_exists: + # 获取嵌入模型的维度 + sample_embedding = self.embeddings.embed_query("sample text") + vector_size = len(sample_embedding) + + self.client.create_collection( + collection_name=self.collection_name, + vectors_config=VectorParams( + size=vector_size, + distance=Distance.COSINE + ) + ) + logger.info(f"已创建新集合，向量维度: {vector_size}") + else: + logger.info("集合已存在") + + async def handle_request(self, request: Dict[str, Any]) -> Dict[str, Any]: + """处理 MCP 请求""" + method = request.get("method") + params = request.get("params", {}) + request_id = request.get("id") + + logger.info(f"处理请求: {method}") + + try: + if method == "initialize": + result = { + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": { + "listChanged": True + }, + "resources": { + "subscribe": True, + "listChanged": True + } + }, + "serverInfo": { + "name": "qdrant-ollama-mcp-server", + "version": "1.0.0" + } + } + elif method == "tools/list": + result = { + "tools": [ + { + "name": "add_document", + "description": "添加文档到向量数据库", + "inputSchema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "文档文本内容" + }, + "metadata": { + "type": "object", + "description": "文档的元数据" + } + }, + "required": ["text"] + } + }, + { + "name": "search_documents", + "description": "在向量数据库中搜索相似文档", + "inputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "搜索查询文本" + }, + "limit": { + "type": "integer", + "description": "返回结果数量限制", + "default": 5 + }, + "filter": { + "type": "object", + "description": "搜索过滤器" + } + }, + "required": ["query"] + } + }, + { + "name": "list_collections", + "description": "列出所有集合", + "inputSchema": { + "type": "object", + "properties": {} + } + }, + { + "name": "get_collection_info", + "description": "获取集合信息", + "inputSchema": { + "type": "object", + "properties": { + "collection_name": { + "type": "string", + "description": "集合名称" + } + }, + "required": ["collection_name"] + } + } + ] + } + elif method == "tools/call": + tool_name = params.get("name") + tool_params = params.get("arguments", {}) + + if tool_name == "add_document": + result = await self._add_document(tool_params) + elif tool_name == "search_documents": + result = await self._search_documents(tool_params) + elif tool_name == "list_collections": + result = await self._list_collections(tool_params) + elif tool_name == "get_collection_info": + result = await self._get_collection_info(tool_params) + else: + raise ValueError(f"未知工具: {tool_name}") + else: + raise ValueError(f"未知方法: {method}") + + response = { + "jsonrpc": "2.0", + "id": request_id, + "result": result + } + + except Exception as e: + logger.error(f"处理请求时出错: {e}") + response = { + "jsonrpc": "2.0", + "id": request_id, + "error": { + "code": -1, + "message": str(e) + } + } + + return response + + async def _add_document(self, params: Dict[str, Any]) -> Dict[str, Any]: + """添加文档到向量数据库""" + text = params.get("text") + metadata = params.get("metadata", {}) + + if not text: + raise ValueError("文档文本不能为空") + + # 生成嵌入 + embedding = self.embeddings.embed_query(text) + + # 创建点 + point = PointStruct( + id=hash(text) % (2 ** 31), # 使用文本哈希作为ID + vector=embedding, + payload={ + "text": text, + "metadata": metadata + } + ) + + # 上传到 Qdrant + self.client.upsert( + collection_name=self.collection_name, + points=[point] + ) + + return {"success": True, "message": "文档已添加"} + + async def _search_documents(self, params: Dict[str, Any]) -> Dict[str, Any]: + """在向量数据库中搜索相似文档""" + query = params.get("query") + limit = params.get("limit", 5) + filter_dict = params.get("filter") + + if not query: + raise ValueError("搜索查询不能为空") + + # 生成查询嵌入 + query_embedding = self.embeddings.embed_query(query) + + # 构建过滤器 + search_filter = None + if filter_dict: + search_filter = Filter(**filter_dict) + + # 执行搜索 + search_result = self.client.search( + collection_name=self.collection_name, + query_vector=query_embedding, + limit=limit, + query_filter=search_filter + ) + + # 格式化结果 + results = [] + for hit in search_result: + results.append({ + "text": hit.payload.get("text", ""), + "metadata": hit.payload.get("metadata", {}), + "score": hit.score + }) + + return {"results": results} + + async def _list_collections(self, params: Dict[str, Any]) -> Dict[str, Any]: + """列出所有集合""" + collections = self.client.get_collections().collections + return { + "collections": [ + {"name": collection.name} for collection in collections + ] + } + + async def _get_collection_info(self, params: Dict[str, Any]) -> Dict[str, Any]: + """获取集合信息""" + collection_name = params.get("collection_name") + + if not collection_name: + raise ValueError("集合名称不能为空") + + try: + collection_info = self.client.get_collection(collection_name) + return { + "name": collection_name, + "vectors_count": collection_info.points_count, + "vectors_config": collection_info.config.params.vectors.dict() + } + except Exception as e: + raise ValueError(f"获取集合信息失败: {str(e)}") + + async def run(self): + """运行 MCP 服务器""" + logger.info("启动 Qdrant-Ollama MCP 服务器") + logger.info(f"Qdrant URL: {self.qdrant_url}") + logger.info(f"Ollama URL: {self.ollama_url}") + logger.info(f"Collection: {self.collection_name}") + + # 从标准输入读取请求 + while True: + try: + line = await asyncio.get_event_loop().run_in_executor( + None, sys.stdin.readline + ) + if not line: + break + + logger.info(f"收到请求: {line.strip()}") + + # 解析 JSON 请求 + request = json.loads(line.strip()) + + # 处理请求 + response = await self.handle_request(request) + + # 发送响应 + response_json = json.dumps(response) + print(response_json, flush=True) + logger.info(f"发送响应: {response_json}") + + except json.JSONDecodeError as e: + logger.error(f"JSON 解析错误: {e}") + except Exception as e: + logger.error(f"处理请求时出错: {e}") + except KeyboardInterrupt: + logger.info("服务器被中断") + break + +async def main(): + """主函数""" + # 设置日志级别 + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + + # 打印环境变量 + print(f"环境变量:") + print(f"QDRANT_URL: {os.getenv('QDRANT_URL', '未设置')}") + print(f"QDRANT_API_KEY: {os.getenv('QDRANT_API_KEY', '未设置')}") + print(f"OLLAMA_URL: {os.getenv('OLLAMA_URL', '未设置')}") + print(f"OLLAMA_MODEL: {os.getenv('OLLAMA_MODEL', '未设置')}") + print(f"COLLECTION_NAME: {os.getenv('COLLECTION_NAME', '未设置')}") + + # 创建服务器实例 + server = QdrantOllamaMCPServer() + + # 运行服务器 + await server.run() + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000..c099d37 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# 项目测试快速执行脚本 +# 从项目根目录快速运行所有MCP服务器测试 + +set -e + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# 获取脚本所在目录 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_DIR="$SCRIPT_DIR/tests" + +# 检查测试目录是否存在 +if [ ! -d "$TEST_DIR" ]; then + echo -e "${RED}错误: 测试目录 $TEST_DIR 不存在${NC}" + exit 1 +fi + +# 检查测试运行器脚本是否存在 +RUNNER_SCRIPT="$TEST_DIR/run_all_tests.sh" +if [ ! -f "$RUNNER_SCRIPT" ]; then + echo -e "${RED}错误: 测试运行器脚本 $RUNNER_SCRIPT 不存在${NC}" + exit 1 +fi + +echo -e "${YELLOW}运行MCP服务器测试套件...${NC}" +echo -e "${YELLOW}测试目录: $TEST_DIR${NC}\n" + +# 运行测试 +exec "$RUNNER_SCRIPT" \ No newline at end of file diff --git a/scripts/deploy_vault.sh b/scripts/deploy_vault.sh new file mode 100755 index 0000000..5f58ac3 --- /dev/null +++ b/scripts/deploy_vault.sh @@ -0,0 +1,143 @@ +#!/bin/bash +# 部署Vault集群的脚本 + +# 检查并安装Vault +if ! which vault >/dev/null; then + echo "==== 安装Vault ====" + VAULT_VERSION="1.20.4" + wget -q https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip + unzip -q vault_${VAULT_VERSION}_linux_amd64.zip + sudo mv vault /usr/local/bin/ + rm vault_${VAULT_VERSION}_linux_amd64.zip +fi + +export PATH=$PATH:/usr/local/bin + +set -e + +echo "===== 开始部署Vault集群 =====" + +# 目录定义 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(dirname "$SCRIPT_DIR")" +ANSIBLE_DIR="$ROOT_DIR/playbooks" +JOBS_DIR="$ROOT_DIR/components/vault/jobs" + +# 颜色定义 +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +# 函数定义 +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# 检查命令是否存在 +check_command() { + if ! command -v $1 &> /dev/null; then + log_error "$1 命令未找到，请先安装" + exit 1 + fi +} + +# 检查必要的命令 +check_command ansible-playbook +check_command nomad +check_command vault + +# 步骤1: 使用Ansible安装Vault +log_info "步骤1: 使用Ansible安装Vault..." +ansible-playbook -i "$ANSIBLE_DIR/inventories/production/vault.ini" "$ANSIBLE_DIR/playbooks/install/install_vault.yml" + +# 步骤2: 部署Vault Nomad作业 +log_info "步骤2: 部署Vault Nomad作业..." +nomad job run "$JOBS_DIR/vault-cluster-exec.nomad" + +# 等待Nomad作业部署完成 +log_info "等待Nomad作业部署完成..." +sleep 10 + +# 检查Nomad作业状态 +nomad_status=$(nomad job status vault-cluster-exec | grep Status | head -1 | awk '{print $2}') +if [ "$nomad_status" != "running" ]; then + log_warn "Vault Nomad作业状态不是'running'，当前状态: $nomad_status" + log_info "请检查Nomad作业状态: nomad job status vault-cluster-exec" +fi + +# 步骤3: 检查Vault状态并初始化（如果需要） +log_info "步骤3: 检查Vault状态..." +export VAULT_ADDR='http://127.0.0.1:8200' + +# 等待Vault启动 +log_info "等待Vault启动..." +for i in {1..30}; do + if curl -s "$VAULT_ADDR/v1/sys/health" > /dev/null; then + break + fi + echo -n "." + sleep 2 +done +echo "" + +# 检查Vault是否已初始化 +init_status=$(curl -s "$VAULT_ADDR/v1/sys/health" | grep -o '"initialized":[^,}]*' | cut -d ':' -f2) +if [ "$init_status" = "false" ]; then + log_info "Vault未初始化，正在初始化..." + + # 初始化Vault并保存密钥 + mkdir -p "$ROOT_DIR/security/secrets/vault" + vault operator init -key-shares=5 -key-threshold=3 -format=json > "$ROOT_DIR/security/secrets/vault/init_keys.json" + + if [ $? -eq 0 ]; then + log_info "Vault初始化成功，解封密钥和根令牌已保存到 $ROOT_DIR/security/secrets/vault/init_keys.json" + log_warn "请确保安全保存这些密钥！" + + # 提取解封密钥 + unseal_key1=$(cat "$ROOT_DIR/security/secrets/vault/init_keys.json" | grep -o '"unseal_keys_b64":\[\([^]]*\)' | sed 's/"unseal_keys_b64":\[//g' | tr ',' '\n' | sed 's/"//g' | head -1) + unseal_key2=$(cat "$ROOT_DIR/security/secrets/vault/init_keys.json" | grep -o '"unseal_keys_b64":\[\([^]]*\)' | sed 's/"unseal_keys_b64":\[//g' | tr ',' '\n' | sed 's/"//g' | head -2 | tail -1) + unseal_key3=$(cat "$ROOT_DIR/security/secrets/vault/init_keys.json" | grep -o '"unseal_keys_b64":\[\([^]]*\)' | sed 's/"unseal_keys_b64":\[//g' | tr ',' '\n' | sed 's/"//g' | head -3 | tail -1) + + # 解封Vault + log_info "正在解封Vault..." + vault operator unseal "$unseal_key1" + vault operator unseal "$unseal_key2" + vault operator unseal "$unseal_key3" + + log_info "Vault已成功解封" + else + log_error "Vault初始化失败" + exit 1 + fi +else + log_info "Vault已初始化" + + # 检查Vault是否已解封 + sealed_status=$(curl -s "$VAULT_ADDR/v1/sys/health" | grep -o '"sealed":[^,}]*' | cut -d ':' -f2) + if [ "$sealed_status" = "true" ]; then + log_warn "Vault已初始化但仍处于密封状态，请手动解封" + log_info "使用以下命令解封Vault:" + log_info "export VAULT_ADDR='http://127.0.0.1:8200'" + log_info "vault operator unseal <解封密钥1>" + log_info "vault operator unseal <解封密钥2>" + log_info "vault operator unseal <解封密钥3>" + else + log_info "Vault已初始化且已解封，可以正常使用" + fi +fi + +# 显示Vault状态 +log_info "Vault状态:" +vault status + +log_info "===== Vault集群部署完成 =====" +log_info "请在其他节点上运行解封操作，确保集群完全可用" \ No newline at end of file diff --git a/start_mcp_server.sh b/start_mcp_server.sh new file mode 100644 index 0000000..eb9f033 --- /dev/null +++ b/start_mcp_server.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# 设置环境变量 +export QDRANT_URL=http://dev1:6333 +export QDRANT_API_KEY=313131 +export OLLAMA_URL=http://dev1:11434 +export OLLAMA_MODEL=nomic-embed-text +export COLLECTION_NAME=ollama_mcp + +# 启动MCP服务器 +python /home/ben/qdrant/qdrant_ollama_mcp_server.py \ No newline at end of file diff --git a/sync_all_mcp_configs.sh b/sync_all_mcp_configs.sh index 79b6259..2faafec 100755 --- a/sync_all_mcp_configs.sh +++ b/sync_all_mcp_configs.sh @@ -17,10 +17,8 @@ echo "✓ 使用NFS共享配置作为基准: $NFS_CONFIG" # 定义所有可能的MCP配置位置 CONFIGS=( - # Kilo Code IDE - "../.trae-cn-server/data/User/globalStorage/kilocode.kilo-code/settings/mcp_settings.json" + # Kilo Code IDE (全局配置，移除了项目级别配置以避免冲突) "../.trae-server/data/User/globalStorage/kilocode.kilo-code/settings/mcp_settings.json" - "../.trae-aicc/data/User/globalStorage/kilocode.kilo-code/settings/mcp_settings.json" # Tencent CodeBuddy "$HOME/.codebuddy-server/data/User/globalStorage/tencent.planning-genie/settings/codebuddy_mcp_settings.json" diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..b8da143 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,88 @@ +# 测试脚本目录 + +本目录包含了项目的所有测试脚本，按照功能进行了分类组织。 + +## 目录结构 + +``` +tests/ +├── mcp_servers/ # MCP服务器相关测试脚本 +│ ├── test_direct_search.sh +│ ├── test_local_mcp_servers.sh +│ ├── test_mcp_interface.sh +│ ├── test_mcp_servers.sh +│ ├── test_mcp_servers_comprehensive.py +│ ├── test_mcp_servers_improved.py +│ ├── test_mcp_servers_simple.py +│ ├── test_qdrant_ollama_server.py +│ ├── test_qdrant_ollama_tools.sh +│ ├── test_qdrant_ollama_tools_fixed.sh +│ ├── test_search_documents.sh +│ └── test_mcp_search_final.sh +├── mcp_server_test_report.md # MCP服务器测试报告 +├── run_all_tests.sh # 自动化测试运行器 +└── legacy/ # 旧的或不再使用的测试脚本 +``` + +## MCP服务器测试脚本说明 + +### Shell脚本 +- `test_direct_search.sh`: 测试search_documents方法，通过SSH执行Python代码 +- `test_local_mcp_servers.sh`: 检查MCP配置，测试服务器可用性（context7, qdrant, qdrant-ollama），验证环境变量 +- `test_mcp_interface.sh`: 通过实际接口测试MCP服务器调用，包括tools/list和qdrant_search方法 +- `test_mcp_servers.sh`: 通过initialize方法调用测试Qdrant和Qdrant-Ollama MCP服务器 +- `test_search_documents.sh`: 添加测试文档并搜索"人工智能"（artificial intelligence） +- `test_qdrant_ollama_tools.sh`: 通过JSON-RPC调用测试search_documents和add_document工具 +- `test_qdrant_ollama_tools_fixed.sh`: 测试search_documents、add_document和list_collections工具 +- `test_mcp_search_final.sh`: 最终版本的MCP搜索测试脚本 + +### Python脚本 +- `test_qdrant_ollama_server.py`: 启动服务器，测试初始化、工具列表、文档添加和搜索功能 +- `test_mcp_servers_comprehensive.py`: 使用asyncio和增强响应处理综合测试MCP服务器 +- `test_mcp_servers_improved.py`: 改进版的MCP服务器测试，使用asyncio和增强响应处理 +- `test_mcp_servers_simple.py`: 简化版MCP服务器测试，使用同步子进程调用 + +## 使用方法 + +### 运行单个测试脚本 +```bash +cd tests/mcp_servers +./test_local_mcp_servers.sh +``` + +或运行Python测试： +```bash +cd tests/mcp_servers +python test_mcp_servers_simple.py +``` + +### 批量运行所有测试 +使用自动化测试运行器脚本，可以一键运行所有测试并生成详细报告： +```bash +cd tests +./run_all_tests.sh +``` + +自动化测试运行器将： +- 自动运行所有Shell和Python测试脚本 +- 彩色输出测试进度和结果 +- 生成详细的测试报告（Markdown格式） +- 统计测试通过率和失败情况 +- 保存测试日志到文件 + +## 注意事项 + +- 所有测试脚本都依赖于正确的环境变量配置 +- 测试前请确保相关服务（context7, qdrant, qdrant-ollama）已启动 +- 某些测试可能需要SSH访问权限 + +## 测试报告 + +`mcp_server_test_report.md` 文件包含了MCP服务器的详细测试结果，包括： +- context7、qdrant和qdrant-ollama三个服务器的测试状态 +- 测试环境和方法说明 +- 发现的问题和解决方案 +- 环境变量配置详情 +- 建议和后续改进方向 + +建议在运行测试脚本前先阅读测试报告，了解当前的测试状态和已知问题。 \ No newline at end of file diff --git a/tests/mcp_server_test_report.md b/tests/mcp_server_test_report.md new file mode 100644 index 0000000..5d1e1b2 --- /dev/null +++ b/tests/mcp_server_test_report.md @@ -0,0 +1,55 @@ +# MCP服务器测试报告 + +## 测试概述 +本报告记录了对context7、qdrant和qdrant-ollama三个MCP服务器的测试结果。 + +## 测试环境 +- 测试时间：2025-06-17 +- 测试方法：通过SSH连接到远程服务器进行测试 +- 测试工具：JSON-RPC协议直接调用MCP服务器 + +## 测试结果 + +### 1. context7服务器 +- **状态**：✅ 正常工作 +- **测试内容**： + - 成功初始化 + - 成功获取工具列表 + - 成功执行搜索功能 +- **备注**：context7服务器运行稳定，所有功能正常 + +### 2. qdrant-ollama服务器 +- **状态**：✅ 正常工作（已修复filter参数问题） +- **测试内容**： + - 成功获取工具列表：add_document、search_documents、list_collections和get_collection_info + - 成功使用add_document工具添加文档 + - 成功使用search_documents工具搜索文档 +- **修复记录**： + - **问题**：search_documents工具使用filter参数时出现"Unknown arguments: ['filter']"错误 + - **原因**：参数名称不匹配，工具定义中使用filter，但实现中使用query_filter + - **解决方案**：将工具定义中的filter参数名改为query_filter + - **验证结果**：修复后search_documents工具正常工作，不再出现错误 + +### 3. qdrant服务器 +- **状态**：✅ 正常工作 +- **测试内容**： + - 成功获取工具列表：qdrant_search、qdrant_add和qdrant_delete + - 成功使用qdrant_add工具添加文档 + - 成功使用qdrant_search工具搜索文档 +- **备注**：qdrant服务器运行稳定，所有功能正常 + +## 环境变量配置 +两个服务器都正确配置了以下环境变量： +- QDRANT_URL: http://dev1:6333 (qdrant-ollama) / http://localhost:6333 (qdrant) +- QDRANT_API_KEY: 313131 +- OLLAMA_URL: http://dev1:11434 (仅qdrant-ollama) +- OLLAMA_MODEL: nomic-embed-text (仅qdrant-ollama) +- COLLECTION_NAME: ollama_mcp (qdrant-ollama) / mcp (qdrant) + +## 结论 +所有三个MCP服务器均已成功测试并正常工作。qdrant-ollama服务器的filter参数问题已修复，不再出现"Unknown arguments: ['filter']"错误。所有服务器的核心功能（添加文档、搜索文档）均正常运行。 + +## 建议 +1. 考虑将qdrant_mcp_server.py中的search方法更新为query_points方法，以消除弃用警告 +2. 可以考虑为qdrant-ollama服务器添加更多过滤选项，增强搜索功能 +3. 建议定期测试MCP服务器的功能，确保持续稳定运行 \ No newline at end of file diff --git a/tests/mcp_servers/test_direct_search.sh b/tests/mcp_servers/test_direct_search.sh new file mode 100755 index 0000000..0b0e099 --- /dev/null +++ b/tests/mcp_servers/test_direct_search.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +echo "直接测试search_documents方法..." + +# 创建一个简单的Python脚本来测试search_documents方法 +ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && python3 -c \" +import asyncio +import json +import sys +sys.path.append('/home/ben/qdrant') + +from qdrant_ollama_mcp_server import QdrantOllamaMCPServer + +async def test_search(): + server = QdrantOllamaMCPServer() + + # 测试search_documents方法 + params = { + 'query': '人工智能', + 'limit': 3 + } + + try: + result = await server._search_documents(params) + print('搜索结果:', json.dumps(result, indent=2, ensure_ascii=False)) + except Exception as e: + print('搜索错误:', str(e)) + import traceback + traceback.print_exc() + +asyncio.run(test_search()) +\"" \ No newline at end of file diff --git a/tests/mcp_servers/test_local_mcp_servers.sh b/tests/mcp_servers/test_local_mcp_servers.sh new file mode 100755 index 0000000..165f7fb --- /dev/null +++ b/tests/mcp_servers/test_local_mcp_servers.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# 测试当前环境中的MCP服务器 + +echo "测试当前环境中的MCP服务器..." + +# 检查当前环境中是否有MCP配置 +echo "检查MCP配置..." +if [ -f "/root/.mcp/mcp_settings.json" ]; then + echo "找到MCP配置文件: /root/.mcp/mcp_settings.json" + cat /root/.mcp/mcp_settings.json +else + echo "未找到MCP配置文件: /root/.mcp/mcp_settings.json" +fi + +echo "" +echo "检查.kilocode/mcp.json..." +if [ -f "/root/mgmt/.kilocode/mcp.json" ]; then + echo "找到MCP配置文件: /root/mgmt/.kilocode/mcp.json" + cat /root/mgmt/.kilocode/mcp.json +else + echo "未找到MCP配置文件: /root/mgmt/.kilocode/mcp.json" +fi + +echo "" +echo "检查是否有可用的MCP服务器..." + +# 检查context7服务器 +echo "测试context7服务器..." +echo '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' | nc localhost 8080 2>/dev/null || echo "context7服务器未在本地运行" + +# 检查qdrant服务器 +echo "测试qdrant服务器..." +if [ -f "/root/mgmt/qdrant_mcp_server.py" ]; then + echo "找到qdrant服务器脚本: /root/mgmt/qdrant_mcp_server.py" + # 尝试直接运行服务器并测试 + echo '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' | python3 /root/mgmt/qdrant_mcp_server.py 2>/dev/null || echo "qdrant服务器无法直接运行" +else + echo "未找到qdrant服务器脚本" +fi + +# 检查qdrant-ollama服务器 +echo "测试qdrant-ollama服务器..." +if [ -f "/root/mgmt/qdrant_ollama_mcp_server.py" ]; then + echo "找到qdrant-ollama服务器脚本: /root/mgmt/qdrant_ollama_mcp_server.py" + # 尝试直接运行服务器并测试 + echo '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' | python3 /root/mgmt/qdrant_ollama_mcp_server.py 2>/dev/null || echo "qdrant-ollama服务器无法直接运行" +else + echo "未找到qdrant-ollama服务器脚本" +fi + +echo "" +echo "检查环境变量..." +echo "QDRANT_URL: ${QDRANT_URL:-未设置}" +echo "QDRANT_API_KEY: ${QDRANT_API_KEY:-未设置}" +echo "OLLAMA_URL: ${OLLAMA_URL:-未设置}" +echo "OLLAMA_MODEL: ${OLLAMA_MODEL:-未设置}" +echo "COLLECTION_NAME: ${COLLECTION_NAME:-未设置}" + +echo "" +echo "测试完成。" \ No newline at end of file diff --git a/tests/mcp_servers/test_mcp_interface.sh b/tests/mcp_servers/test_mcp_interface.sh new file mode 100755 index 0000000..767969d --- /dev/null +++ b/tests/mcp_servers/test_mcp_interface.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# 测试MCP服务器在实际MCP接口中的调用 + +echo "测试Qdrant MCP服务器..." +echo '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' | ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && python qdrant_mcp_server.py" + +echo "" +echo "测试Qdrant-Ollama MCP服务器..." +echo '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' | ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && ./start_mcp_server.sh" + +echo "" +echo "测试Qdrant MCP服务器的搜索功能..." +echo '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"qdrant_search","arguments":{"query":"测试查询","limit":3}}}' | ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && python qdrant_mcp_server.py" + +echo "" +echo "测试Qdrant-Ollama MCP服务器的搜索功能..." +echo '{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"qdrant_search","arguments":{"query":"测试查询","limit":3}}}' | ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && ./start_mcp_server.sh" + +echo "" +echo "测试完成。" \ No newline at end of file diff --git a/tests/mcp_servers/test_mcp_search_final.sh b/tests/mcp_servers/test_mcp_search_final.sh new file mode 100755 index 0000000..475ef18 --- /dev/null +++ b/tests/mcp_servers/test_mcp_search_final.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +echo "测试通过MCP接口调用search_documents工具..." + +# 先添加一个文档 +echo "添加测试文档..." +ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && echo '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"add_document\",\"arguments\":{\"text\":\"机器学习是人工智能的一个子领域，专注于开发能够从数据中学习的算法。\",\"metadata\":{\"source\":\"test\",\"topic\":\"ML\"}}}}' | ./start_mcp_server.sh" + +echo "" +echo "通过MCP接口搜索文档..." +# 测试search_documents工具（不带filter参数） +ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && echo '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"search_documents\",\"arguments\":{\"query\":\"机器学习\",\"limit\":3}}}' | ./start_mcp_server.sh" + +echo "" +echo "测试完成。" \ No newline at end of file diff --git a/tests/mcp_servers/test_mcp_servers.sh b/tests/mcp_servers/test_mcp_servers.sh new file mode 100755 index 0000000..59a0a65 --- /dev/null +++ b/tests/mcp_servers/test_mcp_servers.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# 测试MCP服务器脚本 + +echo "测试Qdrant MCP服务器..." +echo '{"jsonrpc":"2.0","id":1,"method":"initialize"}' | ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && python qdrant_mcp_server.py" + +echo "" +echo "测试Qdrant-Ollama MCP服务器..." +echo '{"jsonrpc":"2.0","id":1,"method":"initialize"}' | ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && ./start_mcp_server.sh" + +echo "" +echo "测试完成。" \ No newline at end of file diff --git a/tests/mcp_servers/test_mcp_servers_comprehensive.py b/tests/mcp_servers/test_mcp_servers_comprehensive.py new file mode 100644 index 0000000..cc31747 --- /dev/null +++ b/tests/mcp_servers/test_mcp_servers_comprehensive.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +""" +测试MCP服务器的脚本 +""" + +import asyncio +import json +import subprocess +import sys +from typing import Dict, Any, List + +async def test_mcp_server(server_name: str, command: List[str], env: Dict[str, str] = None): + """测试MCP服务器""" + print(f"\n=== 测试 {server_name} 服务器 ===") + + # 设置环境变量 + process_env = {} + if env: + process_env.update(env) + + try: + # 启动服务器进程 + process = await asyncio.create_subprocess_exec( + *command, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=process_env + ) + + # 初始化请求 + init_request = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": {} + } + } + } + + # 发送初始化请求 + process.stdin.write((json.dumps(init_request) + "\n").encode()) + await process.stdin.drain() + + # 读取初始化响应 + init_response = await process.stdout.readline() + if init_response: + try: + init_data = json.loads(init_response.decode()) + print(f"初始化响应: {init_data}") + except json.JSONDecodeError: + print(f"初始化响应解析失败: {init_response}") + + # 获取工具列表 + tools_request = { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/list" + } + + # 发送工具列表请求 + process.stdin.write((json.dumps(tools_request) + "\n").encode()) + await process.stdin.drain() + + # 读取工具列表响应 + tools_response = await process.stdout.readline() + if tools_response: + try: + tools_data = json.loads(tools_response.decode()) + print(f"工具列表: {json.dumps(tools_data, indent=2, ensure_ascii=False)}") + + # 如果有搜索工具，测试搜索功能 + if "result" in tools_data and "tools" in tools_data["result"]: + for tool in tools_data["result"]["tools"]: + tool_name = tool.get("name") + if tool_name and ("search" in tool_name or "document" in tool_name): + print(f"\n测试工具: {tool_name}") + + # 测试搜索工具 + search_request = { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": tool_name, + "arguments": { + "query": "测试查询", + "limit": 3 + } + } + } + + # 发送搜索请求 + process.stdin.write((json.dumps(search_request) + "\n").encode()) + await process.stdin.drain() + + # 读取搜索响应 + search_response = await process.stdout.readline() + if search_response: + try: + search_data = json.loads(search_response.decode()) + print(f"搜索结果: {json.dumps(search_data, indent=2, ensure_ascii=False)}") + except json.JSONDecodeError: + print(f"搜索响应解析失败: {search_response}") + break + except json.JSONDecodeError: + print(f"工具列表响应解析失败: {tools_response}") + + # 关闭进程 + process.stdin.close() + await process.wait() + + except Exception as e: + print(f"测试 {server_name} 服务器时出错: {e}") + +async def main(): + """主函数""" + print("开始测试MCP服务器...") + + # 测试context7服务器 + await test_mcp_server( + "context7", + ["npx", "-y", "@upstash/context7-mcp"], + {"DEFAULT_MINIMUM_TOKENS": ""} + ) + + # 测试qdrant服务器 + await test_mcp_server( + "qdrant", + ["ssh", "ben@dev1", "cd /home/ben/qdrant && source venv/bin/activate && python qdrant_mcp_server.py"], + { + "QDRANT_URL": "http://dev1:6333", + "QDRANT_API_KEY": "313131", + "COLLECTION_NAME": "mcp", + "EMBEDDING_MODEL": "bge-m3" + } + ) + + # 测试qdrant-ollama服务器 + await test_mcp_server( + "qdrant-ollama", + ["ssh", "ben@dev1", "cd /home/ben/qdrant && source venv/bin/activate && ./start_mcp_server.sh"], + { + "QDRANT_URL": "http://dev1:6333", + "QDRANT_API_KEY": "313131", + "COLLECTION_NAME": "ollama_mcp", + "OLLAMA_MODEL": "nomic-embed-text", + "OLLAMA_URL": "http://dev1:11434" + } + ) + + print("\n所有测试完成。") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/tests/mcp_servers/test_mcp_servers_improved.py b/tests/mcp_servers/test_mcp_servers_improved.py new file mode 100644 index 0000000..aebff54 --- /dev/null +++ b/tests/mcp_servers/test_mcp_servers_improved.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +""" +改进的MCP服务器测试脚本 +""" + +import asyncio +import json +import subprocess +import sys +from typing import Dict, Any, List, Optional + +async def test_mcp_server(server_name: str, command: List[str], env: Dict[str, str] = None): + """测试MCP服务器""" + print(f"\n=== 测试 {server_name} 服务器 ===") + + # 设置环境变量 + process_env = {} + if env: + process_env.update(env) + + try: + # 启动服务器进程 + process = await asyncio.create_subprocess_exec( + *command, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=process_env + ) + + # 读取并忽略所有非JSON输出 + buffer = "" + while True: + line = await process.stdout.readline() + if not line: + break + + line_str = line.decode().strip() + buffer += line_str + "\n" + + # 尝试解析JSON + try: + data = json.loads(line_str) + if "jsonrpc" in data: + print(f"收到JSON响应: {json.dumps(data, indent=2, ensure_ascii=False)}") + break + except json.JSONDecodeError: + # 不是JSON，继续读取 + continue + + # 如果没有找到JSON响应，显示缓冲区内容 + if "jsonrpc" not in locals(): + print(f"未找到JSON响应，原始输出: {buffer}") + return + + # 初始化请求 + init_request = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": {} + } + } + } + + # 发送初始化请求 + process.stdin.write((json.dumps(init_request) + "\n").encode()) + await process.stdin.drain() + + # 读取初始化响应 + init_response = await read_json_response(process) + if init_response: + print(f"初始化成功") + + # 获取工具列表 + tools_request = { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/list" + } + + # 发送工具列表请求 + process.stdin.write((json.dumps(tools_request) + "\n").encode()) + await process.stdin.drain() + + # 读取工具列表响应 + tools_response = await read_json_response(process) + if tools_response: + print(f"工具列表获取成功") + + # 如果有搜索工具，测试搜索功能 + if "result" in tools_response and "tools" in tools_response["result"]: + for tool in tools_response["result"]["tools"]: + tool_name = tool.get("name") + if tool_name and ("search" in tool_name or "document" in tool_name): + print(f"\n测试工具: {tool_name}") + + # 测试搜索工具 + search_request = { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": tool_name, + "arguments": { + "query": "测试查询", + "limit": 3 + } + } + } + + # 发送搜索请求 + process.stdin.write((json.dumps(search_request) + "\n").encode()) + await process.stdin.drain() + + # 读取搜索响应 + search_response = await read_json_response(process) + if search_response: + print(f"搜索测试成功") + if "result" in search_response and "content" in search_response["result"]: + for content in search_response["result"]["content"]: + if content.get("type") == "text": + print(f"搜索结果: {content.get('text', '')[:100]}...") + break + + # 关闭进程 + process.stdin.close() + await process.wait() + + except Exception as e: + print(f"测试 {server_name} 服务器时出错: {e}") + +async def read_json_response(process): + """读取JSON响应""" + buffer = "" + while True: + line = await process.stdout.readline() + if not line: + break + + line_str = line.decode().strip() + buffer += line_str + "\n" + + # 尝试解析JSON + try: + data = json.loads(line_str) + if "jsonrpc" in data: + return data + except json.JSONDecodeError: + # 不是JSON，继续读取 + continue + + # 如果没有找到JSON响应，返回None + return None + +async def main(): + """主函数""" + print("开始测试MCP服务器...") + + # 测试context7服务器 + await test_mcp_server( + "context7", + ["npx", "-y", "@upstash/context7-mcp"], + {"DEFAULT_MINIMUM_TOKENS": ""} + ) + + # 测试qdrant服务器 + await test_mcp_server( + "qdrant", + ["ssh", "ben@dev1", "cd /home/ben/qdrant && source venv/bin/activate && python qdrant_mcp_server.py"], + { + "QDRANT_URL": "http://dev1:6333", + "QDRANT_API_KEY": "313131", + "COLLECTION_NAME": "mcp", + "EMBEDDING_MODEL": "bge-m3" + } + ) + + # 测试qdrant-ollama服务器 + await test_mcp_server( + "qdrant-ollama", + ["ssh", "ben@dev1", "cd /home/ben/qdrant && source venv/bin/activate && ./start_mcp_server.sh"], + { + "QDRANT_URL": "http://dev1:6333", + "QDRANT_API_KEY": "313131", + "COLLECTION_NAME": "ollama_mcp", + "OLLAMA_MODEL": "nomic-embed-text", + "OLLAMA_URL": "http://dev1:11434" + } + ) + + print("\n所有测试完成。") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/tests/mcp_servers/test_mcp_servers_simple.py b/tests/mcp_servers/test_mcp_servers_simple.py new file mode 100644 index 0000000..6440ed9 --- /dev/null +++ b/tests/mcp_servers/test_mcp_servers_simple.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +""" +简化的MCP服务器测试脚本 +""" + +import json +import subprocess +import sys +import time +from typing import Dict, Any, List + +def test_mcp_server(server_name: str, command: List[str], env: Dict[str, str] = None): + """测试MCP服务器""" + print(f"\n=== 测试 {server_name} 服务器 ===") + + # 设置环境变量 + process_env = {} + if env: + process_env.update(env) + + try: + # 启动服务器进程 + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=process_env, + text=True + ) + + # 等待进程启动 + time.sleep(2) + + # 初始化请求 + init_request = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": {} + } + } + } + + # 发送初始化请求 + process.stdin.write(json.dumps(init_request) + "\n") + process.stdin.flush() + + # 读取初始化响应 + init_response = process.stdout.readline() + if init_response: + try: + init_data = json.loads(init_response.strip()) + print(f"初始化成功: {init_data.get('result', {}).get('serverInfo', {}).get('name', '未知服务器')}") + except json.JSONDecodeError: + print(f"初始化响应解析失败: {init_response}") + + # 获取工具列表 + tools_request = { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/list" + } + + # 发送工具列表请求 + process.stdin.write(json.dumps(tools_request) + "\n") + process.stdin.flush() + + # 读取工具列表响应 + tools_response = process.stdout.readline() + if tools_response: + try: + tools_data = json.loads(tools_response.strip()) + print(f"工具列表获取成功") + + # 如果有搜索工具，测试搜索功能 + if "result" in tools_data and "tools" in tools_data["result"]: + for tool in tools_data["result"]["tools"]: + tool_name = tool.get("name") + if tool_name and ("search" in tool_name or "document" in tool_name): + print(f"\n测试工具: {tool_name}") + + # 测试搜索工具 + search_request = { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": tool_name, + "arguments": { + "query": "测试查询", + "limit": 3 + } + } + } + + # 发送搜索请求 + process.stdin.write(json.dumps(search_request) + "\n") + process.stdin.flush() + + # 读取搜索响应 + search_response = process.stdout.readline() + if search_response: + try: + search_data = json.loads(search_response.strip()) + print(f"搜索测试成功") + if "result" in search_data and "content" in search_data["result"]: + for content in search_data["result"]["content"]: + if content.get("type") == "text": + print(f"搜索结果: {content.get('text', '')[:100]}...") + except json.JSONDecodeError: + print(f"搜索响应解析失败: {search_response}") + break + except json.JSONDecodeError: + print(f"工具列表响应解析失败: {tools_response}") + + # 关闭进程 + process.stdin.close() + process.terminate() + process.wait() + + except Exception as e: + print(f"测试 {server_name} 服务器时出错: {e}") + +def main(): + """主函数""" + print("开始测试MCP服务器...") + + # 测试context7服务器 + test_mcp_server( + "context7", + ["npx", "-y", "@upstash/context7-mcp"], + {"DEFAULT_MINIMUM_TOKENS": ""} + ) + + # 测试qdrant服务器 + test_mcp_server( + "qdrant", + ["ssh", "ben@dev1", "cd /home/ben/qdrant && source venv/bin/activate && python qdrant_mcp_server.py"], + { + "QDRANT_URL": "http://dev1:6333", + "QDRANT_API_KEY": "313131", + "COLLECTION_NAME": "mcp", + "EMBEDDING_MODEL": "bge-m3" + } + ) + + # 测试qdrant-ollama服务器 + test_mcp_server( + "qdrant-ollama", + ["ssh", "ben@dev1", "cd /home/ben/qdrant && source venv/bin/activate && ./start_mcp_server.sh"], + { + "QDRANT_URL": "http://dev1:6333", + "QDRANT_API_KEY": "313131", + "COLLECTION_NAME": "ollama_mcp", + "OLLAMA_MODEL": "nomic-embed-text", + "OLLAMA_URL": "http://dev1:11434" + } + ) + + print("\n所有测试完成。") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/mcp_servers/test_qdrant_ollama_server.py b/tests/mcp_servers/test_qdrant_ollama_server.py new file mode 100644 index 0000000..90b77a7 --- /dev/null +++ b/tests/mcp_servers/test_qdrant_ollama_server.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +""" +专门测试qdrant-ollama服务器的脚本 +""" + +import json +import subprocess +import sys +import time +from typing import Dict, Any, List + +def test_qdrant_ollama_server(): + """测试qdrant-ollama服务器""" + print("\n=== 测试 qdrant-ollama 服务器 ===") + + try: + # 启动服务器进程 + process = subprocess.Popen( + ["ssh", "ben@dev1", "cd /home/ben/qdrant && source venv/bin/activate && ./start_mcp_server.sh"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True + ) + + # 读取并忽略所有非JSON输出 + buffer = "" + json_found = False + + # 等待进程启动并读取初始输出 + for _ in range(10): # 最多尝试10次 + line = process.stdout.readline() + if not line: + time.sleep(0.5) + continue + + line = line.strip() + buffer += line + "\n" + + # 尝试解析JSON + try: + data = json.loads(line) + if "jsonrpc" in data: + json_found = True + print(f"收到JSON响应: {json.dumps(data, indent=2, ensure_ascii=False)}") + break + except json.JSONDecodeError: + # 不是JSON，继续读取 + continue + + if not json_found: + print(f"未找到JSON响应，原始输出: {buffer}") + process.terminate() + process.wait() + return + + # 初始化请求 + init_request = { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": {} + } + } + } + + # 发送初始化请求 + process.stdin.write(json.dumps(init_request) + "\n") + process.stdin.flush() + + # 读取初始化响应 + init_response = process.stdout.readline() + if init_response: + try: + init_data = json.loads(init_response.strip()) + print(f"初始化成功: {init_data.get('result', {}).get('serverInfo', {}).get('name', '未知服务器')}") + except json.JSONDecodeError: + print(f"初始化响应解析失败: {init_response}") + + # 获取工具列表 + tools_request = { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/list" + } + + # 发送工具列表请求 + process.stdin.write(json.dumps(tools_request) + "\n") + process.stdin.flush() + + # 读取工具列表响应 + tools_response = process.stdout.readline() + if tools_response: + try: + tools_data = json.loads(tools_response.strip()) + print(f"工具列表获取成功") + + # 如果有搜索工具，测试搜索功能 + if "result" in tools_data and "tools" in tools_data["result"]: + for tool in tools_data["result"]["tools"]: + tool_name = tool.get("name") + if tool_name and ("search" in tool_name or "document" in tool_name): + print(f"\n测试工具: {tool_name}") + + # 先添加一个文档 + add_request = { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": "add_document", + "arguments": { + "text": "这是一个测试文档，用于验证qdrant-ollama服务器的功能。", + "metadata": { + "source": "test", + "topic": "测试" + } + } + } + } + + # 发送添加文档请求 + process.stdin.write(json.dumps(add_request) + "\n") + process.stdin.flush() + + # 读取添加文档响应 + add_response = process.stdout.readline() + if add_response: + try: + add_data = json.loads(add_response.strip()) + print(f"添加文档测试成功") + except json.JSONDecodeError: + print(f"添加文档响应解析失败: {add_response}") + + # 测试搜索工具 + search_request = { + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": { + "name": tool_name, + "arguments": { + "query": "测试文档", + "limit": 3 + } + } + } + + # 发送搜索请求 + process.stdin.write(json.dumps(search_request) + "\n") + process.stdin.flush() + + # 读取搜索响应 + search_response = process.stdout.readline() + if search_response: + try: + search_data = json.loads(search_response.strip()) + print(f"搜索测试成功") + if "result" in search_data and "content" in search_data["result"]: + for content in search_data["result"]["content"]: + if content.get("type") == "text": + print(f"搜索结果: {content.get('text', '')[:100]}...") + except json.JSONDecodeError: + print(f"搜索响应解析失败: {search_response}") + break + except json.JSONDecodeError: + print(f"工具列表响应解析失败: {tools_response}") + + # 关闭进程 + process.stdin.close() + process.terminate() + process.wait() + + except Exception as e: + print(f"测试 qdrant-ollama 服务器时出错: {e}") + +def main(): + """主函数""" + print("开始测试qdrant-ollama服务器...") + + test_qdrant_ollama_server() + + print("\n测试完成。") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/mcp_servers/test_qdrant_ollama_tools.sh b/tests/mcp_servers/test_qdrant_ollama_tools.sh new file mode 100755 index 0000000..10f2917 --- /dev/null +++ b/tests/mcp_servers/test_qdrant_ollama_tools.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +echo "测试Qdrant-Ollama MCP服务器的search_documents工具..." + +# 测试search_documents工具 +ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && echo '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"search_documents\",\"arguments\":{\"query\":\"测试查询\",\"limit\":3}}}' | ./start_mcp_server.sh" + +echo "" +echo "测试Qdrant-Ollama MCP服务器的add_document工具..." + +# 测试add_document工具 +ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && echo '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"add_document\",\"arguments\":{\"text\":\"这是一个测试文档\",\"metadata\":{\"source\":\"test\"}}}}' | ./start_mcp_server.sh" + +echo "" +echo "测试完成。" \ No newline at end of file diff --git a/tests/mcp_servers/test_qdrant_ollama_tools_fixed.sh b/tests/mcp_servers/test_qdrant_ollama_tools_fixed.sh new file mode 100755 index 0000000..6ab3082 --- /dev/null +++ b/tests/mcp_servers/test_qdrant_ollama_tools_fixed.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +echo "测试Qdrant-Ollama MCP服务器的search_documents工具（不带filter参数）..." + +# 测试search_documents工具（不带filter参数） +ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && echo '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"search_documents\",\"arguments\":{\"query\":\"测试查询\",\"limit\":3}}}' | ./start_mcp_server.sh" + +echo "" +echo "测试Qdrant-Ollama MCP服务器的add_document工具..." + +# 测试add_document工具 +ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && echo '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"add_document\",\"arguments\":{\"text\":\"这是一个测试文档\",\"metadata\":{\"source\":\"test\"}}}}' | ./start_mcp_server.sh" + +echo "" +echo "测试Qdrant-Ollama MCP服务器的list_collections工具..." + +# 测试list_collections工具 +ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && echo '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"list_collections\",\"arguments\":{}}}' | ./start_mcp_server.sh" + +echo "" +echo "测试完成。" \ No newline at end of file diff --git a/tests/mcp_servers/test_search_documents.sh b/tests/mcp_servers/test_search_documents.sh new file mode 100755 index 0000000..b501f35 --- /dev/null +++ b/tests/mcp_servers/test_search_documents.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +echo "测试Qdrant-Ollama MCP服务器的search_documents工具（不带filter参数）..." + +# 先添加一个文档 +echo "添加测试文档..." +ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && echo '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"add_document\",\"arguments\":{\"text\":\"人工智能是计算机科学的一个分支，致力于创建能够执行通常需要人类智能的任务的系统。\",\"metadata\":{\"source\":\"test\",\"topic\":\"AI\"}}}}' | ./start_mcp_server.sh" + +echo "" +echo "搜索文档..." +# 测试search_documents工具（不带filter参数） +ssh ben@dev1 "cd /home/ben/qdrant && source venv/bin/activate && echo '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"search_documents\",\"arguments\":{\"query\":\"人工智能\",\"limit\":3}}}' | ./start_mcp_server.sh" + +echo "" +echo "测试完成。" \ No newline at end of file diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh new file mode 100755 index 0000000..a4a95b1 --- /dev/null +++ b/tests/run_all_tests.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +# MCP服务器测试运行器 +# 自动运行所有MCP服务器测试脚本 + +set -e + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# 测试目录 +TEST_DIR="/root/mgmt/tests/mcp_servers" +REPORT_FILE="/root/mgmt/tests/test_results_$(date +%Y%m%d_%H%M%S).md" + +# 检查测试目录是否存在 +if [ ! -d "$TEST_DIR" ]; then + echo -e "${RED}错误: 测试目录 $TEST_DIR 不存在${NC}" + exit 1 +fi + +# 创建测试报告头部 +cat > "$REPORT_FILE" << EOF +# MCP服务器测试报告 - $(date '+%Y-%m-%d %H:%M:%S') + +## 测试环境 +- 测试时间: $(date '+%Y-%m-%d %H:%M:%S') +- 测试目录: $TEST_DIR +- 测试类型: 自动化批量测试 + +## 测试结果概览 + +EOF + +echo -e "${YELLOW}开始运行MCP服务器测试套件...${NC}" +echo -e "${YELLOW}测试报告将保存到: $REPORT_FILE${NC}\n" + +# 测试计数器 +TOTAL_TESTS=0 +PASSED_TESTS=0 +FAILED_TESTS=0 + +# 运行Shell脚本测试 +echo -e "${YELLOW}运行Shell脚本测试...${NC}" +for test_script in "$TEST_DIR"/*.sh; do + if [ -f "$test_script" ]; then + TEST_NAME=$(basename "$test_script") + echo -e "${YELLOW}运行测试: $TEST_NAME${NC}" + + # 运行测试脚本 + if bash "$test_script" >> "$REPORT_FILE" 2>&1; then + echo -e "${GREEN}✅ $TEST_NAME 通过${NC}" + echo "- ✅ $TEST_NAME: 通过" >> "$REPORT_FILE" + ((PASSED_TESTS++)) + else + echo -e "${RED}❌ $TEST_NAME 失败${NC}" + echo "- ❌ $TEST_NAME: 失败" >> "$REPORT_FILE" + ((FAILED_TESTS++)) + fi + ((TOTAL_TESTS++)) + echo + fi +done + +# 运行Python脚本测试 +echo -e "${YELLOW}运行Python脚本测试...${NC}" +for test_script in "$TEST_DIR"/*.py; do + if [ -f "$test_script" ]; then + TEST_NAME=$(basename "$test_script") + echo -e "${YELLOW}运行测试: $TEST_NAME${NC}" + + # 运行Python测试 + if python3 "$test_script" >> "$REPORT_FILE" 2>&1; then + echo -e "${GREEN}✅ $TEST_NAME 通过${NC}" + echo "- ✅ $TEST_NAME: 通过" >> "$REPORT_FILE" + ((PASSED_TESTS++)) + else + echo -e "${RED}❌ $TEST_NAME 失败${NC}" + echo "- ❌ $TEST_NAME: 失败" >> "$REPORT_FILE" + ((FAILED_TESTS++)) + fi + ((TOTAL_TESTS++)) + echo + fi +done + +# 更新测试报告 +cat >> "$REPORT_FILE" << EOF + +## 测试统计 +- 总测试数: $TOTAL_TESTS +- 通过测试: $PASSED_TESTS +- 失败测试: $FAILED_TESTS +- 通过率: $((PASSED_TESTS * 100 / TOTAL_TESTS))% + +## 详细测试输出 +EOF + +# 显示测试结果摘要 +echo -e "\n${YELLOW}=== 测试完成 ===${NC}" +echo -e "总测试数: $TOTAL_TESTS" +echo -e "通过测试: ${GREEN}$PASSED_TESTS${NC}" +echo -e "失败测试: ${RED}$FAILED_TESTS${NC}" +echo -e "通过率: $((PASSED_TESTS * 100 / TOTAL_TESTS))%" +echo -e "详细报告: $REPORT_FILE" + +# 如果所有测试都通过，返回成功 +if [ $FAILED_TESTS -eq 0 ]; then + echo -e "\n${GREEN}所有测试均通过！${NC}" + exit 0 +else + echo -e "\n${RED}部分测试失败，请查看详细报告。${NC}" + exit 1 +fi \ No newline at end of file diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf new file mode 100644 index 0000000..c7708fe --- /dev/null +++ b/tf/environments/dev/main.tf @@ -0,0 +1,155 @@ +# 开发环境主配置文件 + +# 引入共享版本配置 +terraform { + required_version = ">= 1.6" + + required_providers { + # Oracle Cloud Infrastructure + oci = { + source = "oracle/oci" + version = "~> 7.20" + } + + # 其他常用提供商 + random = { + source = "hashicorp/random" + version = "~> 3.1" + } + + tls = { + source = "hashicorp/tls" + version = "~> 4.0" + } + + local = { + source = "hashicorp/local" + version = "~> 2.1" + } + + # Consul Provider + consul = { + source = "hashicorp/consul" + version = "~> 2.22.0" + } + + # HashiCorp Vault Provider + vault = { + source = "hashicorp/vault" + version = "~> 4.0" + } + } + + # 后端配置 + backend "local" { + path = "terraform.tfstate" + } +} + +# Consul Provider配置 - 使用Tailscale IP而非localhost +provider "consul" { + address = "100.116.158.95:8500" + scheme = "http" + datacenter = "dc1" +} + +# Vault Provider配置 +provider "vault" { + address = var.vault_config.address + token = var.vault_token +} + +# 从Consul获取Oracle Cloud配置 +data "consul_keys" "oracle_config" { + key { + name = "tenancy_ocid" + path = "config/dev/oracle/kr/tenancy_ocid" + } + key { + name = "user_ocid" + path = "config/dev/oracle/kr/user_ocid" + } + key { + name = "fingerprint" + path = "config/dev/oracle/kr/fingerprint" + } + key { + name = "private_key" + path = "config/dev/oracle/kr/private_key" + } +} + +# 从Consul获取Oracle Cloud美国区域配置 +data "consul_keys" "oracle_config_us" { + key { + name = "tenancy_ocid" + path = "config/dev/oracle/us/tenancy_ocid" + } + key { + name = "user_ocid" + path = "config/dev/oracle/us/user_ocid" + } + key { + name = "fingerprint" + path = "config/dev/oracle/us/fingerprint" + } + key { + name = "private_key" + path = "config/dev/oracle/us/private_key" + } +} + +# 使用从Consul获取的配置的OCI Provider +provider "oci" { + tenancy_ocid = data.consul_keys.oracle_config.var.tenancy_ocid + user_ocid = data.consul_keys.oracle_config.var.user_ocid + fingerprint = data.consul_keys.oracle_config.var.fingerprint + private_key = data.consul_keys.oracle_config.var.private_key + region = "ap-chuncheon-1" +} + +# 美国区域的OCI Provider +provider "oci" { + alias = "us" + tenancy_ocid = data.consul_keys.oracle_config_us.var.tenancy_ocid + user_ocid = data.consul_keys.oracle_config_us.var.user_ocid + fingerprint = data.consul_keys.oracle_config_us.var.fingerprint + private_key = data.consul_keys.oracle_config_us.var.private_key + region = "us-ashburn-1" +} + +# Oracle Cloud 基础设施 +module "oracle_cloud" { + source = "../../providers/oracle-cloud" + + # 传递变量 + environment = var.environment + project_name = var.project_name + owner = var.owner + vpc_cidr = var.vpc_cidr + availability_zones = var.availability_zones + common_tags = var.common_tags + + # 使用从Consul获取的配置 + oci_config = { + tenancy_ocid = data.consul_keys.oracle_config.var.tenancy_ocid + user_ocid = data.consul_keys.oracle_config.var.user_ocid + fingerprint = data.consul_keys.oracle_config.var.fingerprint + private_key = data.consul_keys.oracle_config.var.private_key + region = "ap-chuncheon-1" + } + + # 开发环境特定配置 + instance_count = 1 + instance_size = "VM.Standard.E2.1.Micro" # 免费层 + + providers = { + oci = oci + } +} + +# 输出 +output "oracle_cloud_outputs" { + description = "Oracle Cloud 基础设施输出" + value = module.oracle_cloud +} \ No newline at end of file diff --git a/tf/environments/dev/variables.tf b/tf/environments/dev/variables.tf new file mode 100644 index 0000000..35a892e --- /dev/null +++ b/tf/environments/dev/variables.tf @@ -0,0 +1,154 @@ +# 开发环境变量定义 + +variable "environment" { + description = "环境名称" + type = string + default = "dev" +} + +variable "project_name" { + description = "项目名称" + type = string + default = "mgmt" +} + +variable "owner" { + description = "项目所有者" + type = string + default = "ben" +} + +variable "cloud_providers" { + description = "要启用的云服务商列表" + type = list(string) + default = ["oracle"] +} + +variable "vpc_cidr" { + description = "VPC CIDR 块" + type = string + default = "10.0.0.0/16" +} + +variable "availability_zones" { + description = "可用区列表" + type = list(string) + default = ["a", "b"] +} + +variable "common_tags" { + description = "通用标签" + type = map(string) + default = { + Environment = "dev" + Project = "mgmt" + ManagedBy = "terraform" + } +} + +# Oracle Cloud 配置 +variable "oci_config" { + description = "Oracle Cloud 配置" + type = object({ + tenancy_ocid = string + user_ocid = string + fingerprint = string + private_key_path = string + region = string + compartment_ocid = optional(string) + }) + default = { + tenancy_ocid = "" + user_ocid = "" + fingerprint = "" + private_key_path = "" + region = "ap-seoul-1" + compartment_ocid = "" + } +} + +# 华为云配置 +variable "huawei_config" { + description = "华为云配置" + type = object({ + access_key = string + secret_key = string + region = string + project_id = optional(string) + }) + default = { + access_key = "" + secret_key = "" + region = "cn-north-4" + project_id = "" + } + sensitive = true +} + +# Google Cloud 配置 +variable "gcp_config" { + description = "Google Cloud 配置" + type = object({ + project_id = string + region = string + zone = string + credentials_file = string + }) + default = { + project_id = "" + region = "asia-northeast3" + zone = "asia-northeast3-a" + credentials_file = "" + } +} + +# AWS 配置 +variable "aws_config" { + description = "AWS 配置" + type = object({ + region = string + access_key = string + secret_key = string + }) + default = { + region = "ap-northeast-2" + access_key = "" + secret_key = "" + } + sensitive = true +} + +# DigitalOcean 配置 +variable "do_config" { + description = "DigitalOcean 配置" + type = object({ + token = string + region = string + }) + default = { + token = "" + region = "sgp1" + } + sensitive = true +} + +# HashiCorp Vault 配置 - 使用Tailscale IP而非localhost +variable "vault_config" { + description = "HashiCorp Vault 配置" + type = object({ + address = string + token = string + }) + default = { + address = "http://100.116.158.95:8200" + token = "" + } + sensitive = true +} + +variable "vault_token" { + description = "Vault 访问令牌" + type = string + default = "" + sensitive = true +} \ No newline at end of file diff --git a/tf/environments/production/nomad-multi-dc.tf b/tf/environments/production/nomad-multi-dc.tf new file mode 100644 index 0000000..7f0b00f --- /dev/null +++ b/tf/environments/production/nomad-multi-dc.tf @@ -0,0 +1,169 @@ +# Nomad 多数据中心生产环境配置 +# 部署架构: CN(dc1) + KR(dc2) + US(dc3) + +terraform { + required_version = ">= 1.0" + + required_providers { + oci = { + source = "oracle/oci" + version = "~> 7.20" + } + huaweicloud = { + source = "huaweicloud/huaweicloud" + version = "~> 1.60" + } + } +} + +# Oracle Cloud Provider (韩国) +provider "oci" { + alias = "korea" + tenancy_ocid = var.oracle_tenancy_ocid + user_ocid = var.oracle_user_ocid + fingerprint = var.oracle_fingerprint + private_key_path = var.oracle_private_key_path + region = "ap-seoul-1" # 韩国首尔 +} + +# 华为云 Provider (美国) +provider "huaweicloud" { + alias = "us" + access_key = var.huawei_access_key + secret_key = var.huawei_secret_key + region = "us-east-1" # 美国东部 +} + +# 本地变量 +locals { + project_name = "nomad-multi-dc" + environment = "production" + + common_tags = { + Project = local.project_name + Environment = local.environment + ManagedBy = "terraform" + Owner = "devops-team" + } +} + +# 数据源：获取 SSH 公钥 +data "local_file" "ssh_public_key" { + filename = pathexpand("~/.ssh/id_rsa.pub") +} + +# Oracle Cloud 基础设施 (韩国 - dc2) +module "oracle_infrastructure" { + source = "../../providers/oracle-cloud" + + providers = { + oci = oci.korea + } + + project_name = local.project_name + environment = local.environment + vpc_cidr = "10.1.0.0/16" + + oci_config = { + tenancy_ocid = var.oracle_tenancy_ocid + user_ocid = var.oracle_user_ocid + fingerprint = var.oracle_fingerprint + private_key_path = var.oracle_private_key_path + region = "ap-seoul-1" + } + + common_tags = local.common_tags +} + +# 华为云基础设施 (美国 - dc3) +module "huawei_infrastructure" { + source = "../../providers/huawei-cloud" + + providers = { + huaweicloud = huaweicloud.us + } + + project_name = local.project_name + environment = local.environment + vpc_cidr = "10.2.0.0/16" + availability_zones = ["us-east-1a", "us-east-1b"] + + common_tags = local.common_tags +} + +# Nomad 多数据中心集群 +module "nomad_cluster" { + source = "../../modules/nomad-cluster" + + # 部署配置 + deploy_korea_node = var.deploy_korea_node + deploy_us_node = var.deploy_us_node + + # Oracle Cloud 配置 + oracle_config = { + tenancy_ocid = var.oracle_tenancy_ocid + user_ocid = var.oracle_user_ocid + fingerprint = var.oracle_fingerprint + private_key_path = var.oracle_private_key_path + region = "ap-seoul-1" + } + + oracle_subnet_id = module.oracle_infrastructure.public_subnet_ids[0] + oracle_security_group_id = module.oracle_infrastructure.security_group_id + + # 华为云配置 + huawei_config = { + access_key = var.huawei_access_key + secret_key = var.huawei_secret_key + region = "us-east-1" + } + + huawei_subnet_id = module.huawei_infrastructure.public_subnet_ids[0] + huawei_security_group_id = module.huawei_infrastructure.security_group_id + + # 通用配置 + ssh_public_key = data.local_file.ssh_public_key.content + common_tags = local.common_tags + + # Nomad 配置 + nomad_version = "1.10.5" + nomad_encrypt_key = var.nomad_encrypt_key +} + +# 生成 Ansible inventory +resource "local_file" "ansible_inventory" { + filename = "${path.module}/generated/nomad-cluster-inventory.yml" + content = yamlencode({ + all = { + children = { + nomad_servers = { + hosts = module.nomad_cluster.ansible_inventory.all.children.nomad_servers.hosts + } + } + vars = { + ansible_user = "ubuntu" + ansible_ssh_private_key_file = "~/.ssh/id_rsa" + ansible_ssh_common_args = "-o StrictHostKeyChecking=no" + } + } + }) +} + +# 生成部署后配置脚本 +resource "local_file" "post_deploy_script" { + filename = "${path.module}/generated/post-deploy.sh" + content = templatefile("${path.module}/templates/post-deploy.sh", { + cluster_overview = module.nomad_cluster.cluster_overview + endpoints = module.nomad_cluster.cluster_endpoints + }) + + file_permission = "0755" +} + +# 生成跨数据中心测试任务 +resource "local_file" "cross_dc_test_job" { + filename = "${path.module}/generated/cross-dc-test.nomad" + content = templatefile("${path.module}/templates/cross-dc-test.nomad", { + datacenters = ["dc1", "dc2", "dc3"] + }) +} \ No newline at end of file diff --git a/tf/environments/production/outputs.tf b/tf/environments/production/outputs.tf new file mode 100644 index 0000000..2241b89 --- /dev/null +++ b/tf/environments/production/outputs.tf @@ -0,0 +1,46 @@ +# Nomad 多数据中心生产环境输出 + +output "cluster_overview" { + description = "Nomad 多数据中心集群概览" + value = module.nomad_cluster.cluster_overview +} + +output "cluster_endpoints" { + description = "集群连接端点" + value = module.nomad_cluster.cluster_endpoints +} + +output "oracle_korea_node" { + description = "Oracle Cloud 韩国节点信息" + value = module.nomad_cluster.oracle_korea_node +} + +output "huawei_us_node" { + description = "华为云美国节点信息" + value = module.nomad_cluster.huawei_us_node +} + +output "deployment_summary" { + description = "部署摘要" + value = { + total_nodes = module.nomad_cluster.cluster_overview.total_nodes + datacenters = keys(module.nomad_cluster.cluster_overview.datacenters) + + next_steps = [ + "1. 等待所有节点启动完成 (约 5-10 分钟)", + "2. 运行: ./generated/post-deploy.sh", + "3. 验证集群: nomad server members", + "4. 测试跨 DC 调度: nomad job run generated/cross-dc-test.nomad", + "5. 访问 Web UI 查看集群状态" + ] + + web_ui_urls = module.nomad_cluster.cluster_endpoints.nomad_ui_urls + + ssh_commands = module.nomad_cluster.cluster_endpoints.ssh_commands + } +} + +output "verification_commands" { + description = "验证命令" + value = module.nomad_cluster.verification_commands +} \ No newline at end of file diff --git a/tf/environments/production/terraform.tfvars.example b/tf/environments/production/terraform.tfvars.example new file mode 100644 index 0000000..4fc4c7c --- /dev/null +++ b/tf/environments/production/terraform.tfvars.example @@ -0,0 +1,22 @@ +# Nomad 多数据中心生产环境配置示例 +# 复制此文件为 terraform.tfvars 并填入实际值 + +# 部署控制 +deploy_korea_node = true # 是否部署韩国节点 +deploy_us_node = true # 是否部署美国节点 + +# Oracle Cloud 配置 (韩国 - dc2) +# 获取方式: https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm +oracle_tenancy_ocid = "ocid1.tenancy.oc1..aaaaaaaa..." +oracle_user_ocid = "ocid1.user.oc1..aaaaaaaa..." +oracle_fingerprint = "aa:bb:cc:dd:ee:ff:..." +oracle_private_key_path = "~/.oci/oci_api_key.pem" + +# 华为云配置 (美国 - dc3) +# 获取方式: https://console.huaweicloud.com/iam/#/mine/accessKey +huawei_access_key = "YOUR_HUAWEI_ACCESS_KEY" +huawei_secret_key = "YOUR_HUAWEI_SECRET_KEY" + +# Nomad 集群加密密钥 (可选，已有默认值) +# 生成方式: nomad operator keygen +nomad_encrypt_key = "NVOMDvXblgWfhtzFzOUIHnKEOrbXOkPrkIPbRGGf1YQ=" \ No newline at end of file diff --git a/tf/environments/production/variables.tf b/tf/environments/production/variables.tf new file mode 100644 index 0000000..dbe8661 --- /dev/null +++ b/tf/environments/production/variables.tf @@ -0,0 +1,81 @@ +# Nomad 多数据中心生产环境变量 + +# 部署控制 +variable "deploy_korea_node" { + description = "是否部署韩国节点 (Oracle Cloud)" + type = bool + default = true +} + +variable "deploy_us_node" { + description = "是否部署美国节点 (华为云)" + type = bool + default = true +} + +# Oracle Cloud 配置 +variable "oracle_tenancy_ocid" { + description = "Oracle Cloud 租户 OCID" + type = string + sensitive = true +} + +variable "oracle_user_ocid" { + description = "Oracle Cloud 用户 OCID" + type = string + sensitive = true +} + +variable "oracle_fingerprint" { + description = "Oracle Cloud API 密钥指纹" + type = string + sensitive = true +} + +variable "oracle_private_key_path" { + description = "Oracle Cloud 私钥文件路径" + type = string + sensitive = true +} + +# 华为云配置 +variable "huawei_access_key" { + description = "华为云访问密钥" + type = string + sensitive = true +} + +variable "huawei_secret_key" { + description = "华为云秘密密钥" + type = string + sensitive = true +} + +# Nomad 配置 +variable "nomad_encrypt_key" { + description = "Nomad 集群加密密钥" + type = string + sensitive = true + default = "NVOMDvXblgWfhtzFzOUIHnKEOrbXOkPrkIPbRGGf1YQ=" +} + +# Vault 配置 +variable "vault_config" { + description = "Vault 配置" + type = object({ + address = string + token = string + }) + default = { + address = "http://100.116.158.95:8200" + token = "" + } + sensitive = true +} + +variable "vault_token" { + description = "Vault 访问令牌" + type = string + default = "" + sensitive = true +} \ No newline at end of file diff --git a/tf/environments/staging/main.tf b/tf/environments/staging/main.tf new file mode 100644 index 0000000..8ab5958 --- /dev/null +++ b/tf/environments/staging/main.tf @@ -0,0 +1,155 @@ +# Staging环境主配置文件 + +# 引入共享版本配置 +terraform { + required_version = ">= 1.6" + + required_providers { + # Oracle Cloud Infrastructure + oci = { + source = "oracle/oci" + version = "~> 7.20" + } + + # 其他常用提供商 + random = { + source = "hashicorp/random" + version = "~> 3.1" + } + + tls = { + source = "hashicorp/tls" + version = "~> 4.0" + } + + local = { + source = "hashicorp/local" + version = "~> 2.1" + } + + # Consul Provider + consul = { + source = "hashicorp/consul" + version = "~> 2.22.0" + } + + # HashiCorp Vault Provider + vault = { + source = "hashicorp/vault" + version = "~> 4.0" + } + } + + # 后端配置 + backend "local" { + path = "terraform.tfstate" + } +} + +# Consul Provider配置 +provider "consul" { + address = "100.116.158.95:8500" + scheme = "http" + datacenter = "dc1" +} + +# Vault Provider配置 +provider "vault" { + address = var.vault_config.address + token = var.vault_token +} + +# 从Consul获取Oracle Cloud配置 +data "consul_keys" "oracle_config" { + key { + name = "tenancy_ocid" + path = "config/staging/oracle/kr/tenancy_ocid" + } + key { + name = "user_ocid" + path = "config/staging/oracle/kr/user_ocid" + } + key { + name = "fingerprint" + path = "config/staging/oracle/kr/fingerprint" + } + key { + name = "private_key" + path = "config/staging/oracle/kr/private_key" + } +} + +# 从Consul获取Oracle Cloud美国区域配置 +data "consul_keys" "oracle_config_us" { + key { + name = "tenancy_ocid" + path = "config/staging/oracle/us/tenancy_ocid" + } + key { + name = "user_ocid" + path = "config/staging/oracle/us/user_ocid" + } + key { + name = "fingerprint" + path = "config/staging/oracle/us/fingerprint" + } + key { + name = "private_key" + path = "config/staging/oracle/us/private_key" + } +} + +# 使用从Consul获取的配置的OCI Provider +provider "oci" { + tenancy_ocid = data.consul_keys.oracle_config.var.tenancy_ocid + user_ocid = data.consul_keys.oracle_config.var.user_ocid + fingerprint = data.consul_keys.oracle_config.var.fingerprint + private_key = data.consul_keys.oracle_config.var.private_key + region = "ap-chuncheon-1" +} + +# 美国区域的OCI Provider +provider "oci" { + alias = "us" + tenancy_ocid = data.consul_keys.oracle_config_us.var.tenancy_ocid + user_ocid = data.consul_keys.oracle_config_us.var.user_ocid + fingerprint = data.consul_keys.oracle_config_us.var.fingerprint + private_key = data.consul_keys.oracle_config_us.var.private_key + region = "us-ashburn-1" +} + +# Oracle Cloud 基础设施 +module "oracle_cloud" { + source = "../../providers/oracle-cloud" + + # 传递变量 + environment = var.environment + project_name = var.project_name + owner = var.owner + vpc_cidr = var.vpc_cidr + availability_zones = var.availability_zones + common_tags = var.common_tags + + # 使用从Consul获取的配置 + oci_config = { + tenancy_ocid = data.consul_keys.oracle_config.var.tenancy_ocid + user_ocid = data.consul_keys.oracle_config.var.user_ocid + fingerprint = data.consul_keys.oracle_config.var.fingerprint + private_key = data.consul_keys.oracle_config.var.private_key + region = "ap-chuncheon-1" + } + + # Staging环境特定配置 + instance_count = 2 + instance_size = "VM.Standard.E2.1.Micro" + + providers = { + oci = oci + } +} + +# 输出 +output "oracle_cloud_outputs" { + description = "Oracle Cloud 基础设施输出" + value = module.oracle_cloud +} \ No newline at end of file diff --git a/tf/environments/staging/variables.tf b/tf/environments/staging/variables.tf new file mode 100644 index 0000000..72811a9 --- /dev/null +++ b/tf/environments/staging/variables.tf @@ -0,0 +1,157 @@ +# Staging环境变量定义 + +# 环境配置 +variable "environment" { + description = "部署环境" + type = string + default = "staging" +} + +variable "project_name" { + description = "项目名称" + type = string + default = "mgmt" +} + +variable "owner" { + description = "资源所有者" + type = string + default = "ben" +} + +# 网络配置 +variable "vpc_cidr" { + description = "VPC CIDR 块" + type = string + default = "10.1.0.0/16" +} + +variable "availability_zones" { + description = "可用区列表" + type = list(string) + default = ["a", "b", "c"] +} + +# 标签配置 +variable "common_tags" { + description = "通用标签" + type = map(string) + default = { + Project = "mgmt" + ManagedBy = "terraform" + Owner = "ben" + Environment = "staging" + } +} + +# 云服务商特定配置 +variable "cloud_providers" { + description = "启用的云服务商" + type = list(string) + default = ["oracle", "huawei", "google", "digitalocean", "aws"] +} + +# Oracle Cloud 配置 +variable "oci_config" { + description = "Oracle Cloud 配置" + type = object({ + tenancy_ocid = string + user_ocid = string + fingerprint = string + private_key_path = string + region = string + }) + default = { + tenancy_ocid = "" + user_ocid = "" + fingerprint = "" + private_key_path = "~/.oci/oci_api_key.pem" + region = "ap-chuncheon-1" + } + sensitive = true +} + +# 华为云配置 +variable "huawei_config" { + description = "华为云配置" + type = object({ + access_key = string + secret_key = string + region = string + }) + default = { + access_key = "" + secret_key = "" + region = "cn-north-4" + } + sensitive = true +} + +# Google Cloud 配置 +variable "gcp_config" { + description = "Google Cloud 配置" + type = object({ + project_id = string + region = string + zone = string + credentials = string + }) + default = { + project_id = "" + region = "asia-northeast3" + zone = "asia-northeast3-a" + credentials = "" + } + sensitive = true +} + +# DigitalOcean 配置 +variable "do_config" { + description = "DigitalOcean 配置" + type = object({ + token = string + region = string + }) + default = { + token = "" + region = "sgp1" + } + sensitive = true +} + +# AWS 配置 +variable "aws_config" { + description = "AWS 配置" + type = object({ + access_key = string + secret_key = string + region = string + }) + default = { + access_key = "" + secret_key = "" + region = "ap-northeast-1" + } + sensitive = true +} + +# Vault 配置 +variable "vault_config" { + description = "Vault 配置" + type = object({ + address = string + token = string + }) + default = { + address = "http://100.116.158.95:8200" + token = "" + } + sensitive = true +} + +variable "vault_token" { + description = "Vault 访问令牌" + type = string + default = "" + sensitive = true +} \ No newline at end of file diff --git a/tf/modules/nomad-cluster/main.tf b/tf/modules/nomad-cluster/main.tf new file mode 100644 index 0000000..d33a4a3 --- /dev/null +++ b/tf/modules/nomad-cluster/main.tf @@ -0,0 +1,159 @@ +# Nomad 多数据中心集群模块 +# 支持跨地域部署：CN(dc1) + KR(dc2) + US(dc3) + +terraform { + required_providers { + oci = { + source = "oracle/oci" + version = "~> 7.20" + } + huaweicloud = { + source = "huaweicloud/huaweicloud" + version = "~> 1.60" + } + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +# 本地变量 +locals { + nomad_version = "1.10.5" + + # 通用 Nomad 配置 + nomad_encrypt_key = "NVOMDvXblgWfhtzFzOUIHnKEOrbXOkPrkIPbRGGf1YQ=" + + # 数据中心配置 + datacenters = { + dc1 = { + name = "dc1" + region = "cn" + location = "China" + provider = "existing" # 现有的 semaphore 节点 + } + dc2 = { + name = "dc2" + region = "kr" + location = "Korea" + provider = "oracle" + } + dc3 = { + name = "dc3" + region = "us" + location = "US" + provider = "huawei" # 或 aws + } + } + + # 用户数据模板 + user_data_template = templatefile("${path.module}/templates/nomad-userdata.sh", { + nomad_version = local.nomad_version + nomad_encrypt_key = local.nomad_encrypt_key + }) +} + +# 数据源：获取现有的 semaphore 节点信息 +data "external" "semaphore_info" { + program = ["bash", "-c", <<-EOF + echo '{ + "ip": "100.116.158.95", + "datacenter": "dc1", + "status": "existing" + }' + EOF + ] +} + +# Oracle Cloud 韩国节点 (dc2) +module "oracle_korea_node" { + source = "../compute" + + count = var.deploy_korea_node ? 1 : 0 + + # Oracle Cloud 特定配置 + provider_type = "oracle" + + # 实例配置 + instance_config = { + name = "nomad-master-kr" + datacenter = "dc2" + instance_type = "VM.Standard.E2.1.Micro" # 免费层 + image_id = var.oracle_ubuntu_image_id + subnet_id = var.oracle_subnet_id + + # Nomad 配置 + nomad_role = "server" + bootstrap_expect = 1 + bind_addr = "auto" # 自动检测 + + # 网络配置 + security_groups = [var.oracle_security_group_id] + + # 标签 + tags = merge(var.common_tags, { + Name = "nomad-master-kr" + Datacenter = "dc2" + Role = "nomad-server" + Provider = "oracle" + }) + } + + # 用户数据 + user_data = templatefile("${path.module}/templates/nomad-userdata.sh", { + datacenter = "dc2" + nomad_version = local.nomad_version + nomad_encrypt_key = local.nomad_encrypt_key + bootstrap_expect = 1 + bind_addr = "auto" + server_enabled = true + client_enabled = true + }) +} + +# 华为云美国节点 (dc3) +module "huawei_us_node" { + source = "../compute" + + count = var.deploy_us_node ? 1 : 0 + + # 华为云特定配置 + provider_type = "huawei" + + # 实例配置 + instance_config = { + name = "nomad-ash3c-us" + datacenter = "dc3" + instance_type = "s6.small.1" # 1vCPU 1GB + image_id = var.huawei_ubuntu_image_id + subnet_id = var.huawei_subnet_id + + # Nomad 配置 + nomad_role = "server" + bootstrap_expect = 1 + bind_addr = "auto" + + # 网络配置 + security_groups = [var.huawei_security_group_id] + + # 标签 + tags = merge(var.common_tags, { + Name = "nomad-ash3c-us" + Datacenter = "dc3" + Role = "nomad-server" + Provider = "huawei" + }) + } + + # 用户数据 + user_data = templatefile("${path.module}/templates/nomad-userdata.sh", { + datacenter = "dc3" + nomad_version = local.nomad_version + nomad_encrypt_key = local.nomad_encrypt_key + bootstrap_expect = 1 + bind_addr = "auto" + server_enabled = true + client_enabled = true + }) +} \ No newline at end of file diff --git a/tf/modules/nomad-cluster/outputs.tf b/tf/modules/nomad-cluster/outputs.tf new file mode 100644 index 0000000..61148ef --- /dev/null +++ b/tf/modules/nomad-cluster/outputs.tf @@ -0,0 +1,145 @@ +# Nomad 多数据中心集群输出 + +# 集群概览 +output "cluster_overview" { + description = "Nomad 多数据中心集群概览" + value = { + datacenters = { + dc1 = { + name = "dc1" + location = "China (CN)" + provider = "existing" + node = "semaphore" + ip = "100.116.158.95" + status = "existing" + } + dc2 = var.deploy_korea_node ? { + name = "dc2" + location = "Korea (KR)" + provider = "oracle" + node = "master" + ip = try(module.oracle_korea_node[0].public_ip, "pending") + status = "deployed" + } : null + dc3 = var.deploy_us_node ? { + name = "dc3" + location = "US" + provider = "huawei" + node = "ash3c" + ip = try(module.huawei_us_node[0].public_ip, "pending") + status = "deployed" + } : null + } + total_nodes = 1 + (var.deploy_korea_node ? 1 : 0) + (var.deploy_us_node ? 1 : 0) + } +} + +# Oracle Cloud 韩国节点输出 +output "oracle_korea_node" { + description = "Oracle Cloud 韩国节点信息" + value = var.deploy_korea_node ? { + instance_id = try(module.oracle_korea_node[0].instance_id, null) + public_ip = try(module.oracle_korea_node[0].public_ip, null) + private_ip = try(module.oracle_korea_node[0].private_ip, null) + datacenter = "dc2" + provider = "oracle" + region = var.oracle_config.region + + # 连接信息 + ssh_command = try("ssh ubuntu@${module.oracle_korea_node[0].public_ip}", null) + nomad_ui = try("http://${module.oracle_korea_node[0].public_ip}:4646", null) + } : null +} + +# 华为云美国节点输出 +output "huawei_us_node" { + description = "华为云美国节点信息" + value = var.deploy_us_node ? { + instance_id = try(module.huawei_us_node[0].instance_id, null) + public_ip = try(module.huawei_us_node[0].public_ip, null) + private_ip = try(module.huawei_us_node[0].private_ip, null) + datacenter = "dc3" + provider = "huawei" + region = var.huawei_config.region + + # 连接信息 + ssh_command = try("ssh ubuntu@${module.huawei_us_node[0].public_ip}", null) + nomad_ui = try("http://${module.huawei_us_node[0].public_ip}:4646", null) + } : null +} + +# 集群连接信息 +output "cluster_endpoints" { + description = "集群连接端点" + value = { + nomad_ui_urls = compact([ + "http://100.116.158.95:4646", # dc1 - semaphore + var.deploy_korea_node ? try("http://${module.oracle_korea_node[0].public_ip}:4646", null) : null, # dc2 + var.deploy_us_node ? try("http://${module.huawei_us_node[0].public_ip}:4646", null) : null # dc3 + ]) + + ssh_commands = compact([ + "ssh root@100.116.158.95", # dc1 - semaphore + var.deploy_korea_node ? try("ssh ubuntu@${module.oracle_korea_node[0].public_ip}", null) : null, # dc2 + var.deploy_us_node ? try("ssh ubuntu@${module.huawei_us_node[0].public_ip}", null) : null # dc3 + ]) + } +} + +# Ansible inventory 生成 +output "ansible_inventory" { + description = "生成的 Ansible inventory" + value = { + all = { + children = { + nomad_servers = { + hosts = merge( + { + semaphore = { + ansible_host = "100.116.158.95" + datacenter = "dc1" + provider = "existing" + } + }, + var.deploy_korea_node ? { + master = { + ansible_host = try(module.oracle_korea_node[0].public_ip, "pending") + datacenter = "dc2" + provider = "oracle" + } + } : {}, + var.deploy_us_node ? { + ash3c = { + ansible_host = try(module.huawei_us_node[0].public_ip, "pending") + datacenter = "dc3" + provider = "huawei" + } + } : {} + ) + } + } + } + } +} + +# 部署后验证命令 +output "verification_commands" { + description = "部署后验证命令" + value = [ + "# 检查集群状态", + "nomad server members", + "", + "# 检查各数据中心节点", + "nomad node status -verbose", + "", + "# 跨数据中心任务调度测试", + "nomad job run examples/cross-dc-test.nomad", + "", + "# 访问 UI", + join("\n", [for url in compact([ + "http://100.116.158.95:4646", + var.deploy_korea_node ? try("http://${module.oracle_korea_node[0].public_ip}:4646", null) : null, + var.deploy_us_node ? try("http://${module.huawei_us_node[0].public_ip}:4646", null) : null + ]) : "curl -s ${url}/v1/status/leader"]) + ] +} \ No newline at end of file diff --git a/tf/modules/nomad-cluster/templates/nomad-userdata.sh b/tf/modules/nomad-cluster/templates/nomad-userdata.sh new file mode 100644 index 0000000..ad12971 --- /dev/null +++ b/tf/modules/nomad-cluster/templates/nomad-userdata.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# Nomad 多数据中心节点自动配置脚本 +# 数据中心: ${datacenter} + +set -e + +# 日志函数 +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a /var/log/nomad-setup.log +} + +log "开始配置 Nomad 节点 - 数据中心: ${datacenter}" + +# 更新系统 +log "更新系统包..." +apt-get update -y +apt-get upgrade -y + +# 安装必要的包 +log "安装必要的包..." +apt-get install -y \ + curl \ + wget \ + unzip \ + jq \ + podman \ + htop \ + net-tools \ + vim + +# 启动 Podman +log "启动 Podman 服务..." +systemctl enable podman +systemctl start podman +usermod -aG podman ubuntu + +# 安装 Nomad +log "安装 Nomad ${nomad_version}..." +cd /tmp +wget -q https://releases.hashicorp.com/nomad/${nomad_version}/nomad_${nomad_version}_linux_amd64.zip +unzip nomad_${nomad_version}_linux_amd64.zip +mv nomad /usr/local/bin/ +chmod +x /usr/local/bin/nomad + +# 创建 Nomad 用户和目录 +log "创建 Nomad 用户和目录..." +useradd --system --home /etc/nomad.d --shell /bin/false nomad +mkdir -p /opt/nomad/data +mkdir -p /etc/nomad.d +mkdir -p /var/log/nomad +chown -R nomad:nomad /opt/nomad /etc/nomad.d /var/log/nomad + +# 获取本机 IP 地址 +if [ "${bind_addr}" = "auto" ]; then + # 尝试多种方法获取 IP + BIND_ADDR=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4 2>/dev/null || \ + curl -s http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip -H "Metadata-Flavor: Google" 2>/dev/null || \ + ip route get 8.8.8.8 | awk '{print $7; exit}' || \ + hostname -I | awk '{print $1}') +else + BIND_ADDR="${bind_addr}" +fi + +log "检测到 IP 地址: $BIND_ADDR" + +# 创建 Nomad 配置文件 +log "创建 Nomad 配置文件..." +cat > /etc/nomad.d/nomad.hcl << EOF +datacenter = "${datacenter}" +region = "global" +data_dir = "/opt/nomad/data" + +bind_addr = "$BIND_ADDR" + +%{ if server_enabled } +server { + enabled = true + bootstrap_expect = ${bootstrap_expect} + encrypt = "${nomad_encrypt_key}" +} +%{ endif } + +%{ if client_enabled } +client { + enabled = true + + host_volume "podman-sock" { + path = "/run/podman/podman.sock" + read_only = false + } +} +%{ endif } + +ui { + enabled = true +} + +addresses { + http = "0.0.0.0" + rpc = "$BIND_ADDR" + serf = "$BIND_ADDR" +} + +ports { + http = 4646 + rpc = 4647 + serf = 4648 +} + +plugin "podman" { + config { + volumes { + enabled = true + } + } +} + +telemetry { + collection_interval = "10s" + disable_hostname = false + prometheus_metrics = true + publish_allocation_metrics = true + publish_node_metrics = true +} + +log_level = "INFO" +log_file = "/var/log/nomad/nomad.log" +EOF + +# 创建 systemd 服务文件 +log "创建 systemd 服务文件..." +cat > /etc/systemd/system/nomad.service << EOF +[Unit] +Description=Nomad +Documentation=https://www.nomadproject.io/ +Requires=network-online.target +After=network-online.target +ConditionFileNotEmpty=/etc/nomad.d/nomad.hcl + +[Service] +Type=notify +User=nomad +Group=nomad +ExecStart=/usr/local/bin/nomad agent -config=/etc/nomad.d/nomad.hcl +ExecReload=/bin/kill -HUP \$MAINPID +KillMode=process +Restart=on-failure +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +EOF + +# 启动 Nomad 服务 +log "启动 Nomad 服务..." +systemctl daemon-reload +systemctl enable nomad +systemctl start nomad + +# 等待服务启动 +log "等待 Nomad 服务启动..." +sleep 10 + +# 验证安装 +log "验证 Nomad 安装..." +if systemctl is-active --quiet nomad; then + log "✅ Nomad 服务运行正常" + log "📊 节点信息:" + /usr/local/bin/nomad node status -self || true +else + log "❌ Nomad 服务启动失败" + systemctl status nomad --no-pager || true + journalctl -u nomad --no-pager -n 20 || true +fi + +# 配置防火墙（如果需要） +log "配置防火墙规则..." +if command -v ufw >/dev/null 2>&1; then + ufw allow 4646/tcp # HTTP API + ufw allow 4647/tcp # RPC + ufw allow 4648/tcp # Serf + ufw allow 22/tcp # SSH +fi + +# 创建有用的别名和脚本 +log "创建管理脚本..." +cat > /usr/local/bin/nomad-status << 'EOF' +#!/bin/bash +echo "=== Nomad 服务状态 ===" +systemctl status nomad --no-pager + +echo -e "\n=== Nomad 集群成员 ===" +nomad server members 2>/dev/null || echo "无法连接到集群" + +echo -e "\n=== Nomad 节点状态 ===" +nomad node status 2>/dev/null || echo "无法获取节点状态" + +echo -e "\n=== 最近日志 ===" +journalctl -u nomad --no-pager -n 5 +EOF + +chmod +x /usr/local/bin/nomad-status + +log "✅ Nomad 节点配置完成" +log "🌐 Nomad UI: http://$BIND_ADDR:4646" +log "📋 查看状态: nomad-status" \ No newline at end of file diff --git a/tf/modules/nomad-cluster/variables.tf b/tf/modules/nomad-cluster/variables.tf new file mode 100644 index 0000000..1ce9c38 --- /dev/null +++ b/tf/modules/nomad-cluster/variables.tf @@ -0,0 +1,118 @@ +# Nomad 多数据中心集群变量定义 + +variable "deploy_korea_node" { + description = "是否部署韩国节点 (Oracle Cloud)" + type = bool + default = true +} + +variable "deploy_us_node" { + description = "是否部署美国节点 (华为云)" + type = bool + default = true +} + +# Oracle Cloud 配置 +variable "oracle_config" { + description = "Oracle Cloud 配置" + type = object({ + tenancy_ocid = string + user_ocid = string + fingerprint = string + private_key_path = string + region = string + }) + sensitive = true +} + +variable "oracle_ubuntu_image_id" { + description = "Oracle Cloud Ubuntu 镜像 ID" + type = string + default = "" # 将通过数据源自动获取 +} + +variable "oracle_subnet_id" { + description = "Oracle Cloud 子网 ID" + type = string +} + +variable "oracle_security_group_id" { + description = "Oracle Cloud 安全组 ID" + type = string +} + +# 华为云配置 +variable "huawei_config" { + description = "华为云配置" + type = object({ + access_key = string + secret_key = string + region = string + }) + sensitive = true +} + +variable "huawei_ubuntu_image_id" { + description = "华为云 Ubuntu 镜像 ID" + type = string + default = "" # 将通过数据源自动获取 +} + +variable "huawei_subnet_id" { + description = "华为云子网 ID" + type = string +} + +variable "huawei_security_group_id" { + description = "华为云安全组 ID" + type = string +} + +# 通用配置 +variable "common_tags" { + description = "通用标签" + type = map(string) + default = { + Project = "nomad-multi-dc" + Environment = "production" + ManagedBy = "terraform" + } +} + +variable "ssh_public_key" { + description = "SSH 公钥" + type = string +} + +variable "allowed_cidr_blocks" { + description = "允许访问的 CIDR 块" + type = list(string) + default = ["0.0.0.0/0"] # 生产环境应该限制 +} + +# Nomad 特定配置 +variable "nomad_version" { + description = "Nomad 版本" + type = string + default = "1.10.5" +} + +variable "nomad_encrypt_key" { + description = "Nomad 集群加密密钥" + type = string + sensitive = true + default = "NVOMDvXblgWfhtzFzOUIHnKEOrbXOkPrkIPbRGGf1YQ=" +} + +# 网络配置 +variable "vpc_cidr" { + description = "VPC CIDR 块" + type = string + default = "10.0.0.0/16" +} + +variable "availability_zones" { + description = "可用区列表" + type = list(string) + default = ["a", "b"] +} \ No newline at end of file diff --git a/tf/providers/huawei-cloud/main.tf b/tf/providers/huawei-cloud/main.tf new file mode 100644 index 0000000..83446a5 --- /dev/null +++ b/tf/providers/huawei-cloud/main.tf @@ -0,0 +1,137 @@ +# 华为云模块 + +terraform { + required_providers { + huaweicloud = { + source = "huaweicloud/huaweicloud" + version = "~> 1.60" + } + } +} + +# 获取可用区 +data "huaweicloud_availability_zones" "zones" {} + +# 获取镜像 +data "huaweicloud_images_image" "ubuntu" { + name = "Ubuntu 22.04 server 64bit" + most_recent = true +} + +# VPC +resource "huaweicloud_vpc" "main" { + name = "${var.project_name}-${var.environment}-vpc" + cidr = var.vpc_cidr + + tags = merge(var.common_tags, { + Name = "${var.project_name}-${var.environment}-vpc" + }) +} + +# 子网 +resource "huaweicloud_vpc_subnet" "public" { + count = length(var.availability_zones) + name = "${var.project_name}-${var.environment}-public-${var.availability_zones[count.index]}" + cidr = cidrsubnet(var.vpc_cidr, 8, count.index) + gateway_ip = cidrhost(cidrsubnet(var.vpc_cidr, 8, count.index), 1) + vpc_id = huaweicloud_vpc.main.id + + tags = merge(var.common_tags, { + Name = "${var.project_name}-${var.environment}-public-${var.availability_zones[count.index]}" + Type = "public" + }) +} + +# 安全组 +resource "huaweicloud_networking_secgroup" "main" { + name = "${var.project_name}-${var.environment}-sg" + description = "Security group for ${var.project_name} ${var.environment}" + + tags = merge(var.common_tags, { + Name = "${var.project_name}-${var.environment}-sg" + }) +} + +# 安全组规则 - SSH +resource "huaweicloud_networking_secgroup_rule" "ssh" { + direction = "ingress" + ethertype = "IPv4" + protocol = "tcp" + port_range_min = 22 + port_range_max = 22 + remote_ip_prefix = "0.0.0.0/0" + security_group_id = huaweicloud_networking_secgroup.main.id +} + +# 安全组规则 - HTTP +resource "huaweicloud_networking_secgroup_rule" "http" { + direction = "ingress" + ethertype = "IPv4" + protocol = "tcp" + port_range_min = 80 + port_range_max = 80 + remote_ip_prefix = "0.0.0.0/0" + security_group_id = huaweicloud_networking_secgroup.main.id +} + +# 安全组规则 - HTTPS +resource "huaweicloud_networking_secgroup_rule" "https" { + direction = "ingress" + ethertype = "IPv4" + protocol = "tcp" + port_range_min = 443 + port_range_max = 443 + remote_ip_prefix = "0.0.0.0/0" + security_group_id = huaweicloud_networking_secgroup.main.id +} + +# 弹性IP +resource "huaweicloud_vpc_eip" "main" { + count = var.environment == "production" ? 2 : 1 + + publicip { + type = "5_bgp" + } + + bandwidth { + name = "${var.project_name}-${var.environment}-bandwidth-${count.index}" + size = var.environment == "production" ? 10 : 5 + share_type = "PER" + charge_mode = "traffic" + } + + tags = merge(var.common_tags, { + Name = "${var.project_name}-${var.environment}-eip-${count.index}" + }) +} + +# 输出 +output "vpc_id" { + description = "VPC ID" + value = huaweicloud_vpc.main.id +} + +output "subnet_ids" { + description = "子网 ID 列表" + value = huaweicloud_vpc_subnet.public[*].id +} + +output "security_group_id" { + description = "安全组 ID" + value = huaweicloud_networking_secgroup.main.id +} + +output "availability_zones" { + description = "可用区列表" + value = data.huaweicloud_availability_zones.zones.names +} + +output "ubuntu_image_id" { + description = "Ubuntu 镜像 ID" + value = data.huaweicloud_images_image.ubuntu.id +} + +output "eip_addresses" { + description = "弹性IP地址列表" + value = huaweicloud_vpc_eip.main[*].address +} \ No newline at end of file diff --git a/tf/providers/huawei-cloud/variables.tf b/tf/providers/huawei-cloud/variables.tf new file mode 100644 index 0000000..ff866f6 --- /dev/null +++ b/tf/providers/huawei-cloud/variables.tf @@ -0,0 +1,54 @@ +# 华为云提供商变量定义 + +variable "environment" { + description = "环境名称" + type = string +} + +variable "project_name" { + description = "项目名称" + type = string +} + +variable "owner" { + description = "项目所有者" + type = string +} + +variable "vpc_cidr" { + description = "VPC CIDR 块" + type = string +} + +variable "availability_zones" { + description = "可用区列表" + type = list(string) +} + +variable "common_tags" { + description = "通用标签" + type = map(string) +} + +variable "huawei_config" { + description = "华为云配置" + type = object({ + access_key = string + secret_key = string + region = string + project_id = string + }) + sensitive = true +} + +variable "instance_count" { + description = "实例数量" + type = number + default = 1 +} + +variable "instance_size" { + description = "实例规格" + type = string + default = "s6.small.1" +} \ No newline at end of file diff --git a/tf/providers/oracle-cloud/main.tf b/tf/providers/oracle-cloud/main.tf new file mode 100644 index 0000000..cb8fd2e --- /dev/null +++ b/tf/providers/oracle-cloud/main.tf @@ -0,0 +1,151 @@ +# Oracle Cloud Infrastructure 模块 + +terraform { + required_providers { + oci = { + source = "oracle/oci" + version = "~> 7.20" + } + } +} + +# 获取可用域 +data "oci_identity_availability_domains" "ads" { + compartment_id = var.oci_config.tenancy_ocid +} + +# 获取镜像 +data "oci_core_images" "ubuntu_images" { + compartment_id = var.oci_config.tenancy_ocid + operating_system = "Canonical Ubuntu" + operating_system_version = "22.04" + shape = "VM.Standard.E2.1.Micro" + sort_by = "TIMECREATED" + sort_order = "DESC" +} + +# VCN (虚拟云网络) +resource "oci_core_vcn" "main" { + compartment_id = var.oci_config.tenancy_ocid + cidr_blocks = [var.vpc_cidr] + display_name = "${var.project_name}-${var.environment}-vcn" + dns_label = "${var.project_name}${var.environment}" + + freeform_tags = merge(var.common_tags, { + Name = "${var.project_name}-${var.environment}-vcn" + }) +} + +# 互联网网关 +resource "oci_core_internet_gateway" "main" { + compartment_id = var.oci_config.tenancy_ocid + vcn_id = oci_core_vcn.main.id + display_name = "${var.project_name}-${var.environment}-igw" + enabled = true + + freeform_tags = merge(var.common_tags, { + Name = "${var.project_name}-${var.environment}-igw" + }) +} + +# 路由表 +resource "oci_core_route_table" "main" { + compartment_id = var.oci_config.tenancy_ocid + vcn_id = oci_core_vcn.main.id + display_name = "${var.project_name}-${var.environment}-rt" + + route_rules { + destination = "0.0.0.0/0" + destination_type = "CIDR_BLOCK" + network_entity_id = oci_core_internet_gateway.main.id + } + + freeform_tags = merge(var.common_tags, { + Name = "${var.project_name}-${var.environment}-rt" + }) +} + +# 安全列表 +resource "oci_core_security_list" "main" { + compartment_id = var.oci_config.tenancy_ocid + vcn_id = oci_core_vcn.main.id + display_name = "${var.project_name}-${var.environment}-sl" + + # 出站规则 + egress_security_rules { + destination = "0.0.0.0/0" + protocol = "all" + } + + # 入站规则 - SSH + ingress_security_rules { + protocol = "6" # TCP + source = "0.0.0.0/0" + tcp_options { + min = 22 + max = 22 + } + } + + # 入站规则 - HTTP + ingress_security_rules { + protocol = "6" # TCP + source = "0.0.0.0/0" + tcp_options { + min = 80 + max = 80 + } + } + + # 入站规则 - HTTPS + ingress_security_rules { + protocol = "6" # TCP + source = "0.0.0.0/0" + tcp_options { + min = 443 + max = 443 + } + } + + freeform_tags = merge(var.common_tags, { + Name = "${var.project_name}-${var.environment}-sl" + }) +} + +# 子网 +resource "oci_core_subnet" "public" { + count = length(var.availability_zones) + compartment_id = var.oci_config.tenancy_ocid + vcn_id = oci_core_vcn.main.id + cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index) + display_name = "${var.project_name}-${var.environment}-public-${var.availability_zones[count.index]}" + dns_label = "public${var.availability_zones[count.index]}" + route_table_id = oci_core_route_table.main.id + security_list_ids = [oci_core_security_list.main.id] + + freeform_tags = merge(var.common_tags, { + Name = "${var.project_name}-${var.environment}-public-${var.availability_zones[count.index]}" + Type = "public" + }) +} + +# 输出 +output "vcn_id" { + description = "VCN ID" + value = oci_core_vcn.main.id +} + +output "subnet_ids" { + description = "子网 ID 列表" + value = oci_core_subnet.public[*].id +} + +output "availability_domains" { + description = "可用域列表" + value = data.oci_identity_availability_domains.ads.availability_domains[*].name +} + +output "ubuntu_image_id" { + description = "Ubuntu 镜像 ID" + value = data.oci_core_images.ubuntu_images.images[0].id +} \ No newline at end of file diff --git a/tf/providers/oracle-cloud/variables.tf b/tf/providers/oracle-cloud/variables.tf new file mode 100644 index 0000000..d6254fa --- /dev/null +++ b/tf/providers/oracle-cloud/variables.tf @@ -0,0 +1,55 @@ +# Oracle Cloud 提供商变量定义 + +variable "environment" { + description = "环境名称" + type = string +} + +variable "project_name" { + description = "项目名称" + type = string +} + +variable "owner" { + description = "项目所有者" + type = string +} + +variable "vpc_cidr" { + description = "VPC CIDR 块" + type = string +} + +variable "availability_zones" { + description = "可用区列表" + type = list(string) +} + +variable "common_tags" { + description = "通用标签" + type = map(string) +} + +variable "oci_config" { + description = "Oracle Cloud 配置" + type = object({ + tenancy_ocid = string + user_ocid = string + fingerprint = string + private_key = string + region = string + compartment_ocid = string + }) +} + +variable "instance_count" { + description = "实例数量" + type = number + default = 1 +} + +variable "instance_size" { + description = "实例规格" + type = string + default = "VM.Standard.E2.1.Micro" +} \ No newline at end of file diff --git a/tf/shared/outputs.tf b/tf/shared/outputs.tf new file mode 100644 index 0000000..0c30ee9 --- /dev/null +++ b/tf/shared/outputs.tf @@ -0,0 +1,39 @@ +# 全局输出定义 + +# 环境信息 +output "environment" { + description = "当前部署环境" + value = var.environment +} + +output "project_name" { + description = "项目名称" + value = var.project_name +} + +# 网络信息 +output "vpc_cidr" { + description = "VPC CIDR 块" + value = var.vpc_cidr +} + +# 通用标签 +output "common_tags" { + description = "通用资源标签" + value = merge(var.common_tags, { + Environment = var.environment + Timestamp = timestamp() + }) +} + +# 云服务商配置状态 +output "enabled_providers" { + description = "启用的云服务商列表" + value = var.cloud_providers +} + +# 实例类型配置 +output "instance_types" { + description = "当前环境的实例类型配置" + value = var.instance_types[var.environment] +} \ No newline at end of file diff --git a/tf/shared/variables.tf b/tf/shared/variables.tf new file mode 100644 index 0000000..6bcbc60 --- /dev/null +++ b/tf/shared/variables.tf @@ -0,0 +1,169 @@ +# 全局变量定义 + +# 环境配置 +variable "environment" { + description = "部署环境 (dev, staging, production)" + type = string + validation { + condition = contains(["dev", "staging", "production"], var.environment) + error_message = "环境必须是 dev, staging, 或 production 之一。" + } +} + +variable "project_name" { + description = "项目名称" + type = string + default = "mgmt" +} + +variable "owner" { + description = "资源所有者" + type = string + default = "ben" +} + +# 网络配置 +variable "vpc_cidr" { + description = "VPC CIDR 块" + type = string + default = "10.0.0.0/16" +} + +variable "availability_zones" { + description = "可用区列表" + type = list(string) + default = ["a", "b", "c"] +} + +# 计算资源配置 +variable "instance_types" { + description = "不同环境的实例类型" + type = map(object({ + web = string + app = string + db = string + cache = string + })) + default = { + dev = { + web = "t3.micro" + app = "t3.small" + db = "t3.micro" + cache = "t3.micro" + } + staging = { + web = "t3.small" + app = "t3.medium" + db = "t3.small" + cache = "t3.small" + } + production = { + web = "t3.medium" + app = "t3.large" + db = "t3.medium" + cache = "t3.medium" + } + } +} + +# 标签配置 +variable "common_tags" { + description = "通用标签" + type = map(string) + default = { + Project = "mgmt" + ManagedBy = "terraform" + Owner = "ben" + } +} + +# 云服务商特定配置 +variable "cloud_providers" { + description = "启用的云服务商" + type = list(string) + default = ["oracle", "huawei", "google", "digitalocean", "aws"] +} + +# Oracle Cloud 配置 +variable "oci_config" { + description = "Oracle Cloud 配置" + type = object({ + tenancy_ocid = string + user_ocid = string + fingerprint = string + private_key_path = string + region = string + }) + default = { + tenancy_ocid = "" + user_ocid = "" + fingerprint = "" + private_key_path = "~/.oci/oci_api_key.pem" + region = "ap-seoul-1" + } + sensitive = true +} + +# 华为云配置 +variable "huawei_config" { + description = "华为云配置" + type = object({ + access_key = string + secret_key = string + region = string + }) + default = { + access_key = "" + secret_key = "" + region = "cn-north-4" + } + sensitive = true +} + +# Google Cloud 配置 +variable "gcp_config" { + description = "Google Cloud 配置" + type = object({ + project_id = string + region = string + zone = string + credentials = string + }) + default = { + project_id = "" + region = "asia-northeast3" + zone = "asia-northeast3-a" + credentials = "" + } + sensitive = true +} + +# DigitalOcean 配置 +variable "do_config" { + description = "DigitalOcean 配置" + type = object({ + token = string + region = string + }) + default = { + token = "" + region = "sgp1" + } + sensitive = true +} + +# AWS 配置 +variable "aws_config" { + description = "AWS 配置" + type = object({ + access_key = string + secret_key = string + region = string + }) + default = { + access_key = "" + secret_key = "" + region = "ap-northeast-1" + } + sensitive = true +} \ No newline at end of file diff --git a/tf/shared/versions.tf b/tf/shared/versions.tf new file mode 100644 index 0000000..9c43f6f --- /dev/null +++ b/tf/shared/versions.tf @@ -0,0 +1,63 @@ +# Terraform 版本和提供商配置 +terraform { + required_version = ">= 1.0" + + required_providers { + # Oracle Cloud Infrastructure + oci = { + source = "oracle/oci" + version = "7.20.0" + } + + # 华为云 + huaweicloud = { + source = "huaweicloud/huaweicloud" + version = "~> 1.60" + } + + # Google Cloud Platform + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + + # DigitalOcean + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + + # Amazon Web Services + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + + # 其他常用提供商 + random = { + source = "hashicorp/random" + version = "3.7.2" + } + + tls = { + source = "hashicorp/tls" + version = "4.1.0" + } + + local = { + source = "hashicorp/local" + version = "2.5.3" + } + + # HashiCorp Vault + vault = { + source = "hashicorp/vault" + version = "~> 4.0" + } + } + + # 后端配置 - 可以使用 S3, GCS, 或本地 + backend "local" { + path = "terraform.tfstate" + } +} \ No newline at end of file diff --git a/tofu/environments/dev/main.tf b/tofu/environments/dev/main.tf index bafe77e..8b3e17c 100644 --- a/tofu/environments/dev/main.tf +++ b/tofu/environments/dev/main.tf @@ -46,17 +46,6 @@ terraform { } } -# 将从Consul获取的私钥保存到临时文件 -resource "local_file" "oci_kr_private_key" { - content = data.consul_keys.oracle_config.var.private_key - filename = "/tmp/oci_kr_private_key.pem" -} - -resource "local_file" "oci_us_private_key" { - content = data.consul_keys.oracle_config_us.var.private_key - filename = "/tmp/oci_us_private_key.pem" -} - # Consul Provider配置 provider "consul" { address = "localhost:8500" @@ -115,7 +104,7 @@ provider "oci" { tenancy_ocid = data.consul_keys.oracle_config.var.tenancy_ocid user_ocid = data.consul_keys.oracle_config.var.user_ocid fingerprint = data.consul_keys.oracle_config.var.fingerprint - private_key_path = local_file.oci_kr_private_key.filename + private_key = data.consul_keys.oracle_config.var.private_key region = "ap-chuncheon-1" } @@ -125,7 +114,7 @@ provider "oci" { tenancy_ocid = data.consul_keys.oracle_config_us.var.tenancy_ocid user_ocid = data.consul_keys.oracle_config_us.var.user_ocid fingerprint = data.consul_keys.oracle_config_us.var.fingerprint - private_key_path = local_file.oci_us_private_key.filename + private_key = data.consul_keys.oracle_config_us.var.private_key region = "us-ashburn-1" } @@ -146,7 +135,7 @@ module "oracle_cloud" { tenancy_ocid = data.consul_keys.oracle_config.var.tenancy_ocid user_ocid = data.consul_keys.oracle_config.var.user_ocid fingerprint = data.consul_keys.oracle_config.var.fingerprint - private_key_path = local_file.oci_kr_private_key.filename + private_key = data.consul_keys.oracle_config.var.private_key region = "ap-chuncheon-1" } diff --git a/tofu/providers/oracle-cloud/variables.tf b/tofu/providers/oracle-cloud/variables.tf index 5bf2b3f..d6254fa 100644 --- a/tofu/providers/oracle-cloud/variables.tf +++ b/tofu/providers/oracle-cloud/variables.tf @@ -36,7 +36,7 @@ variable "oci_config" { tenancy_ocid = string user_ocid = string fingerprint = string - private_key_path = string + private_key = string region = string compartment_ocid = string }) diff --git a/vault_1.20.4_linux_amd64.zip b/vault_1.20.4_linux_amd64.zip new file mode 100644 index 0000000..9ec8e9e Binary files /dev/null and b/vault_1.20.4_linux_amd64.zip differ