mgmt/pve/deep-595-investigation.yml

---
- name: Deep 595 Error Investigation
  hosts: pve_cluster
  gather_facts: yes
  tasks:
    - name: Check PVE proxy detailed configuration
      command: ps aux | grep pveproxy
      register: pveproxy_processes

    - name: Display PVE proxy processes
      debug:
        msg: "{{ pveproxy_processes.stdout_lines }}"

    - name: Check PVE proxy configuration file
      stat:
        path: /etc/pveproxy.conf
      register: proxy_config_file

    - name: Display proxy config file status
      debug:
        msg: "Proxy config file exists: {{ proxy_config_file.stat.exists }}"

    - name: Check PVE proxy logs for connection errors
      command: journalctl -u pveproxy -n 50 --no-pager | grep -i "error\|fail\|refuse\|deny\|595"
      register: proxy_error_logs
      ignore_errors: yes

    - name: Display proxy error logs
      debug:
        msg: "{{ proxy_error_logs.stdout_lines }}"
      when: proxy_error_logs.rc == 0

    - name: Check system logs for network errors
      command: journalctl -n 100 --no-pager | grep -i "595\|no route\|network\|connection"
      register: system_network_logs
      ignore_errors: yes

    - name: Display system network logs
      debug:
        msg: "{{ system_network_logs.stdout_lines }}"
      when: system_network_logs.rc == 0

    - name: Check network interface details
      command: ip addr show
      register: network_interfaces

    - name: Display network interfaces
      debug:
        msg: "{{ network_interfaces.stdout_lines }}"

    - name: Check routing table details
      command: ip route show
      register: routing_table

    - name: Display routing table
      debug:
        msg: "{{ routing_table.stdout_lines }}"

    - name: Check ARP table
      command: arp -a
      register: arp_table
      ignore_errors: yes

    - name: Display ARP table
      debug:
        msg: "{{ arp_table.stdout_lines }}"
      when: arp_table.rc == 0

    - name: Test connectivity with different methods
      shell: |
        echo "=== Testing connectivity to PVE ==="
        echo "1. Ping test:"
        ping -c 3 pve
        echo "2. Telnet test:"
        timeout 5 telnet pve 8006 || echo "Telnet failed"
        echo "3. nc test:"
        nc -zv pve 8006
        echo "4. curl test:"
        curl -k -s -o /dev/null -w "HTTP Status: %{http_code}, Time: %{time_total}s\n" https://pve:8006        
      register: connectivity_tests
      when: inventory_hostname != 'pve'

    - name: Display connectivity test results
      debug:
        msg: "{{ connectivity_tests.stdout_lines }}"
      when: inventory_hostname != 'pve'

    - name: Check PVE proxy binding details
      command: ss -tlnp | grep 8006
      register: port_binding

    - name: Display port binding details
      debug:
        msg: "{{ port_binding.stdout_lines }}"

    - name: Check if PVE proxy is binding to specific interfaces
      command: netstat -tlnp | grep 8006
      register: netstat_binding
      ignore_errors: yes

    - name: Display netstat binding details
      debug:
        msg: "{{ netstat_binding.stdout_lines }}"
      when: netstat_binding.rc == 0

    - name: Check PVE cluster communication
      command: pvecm status
      register: cluster_status
      ignore_errors: yes

    - name: Display cluster status
      debug:
        msg: "{{ cluster_status.stdout_lines }}"
      when: cluster_status.rc == 0

    - name: Check PVE cluster nodes
      command: pvecm nodes
      register: cluster_nodes
      ignore_errors: yes

    - name: Display cluster nodes
      debug:
        msg: "{{ cluster_nodes.stdout_lines }}"
      when: cluster_nodes.rc == 0

    - name: Test PVE API access
      uri:
        url: "https://localhost:8006/api2/json/version"
        method: GET
        validate_certs: no
        timeout: 10
      register: pve_api_test
      ignore_errors: yes

    - name: Display PVE API test result
      debug:
        msg: "PVE API access: {{ 'SUCCESS' if pve_api_test.status == 200 else 'FAILED' }}"
      when: inventory_hostname == 'pve'

    - name: Check PVE proxy configuration in detail
      shell: |
        echo "=== PVE Proxy Configuration ==="
        if [ -f /etc/pveproxy.conf ]; then
          cat /etc/pveproxy.conf
        else
          echo "No /etc/pveproxy.conf found"
        fi
        echo "=== PVE Proxy Service Status ==="
        systemctl status pveproxy --no-pager
        echo "=== PVE Proxy Logs (last 20 lines) ==="
        journalctl -u pveproxy -n 20 --no-pager        
      register: pve_proxy_details

    - name: Display PVE proxy details
      debug:
        msg: "{{ pve_proxy_details.stdout_lines }}"

    - name: Check network connectivity from PVE to other nodes
      shell: |
        echo "=== Testing connectivity FROM PVE to other nodes ==="
        for node in nuc12 xgp; do
          if [ "$node" != "pve" ]; then
            echo "Testing to $node:"
            ping -c 2 $node
            nc -zv $node 8006
          fi
        done        
      register: pve_outbound_test
      when: inventory_hostname == 'pve'

    - name: Display PVE outbound test results
      debug:
        msg: "{{ pve_outbound_test.stdout_lines }}"
      when: inventory_hostname == 'pve'