mgmt/pve/complete-user-verification-test.yml

---
- name: Complete User Verification Test for 595 Error
  hosts: pve_cluster
  gather_facts: yes
  tasks:
    - name: Test web access from xgp to pve
      uri:
        url: "https://pve:8006"
        method: GET
        validate_certs: no
        timeout: 10
      register: xgp_to_pve_test
      ignore_errors: yes
      when: inventory_hostname == 'xgp'

    - name: Display xgp to pve test result
      debug:
        msg: "xgp -> pve web access: {{ 'SUCCESS' if xgp_to_pve_test.status == 200 else 'FAILED' }} (Status: {{ xgp_to_pve_test.status | default('N/A') }})"
      when: inventory_hostname == 'xgp'

    - name: Test web access from nuc12 to pve
      uri:
        url: "https://pve:8006"
        method: GET
        validate_certs: no
        timeout: 10
      register: nuc12_to_pve_test
      ignore_errors: yes
      when: inventory_hostname == 'nuc12'

    - name: Display nuc12 to pve test result
      debug:
        msg: "nuc12 -> pve web access: {{ 'SUCCESS' if nuc12_to_pve_test.status == 200 else 'FAILED' }} (Status: {{ nuc12_to_pve_test.status | default('N/A') }})"
      when: inventory_hostname == 'nuc12'

    - name: Test local web access on pve
      uri:
        url: "https://localhost:8006"
        method: GET
        validate_certs: no
        timeout: 10
      register: pve_local_test
      ignore_errors: yes
      when: inventory_hostname == 'pve'

    - name: Display pve local test result
      debug:
        msg: "pve local web access: {{ 'SUCCESS' if pve_local_test.status == 200 else 'FAILED' }} (Status: {{ pve_local_test.status | default('N/A') }})"
      when: inventory_hostname == 'pve'

    - name: Check PVE cluster status
      shell: |
        echo "=== PVE Cluster Status ==="
        pvecm status
        echo "=== PVE Cluster Nodes ==="
        pvecm nodes
        echo "=== PVE Cluster Quorum ==="
        pvecm quorum status        
      register: cluster_status
      ignore_errors: yes

    - name: Display cluster status
      debug:
        msg: "{{ cluster_status.stdout_lines }}"

    - name: Check PVE services status
      shell: |
        echo "=== PVE Services Status ==="
        systemctl is-active pve-cluster pveproxy pvedaemon pvestatd
        echo "=== PVE Proxy Status ==="
        systemctl status pveproxy --no-pager -l        
      register: pve_services_status

    - name: Display PVE services status
      debug:
        msg: "{{ pve_services_status.stdout_lines }}"

    - name: Check recent error logs
      shell: |
        echo "=== Recent Error Logs ==="
        journalctl -n 50 --no-pager | grep -i "error\|fail\|refuse\|deny\|timeout\|595"
        echo "=== PVE Proxy Error Logs ==="
        journalctl -u pveproxy -n 20 --no-pager | grep -i "error\|fail\|refuse\|deny"
        echo "=== PVE Status Daemon Error Logs ==="
        journalctl -u pvestatd -n 20 --no-pager | grep -i "error\|fail\|refuse\|deny"        
      register: error_logs
      ignore_errors: yes

    - name: Display error logs
      debug:
        msg: "{{ error_logs.stdout_lines }}"

    - name: Test InfluxDB connection
      shell: |
        echo "=== Testing InfluxDB Connection ==="
        nc -zv 192.168.31.3 8086
        echo "=== Testing InfluxDB HTTP ==="
        curl -s -o /dev/null -w "HTTP Status: %{http_code}\n" http://192.168.31.3:8086/ping        
      register: influxdb_test
      ignore_errors: yes

    - name: Display InfluxDB test results
      debug:
        msg: "{{ influxdb_test.stdout_lines }}"

    - name: Check network connectivity between nodes
      shell: |
        echo "=== Network Connectivity Test ==="
        for node in nuc12 xgp pve; do
          if [ "$node" != "{{ inventory_hostname }}" ]; then
            echo "Testing connectivity to $node:"
            ping -c 2 $node
            nc -zv $node 8006
          fi
        done        
      register: network_connectivity

    - name: Display network connectivity results
      debug:
        msg: "{{ network_connectivity.stdout_lines }}"

    - name: Check PVE proxy port binding
      shell: |
        echo "=== PVE Proxy Port Binding ==="
        ss -tlnp | grep 8006
        echo "=== PVE Proxy Process ==="
        ps aux | grep pveproxy | grep -v grep        
      register: pve_proxy_binding

    - name: Display PVE proxy binding
      debug:
        msg: "{{ pve_proxy_binding.stdout_lines }}"

    - name: Test PVE API access
      uri:
        url: "https://localhost:8006/api2/json/version"
        method: GET
        validate_certs: no
        timeout: 10
      register: pve_api_test
      ignore_errors: yes

    - name: Display PVE API test result
      debug:
        msg: "PVE API access: {{ 'SUCCESS' if pve_api_test.status == 200 else 'FAILED' }} (Status: {{ pve_api_test.status | default('N/A') }})"

    - name: Check system resources
      shell: |
        echo "=== System Resources ==="
        free -h
        echo "=== Load Average ==="
        uptime
        echo "=== Disk Usage ==="
        df -h | head -5        
      register: system_resources

    - name: Display system resources
      debug:
        msg: "{{ system_resources.stdout_lines }}"

    - name: Final verification test
      shell: |
        echo "=== Final Verification Test ==="
        echo "Testing web access with curl:"
        curl -k -s -o /dev/null -w "HTTP Status: %{http_code}, Time: %{time_total}s\n" https://pve:8006
        echo "Testing with different hostnames:"
        curl -k -s -o /dev/null -w "pve.tailnet-68f9.ts.net: %{http_code}\n" https://pve.tailnet-68f9.ts.net:8006
        curl -k -s -o /dev/null -w "100.71.59.40: %{http_code}\n" https://100.71.59.40:8006
        curl -k -s -o /dev/null -w "192.168.31.4: %{http_code}\n" https://192.168.31.4:8006        
      register: final_verification
      when: inventory_hostname != 'pve'

    - name: Display final verification results
      debug:
        msg: "{{ final_verification.stdout_lines }}"
      when: inventory_hostname != 'pve'