liurenchaxin/tests/validate_models.py

90 lines
2.5 KiB
Python

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Script to validate model availability on OpenRouter
"""
import asyncio
import aiohttp
import os
# Read models from .env file
models = []
with open(".env", "r") as f:
for line in f:
line = line.strip()
if line.endswith(":free"):
models.append(line)
# Get API key from environment or .env file
api_key = os.getenv('OPENROUTER_API_KEY')
if not api_key:
with open(".env", "r") as f:
for line in f:
line = line.strip()
if line.startswith("sk-or-v1-"):
api_key = line
break
if not api_key:
print("❌ No API key found")
exit(1)
async def test_model(session, model):
"""Test if a model is available on OpenRouter"""
# Remove :free tag for API call
clean_model = model.split(":")[0]
url = "https://openrouter.ai/api/v1/chat/completions"
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}
payload = {
"model": clean_model,
"messages": [
{"role": "user", "content": "Hello, world!"}
],
"max_tokens": 10
}
try:
async with session.post(url, headers=headers, json=payload, timeout=10) as response:
if response.status == 200:
return model, True, "Available"
else:
error_text = await response.text()
return model, False, f"Status {response.status}: {error_text[:100]}"
except Exception as e:
return model, False, f"Exception: {str(e)[:100]}"
async def main():
"""Main function"""
print(f"🔍 Testing {len(models)} models from .env file...")
async with aiohttp.ClientSession() as session:
tasks = [test_model(session, model) for model in models]
results = await asyncio.gather(*tasks)
print("\n📊 Results:")
valid_models = []
invalid_models = []
for model, is_valid, message in results:
if is_valid:
print(f"{model:<50} - {message}")
valid_models.append(model)
else:
print(f"{model:<50} - {message}")
invalid_models.append(model)
print(f"\n✅ Valid models ({len(valid_models)}):")
for model in valid_models:
print(f" {model}")
print(f"\n❌ Invalid models ({len(invalid_models)}):")
for model in invalid_models:
print(f" {model}")
if __name__ == "__main__":
asyncio.run(main())