268 lines
9.6 KiB
Python
268 lines
9.6 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
RSS新闻收集器
|
||
收集RSS新闻并存储到MongoDB，为辩论系统提供数据源
|
||
"""
|
||
|
||
import asyncio
|
||
import feedparser
|
||
import json
|
||
import logging
|
||
from datetime import datetime, timezone
|
||
from typing import Dict, List, Any, Optional
|
||
from urllib.parse import urlparse
|
||
import hashlib
|
||
import requests
|
||
from src.mcp.swarm_mongodb_client import SwarmMongoDBClient
|
||
|
||
class RSSNewsCollector:
|
||
"""RSS新闻收集器"""
|
||
|
||
def __init__(self, mongodb_client: SwarmMongoDBClient):
|
||
self.mongodb_client = mongodb_client
|
||
self.logger = logging.getLogger(__name__)
|
||
|
||
# 默认RSS源配置
|
||
self.rss_sources = {
|
||
'财经新闻': [
|
||
'https://feeds.finance.yahoo.com/rss/2.0/headline',
|
||
'https://www.cnbc.com/id/100003114/device/rss/rss.html',
|
||
'https://feeds.reuters.com/reuters/businessNews'
|
||
],
|
||
'科技新闻': [
|
||
'https://feeds.feedburner.com/TechCrunch',
|
||
'https://www.wired.com/feed/rss',
|
||
'https://feeds.arstechnica.com/arstechnica/index'
|
||
],
|
||
'市场分析': [
|
||
'https://feeds.marketwatch.com/marketwatch/marketpulse/',
|
||
'https://feeds.bloomberg.com/markets/news.rss'
|
||
]
|
||
}
|
||
|
||
def generate_article_id(self, url: str, title: str) -> str:
|
||
"""生成文章唯一ID"""
|
||
content = f"{url}_{title}"
|
||
return hashlib.md5(content.encode()).hexdigest()
|
||
|
||
def parse_rss_feed(self, rss_url: str) -> List[Dict[str, Any]]:
|
||
"""解析RSS源"""
|
||
try:
|
||
feed = feedparser.parse(rss_url)
|
||
articles = []
|
||
|
||
for entry in feed.entries:
|
||
# 提取文章信息
|
||
article = {
|
||
'article_id': self.generate_article_id(entry.link, entry.title),
|
||
'title': entry.title,
|
||
'link': entry.link,
|
||
'description': getattr(entry, 'description', ''),
|
||
'summary': getattr(entry, 'summary', ''),
|
||
'published': self._parse_date(getattr(entry, 'published', '')),
|
||
'author': getattr(entry, 'author', ''),
|
||
'tags': [tag.term for tag in getattr(entry, 'tags', [])],
|
||
'source_url': rss_url,
|
||
'source_title': feed.feed.get('title', ''),
|
||
'collected_at': datetime.now(timezone.utc),
|
||
'content_hash': hashlib.md5(entry.title.encode()).hexdigest()
|
||
}
|
||
articles.append(article)
|
||
|
||
return articles
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"解析RSS源失败 {rss_url}: {e}")
|
||
return []
|
||
|
||
def _parse_date(self, date_str: str) -> Optional[datetime]:
|
||
"""解析日期字符串"""
|
||
if not date_str:
|
||
return None
|
||
|
||
try:
|
||
# feedparser通常会解析时间
|
||
import time
|
||
parsed_time = feedparser._parse_date(date_str)
|
||
if parsed_time:
|
||
return datetime.fromtimestamp(time.mktime(parsed_time), tz=timezone.utc)
|
||
except:
|
||
pass
|
||
|
||
return datetime.now(timezone.utc)
|
||
|
||
async def collect_news_from_category(self, category: str) -> List[Dict[str, Any]]:
|
||
"""从指定类别收集新闻"""
|
||
if category not in self.rss_sources:
|
||
self.logger.warning(f"未知新闻类别: {category}")
|
||
return []
|
||
|
||
all_articles = []
|
||
for rss_url in self.rss_sources[category]:
|
||
self.logger.info(f"正在收集新闻: {rss_url}")
|
||
articles = self.parse_rss_feed(rss_url)
|
||
|
||
# 添加类别标签
|
||
for article in articles:
|
||
article['category'] = category
|
||
|
||
all_articles.extend(articles)
|
||
|
||
return all_articles
|
||
|
||
async def collect_all_news(self) -> Dict[str, List[Dict[str, Any]]]:
|
||
"""收集所有类别的新闻"""
|
||
all_news = {}
|
||
|
||
for category in self.rss_sources.keys():
|
||
news = await self.collect_news_from_category(category)
|
||
all_news[category] = news
|
||
self.logger.info(f"收集到 {len(news)} 条 {category} 新闻")
|
||
|
||
return all_news
|
||
|
||
async def store_news_to_mongodb(self, articles: List[Dict[str, Any]], collection_name: str = "news_articles") -> Dict[str, Any]:
|
||
"""将新闻存储到MongoDB"""
|
||
if not articles:
|
||
return {'success': True, 'inserted_count': 0, 'updated_count': 0}
|
||
|
||
inserted_count = 0
|
||
updated_count = 0
|
||
|
||
for article in articles:
|
||
# 检查文章是否已存在
|
||
existing = self.mongodb_client.find_documents(
|
||
collection_name,
|
||
query={'article_id': article['article_id']},
|
||
limit=1
|
||
)
|
||
|
||
if existing.get('success') and existing.get('documents'):
|
||
# 更新现有文章
|
||
update_result = self.mongodb_client.update_document(
|
||
collection_name,
|
||
query={'article_id': article['article_id']},
|
||
update={'$set': article}
|
||
)
|
||
if update_result.get('success'):
|
||
updated_count += 1
|
||
else:
|
||
# 插入新文章
|
||
insert_result = self.mongodb_client.insert_document(
|
||
collection_name,
|
||
document=article
|
||
)
|
||
if insert_result.get('success'):
|
||
inserted_count += 1
|
||
|
||
return {
|
||
'success': True,
|
||
'inserted_count': inserted_count,
|
||
'updated_count': updated_count,
|
||
'total_processed': len(articles)
|
||
}
|
||
|
||
async def get_latest_news(self, category: Optional[str] = None, limit: int = 10) -> List[Dict[str, Any]]:
|
||
"""获取最新新闻"""
|
||
query = {}
|
||
if category:
|
||
query['category'] = category
|
||
|
||
result = self.mongodb_client.find_documents(
|
||
'news_articles',
|
||
query=query,
|
||
sort={'collected_at': -1},
|
||
limit=limit
|
||
)
|
||
|
||
if result.get('success'):
|
||
return result.get('documents', [])
|
||
return []
|
||
|
||
async def get_news_for_debate(self, topic_keywords: List[str], limit: int = 5) -> List[Dict[str, Any]]:
|
||
"""根据关键词获取相关新闻用于辩论"""
|
||
# 构建搜索查询
|
||
search_conditions = []
|
||
for keyword in topic_keywords:
|
||
search_conditions.extend([
|
||
{'title': {'$regex': keyword, '$options': 'i'}},
|
||
{'description': {'$regex': keyword, '$options': 'i'}},
|
||
{'summary': {'$regex': keyword, '$options': 'i'}}
|
||
])
|
||
|
||
query = {'$or': search_conditions} if search_conditions else {}
|
||
|
||
result = self.mongodb_client.find_documents(
|
||
'news_articles',
|
||
query=query,
|
||
sort={'published': -1},
|
||
limit=limit
|
||
)
|
||
|
||
if result.get('success'):
|
||
return result.get('documents', [])
|
||
return []
|
||
|
||
async def run_collection_cycle(self):
|
||
"""运行一次完整的新闻收集周期"""
|
||
self.logger.info("开始新闻收集周期")
|
||
|
||
# 收集所有新闻
|
||
all_news = await self.collect_all_news()
|
||
|
||
# 存储到数据库
|
||
total_inserted = 0
|
||
total_updated = 0
|
||
|
||
for category, articles in all_news.items():
|
||
if articles:
|
||
result = await self.store_news_to_mongodb(articles)
|
||
total_inserted += result.get('inserted_count', 0)
|
||
total_updated += result.get('updated_count', 0)
|
||
self.logger.info(f"{category}: 新增 {result.get('inserted_count', 0)}, 更新 {result.get('updated_count', 0)}")
|
||
|
||
self.logger.info(f"新闻收集完成: 总新增 {total_inserted}, 总更新 {total_updated}")
|
||
|
||
return {
|
||
'success': True,
|
||
'total_inserted': total_inserted,
|
||
'total_updated': total_updated,
|
||
'categories_processed': len(all_news)
|
||
}
|
||
|
||
async def main():
|
||
"""主函数 - 演示RSS新闻收集"""
|
||
# 初始化MongoDB客户端
|
||
mongodb_client = SwarmMongoDBClient(
|
||
mcp_server_url="http://localhost:8080",
|
||
default_database="news_debate_db"
|
||
)
|
||
|
||
# 连接数据库
|
||
connect_result = mongodb_client.connect("news_debate_db")
|
||
if not connect_result.get('success'):
|
||
print(f"数据库连接失败: {connect_result}")
|
||
return
|
||
|
||
# 创建新闻收集器
|
||
collector = RSSNewsCollector(mongodb_client)
|
||
|
||
# 运行收集周期
|
||
result = await collector.run_collection_cycle()
|
||
print(f"收集结果: {result}")
|
||
|
||
# 获取最新新闻示例
|
||
latest_news = await collector.get_latest_news(limit=5)
|
||
print(f"\n最新新闻 ({len(latest_news)} 条):")
|
||
for news in latest_news:
|
||
print(f"- {news.get('title', 'N/A')} [{news.get('category', 'N/A')}]")
|
||
|
||
# 根据关键词搜索新闻示例
|
||
debate_news = await collector.get_news_for_debate(['投资', '市场', '经济'], limit=3)
|
||
print(f"\n辩论相关新闻 ({len(debate_news)} 条):")
|
||
for news in debate_news:
|
||
print(f"- {news.get('title', 'N/A')} [{news.get('category', 'N/A')}]")
|
||
|
||
if __name__ == "__main__":
|
||
logging.basicConfig(level=logging.INFO)
|
||
asyncio.run(main()) |