mirror of
https://github.com/ucmao/media-parser.git
synced 2026-11-20 08:04:56 +08:00
feat: 添加闲鱼平台解析支持
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
|
||||
**基于 Python 的多平台媒体原生本地解析系统**
|
||||
|
||||
[](LICENSE) [](https://www.python.org/) [](#💾-支持的平台矩阵)
|
||||
[](LICENSE) [](https://www.python.org/) [](#💾-支持的平台矩阵)
|
||||
|
||||
<p align="center">
|
||||
<a href="#-核心解析逻辑">解析逻辑</a> •
|
||||
@@ -49,6 +49,7 @@
|
||||
| **即梦 AI** | ✓ | ✓ | ✓ | ✓ | | | |
|
||||
| **可灵 AI** | ✓ | ✓ | ✓ | ✓ | | | |
|
||||
| **通义千问** | ✓ | ✓ | ✓ | ✓ | ✓ | | |
|
||||
| **闲鱼** | | ✓ | ✓ | | ✓ | | |
|
||||
| **Soul** | ✓ | ✓ | ✓ | ✓ | ✓ | | |
|
||||
| **汽水音乐** | ✓ | ✓ | ✓ | ✓ | | ✓ | |
|
||||
| **腾讯频道** | ✓ | ✓ | ✓ | ✓ | | | |
|
||||
|
||||
@@ -52,7 +52,12 @@
|
||||
"channels.weixin.qq.com": "微信视频号",
|
||||
"activity.qianwen.com": "通义千问",
|
||||
"qianwen.aliyun.com": "通义千问",
|
||||
"tongyi.aliyun.com": "通义千问"
|
||||
"tongyi.aliyun.com": "通义千问",
|
||||
"e.tb.cn": "闲鱼",
|
||||
"m.tb.cn": "闲鱼",
|
||||
"tb.cn": "闲鱼",
|
||||
"h5.m.goofish.com": "闲鱼",
|
||||
"2.taobao.com": "闲鱼"
|
||||
},
|
||||
"USER_AGENT_PC": [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
||||
|
||||
@@ -26,6 +26,7 @@ from src.parsers.qsmusic_parser import QSMusicParser
|
||||
from src.parsers.tencent_channel_parser import TencentChannelParser
|
||||
from src.parsers.jianying_parser import JianyingParser
|
||||
from src.parsers.qianwen_parser import QianwenParser
|
||||
from src.parsers.xianyu_parser import XianyuParser
|
||||
|
||||
class ParserFactory:
|
||||
platform_to_parser = {
|
||||
@@ -57,6 +58,7 @@ class ParserFactory:
|
||||
"腾讯频道": TencentChannelParser,
|
||||
"剪映": JianyingParser,
|
||||
"通义千问": QianwenParser,
|
||||
"闲鱼": XianyuParser,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import re
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from configs.general_constants import USER_AGENT_M
|
||||
from configs.logging_config import get_logger
|
||||
from src.parsers.base_parser import BaseParser
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class XianyuParser(BaseParser):
|
||||
"""闲鱼/淘宝短链解析器,提取商品 ID、标价与真实目标链接。"""
|
||||
|
||||
def __init__(self, real_url):
|
||||
super().__init__(real_url)
|
||||
self.headers = {
|
||||
"User-Agent": USER_AGENT_M[0],
|
||||
"Referer": "https://e.tb.cn/",
|
||||
}
|
||||
self.item_id = None
|
||||
self.price = None
|
||||
self.target_url = None
|
||||
self.title = None
|
||||
self.cover_url = None
|
||||
self.video_url = None
|
||||
self.image_list = []
|
||||
self.author = None
|
||||
self._parse()
|
||||
|
||||
def _parse(self):
|
||||
if not self.real_url:
|
||||
return
|
||||
try:
|
||||
resp = self.session.get(self.real_url, headers=self.headers, timeout=10)
|
||||
resp.raise_for_status()
|
||||
|
||||
match = re.search(r"var\s+url\s*=\s*'([^']+)'", resp.text)
|
||||
if match:
|
||||
self.target_url = match.group(1)
|
||||
parsed = urlparse(self.target_url)
|
||||
params = parse_qs(parsed.query)
|
||||
self.item_id = (params.get("id") or [None])[0]
|
||||
self.price = (params.get("price") or [None])[0]
|
||||
|
||||
if not self.item_id:
|
||||
parsed_real = urlparse(self.real_url)
|
||||
params_real = parse_qs(parsed_real.query)
|
||||
self.item_id = (params_real.get("id") or [None])[0]
|
||||
|
||||
if self.target_url:
|
||||
self.image_list = [self.target_url]
|
||||
self.cover_url = self.target_url
|
||||
|
||||
if self.item_id:
|
||||
if self.price:
|
||||
self.title = f"闲鱼商品 (商品ID: {self.item_id}, 标价: ¥{self.price})"
|
||||
else:
|
||||
self.title = f"闲鱼商品 (商品ID: {self.item_id})"
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to parse Xianyu share URL: %s", exc)
|
||||
|
||||
def get_real_video_url(self):
|
||||
return self.video_url
|
||||
|
||||
def get_title_content(self):
|
||||
return self.title or ""
|
||||
|
||||
def get_cover_photo_url(self):
|
||||
return self.cover_url
|
||||
|
||||
def get_author_info(self):
|
||||
return self.author
|
||||
|
||||
def get_image_list(self):
|
||||
return self.image_list
|
||||
@@ -28,6 +28,7 @@
|
||||
{"platform": "虎牙", "url": "https://hy.fan/v3cwMK", "media_types": ["video"], "expected_fields": ["video", "cover", "title"], "note": "2026-08-27 已验证"},
|
||||
{"platform": "西瓜视频", "url": "https://www.iesdouyin.com/xg/video/7676450021063735414/?app=video_article×tamp=1787834623&device_id=973518813013579&tt_from=copy_link&utm_source=copy_link&utm_medium=ios&utm_campaign=client_share&test_group=v1", "alternate_urls": ["https://v.douyin.com/Nid-fFF_sdI/"], "media_types": ["video"], "expected_fields": ["video", "cover", "title", "author"], "note": "2026-08-27 更新;App 短链会跳转到同类西瓜视频页面"},
|
||||
{"platform": "豆包", "url": "https://www.doubao.com/video-sharing?share_id=41356597786354690&source_type=mobile&video_id=v0d69cg10004d6978e2ljht0i4fdpp00&share_scene=video_viewer", "media_types": ["video"], "expected_fields": ["video", "cover", "title", "author"], "note": "待人工复核"},
|
||||
{"platform": "通义千问", "url": "https://activity.qianwen.com/r/ai-studio-mobile/qwen-external-share?&shareId=ZeeOedXncnGlElkluRMA&authorId=OjTAcGHgZgyeCwY4jeMDuH9JkGZ10BNYFM1HfW4CcLhxRA%3D%3D&enter_from=create_share&fp_from=create_share&channel_from=qwen&image_index=0", "media_types": ["images"], "expected_fields": ["images", "cover", "title", "author"], "note": "2026-08-27 已验证"}
|
||||
{"platform": "通义千问", "url": "https://activity.qianwen.com/r/ai-studio-mobile/qwen-external-share?&shareId=ZeeOedXncnGlElkluRMA&authorId=OjTAcGHgZgyeCwY4jeMDuH9JkGZ10BNYFM1HfW4CcLhxRA%3D%3D&enter_from=create_share&fp_from=create_share&channel_from=qwen&image_index=0", "media_types": ["images"], "expected_fields": ["images", "cover", "title", "author"], "note": "2026-08-27 已验证"},
|
||||
{"platform": "闲鱼", "url": "https://e.tb.cn/h.87fj9SNrqHW8kfC?tk=y1E5Tb4wkGd", "media_types": ["images"], "expected_fields": ["title"], "note": "2026-08-27 已验证"}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from src.parsers.xianyu_parser import XianyuParser
|
||||
|
||||
|
||||
class XianyuParserTest(unittest.TestCase):
|
||||
def test_parses_xianyu_short_link(self):
|
||||
page_html = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head><title>闲鱼分享</title></head>
|
||||
<body>
|
||||
<script>
|
||||
var shortName = "h.87fj9SNrqHW8kfC";
|
||||
var url = 'https://item.taobao.com/item.htm?id=967494320598&price=12.2&short_name=h.87fj9SNrqHW8kfC';
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
page_resp = Mock()
|
||||
page_resp.raise_for_status.return_value = None
|
||||
page_resp.text = page_html
|
||||
|
||||
url = "https://e.tb.cn/h.87fj9SNrqHW8kfC?tk=y1E5Tb4wkGd"
|
||||
with patch("requests.Session.get", return_value=page_resp):
|
||||
parser = XianyuParser(url)
|
||||
|
||||
self.assertEqual(parser.get_title_content(), "闲鱼商品 (商品ID: 967494320598, 标价: ¥12.2)")
|
||||
|
||||
def test_invalid_url_handles_gracefully(self):
|
||||
parser = XianyuParser("https://e.tb.cn/invalid")
|
||||
self.assertEqual(parser.get_title_content(), "")
|
||||
self.assertIsNone(parser.get_real_video_url())
|
||||
self.assertEqual(parser.get_image_list(), [])
|
||||
self.assertIsNone(parser.get_cover_photo_url())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -221,6 +221,15 @@ class UrlParser:
|
||||
preserved_params.append((key, value))
|
||||
if preserved_params:
|
||||
address = f"{address}?{urlencode(preserved_params)}"
|
||||
elif platform == "闲鱼":
|
||||
query_params = parse_qs(parsed_url.query)
|
||||
preserved_params = []
|
||||
for key in ("tk", "id", "price", "shareurl", "short_name", "sp_tk"):
|
||||
value = query_params.get(key, [None])[0]
|
||||
if value is not None:
|
||||
preserved_params.append((key, value))
|
||||
if preserved_params:
|
||||
address = f"{address}?{urlencode(preserved_params)}"
|
||||
return address
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user