mirror of
https://github.com/ucmao/media-parser.git
synced 2026-11-20 08:04:56 +08:00
fix(fanqie): 分享页只剩30秒试看时改从官网播放器取完整分集
This commit is contained in:
+3
-3
@@ -68,9 +68,9 @@
|
||||
| 35 | **快影** | 模板视频 / 原声音乐 | ✅ 支持 | 🟢 免配置 | 快影 OpenAPI + 动态签名 | [查看指南](parsers/kwaiying.md) |
|
||||
| 36 | **微信公众号** | 视频 / 文章插图图集 / 音频 | ✅ 支持 | 🟢 免配置 | SSR 正文解析 + 1080P 视频流提取与原画升级 | [查看指南](parsers/wechat-mp.md) |
|
||||
| 37 | **拼多多** | 多多视频 / 商品图集 / 实物原图 | ✅ 支持 | 🟡 视频需 Cookie / 商品免配置 | anti-content 动态验签 + 307 素材提取 | [查看指南](parsers/pinduoduo.md) |
|
||||
| 38 | **番茄小说** | 短剧视频 | ✅ 支持 | 🟢 免配置 | H5 推广页 HTML 提取 | [查看指南](parsers/fanqie.md) |
|
||||
| 39 | **红果短剧** | 短剧视频 | ✅ 支持 | 🟢 免配置 | H5 推广页 HTML 提取 | [查看指南](parsers/fanqie.md) |
|
||||
| 40 | **红果漫剧** | 漫剧/短剧视频 | ✅ 支持 | 🟢 免配置 | H5 推广页 HTML 提取 | [查看指南](parsers/fanqie.md) |
|
||||
| 38 | **番茄小说** | 短剧视频 | ✅ 支持 | 🟢 免配置 | H5 推广页提取 + 官网播放器完整分集置换 | [查看指南](parsers/fanqie.md) |
|
||||
| 39 | **红果短剧** | 短剧视频 | ✅ 支持 | 🟢 免配置 | H5 推广页提取 + 官网播放器完整分集置换 | [查看指南](parsers/fanqie.md) |
|
||||
| 40 | **红果漫剧** | 漫剧/短剧视频 | ✅ 支持 | 🟢 免配置 | H5 推广页提取 + 官网播放器完整分集置换 | [查看指南](parsers/fanqie.md) |
|
||||
| 41 | **QQ音乐** | 歌曲音频 / LRC 歌词 / MV / 多档 MP4 | ✅ 支持 | 🟢 免配置 | PC Web 协议 + MusicU 播放与歌词接口 | [查看指南](parsers/qqmusic.md) |
|
||||
| 42 | **网易云音乐** | MV / Mlog / 歌曲 / 动态图集与 LivePhoto | ✅ 支持 | 🟢 公开内容免配置 | SSR / Event 数据 + 公开播放接口 | [查看指南](parsers/netease-music.md) |
|
||||
| 43 | **酷狗音乐** | MV / 免费歌曲 / 多档 MP4 | ✅ 支持 | 🟢 公开内容免配置 | H5 公开接口签名 + 分享页数据 | [查看指南](parsers/kugou-music.md) |
|
||||
|
||||
@@ -43,3 +43,23 @@
|
||||
<!-- 或在 video 标签显式声明 referrerpolicy -->
|
||||
<video src="parsed_video_url" referrerpolicy="no-referrer" controls></video>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. 推广落地页只给 30 秒试看处理机制
|
||||
|
||||
### 现象与原理
|
||||
* 推广落地页中 `series_data.play_url` 携带 `&start=0&end=30`,且 CDN 上的实体切片本身仅 30 秒(约 1.85 MB)。
|
||||
* 鉴权签名覆盖整个 Query 参数,修改 `end`、删除参数或追加 `&full=1` 均会导致 CDN 返回 **HTTP 403**。
|
||||
|
||||
### 完整分集置换方案
|
||||
当检测到 `play_url` 包含试看标识(`[?&]end=\d+`)时,通过以下逻辑获取完整分集:
|
||||
1. 从 `window._ROUTER_DATA` 提取:
|
||||
* **剧集 ID**:`linkParams.schemeParams.video_id`
|
||||
* **分集 ID**:`pageData.used_chapter_id`
|
||||
2. 请求官网网页播放器:`https://hongguoduanju.com/player/<剧集 ID>/<分集 ID>`。
|
||||
3. 从页面 `<script type="application/ld+json">` 中提取 `VideoObject.contentUrl` 作为完整无截断视频链接。
|
||||
|
||||
### 容错与降级策略
|
||||
* 若官网播放器请求失败(404、超时)或未找到 `contentUrl`,自动降级沿用分享页原有链接,确保接口整体可用性。
|
||||
* 严禁回退到 `/player/<剧集 ID>`(默认返回第 1 集),避免造成分集错乱。
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
from src.parsers.base_parser import BaseParser
|
||||
from src.parser_factory import register_parser
|
||||
from utils.html_video_extractor import HtmlVideoExtractor
|
||||
@@ -7,6 +9,18 @@ from configs.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# 官网网页播放器。推广落地页只给 30 秒试看(play_url 里带 &start=0&end=30),
|
||||
# 完整分集要打这个站的 /player/<剧集 id>/<分集 id>,页面 ld+json 里的
|
||||
# contentUrl 就是完整视频。红果短剧、红果漫剧、番茄小说的剧都挂在这一个站上。
|
||||
PLAYER_HOST = "https://hongguoduanju.com"
|
||||
# 试看链接的特征:查询串里有 end=<秒数>
|
||||
PREVIEW_MARK = re.compile(r'[?&]end=\d+')
|
||||
CONTENT_URL_PATTERN = re.compile(r'"contentUrl"\s*:\s*"(https[^"]+)"')
|
||||
# 分享页 _ROUTER_DATA 里的 linkParams.schemeParams.video_id 是剧集 id,
|
||||
# pageData.used_chapter_id 是当前分集 id。
|
||||
SCHEME_PARAMS_PATTERN = re.compile(r'"schemeParams"\s*:\s*(\{.*?\})', re.DOTALL)
|
||||
CHAPTER_ID_PATTERN = re.compile(r'"used_chapter_id"\s*:\s*"?(\d+)"?')
|
||||
|
||||
|
||||
@register_parser("番茄小说", "红果短剧", "红果漫剧")
|
||||
class FanqieParser(BaseParser):
|
||||
@@ -23,11 +37,60 @@ class FanqieParser(BaseParser):
|
||||
def _parse_html(self):
|
||||
try:
|
||||
html = self.fetch_html_content()
|
||||
return HtmlVideoExtractor.parse_page(html)
|
||||
parsed = HtmlVideoExtractor.parse_page(html)
|
||||
full_url = self._fetch_full_video_url(html, parsed.get('video_url'))
|
||||
if full_url:
|
||||
parsed['video_url'] = full_url
|
||||
return parsed
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch or parse HTML for {self.real_url}: {e}")
|
||||
return {'title': None, 'video_url': None, 'cover_url': None, 'author': None}
|
||||
|
||||
def _fetch_full_video_url(self, html, share_video_url):
|
||||
"""把试看链接换成完整分集链接;拿不到就返回 None,继续用分享页那条。"""
|
||||
if not html or not share_video_url or not PREVIEW_MARK.search(share_video_url):
|
||||
return None
|
||||
|
||||
series_id, chapter_id = self._extract_ids(html)
|
||||
if not (series_id and chapter_id):
|
||||
logger.warning(f"试看链接但没解析出剧集/分集 id: {self.real_url}")
|
||||
return None
|
||||
|
||||
player_url = f"{PLAYER_HOST}/player/{series_id}/{chapter_id}"
|
||||
try:
|
||||
page = self.fetch_player_html(player_url)
|
||||
except Exception as e:
|
||||
logger.warning(f"官网播放器页取不到({player_url}): {e}")
|
||||
return None
|
||||
|
||||
match = CONTENT_URL_PATTERN.search(page or '')
|
||||
if not match:
|
||||
logger.warning(f"官网播放器页没有 contentUrl({player_url})")
|
||||
return None
|
||||
return HtmlVideoExtractor._clean_url(match.group(1)) or match.group(1).replace(r'\u002F', '/')
|
||||
|
||||
@staticmethod
|
||||
def _extract_ids(html):
|
||||
"""从分享页 _ROUTER_DATA 抽 (剧集 id, 分集 id)。"""
|
||||
series_id = None
|
||||
scheme_match = SCHEME_PARAMS_PATTERN.search(html)
|
||||
if scheme_match:
|
||||
try:
|
||||
series_id = json.loads(scheme_match.group(1)).get('video_id')
|
||||
except ValueError:
|
||||
logger.warning(f"schemeParams 不是合法 JSON: {scheme_match.group(1)[:80]}")
|
||||
if not series_id:
|
||||
vid_match = re.search(r'"video_id"\s*:\s*"?(\d+)"?', html)
|
||||
if vid_match:
|
||||
series_id = vid_match.group(1)
|
||||
chapter_match = CHAPTER_ID_PATTERN.search(html)
|
||||
return series_id, (chapter_match.group(1) if chapter_match else None)
|
||||
|
||||
def fetch_player_html(self, url):
|
||||
resp = self.session.get(url, headers=self.headers, timeout=5)
|
||||
resp.raise_for_status()
|
||||
return resp.text
|
||||
|
||||
def get_real_video_url(self):
|
||||
return self.parsed_data.get('video_url')
|
||||
|
||||
|
||||
@@ -33,6 +33,46 @@ MOCK_NOVEL_HTML = """
|
||||
</html>
|
||||
"""
|
||||
|
||||
# 现在的分享落地页只给 30 秒试看:play_url 里带 &start=0&end=30
|
||||
MOCK_PREVIEW_HTML = """
|
||||
<!doctype html>
|
||||
<html>
|
||||
<body>
|
||||
<script>
|
||||
window._ROUTER_DATA = {
|
||||
"loaderData": {
|
||||
"video-animation-share_page": {
|
||||
"pageData": {
|
||||
"series_data": {
|
||||
"title": "全家把我当乡下老太,都吓傻了",
|
||||
"play_url": "https://v3-share.qznovel.com\\u002Fpreview.mp4?a=8662&start=0&end=30"
|
||||
},
|
||||
"used_chapter_id": "7679061418129247257"
|
||||
},
|
||||
"linkParams": {
|
||||
"schemeParams": {"vid": "7679061418129247257", "video_id": "7679018384272395289"}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
MOCK_PLAYER_HTML = """
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<script type="application/ld+json">
|
||||
{"@type": "VideoObject", "name": "全家把我当乡下老太,都吓傻了 第1集",
|
||||
"contentUrl": "https://v3-hgweb.qznovelvod.com\\u002Ffull.mp4?mime_type=video_mp4",
|
||||
"duration": "PT2M51S"}
|
||||
</script>
|
||||
</head>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
class FanqieParserTest(unittest.TestCase):
|
||||
|
||||
@@ -66,6 +106,23 @@ class FanqieParserTest(unittest.TestCase):
|
||||
self.assertEqual(parser.get_real_video_url(), "https://v3-share.qznovel.com/real_play.mp4")
|
||||
self.assertEqual(parser.get_cover_photo_url(), "https://p3-novel.byteimg.com/cover.image")
|
||||
|
||||
@patch.object(FanqieParser, 'fetch_player_html', return_value=MOCK_PLAYER_HTML)
|
||||
@patch.object(FanqieParser, 'fetch_html_content', return_value=MOCK_PREVIEW_HTML)
|
||||
def test_preview_link_replaced_by_full_video(self, mock_fetch, mock_player):
|
||||
"""分享页给试看链接时,要去官网播放器换成完整分集链接。"""
|
||||
parser = FanqieParser("https://novelquickapp.com/s/wQ-MK1vuras/")
|
||||
self.assertEqual(parser.get_real_video_url(), "https://v3-hgweb.qznovelvod.com/full.mp4?mime_type=video_mp4")
|
||||
self.assertEqual(parser.get_title_content(), "全家把我当乡下老太,都吓傻了")
|
||||
mock_player.assert_called_once_with(
|
||||
"https://hongguoduanju.com/player/7679018384272395289/7679061418129247257")
|
||||
|
||||
@patch.object(FanqieParser, 'fetch_player_html', side_effect=RuntimeError("404"))
|
||||
@patch.object(FanqieParser, 'fetch_html_content', return_value=MOCK_PREVIEW_HTML)
|
||||
def test_fallback_to_share_url_when_player_fails(self, mock_fetch, mock_player):
|
||||
"""官网播放器拿不到时,不能把视频解析弄失败,继续用分享页那条。"""
|
||||
parser = FanqieParser("https://novelquickapp.com/s/wQ-MK1vuras/")
|
||||
self.assertTrue(parser.get_real_video_url().endswith("start=0&end=30"))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user