From cdc5d817e64d68b3f3cd19f6932a475eaa8d8adb Mon Sep 17 00:00:00 2001 From: ucmao <294323976@qq.com> Date: Wed, 18 Nov 2026 23:55:58 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81B=E7=AB=99=20AV=20=E5=8F=B7?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E5=B9=B6=E5=AE=8C=E5=96=84=E5=B9=B3=E5=8F=B0?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- configs/business_config.json | 3 ++ docs/api.md | 2 ++ docs/parsers/bilibili.md | 13 +++++--- docs/parsers/toutiao.md | 1 + docs/parsers/wechat-mp.md | 17 +++++++++-- docs/parsers/weibo.md | 1 + docs/parsers/zuiyou.md | 1 + src/api/parse.py | 6 +++- src/parsers/bilibili_parser.py | 48 +++++++++++++++++++++++++----- src/parsers/wechat_mp_parser.py | 15 ++++++++++ src/parsers/xianyu_parser.py | 5 ++-- templates/components/api_docs.html | 12 ++++++++ templates/landing.html | 16 ++++++++++ tests/test_bilibili_parser.py | 39 ++++++++++++++++++++++++ tests/test_web_fetcher.py | 18 +++++++++++ tests/test_wechat_mp_parser.py | 22 ++++++++++++++ tests/test_xianyu_parser.py | 2 ++ utils/web_fetcher.py | 10 +++++-- 18 files changed, 212 insertions(+), 19 deletions(-) diff --git a/configs/business_config.json b/configs/business_config.json index 33fc73f..ee62730 100755 --- a/configs/business_config.json +++ b/configs/business_config.json @@ -52,6 +52,7 @@ "video.weibo.com": "微博", "h5.video.weibo.com": "微博", "t.cn": "微博", + "mapp.api.weibo.cn": "微博", "www.ixigua.com": "西瓜视频", "v.ixigua.com": "西瓜视频", "m.ixigua.com": "西瓜视频", @@ -60,6 +61,7 @@ "www.toutiao.com": "今日头条", "toutiao.com": "今日头条", "is.snssdk.com": "今日头条", + "adshare.toutiao.com": "今日头条", "zhihu.com": "知乎", "www.zhihu.com": "知乎", "zhuanlan.zhihu.com": "知乎", @@ -80,6 +82,7 @@ "www.xinpianchang.com": "新片场", "xinpianchang.com": "新片场", "izuiyou.com": "最右", + "s.izuiyou.com": "最右", "share.xiaochuankeji.cn": "最右", "doubao.com": "豆包", "www.doubao.com": "豆包", diff --git a/docs/api.md b/docs/api.md index 054dd6b..409914b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -136,6 +136,8 @@ curl -X POST "http://localhost:5000/api/v1/parse" \ | `retcode` | `error_code` | `retdesc` 描述 | 说明与处理建议 | | :---: | :--- | :--- | :--- | | **400** | `MEDIA_DELETED_OR_PRIVATE` | `因作品权限或已被删除,无法观看...` | 作品已被作者删除、设置为仅自己可见、朋友可见或日常权限限制(终态不可重试) | +| **400** | `ITEM_NOT_FOUND` | `该宝贝不存在或已被卖家删除/下架` | 闲鱼商品已被卖家下架或删除(终态不可重试) | +| **400** | `WECHAT_MP_VERIFY_REQUIRED` | `微信访问触发安全验证,请稍后重试` | 微信公众号页面触发环境异常反爬拦截,需稍后重试或切换 IP | | **400** | `MEDIA_UNDER_REVIEW` | `该作品刚发布,正在抖音审核中,暂时无法查看,请几分钟后重试` | 抖音作品刚发布(2 小时内)仍处于审核期,接口返回仅自己可见;稍后重试通常即可成功 | | **400** | `NO_MEDIA_IN_CONTENT` | `该分享内容仅包含文本对话,未包含图片或视频资源` | 豆包等 AI 对话分享仅有文字问答,无多媒体附件 | | **400** | `XIAOHONGSHU_COOKIE_REQUIRED` | `解析失败:该链接需要小红书登录 Cookie 校验...` | 小红书触发安全反爬校验(含 `/404/sec_*`、`error_code=300031`「当前笔记暂时无法浏览」风控页),需更新有效 Cookie 或稍后重试 | diff --git a/docs/parsers/bilibili.md b/docs/parsers/bilibili.md index 6e69721..33b8834 100644 --- a/docs/parsers/bilibili.md +++ b/docs/parsers/bilibili.md @@ -14,7 +14,7 @@ * 视频标题与 UP 主/官方账号信息 * **常见链接形态**: * 短链接:`https://b23.tv/xxxx`、`https://b23.tv/ep3372237` - * 网页端长链:`https://www.bilibili.com/video/BV1asTR6FEWu` + * 网页端长链 (BV/AV):`https://www.bilibili.com/video/BV1asTR6FEWu`、`https://www.bilibili.com/video/av170001` * 番剧/剧集长链:`https://www.bilibili.com/bangumi/play/ep1231565`、`https://www.bilibili.com/bangumi/play/ss12345` * 动态/Opus 链接:`https://t.bilibili.com/1245189054385881096` 或 `https://www.bilibili.com/opus/1245189054385881096` * **Cookie 依赖**: @@ -49,9 +49,9 @@ sequenceDiagram ``` ### 2.1 步骤 1:获取视频/番剧详情与 `cid` -* **常规视频**:`https://api.bilibili.com/x/web-interface/view?bvid={bvid}` -* **番剧剧集**:`https://api.bilibili.com/pgc/view/web/season?ep_id={ep_id}` -* **产出**:获取视频的 `cid`(内容分块 ID)、`title`、`pic`/`cover`(封面图)、`owner`/`up_info`(UP主)。 +* **常规视频**:支持 BV 号(`https://api.bilibili.com/x/web-interface/view?bvid={bvid}`)与旧版 AV 号(`https://api.bilibili.com/x/web-interface/view?aid={aid}`,如 `/video/av170001` 或 URL 中的 `aid=...`); +* **番剧剧集**:`https://api.bilibili.com/pgc/view/web/season?ep_id={ep_id}`; +* **产出**:获取视频的标准 `bvid`、`cid`(内容分块 ID)、`title`、`pic`/`cover`(封面图)、`owner`/`up_info`(UP主)。 ### 2.2 步骤 2:获取无需合成的单文件 MP4 流 (`PlayURL`) * **常规视频**:`https://api.bilibili.com/x/player/playurl?bvid={bvid}&cid={cid}&platform=html5&fnval=3&qn=112` @@ -70,6 +70,11 @@ sequenceDiagram 3. **云服务器/机房 IP 触发 412 (Precondition Failed) 风控拦截**: * 在云服务器/数据中心(如 Docker 部署)环境下,使用 PC 浏览器(Chrome/Firefox 等)User-Agent 访问视频详情接口 `https://api.bilibili.com/x/web-interface/view` 会被 B 站 WAF 拦截并返回 `HTTP 412 Precondition Failed`。 * **解法**:视频详情接口单独使用非浏览器/客户端标识(如 `okhttp/4.9.3`),即可绕过该 WAF 拦截正常获取视频信息。 +4. **不可见/删除/付费等终态错误映射 (`MEDIA_DELETED_OR_PRIVATE`)**: + * **稿件不可见/已删除**:B 站详情接口返回错误码 `62002`(稿件不可见)、`62004`、`-404`(啥都木有); + * **动态不可见/已删除**:动态接口返回错误码 `4101152`(动态不可见)、`4101153`; + * **充电专属/付费视频**:接口返回错误码 `87008`(需购买充电专属视频); + * 解析器捕获上述状态码后会自动将错误提升并映射为 `MEDIA_DELETED_OR_PRIVATE` 终态错误,提示调用方内容不可查看,阻断客户端无意义重试。 --- diff --git a/docs/parsers/toutiao.md b/docs/parsers/toutiao.md index 068213c..ec17ce6 100644 --- a/docs/parsers/toutiao.md +++ b/docs/parsers/toutiao.md @@ -10,6 +10,7 @@ * **支持媒体类型**:高清视频 (MP4,最高 1080P/720P) / 微头条图文图集 / 长文正文内嵌配图 / 封面图 / 标题与清洗正文 / 创作者信息 * **常见链接形态**: * 短链接:`https://m.toutiao.com/is/fLrXD62Zo2U/` 或 `https://m.toutiao.com/is/geu37y7CA4w/` + * 推广/分享链接:`https://adshare.toutiao.com/s/...` 或 `https://adshare.toutiao.com/...` * 网页链接:`https://www.toutiao.com/video/7680960670263493172/` 或 `https://www.toutiao.com/article/7680960670263493172/` * 移动端落地页:`https://m.toutiao.com/video/7680960670263493172/` 或 `https://m.toutiao.com/i7680960670263493172/` * 微头条:`https://m.toutiao.com/w/1876628745217031/`(短链解析后通常落到该形态) diff --git a/docs/parsers/wechat-mp.md b/docs/parsers/wechat-mp.md index 9e9e3c3..12519ac 100644 --- a/docs/parsers/wechat-mp.md +++ b/docs/parsers/wechat-mp.md @@ -20,6 +20,9 @@ * **多视频文章长链**:`https://mp.weixin.qq.com/s/aNmLs3qJVhFGcgtGxAVlig` * **视频消息动态**:`https://mp.weixin.qq.com/s/GQiMB7CC6r7wil54I0TMwQ` * **历史参数长链**:`https://mp.weixin.qq.com/s?__biz=MzA5...&mid=...&idx=1&sn=...` + * **视频详情落地页**:`https://mp.weixin.qq.com/s?__biz=...&mid=...&idx=1&sn=...&t=pages/video_detail_new&vid=...` + * **音频详情落地页**:`https://mp.weixin.qq.com/s?__biz=...&mid=...&idx=1&sn=...&t=pages/audio_detail&voice_id=...` + * **带校验和参数长链**:`https://mp.weixin.qq.com/s?__biz=...&mid=...&idx=1&sn=...&chksm=...` * **Cookie 依赖**:🟢 免配置(开箱即用)。 --- @@ -55,11 +58,16 @@ * 从正文 `` 标签提取语音文件 ID; * 组装微信原生音频直链:`https://res.wx.qq.com/voice/getvoice?mediaid={voice_encode_fileid}`。 +### 6. 参数保真与 Base64 规范化处理 +微信公众号独立视频/音频页(如 `t=pages/video_detail_new&vid=...`、`t=pages/audio_detail&voice_id=...`)以及历史长链高度依赖 URL Query 参数。在预处理与清洗环节: +* **关键参数全量保留**:系统定向保留 `__biz`、`mid`、`idx`、`sn`、`chksm`、`t`、`vid`、`vidsn`、`voice_id` 等业务参数,防止因参数被过度剔除导致 404; +* **Base64 与路径符号防转义**:在重编码 URL 时指定 `safe='=/'`,避免 Base64 尾部填充等号 `=` 被错误转码为 `%3D` 或路径分隔符 `/` 被转为 `%2F` 引发微信服务反爬拦截或跳转异常。 + --- -## 3. 视频直链访问与下载注意事项(防盗链与 302 重定向) +## 3. 视频直链访问与状态识别注意事项 -腾讯视频 CDN(`mpvideo.qpic.cn`)具有以下访问机制,下载或播放时需注意: +腾讯视频 CDN(`mpvideo.qpic.cn`)与公众号安全机制如下,调用或下载时需注意: 1. **防盗链校验 (Referer)**: * 微信 CDN 强制校验来源。浏览器直接在地址栏打开直链会触发 `403 Forbidden`。 @@ -72,6 +80,11 @@ ``` 3. **动态凭证时效性**: * 视频链接中的 `dis_k`、`dis_t`、`auth_key` 参数为动态限时签名,过期需重新调用解析接口获取最新链接。 +4. **环境异常反爬拦截识别 (`WECHAT_MP_VERIFY_REQUIRED`)**: + * 当微信服务端对频繁访问或特定机房 IP 触发安全验证时,页面会跳转至 `secitptpage/verify`(“环境异常,完成验证后即可继续访问”)。 + * 解析器会自动识别该拦截页并抛出标准错误码 `WECHAT_MP_VERIFY_REQUIRED`(HTTP 400),提示调用方稍后重试或切换代理 IP,避免误判为空内容。 +5. **文章删除与违规终态识别 (`MEDIA_DELETED_OR_PRIVATE`)**: + * 当微信页面包含「该内容已被发布者删除」或「此内容因违规无法查看」等提示时,解析器直接将其标记为 `MEDIA_DELETED_OR_PRIVATE` 终态错误,阻断无意义的频繁轮询。 --- diff --git a/docs/parsers/weibo.md b/docs/parsers/weibo.md index 2bc03ba..555343c 100644 --- a/docs/parsers/weibo.md +++ b/docs/parsers/weibo.md @@ -19,6 +19,7 @@ * 实况/图文博文:`https://weibo.com/6288169783/5344438765488239` * 网页长链:`https://weibo.com/1234567890/Mabcdef` * 移动端带渠道后缀:`https://m.weibo.cn/7753941940/5332536008119716/qq?wm=3333_2001` + * 移动客户端分享链接:`https://mapp.api.weibo.cn/fx/...` * **Cookie 依赖**:常规内容免登录 Cookie(内置自动生成临时 Visitor 访客会话,机房 IP 或高规格视频支持配置 `WEIBO_COOKIE`)。 --- diff --git a/docs/parsers/zuiyou.md b/docs/parsers/zuiyou.md index 95a5df9..a9bff27 100644 --- a/docs/parsers/zuiyou.md +++ b/docs/parsers/zuiyou.md @@ -9,6 +9,7 @@ * **平台标识**:`最右` * **支持媒体类型**:高清视频 (MP4) / 多图图集 (原图 / WebP) / 实况图 (Live Photo 封面 + 动态片段) / 封面 / 帖子正文 / 作者信息 * **常见链接形态**: + * 短链接:`https://s.izuiyou.com/s/ABCDEF` 或 `https://s.izuiyou.com/...` * 分享落地页:`https://share.xiaochuankeji.cn/hybrid/share/post?pid=423835942&vid=2542343457` * 图文帖子页:`https://share.xiaochuankeji.cn/hybrid/share/post?pid=419346905` * **Cookie 依赖**:无需 Cookie。 diff --git a/src/api/parse.py b/src/api/parse.py index c567c30..e6d54d4 100644 --- a/src/api/parse.py +++ b/src/api/parse.py @@ -269,10 +269,14 @@ def _execute_parse(text, access): def _terminal_detail(parser): - for name in ('terminal_error', '_terminal_filter_detail'): + for name in ('terminal_error', '_terminal_filter_detail', 'terminal_detail'): detail = getattr(parser, name, None) if isinstance(detail, dict) and detail: return detail + if callable(getattr(parser, 'get_terminal_detail', None)): + detail = parser.get_terminal_detail() + if isinstance(detail, dict) and detail: + return detail return None diff --git a/src/parsers/bilibili_parser.py b/src/parsers/bilibili_parser.py index c649939..6d08ccc 100644 --- a/src/parsers/bilibili_parser.py +++ b/src/parsers/bilibili_parser.py @@ -34,9 +34,10 @@ class BilibiliParser(BaseParser): "Referer": "https://www.bilibili.com/", } self.bvid = self._extract_bvid(real_url) + self.aid = self._extract_aid(real_url) if not self.bvid else None self.ep_id = self._extract_ep_id(real_url) self.season_id = self._extract_season_id(real_url) if not self.ep_id else None - self.dynamic_id = None if (self.bvid or self.ep_id or self.season_id) else self._extract_dynamic_id(real_url) + self.dynamic_id = None if (self.bvid or self.aid or self.ep_id or self.season_id) else self._extract_dynamic_id(real_url) self.dynamic_info = {} self.season_info = {} self.bangumi_ep_info = {} @@ -59,7 +60,7 @@ class BilibiliParser(BaseParser): if bvid := self._extract_bvid_from_dynamic(self.dynamic_info): self.bvid = bvid - self.video_info = self._fetch_video_info() if self.bvid else {} + self.video_info = self._fetch_video_info() if (self.bvid or self.aid) else {} self._play_info_by_cid = {} @staticmethod @@ -70,6 +71,14 @@ class BilibiliParser(BaseParser): return match.group(1) return None + @staticmethod + def _extract_aid(url): + """从 URL 中提取 AV 号,例如 ``av117586188636547`` 或 ``aid=123``。""" + match = re.search(r"(?:/video/av|aid=)(\d+)", url or "", re.IGNORECASE) + if match: + return match.group(1) + return None + @staticmethod def _extract_ep_id(url): """从 URL 中提取番剧/剧集 EP 号,例如 ``ep1231565``。""" @@ -144,6 +153,13 @@ class BilibiliParser(BaseParser): if result.get("code") == 0: return result.get("data", {}).get("item") or {} + code = result.get("code") + msg = result.get("message") or "" + if code in (4101152, 4101153) or "不可见" in msg or "不存在" in msg: + self.terminal_error = { + "error_code": "MEDIA_DELETED_OR_PRIVATE", + "detail_msg": "该B站动态不存在、已被删除或不可见", + } logger.error( "B站动态 API 返回错误: code=%s, message=%s", result.get("code"), @@ -164,12 +180,13 @@ class BilibiliParser(BaseParser): return None def _fetch_video_info(self): - if not self.bvid: + if not self.bvid and not getattr(self, "aid", None): return {} + params = {"bvid": self.bvid} if self.bvid else {"aid": self.aid} try: response = self.session.get( self.API_VIEW, - params={"bvid": self.bvid}, + params=params, headers=self.view_headers, timeout=10, ) @@ -180,11 +197,22 @@ class BilibiliParser(BaseParser): return {} if result.get("code") == 0: - return result.get("data") or {} + data = result.get("data") or {} + if not self.bvid and data.get("bvid"): + self.bvid = data["bvid"] + return data + + code = result.get("code") + msg = result.get("message") or "" + if code in (62002, 62004, -404) or "不可见" in msg or "不存在" in msg: + self.terminal_error = { + "error_code": "MEDIA_DELETED_OR_PRIVATE", + "detail_msg": "该B站稿件不存在、已被作者删除或不可见", + } logger.error( "B站 API 返回错误: code=%s, message=%s", - result.get("code"), - result.get("message"), + code, + msg, ) return {} @@ -253,6 +281,12 @@ class BilibiliParser(BaseParser): return {} if result.get("code") != 0: + code = result.get("code") + if code == 87008 or "充电" in str(result.get("message")): + self.terminal_error = { + "error_code": "MEDIA_DELETED_OR_PRIVATE", + "detail_msg": "该视频为B站充电专属或付费视频,无法直接提取", + } logger.error( "B站 API playurl 返回错误: code=%s, message=%s", result.get("code"), diff --git a/src/parsers/wechat_mp_parser.py b/src/parsers/wechat_mp_parser.py index 6be162d..07c33e8 100644 --- a/src/parsers/wechat_mp_parser.py +++ b/src/parsers/wechat_mp_parser.py @@ -36,6 +36,21 @@ class WechatMpParser(BaseParser): response = self.session.get(self.real_url, headers=headers, timeout=12) response.raise_for_status() html = response.text + + if "环境异常" in html and ("secitptpage/verify" in html or "完成验证后即可继续访问" in html): + self.terminal_error = { + "error_code": "WECHAT_MP_VERIFY_REQUIRED", + "detail_msg": "微信公众号访问触发安全验证,请稍后重试", + } + return {"title": None, "desc": None, "video_url": None, "video_list": [], "cover_url": None, "author": None, "image_list": []} + + if any(kw in html for kw in ("该内容已被发布者删除", "此内容因违规无法查看", "此内容已被发布者删除", "该内容因违规无法查看")): + self.terminal_error = { + "error_code": "MEDIA_DELETED_OR_PRIVATE", + "detail_msg": "该公众号文章已被作者删除或违规下架", + } + return {"title": None, "desc": None, "video_url": None, "video_list": [], "cover_url": None, "author": None, "image_list": []} + soup = BeautifulSoup(html, "html.parser") # 1. 标题提取 (优先 JS 变量,其次 meta,再次 h1) diff --git a/src/parsers/xianyu_parser.py b/src/parsers/xianyu_parser.py index 5b0da99..4eb253e 100644 --- a/src/parsers/xianyu_parser.py +++ b/src/parsers/xianyu_parser.py @@ -145,10 +145,11 @@ class XianyuParser(BaseParser): ret_code = ret[0] if isinstance(ret, list) and ret else "" if "FAIL_BIZ_ITEM_DEL_NOT_FOUND" in ret_code or "FAIL_BIZ_ITEM_QUERY_FAILED" in ret_code: - self.terminal_detail = { + self.terminal_error = { "detail_msg": "该宝贝不存在或已被卖家删除/下架", "error_code": "ITEM_NOT_FOUND", } + self.terminal_detail = self.terminal_error return if "RGV587_ERROR" in ret_code: @@ -320,4 +321,4 @@ class XianyuParser(BaseParser): return self.image_list def get_terminal_detail(self): - return self.terminal_detail + return self.terminal_error or self.terminal_detail diff --git a/templates/components/api_docs.html b/templates/components/api_docs.html index 70ba64b..18c2cb3 100644 --- a/templates/components/api_docs.html +++ b/templates/components/api_docs.html @@ -284,6 +284,16 @@ MEDIA_DELETED_OR_PRIVATE 目标内容为私密作品、仅好友可见的日常/动态,或已被作者删除下架。 + + 400 + ITEM_NOT_FOUND + 闲鱼商品已被卖家下架、删除或不存在(终态不可重试)。 + + + 400 + WECHAT_MP_VERIFY_REQUIRED + 微信公众号页面触发环境异常与安全验证拦截,需稍后重试或切换代理 IP。 + 400 MEDIA_UNDER_REVIEW @@ -521,6 +531,8 @@ // 智能排障建议字典 (In-context Error Help) const ERROR_TIPS = { 'COOKIE_REQUIRED': '💡 排查建议:该平台(如小红书、拼多多、微信)需要有效登录凭证。请在「系统设置 - Cookie管理」中配置有效 Cookie。', + 'ITEM_NOT_FOUND': '💡 排查建议:该闲鱼宝贝已被卖家下架、删除或不存在,属于终态错误,无需重试。', + 'WECHAT_MP_VERIFY_REQUIRED': '💡 排查建议:微信公众号页面触发了环境异常安全验证拦截,请稍后重试或切换代理 IP。', 'INSUFFICIENT_CREDITS': '💡 排查建议:当前账号可用点数已耗尽(剩余 0 点)。请联系系统管理员充值点数或修改额度。', 'EXPIRED_USER': '💡 排查建议:您的账号使用期限已到期。请联系系统超级管理员进行账号延期。', 'ACCOUNT_EXPIRED': '💡 排查建议:您的账号尚未开通或已到期。请联系系统超级管理员进行账号延期。', diff --git a/templates/landing.html b/templates/landing.html index b0afb70..6f60d20 100644 --- a/templates/landing.html +++ b/templates/landing.html @@ -1099,6 +1099,22 @@ tag: '作品状态' }; } + if (code === 'ITEM_NOT_FOUND') { + return { + title: '宝贝已下架或删除', + message: '该闲鱼商品已被卖家删除或下架,无法获取宝贝详情。', + hint: '请确认商品链接有效并在闲鱼客户端内处于在售状态。', + tag: '商品下架' + }; + } + if (code === 'WECHAT_MP_VERIFY_REQUIRED') { + return { + title: '触发安全验证', + message: '微信公众号访问触发了环境异常安全验证,暂时无法直接抓取页面。', + hint: '建议稍候几分钟再试,或更换网络环境后重试。', + tag: '安全验证' + }; + } if (code === 'NO_MEDIA_IN_CONTENT') { return { title: '未包含多媒体资源', diff --git a/tests/test_bilibili_parser.py b/tests/test_bilibili_parser.py index f65d328..10fda79 100644 --- a/tests/test_bilibili_parser.py +++ b/tests/test_bilibili_parser.py @@ -194,6 +194,45 @@ class BilibiliParserTest(unittest.TestCase): timeout=10, ) + def test_parses_aid_url(self): + mock_response = Mock() + mock_response.json.return_value = { + "code": 0, + "data": {"title": "AV视频测试", "bvid": "BV1xx411c7mD", "pages": [{"cid": 1001}]} + } + mock_response.raise_for_status = Mock() + + with patch("requests.Session.get", return_value=mock_response) as mock_get: + parser = BilibiliParser("https://www.bilibili.com/video/av117586188636547") + self.assertEqual(parser.aid, "117586188636547") + self.assertEqual(parser.bvid, "BV1xx411c7mD") + mock_get.assert_called_with( + parser.API_VIEW, + params={"aid": "117586188636547"}, + headers=parser.view_headers, + timeout=10, + ) + + def test_video_deleted_or_invisible_sets_terminal_error(self): + mock_response = Mock() + mock_response.json.return_value = {"code": 62002, "message": "稿件不可见"} + mock_response.raise_for_status = Mock() + + with patch("requests.Session.get", return_value=mock_response): + parser = BilibiliParser("https://www.bilibili.com/video/BV1asTR6FEWu") + self.assertIsNotNone(parser.terminal_error) + self.assertEqual(parser.terminal_error["error_code"], "MEDIA_DELETED_OR_PRIVATE") + + def test_dynamic_invisible_sets_terminal_error(self): + mock_response = Mock() + mock_response.json.return_value = {"code": 4101152, "message": "动态不可见"} + mock_response.raise_for_status = Mock() + + with patch("requests.Session.get", return_value=mock_response): + parser = BilibiliParser("https://t.bilibili.com/1082636825840844823") + self.assertIsNotNone(parser.terminal_error) + self.assertEqual(parser.terminal_error["error_code"], "MEDIA_DELETED_OR_PRIVATE") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_web_fetcher.py b/tests/test_web_fetcher.py index 1bf2ad6..4bb3b03 100644 --- a/tests/test_web_fetcher.py +++ b/tests/test_web_fetcher.py @@ -432,6 +432,24 @@ class WebFetcherTest(unittest.TestCase): self.assertIsNone(WebFetcher.fetch_redirect_url("https://v.douyin.com/start", max_redirects=5)) self.assertEqual(get.call_count, 1) + def test_recognizes_newly_supported_subdomains(self): + self.assertEqual(UrlParser.get_platform("https://s.izuiyou.com/s/23Qeo"), "最右") + self.assertEqual(UrlParser.get_platform("https://adshare.toutiao.com/s/123"), "今日头条") + self.assertEqual(UrlParser.get_platform("https://mapp.api.weibo.cn/fx/abc.html"), "微博") + + def test_preserves_wechat_mp_url_parameters(self): + v_url = "https://mp.weixin.qq.com/s?t=pages/video_detail_new&item_show_type=16&vid=wxv_123&__biz=MzAwMjMwMDAzMQ==&sn=abc" + res_v = UrlParser.extract_video_address(v_url) + self.assertIn("t=pages/video_detail_new", res_v) + self.assertIn("vid=wxv_123", res_v) + self.assertIn("item_show_type=16", res_v) + self.assertIn("__biz=MzAwMjMwMDAzMQ==", res_v) + + art_url = "https://mp.weixin.qq.com/s?__biz=MzAwMjMwMDAzMQ==&mid=123&idx=1&sn=abc&chksm=xyz" + res_a = UrlParser.extract_video_address(art_url) + self.assertIn("chksm=xyz", res_a) + self.assertIn("__biz=MzAwMjMwMDAzMQ==", res_a) + def test_returns_none_on_request_error_or_invalid_input(self): with patch( "utils.web_fetcher.requests.get", diff --git a/tests/test_wechat_mp_parser.py b/tests/test_wechat_mp_parser.py index 58b31ba..257eda0 100644 --- a/tests/test_wechat_mp_parser.py +++ b/tests/test_wechat_mp_parser.py @@ -189,6 +189,28 @@ class WechatMpParserTest(unittest.TestCase): self.assertIsNone(parser.get_real_video_url()) self.assertEqual(parser.get_video_list(), []) + def test_detects_environment_verify_page(self): + response = Mock() + response.raise_for_status.return_value = None + response.text = '

环境异常

当前环境异常,完成验证后即可继续访问。

' + + with patch("requests.Session.get", return_value=response): + parser = WechatMpParser(self.URL) + + self.assertIsNotNone(parser.terminal_error) + self.assertEqual(parser.terminal_error["error_code"], "WECHAT_MP_VERIFY_REQUIRED") + + def test_detects_deleted_article(self): + response = Mock() + response.raise_for_status.return_value = None + response.text = '

该内容已被发布者删除

' + + with patch("requests.Session.get", return_value=response): + parser = WechatMpParser(self.URL) + + self.assertIsNotNone(parser.terminal_error) + self.assertEqual(parser.terminal_error["error_code"], "MEDIA_DELETED_OR_PRIVATE") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_xianyu_parser.py b/tests/test_xianyu_parser.py index 93e7c8b..c462312 100644 --- a/tests/test_xianyu_parser.py +++ b/tests/test_xianyu_parser.py @@ -117,7 +117,9 @@ class XianyuParserTest(unittest.TestCase): self.assertEqual(parser.get_image_list(), []) self.assertIsNotNone(parser.get_terminal_detail()) + self.assertIsNotNone(parser.terminal_error) self.assertEqual(parser.get_terminal_detail().get("error_code"), "ITEM_NOT_FOUND") + self.assertEqual(parser.terminal_error.get("error_code"), "ITEM_NOT_FOUND") def test_invalid_url_handles_gracefully(self): parser = XianyuParser("https://e.tb.cn/invalid") diff --git a/utils/web_fetcher.py b/utils/web_fetcher.py index d76dd65..e49fbdd 100755 --- a/utils/web_fetcher.py +++ b/utils/web_fetcher.py @@ -43,7 +43,7 @@ class WebFetcher: # 绿洲分享页会对通用桌面请求头返回 500,由解析器使用移动端请求头抓取。 # 微博博文可通过 API 直接根据 ID 解析,直接访问网页端常触发访客系统重定向。 domain = UrlParser.get_domain(url) - if domain not in {"t.cn", "b23.tv", "xhslink.cn", "xhslink.com", "hy.fan", "dw4.co", "yspapp.cn"}: + if domain not in {"t.cn", "b23.tv", "xhslink.cn", "xhslink.com", "hy.fan", "dw4.co", "yspapp.cn", "mapp.api.weibo.cn", "s.izuiyou.com", "adshare.toutiao.com"}: if UrlParser.get_platform(url) in {"知乎", "绿洲", "新片场", "夸克AI", "通义千问", "微博", "小云雀AI", "哔哩哔哩", "快影", "微信公众号", "海螺AI", "网易LOFTER", "得物", "央视"}: return UrlParser.extract_video_address(url) try: @@ -380,12 +380,16 @@ class UrlParser: elif platform == "微信公众号": query_params = parse_qs(parsed_url.query) preserved_params = [] - for key in ("__biz", "mid", "idx", "sn", "scene", "subscene", "clicktime", "enterid"): + for key in ( + "__biz", "mid", "idx", "sn", "chksm", "t", "vid", "vidsn", + "voice_id", "item_show_type", "scene", "subscene", "clicktime", + "enterid", "sharer_shareinfo", "sharer_shareinfo_first" + ): value = query_params.get(key, [None])[0] if value is not None: preserved_params.append((key, value)) if preserved_params: - address = f"{address}?{urlencode(preserved_params)}" + address = f"{address}?{urlencode(preserved_params, safe='=/')}" elif platform == "视频号": query_params = parse_qs(parsed_url.query) short_uri = query_params.get('id', [None])[0]