From 8ad3debf9d8031395f576756c61a2cf9c44de401 Mon Sep 17 00:00:00 2001 From: ucmao <294323976@qq.com> Date: Sun, 20 Sep 2026 09:23:11 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A4=AE=E8=A7=86=E9=A2=91?= =?UTF-8?q?=E5=9B=BE=E6=96=87=E8=B5=84=E8=AE=AF/=E6=96=B0=E9=97=BB?= =?UTF-8?q?=E6=96=87=E7=AB=A0=E5=8F=8A=E5=86=85=E5=B5=8C=E5=A4=9A=E5=AA=92?= =?UTF-8?q?=E4=BD=93=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- docs/parsers/yangshipin.md | 19 +++- src/parsers/yangshipin_parser.py | 154 +++++++++++++++++++++++++++++-- tests/live_parser_samples.json | 15 +++ tests/test_yangshipin_parser.py | 92 ++++++++++++++++-- utils/web_fetcher.py | 4 +- 6 files changed, 260 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index fe53839..26361e9 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ Media-Parser是一款专为短视频创作者与开发者打造的**100%原生 | **网易LOFTER** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | | | | [查看](docs/parsers/lofter.md) | | **星绘AI** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | | | | [查看](docs/parsers/butterflyai.md) | | **央视** | ✓ | ✓ | | ✓ | ✓ | | | | | | [查看](docs/parsers/cctv.md) | -| **央视频** | ✓ | ✓ | | ✓ | ✓ | ✓ | | | | | [查看](docs/parsers/yangshipin.md) | +| **央视频** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | ✓ | | [查看](docs/parsers/yangshipin.md) | 注:带 `*` 的项表示该素材保留官方原生水印,未带 `*` 的项均为纯净无水印素材。 > **Cookie凭据配置**:绝大多数平台支持免登录匿名解析;如需配置小红书、视频号等平台凭证,可参考 [`.env.example`](.env.example),详细方法见 [📖 平台Cookie配置指南](docs/cookie-config.md)。 diff --git a/docs/parsers/yangshipin.md b/docs/parsers/yangshipin.md index c2c9565..ece194a 100644 --- a/docs/parsers/yangshipin.md +++ b/docs/parsers/yangshipin.md @@ -1,15 +1,16 @@ # 央视频 (Yangshipin) 逆向解析指南 -本篇详细记录中央广播电视总台 5G 新媒体旗舰平台 **央视频 (Yangshipin)** 的微视频、微短剧与体育资讯作品解析方案。 +本篇详细记录中央广播电视总台 5G 新媒体旗舰平台 **央视频 (Yangshipin)** 的微视频、微短剧、精选栏目与图文新闻资讯作品解析方案。 --- ## 1. 平台特征与支持能力 * **平台标识**:`央视频` -* **支持媒体类型**:原画/蓝光/超清无水印视频源 (`video_url`) / 高分辨率封面 / 完整视频标题 / 创作者与机构作者信息(头像与名称)/ 视频 ID (`vid`) +* **支持媒体类型**:原画/蓝光/超清无水印视频源 (`video_url`) / 文章内嵌多视频 (`video_list`) / 高分辨率封面 (`cover_url`) / 新闻图文高清插图列表 (`image_list`) / 自然段落正文文案 (`description`) / 语音播报音频 (`audio_url`) / 创作者与机构来源信息 / 视频 ID (`vid`) 与 文章 ID (`article_id`) * **常见链接形态**: - * 官方分享短链:`https://www.yspapp.cn/5Sqx`, `https://www.yspapp.cn/d1o` + * 官方分享短链:`https://www.yspapp.cn/5Sqx`, `https://www.yspapp.cn/6j3j`, `https://www.yspapp.cn/d1o` + * 图文资讯 / 新闻文章:`https://m.yangshipin.cn/static/article.html?articleid=e05kmjv3gty29` * 移动端竖屏微短剧/小视频:`https://m.yangshipin.cn/portrait_video?vid=l00005817wl` * 移动端常规横屏视频:`https://m.yangshipin.cn/video?type=0&vid=v000007pgfu&cid=...` * PC/网页端:`https://yangshipin.cn/...` @@ -21,11 +22,19 @@ 1. **Meta Refresh 重定向自动跟随**: * 央视频短链 `yspapp.cn/{code}` 采用 HTML `` 形式执行页面跳转。 - * 解析器在 `WebFetcher` 与 `YangshipinParser` 中双重内置了针对 `` 的无感自动追踪,直接锁定最终目标落地页。 + * 解析器在 `WebFetcher` 与 `YangshipinParser` 中双重内置了针对 `` 的无感自动追踪,直接锁定最终目标落地页(视频页或图文资讯页)。 2. **SSR 双状态机数据提取**: * **横屏常规视频**:页面注入全局变量 `window.__STATE_video__`,解析 `payloads.sharevideo` 结构,获取 `title`、`cover_pic`、`cid`、`vid` 及 `om_info.title`(发布机构或频道名)。 * **竖屏微短剧/短视频**:页面注入全局变量 `window.__STATE_portrait_video__`,解析 `payloads.videoDataList.items[0].videoData`,获取微短剧标题、`shareItem.shareImgUrl`(超清封面)、以及 `detailFollowItem.actorItem` 中的创作者昵称与头像。 -3. **cKey 8.1 签名与 VOD 视频流提取**: +3. **图文资讯与文章 OpenAPI 逆向**: + * 文章页(`static/article.html?articleid={id}`)采用 Vue 单页渲染架构,通过请求官方开放接口 `https://comment.yangshipin.cn/web/article/article_info` 获取结构化数据。 + * 请求携带固定客户端身份校验参数(`vappid: "59306155"`, `vsecret: "b42702bf7309a179d102f3d51b1add2fda0bc7ada64cb801"`, `targetId: 1`),免登录获取完整文章元数据: + * `head`:文章标题、发布来源机构(`source`)、发布时间与高清主封面(`coverImage`)。 + * `content.images`:提取文章包含的所有原始无损超清配图直链。 + * `content.content`:将富文本 HTML 解析清洗为自然段落纯文本(`description`)。 + * `audioCapsule`:提取文章伴随的语音播报音频(`audio_url`)。 + * `content.videos` / ``:提取文章内嵌的视频流并自动触发 VOD 签名流解析。 +4. **cKey 8.1 签名与 VOD 视频流提取**: * 央视频播放器使用 `cKey 8.1` 鉴权算法与 `playvv.yangshipin.cn/playvinfo` 接口交互。 * 解析器通过先探测获取服务端实时时间戳 `curTime`,然后使用内置纯 Python 实现的 AES-128-CBC + PKCS7 算法生成签名 `cKey`,向 `playvinfo` 接口请求高清/蓝光(1080P/720P)视频下载地址与 `fvkey`,拼接构造得到完整可直接播放的 `.mp4` 直链。 diff --git a/src/parsers/yangshipin_parser.py b/src/parsers/yangshipin_parser.py index 9d8a347..8f58e9d 100644 --- a/src/parsers/yangshipin_parser.py +++ b/src/parsers/yangshipin_parser.py @@ -28,10 +28,14 @@ class YangshipinParser(BaseParser): "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", } self.vid = None + self.article_id = None self.title = "" self.cover_url = None self.video_url = None + self.description = None + self.audio_url = None self.image_list = [] + self.video_list = [] self.author = None self._parse() @@ -128,19 +132,125 @@ class YangshipinParser(BaseParser): except Exception as e: logger.warning("Failed to fetch video stream from Yangshipin playvinfo for vid %s: %s", vid, e) + @staticmethod + def _clean_article_html(html_content: str) -> str | None: + """将文章富文本 HTML 解析转换为保留自然段落的纯文本。""" + if not html_content: + return None + soup = BeautifulSoup(html_content, "html.parser") + for tag in soup.find_all(["script", "style"]): + tag.decompose() + for tag in soup.find_all("br"): + tag.replace_with("\n") + for tag in soup.find_all(["p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "section", "article"]): + tag.append("\n") + lines = [] + for line in soup.get_text().replace("\xa0", " ").splitlines(): + clean_line = re.sub(r"[\t\f\v ]+", " ", line).strip() + if clean_line: + lines.append(clean_line) + return "\n\n".join(lines) if lines else None + + def _fetch_article_info(self, article_id: str) -> None: + """通过央视频开放接口获取图文文章/资讯数据及内嵌多媒体。""" + if not article_id: + return + + api_urls = [ + "https://comment.yangshipin.cn/web/article/article_info", + "https://h5access.yangshipin.cn/web/article/article_info", + ] + params = { + "vappid": "59306155", + "vsecret": "b42702bf7309a179d102f3d51b1add2fda0bc7ada64cb801", + "raw": "1", + "targetId": "1", + "id": article_id, + } + headers = { + "User-Agent": USER_AGENT_M[0], + "Referer": "https://m.yangshipin.cn/", + "Accept": "application/json, text/plain, */*", + } + + for api_url in api_urls: + try: + resp = self.session.get(api_url, params=params, headers=headers, timeout=5) + if resp.status_code != 200: + continue + res_json = resp.json() + data = res_json.get("data", {}) + if not data or data.get("errCode") != 0: + continue + + head = data.get("head") or {} + if not self.title: + self.title = (head.get("title") or "").strip() + if not self.cover_url: + self.cover_url = head.get("coverImage") + source = head.get("source") + if source and not self.author: + self.author = { + "name": source, + "avatar": None, + } + + content_obj = data.get("content") or {} + raw_images = content_obj.get("images") or [] + for img in raw_images: + if img and img not in self.image_list: + self.image_list.append(img) + + raw_html = content_obj.get("content") or "" + if raw_html: + self.description = self._clean_article_html(raw_html) + + audio_capsule = data.get("audioCapsule") + if isinstance(audio_capsule, dict) and audio_capsule.get("audioUrl"): + self.audio_url = audio_capsule.get("audioUrl") + elif isinstance(audio_capsule, str) and audio_capsule.startswith("http"): + self.audio_url = audio_capsule + + # 内嵌视频提取 + videos = content_obj.get("videos") or [] + embedded_vids = [] + for v in videos: + if isinstance(v, dict) and v.get("vid"): + embedded_vids.append(v.get("vid")) + elif isinstance(v, str): + embedded_vids.append(v) + + if not embedded_vids and raw_html: + embedded_vids = re.findall(r']*id=[\'"]([^\'"]+)[\'"]', raw_html, re.I) + + if embedded_vids: + if not self.vid: + self.vid = embedded_vids[0] + for vid in embedded_vids: + self._fetch_video_url_by_vid(vid) + if self.video_url and self.video_url not in self.video_list: + self.video_list.append(self.video_url) + + break + except Exception as e: + logger.warning("Failed to fetch article info for Yangshipin article %s from %s: %s", article_id, api_url, e) + def _parse(self): if not self.real_url: return - # 尝试先从 URL 中提取 vid + # 尝试先从 URL 中提取 vid 与 article_id parsed_url = urlparse(self.real_url) qs = parse_qs(parsed_url.query) self.vid = qs.get("vid", [None])[0] + self.article_id = qs.get("articleid", [None])[0] or qs.get("articleId", [None])[0] or qs.get("id", [None])[0] html = self.fetch_html_content() if not html: logger.warning("Failed to fetch HTML content for Yangshipin URL: %s", self.real_url) - if self.vid: + if self.article_id: + self._fetch_article_info(self.article_id) + elif self.vid: self._fetch_video_url_by_vid(self.vid) return @@ -155,12 +265,25 @@ class YangshipinParser(BaseParser): qs = parse_qs(parsed_url.query) if not self.vid or self.vid == "5Sqx": self.vid = qs.get("vid", [None])[0] + if not self.article_id: + self.article_id = qs.get("articleid", [None])[0] or qs.get("articleId", [None])[0] or qs.get("id", [None])[0] html = self.fetch_html_content() if not html: - if self.vid: + if self.article_id: + self._fetch_article_info(self.article_id) + elif self.vid: self._fetch_video_url_by_vid(self.vid) return + # 若目标为文章页,直接调用文章接口解析 + if not self.article_id and "article" in parsed_url.path: + article_match = re.search(r'article(?:id)?=([0-9a-zA-Z_]+)', self.real_url) + if article_match: + self.article_id = article_match.group(1) + + if self.article_id: + self._fetch_article_info(self.article_id) + try: # 1. 尝试从横屏视频 STATE 提取 (__STATE_video__) data_vid = self._extract_state_json(html, "window.__STATE_video__") @@ -168,11 +291,13 @@ class YangshipinParser(BaseParser): sv = data_vid.get("payloads", {}).get("sharevideo", {}) if not self.vid: self.vid = sv.get("vid") - self.title = (sv.get("title") or "").strip() - self.cover_url = sv.get("cover_pic") + if not self.title: + self.title = (sv.get("title") or "").strip() + if not self.cover_url: + self.cover_url = sv.get("cover_pic") om_info = sv.get("om_info", {}) author_name = om_info.get("title") - if author_name: + if author_name and not self.author: self.author = { "name": author_name, "avatar": None, @@ -194,7 +319,7 @@ class YangshipinParser(BaseParser): or vd.get("poster", {}).get("poster", {}).get("imageUrl") ) actor = vd.get("detailFollowItem", {}).get("actorItem", {}) - if actor: + if actor and not self.author: nick_name = actor.get("nickName", {}).get("text") head_url = actor.get("headUrl") if nick_name or head_url: @@ -218,7 +343,7 @@ class YangshipinParser(BaseParser): if og_img and og_img.get("content"): self.cover_url = og_img["content"].strip() - if not self.vid: + if not self.vid and not self.article_id: vid_match = re.search(r'[?&]vid=([0-9a-zA-Z_]+)', self.real_url) or re.search(r'["\']vid["\']\s*:\s*["\']([0-9a-zA-Z_]+)["\']', html) if vid_match: self.vid = vid_match.group(1) @@ -227,13 +352,13 @@ class YangshipinParser(BaseParser): logger.exception("Error parsing Yangshipin page: %s", exc) # 4. 根据提取到的 vid 请求视频播放地址 - if self.vid: + if self.vid and not self.video_url: self._fetch_video_url_by_vid(self.vid) if self.title: self.title = re.sub(r'<[^>]+>', '', self.title).strip() - if not self.video_url and self.cover_url and self.cover_url not in self.image_list: + if not self.video_url and not self.image_list and self.cover_url: self.image_list.append(self.cover_url) def _extract_state_json(self, html, state_key): @@ -258,11 +383,20 @@ class YangshipinParser(BaseParser): def get_title_content(self): return self.title or "" + def get_description(self): + return self.description + def get_cover_photo_url(self): return self.cover_url def get_author_info(self): return self.author + def get_audio_url(self): + return self.audio_url + def get_image_list(self): return self.image_list + + def get_video_list(self): + return self.video_list diff --git a/tests/live_parser_samples.json b/tests/live_parser_samples.json index b92eb44..0a7fc5f 100644 --- a/tests/live_parser_samples.json +++ b/tests/live_parser_samples.json @@ -4571,6 +4571,21 @@ "note": "2026-09-06 已验证: 央视频短链与微视频详情", "pattern": "微视频及短剧详情" }, + { + "platform": "央视频", + "url": "https://www.yspapp.cn/6j3j", + "media_types": [ + "images" + ], + "expected_fields": [ + "cover", + "title", + "images", + "author" + ], + "note": "2026-09-20 已验证: 央视频图文资讯与新闻文章", + "pattern": "图文资讯与文章详情" + }, { "platform": "豆包", "url": "https://www.doubao.com/music-sharing?vid=v02292g10006dadu4kq7dld48hgk0dn0&share_id=55311664453668866", diff --git a/tests/test_yangshipin_parser.py b/tests/test_yangshipin_parser.py index d0e740e..3accbea 100644 --- a/tests/test_yangshipin_parser.py +++ b/tests/test_yangshipin_parser.py @@ -148,14 +148,90 @@ class YangshipinParserTest(unittest.TestCase): self.assertEqual(parser.get_cover_photo_url(), "https://jietufengmian.yangshipin.cn/cover3.jpg") self.assertEqual(parser.get_image_list(), ["https://jietufengmian.yangshipin.cn/cover3.jpg"]) - def test_handles_empty_or_broken_html(self): - with patch.object(YangshipinParser, "fetch_html_content", return_value="404 Not Found"), \ - patch.object(YangshipinParser, "_fetch_video_url_by_vid", return_value=None): - parser = YangshipinParser("https://m.yangshipin.cn/video?vid=empty") - self.assertEqual(parser.get_title_content(), "") - self.assertIsNone(parser.get_cover_photo_url()) - self.assertIsNone(parser.get_author_info()) - self.assertEqual(parser.get_image_list(), []) + def test_parses_article_successfully(self): + meta_html = """ + + + + 央视频 + """ + article_page_html = "文章" + mock_article_resp = MagicMock() + mock_article_resp.status_code = 200 + mock_article_resp.json.return_value = { + "data": { + "errCode": 0, + "head": { + "title": "亚运乒乓球签表:上届冠亚军孙颖莎早田希娜同半区", + "source": "体坛网", + "publishTime": "2026-09-18 21:04", + "coverImage": "https://jietufengmian.yangshipin.cn/cover_article.jpg" + }, + "content": { + "content": "

体坛周报全媒体原创

北京时间9月18日晚上,2026爱知-名古屋亚运会乒乓球项目各项签表正式出炉。

", + "images": [ + "https://jietufengmian.yangshipin.cn/img1.jpg", + "https://jietufengmian.yangshipin.cn/img2.jpg" + ] + } + }, + "ret": 0 + } + + with patch.object(YangshipinParser, "fetch_html_content", side_effect=[meta_html, article_page_html]), \ + patch("requests.Session.get", return_value=mock_article_resp): + parser = YangshipinParser("https://www.yspapp.cn/6j3j") + self.assertEqual(parser.get_title_content(), "亚运乒乓球签表:上届冠亚军孙颖莎早田希娜同半区") + self.assertEqual(parser.get_cover_photo_url(), "https://jietufengmian.yangshipin.cn/cover_article.jpg") + self.assertEqual(parser.get_author_info(), {"name": "体坛网", "avatar": None}) + self.assertEqual( + parser.get_description(), + "体坛周报全媒体原创\n\n北京时间9月18日晚上,2026爱知-名古屋亚运会乒乓球项目各项签表正式出炉。" + ) + self.assertEqual( + parser.get_image_list(), + [ + "https://jietufengmian.yangshipin.cn/img1.jpg", + "https://jietufengmian.yangshipin.cn/img2.jpg" + ] + ) + self.assertIsNone(parser.get_real_video_url()) + self.assertIsNone(parser.get_audio_url()) + + def test_parses_article_with_embedded_video_and_audio(self): + mock_article_resp = MagicMock() + mock_article_resp.status_code = 200 + mock_article_resp.json.return_value = { + "data": { + "errCode": 0, + "head": { + "title": "新闻联播重点快讯", + "source": "央视新闻", + "coverImage": "https://jietufengmian.yangshipin.cn/cover_news.jpg" + }, + "content": { + "content": "

今日新闻要点如下:

", + "images": [] + }, + "audioCapsule": { + "audioUrl": "https://audio.ysp.cctv.cn/news.mp3" + } + }, + "ret": 0 + } + + def mock_fetch_video(self, vid): + self.video_url = f"https://mp4playcloud-cdn.ysp.cctv.cn/{vid}.mp4" + + with patch.object(YangshipinParser, "fetch_html_content", return_value=""), \ + patch("requests.Session.get", return_value=mock_article_resp), \ + patch.object(YangshipinParser, "_fetch_video_url_by_vid", side_effect=mock_fetch_video, autospec=True): + parser = YangshipinParser("https://m.yangshipin.cn/static/article.html?articleid=art123") + self.assertEqual(parser.get_title_content(), "新闻联播重点快讯") + self.assertEqual(parser.get_author_info(), {"name": "央视新闻", "avatar": None}) + self.assertEqual(parser.get_audio_url(), "https://audio.ysp.cctv.cn/news.mp3") + self.assertEqual(parser.get_real_video_url(), "https://mp4playcloud-cdn.ysp.cctv.cn/v00000embedded.mp4") + self.assertEqual(parser.get_video_list(), ["https://mp4playcloud-cdn.ysp.cctv.cn/v00000embedded.mp4"]) if __name__ == "__main__": diff --git a/utils/web_fetcher.py b/utils/web_fetcher.py index 9d09403..81523d1 100755 --- a/utils/web_fetcher.py +++ b/utils/web_fetcher.py @@ -476,7 +476,7 @@ class UrlParser: elif platform == "央视频": query_params = parse_qs(parsed_url.query) preserved_params = [] - for key in ("vid", "cid", "type", "serverFrom"): + for key in ("vid", "cid", "type", "serverFrom", "articleid", "articleId", "id"): value = query_params.get(key, [None])[0] if value is not None: preserved_params.append((key, value)) @@ -586,7 +586,7 @@ class UrlParser: params_item_id = query_params.get('item_id', [None])[0] if params_item_id: return params_item_id - params_article_id = query_params.get('articleId', [None])[0] + params_article_id = query_params.get('articleId', [None])[0] or query_params.get('articleid', [None])[0] if params_article_id: return params_article_id params_guid = query_params.get('guid', [None])[0]