diff --git a/docs/api.md b/docs/api.md index 9910b00..9314689 100644 --- a/docs/api.md +++ b/docs/api.md @@ -123,6 +123,10 @@ curl -X POST "http://localhost:5000/api/v1/parse" \ - 单视频作品:主视频直链放在 `video_url` 中; - 多视频/分页视频/合集作品(如微信公众号多视频、网易云Event多视频):除 `video_url` 返回首个主视频外,`video_list` 会返回全部视频直链数组(首项与 `video_url` 保持一致)。 +5. **试听截断标记 (`is_preview` / `full_duration`)**: + - 仅当平台下发截断的试听片段时出现(如汽水音乐 VIP 会员曲目匿名请求下发 30~60 秒试听); + - `is_preview: true` 表示当前 `audio_url` 为试听片段,`full_duration` 为完整曲目时长(秒)。 + --- ## 4. 全局错误码定义 (Error Codes) diff --git a/docs/parsers/qsmusic.md b/docs/parsers/qsmusic.md index c276fee..ce5b499 100644 --- a/docs/parsers/qsmusic.md +++ b/docs/parsers/qsmusic.md @@ -1,31 +1,40 @@ # 汽水音乐 (Qishui Music) 逆向解析指南 -本篇详细记录字节跳动旗下 **汽水音乐** UGC 视频与背景原声音频的提取方案。 +本篇详细记录字节跳动旗下 **汽水音乐** UGC 视频与歌曲音频的提取方案。 --- ## 1. 平台特征与支持能力 * **平台标识**:`汽水音乐` -* **支持媒体类型**:高清 UGC 视频 (MP4) / 歌曲完整音频 (MP3) / 专辑封面 / 歌曲名称与作者 +* **支持媒体类型**:高清 UGC 视频 (MP4) / 歌曲音频 (audio_mp4) / 专辑封面 / 歌曲名称与作者 / 歌词 * **常见链接形态**: * 分享短链:`https://qishui.douyin.com/s/iX21ep91/` + * 落地页:`https://music.douyin.com/qishui/share/track?track_id=xxx` * **Cookie 依赖**:无需 Cookie。 --- ## 2. 核心逆向流程 -1. **302 跳转获取真实 Item ID**:跟随短链跳转至 `qishui.douyin.com/track/...` 或 `video/...`。 -2. **字节系音乐接口调用**: - * 请求汽水音乐 Web 端曲目详情接口。 -3. **媒体分离提取**: - * `audio_url`:提取无损或高码率音频直链; - * `video_url`:提取 UGC 伴随视频直链。 +1. **短链跳转**:跟随 302 跳到 `music.douyin.com/qishui/share/track?track_id=...`,优先取 `track_id` 查询参数,其次兼容 `/track/`、`/video/` 路径。 +2. **SSR 页面提取**:解析 `_ROUTER_DATA` 里 `loaderData.track_page.audioWithLyricsOption`,取出 `url`(音频直链)、`trackName`、`artistName`、`artistIdStr`、`coverURL`、`duration`、`offsetDuration`。 +3. **播放接口兜底**:页面拿不到有效音频时,依次请求 `https://beta-luna.douyin.com/luna/h5/track_v2`、`.../luna/h5/seo_track`,从 `track_player.video_model.video_list[0]` 取 `main_url`。 +4. **UGC 视频分支**:`videoOptions` 页面(`/share/ugc_video`)走原逻辑,取视频直链。 --- -## 3. 测试与验证 +## 3. VIP 曲目试听片段识别 + +汽水对**会员曲目**(`label_info.only_vip_playable = true`)向未登录请求下发 30~60 秒试听片段: + +* 声明时长:`audioWithLyricsOption.duration` 或 `track.duration`; +* 实际流时长:`audioWithLyricsOption.offsetDuration` 或 `video_model.video_duration`; +* 当流时长显著小于完整时长时,解析器在响应中明确标记 `is_preview: true` 并返回 `full_duration`(秒),防止下游误认为完整音频。 + +--- + +## 4. 测试与验证 * **单元测试**:[tests/test_qsmusic_parser.py](file:///Users/leo/Projects/media-parser/tests/test_qsmusic_parser.py) * **执行命令**:`pytest tests/test_qsmusic_parser.py` diff --git a/src/api/parse.py b/src/api/parse.py index a156222..f615de9 100644 --- a/src/api/parse.py +++ b/src/api/parse.py @@ -248,6 +248,10 @@ def _execute_parse(text, access): data_dict['video_list'] = processed_video_list if content_data.get('subtitles'): data_dict['subtitles'] = content_data['subtitles'] + if content_data.get('is_preview') is True: + data_dict['is_preview'] = True + if content_data.get('full_duration'): + data_dict['full_duration'] = content_data['full_duration'] logger.debug(f'Parse Success for platform {platform}') response, status = make_response(200, '成功', data_dict, True), 200 @@ -293,7 +297,9 @@ def _fetch_with_retry(parser, platform): 'author': safe_execute(getattr(parser, 'get_author_info', None)), 'image_list': safe_execute(getattr(parser, 'get_image_list', None), default=[]), 'audio_url': safe_execute(getattr(parser, 'get_audio_url', None)), - 'subtitles': safe_execute(getattr(parser, 'get_subtitles', None)) + 'subtitles': safe_execute(getattr(parser, 'get_subtitles', None)), + 'is_preview': safe_execute(lambda: parser.is_preview, default=False), + 'full_duration': safe_execute(lambda: parser.full_duration), } if res['video_url'] or res['video_list'] or res['image_list'] or res['audio_url']: return res diff --git a/src/parsers/qsmusic_parser.py b/src/parsers/qsmusic_parser.py index 0af3a40..3025606 100644 --- a/src/parsers/qsmusic_parser.py +++ b/src/parsers/qsmusic_parser.py @@ -30,6 +30,8 @@ class QSMusicParser(BaseParser): self.audio_url = None self.author = {"nickname": "", "author_id": "", "avatar": ""} self.subtitles = None + self.is_preview = False + self.full_duration = None self.track_id = self._extract_track_id(real_url) self._parse_page() @@ -45,7 +47,7 @@ class QSMusicParser(BaseParser): logger.warning("Failed to fetch QSMusic HTML page: %s", exc) if not self.title or not (self.video_url or self.audio_url): - self._parse_seo_payload() + self._parse_track_payload() def _parse_router_data(self, html): match = re.search(r"_ROUTER_DATA\s*=\s*(\{.*?\});", html or "", re.DOTALL) @@ -72,58 +74,103 @@ class QSMusicParser(BaseParser): self.audio_url = self.audio_url or stream_url self.subtitles = self.subtitles or self._extract_subtitles_from_dict(options) + lyrics_opt = page_data.get("audioWithLyricsOption") or {} + if isinstance(lyrics_opt, dict) and lyrics_opt.get("url"): + self.audio_url = self.audio_url or lyrics_opt["url"] + self.title = self.title or lyrics_opt.get("trackName") or "" + self.author["nickname"] = self.author["nickname"] or lyrics_opt.get("artistName") or "" + self.author["author_id"] = self.author["author_id"] or str(lyrics_opt.get("artistIdStr") or "") + self.cover_url = self.cover_url or lyrics_opt.get("coverURL") + self._apply_duration( + lyrics_opt.get("duration"), + lyrics_opt.get("offsetDuration") or lyrics_opt.get("duration"), + ) + track = page_data.get("trackOptions") or page_data.get("track") or page_data.get("seo_track") or {} track = track.get("track") if isinstance(track.get("track"), dict) else track if not isinstance(track, dict) or not track: continue self.title = self.title or track.get("name") or track.get("title") or "" - artist = track.get("artist") or ((track.get("artists") or [{}])[0] or {}).get("user_info") or {} + artist = track.get("artist") or ((track.get("artists") or [{}])[0] or {}).get("user_info") or (track.get("artists") or [{}])[0] or {} if isinstance(artist, dict): self.author["nickname"] = self.author["nickname"] or artist.get("nickname") or artist.get("name") or "" self.author["author_id"] = self.author["author_id"] or str(artist.get("id") or artist.get("user_id") or "") - self.author["avatar"] = self.author["avatar"] or self._first_url(artist.get("avatar_url")) or "" + self.author["avatar"] = self.author["avatar"] or self._image_url(artist.get("avatar_url") or artist.get("url_avatar")) or "" album = track.get("album") or {} - self.cover_url = self.cover_url or self._first_url(album.get("cover_url")) or self._first_url(track.get("cover_url")) + self.cover_url = self.cover_url or self._image_url(album.get("cover_url") or album.get("url_cover")) or self._image_url(track.get("cover_url")) self.audio_url = self.audio_url or track.get("audio_url") or track.get("play_url") or track.get("main_url") - lyrics_opt = page_data.get("audioWithLyricsOption") or track - self.subtitles = self.subtitles or self._extract_subtitles_from_dict(lyrics_opt) + self.subtitles = self.subtitles or self._extract_subtitles_from_dict(lyrics_opt or track) except (json.JSONDecodeError, TypeError) as exc: logger.warning("Error parsing QSMusic router data: %s", exc) - def _parse_seo_payload(self): + def _parse_track_payload(self): + """走 Web 播放接口兜底。""" if not self.track_id: return - try: - response = self.session.get( - "https://beta-luna.douyin.com/luna/h5/seo_track", - params={"track_id": self.track_id, "device_platform": "web"}, - headers={"User-Agent": self.MOBILE_UA, "X-Requested-With": "XMLHttpRequest"}, - timeout=15, - ) - response.raise_for_status() - payload = response.json() - if not isinstance(payload, dict): - return - except Exception as exc: - logger.warning("Failed to fetch QSMusic SEO payload: %s", exc) + headers = { + "User-Agent": self.MOBILE_UA, + "X-Requested-With": "XMLHttpRequest", + "Referer": f"https://music.douyin.com/qishui/share/track?track_id={self.track_id}", + } + + payload = None + for path in ("/luna/h5/track_v2", "/luna/h5/seo_track"): + try: + response = self.session.get( + "https://beta-luna.douyin.com" + path, + params={"track_id": self.track_id, "device_platform": "web"}, + headers=headers, + timeout=15, + ) + response.raise_for_status() + candidate = response.json() + except Exception as exc: + logger.warning("Failed to fetch QSMusic track payload (%s): %s", path, exc) + continue + if isinstance(candidate, dict): + payload = candidate + break + if not payload: return - track = ((payload.get("seo_track") or {}).get("track") or {}) + track = payload.get("track") or ((payload.get("seo_track") or {}).get("track") or {}) + if not isinstance(track, dict): + track = {} self.title = self.title or track.get("name") or "" - artist = ((track.get("artists") or [{}])[0] or {}).get("user_info") or {} - self.author["nickname"] = self.author["nickname"] or artist.get("nickname") or "" - self.author["author_id"] = self.author["author_id"] or str(artist.get("id") or "") - self.author["avatar"] = self.author["avatar"] or self._first_url(artist.get("medium_avatar_url")) or "" + artist = ((track.get("artists") or [{}])[0] or {}).get("user_info") or (track.get("artists") or [{}])[0] or {} + if isinstance(artist, dict): + self.author["nickname"] = self.author["nickname"] or artist.get("nickname") or artist.get("name") or "" + self.author["author_id"] = self.author["author_id"] or str(artist.get("id") or artist.get("user_id") or "") + self.author["avatar"] = self.author["avatar"] or self._image_url(artist.get("medium_avatar_url") or artist.get("url_avatar")) or "" album = track.get("album") or {} - self.cover_url = self.cover_url or self._first_url(album.get("cover_url")) + self.cover_url = self.cover_url or self._image_url(album.get("cover_url") or album.get("url_cover")) + player = payload.get("track_player") or {} - if not self.audio_url and player.get("video_model"): - try: - video = (json.loads(player["video_model"]).get("video_list") or [{}])[0] - self.audio_url = video.get("main_url") or video.get("backup_url") - except (json.JSONDecodeError, TypeError): - pass + model = self._load_video_model(player.get("video_model")) + if model: + video = (model.get("video_list") or [{}])[0] or {} + self.audio_url = self.audio_url or video.get("main_url") or video.get("backup_url") + self._apply_duration((track.get("duration") or 0) / 1000, model.get("video_duration")) + + @staticmethod + def _load_video_model(raw): + if not raw: + return None + try: + model = json.loads(raw) if isinstance(raw, str) else raw + except (json.JSONDecodeError, TypeError): + return None + return model if isinstance(model, dict) else None + + def _apply_duration(self, full_seconds, clip_seconds): + """比对曲目声明时长与实际下发流时长,识别汽水 VIP 试听片段。""" + if not isinstance(full_seconds, (int, float)) or full_seconds <= 0: + return + self.full_duration = round(float(full_seconds), 3) + if not isinstance(clip_seconds, (int, float)) or clip_seconds <= 0: + return + self.is_preview = clip_seconds < full_seconds - 1 @staticmethod def _extract_track_id(url): @@ -149,6 +196,17 @@ class QSMusicParser(BaseParser): return next((value.get(key) for key in ("url", "origin_url", "large_url") if value.get(key)), None) return None + @classmethod + def _image_url(cls, value): + """兼容汽水图片载荷:直链、URL 列表,以及 urls[0] + uri 的模板形式。""" + if isinstance(value, dict) and value.get("urls"): + base = cls._first_url(value.get("urls")) + uri = value.get("uri") + if base and uri: + return base.rstrip("/") + "/" + str(uri).lstrip("/") + return base + return cls._first_url(value) + def _extract_subtitles_from_dict(self, data): """从字典数据中提炼字幕/歌词列表。""" if not isinstance(data, dict): diff --git a/tests/test_qsmusic_parser.py b/tests/test_qsmusic_parser.py index b8ac472..7d5ed68 100644 --- a/tests/test_qsmusic_parser.py +++ b/tests/test_qsmusic_parser.py @@ -80,6 +80,76 @@ class QSMusicParserTest(unittest.TestCase): ] ) + def test_flags_vip_preview(self): + router_data = {"loaderData": {"track_page": {"audioWithLyricsOption": { + "url": "https://audio.example.com/clip.mp4", + "trackName": "会员歌曲", + "artistName": "会员歌手", + "artistIdStr": "10086", + "coverURL": "https://image.example.com/cover.jpg", + "duration": 264.333, + "offsetDuration": 30.001, + }}}} + response = Mock(url="https://music.douyin.com/qishui/share/track?track_id=6845") + response.text = f"" + + with patch("requests.Session.get", return_value=response): + parser = QSMusicParser("https://qishui.douyin.com/s/code/") + + self.assertTrue(parser.is_preview) + self.assertEqual(parser.full_duration, 264.333) + self.assertEqual(parser.get_title_content(), "会员歌曲") + self.assertEqual(parser.get_author_info()["author_id"], "10086") + self.assertEqual(parser.get_cover_photo_url(), "https://image.example.com/cover.jpg") + + def test_free_track_is_not_flagged_as_preview(self): + router_data = {"loaderData": {"track_page": {"audioWithLyricsOption": { + "url": "https://audio.example.com/full.mp4", + "trackName": "免费歌曲", + "duration": 388.728, + "previewStart": 0, + "previewEnd": 388.728, + }}}} + response = Mock(url="https://music.douyin.com/qishui/share/track?track_id=7225") + response.text = f"" + + with patch("requests.Session.get", return_value=response): + parser = QSMusicParser("https://qishui.douyin.com/s/code/") + + self.assertFalse(parser.is_preview) + self.assertEqual(parser.full_duration, 388.728) + self.assertEqual(parser.get_audio_url(), "https://audio.example.com/full.mp4") + + def test_flags_preview_from_player_payload(self): + page = Mock(url="https://music.douyin.com/qishui/share/track?track_id=6845376862848321538") + page.text = "" + player = Mock() + player.raise_for_status = Mock() + player.json.return_value = { + "track": { + "name": "Just to Be in Love", + "duration": 264333, + "artists": [{"id": 6769796358737532929, "name": "Alex Rasov", "url_avatar": {"urls": ["https://img.example.com"], "uri": "avatar.jpg"}}], + "album": {"url_cover": {"urls": ["https://img.example.com"], "uri": "cover.jpg"}}, + }, + "track_player": {"video_model": json.dumps({ + "video_duration": 30.001, + "video_list": [{"main_url": "https://audio.example.com/clip.mp4"}], + })}, + } + + with patch("requests.Session.get", side_effect=[page, player]): + parser = QSMusicParser( + "https://music.douyin.com/qishui/share/track?track_id=6845376862848321538" + ) + + self.assertTrue(parser.is_preview) + self.assertEqual(parser.full_duration, 264.333) + self.assertEqual(parser.get_audio_url(), "https://audio.example.com/clip.mp4") + self.assertEqual(parser.get_author_info()["author_id"], "6769796358737532929") + self.assertEqual(parser.get_author_info()["avatar"], "https://img.example.com/avatar.jpg") + self.assertEqual(parser.get_cover_photo_url(), "https://img.example.com/cover.jpg") + if __name__ == "__main__": unittest.main()