From fd3ac0cd4cae658f3b8cb1e19e66f7265df83b3c Mon Sep 17 00:00:00 2001 From: ucmao <294323976@qq.com> Date: Tue, 8 Sep 2026 16:34:52 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B0=8F=E7=BA=A2=E4=B9=A6Ap?= =?UTF-8?q?p=E7=9F=AD=E9=93=BE=E4=B8=8ELivePhoto=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=EF=BC=8C=E4=B8=8EUI=E4=BD=93=E9=AA=8C=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app.py | 2 + docs/index.md | 2 +- docs/parsers/xiaohongshu.md | 67 ++++++++---- src/auth.py | 1 + src/parsers/xiaohongshu_parser.py | 169 ++++++++++++++++++++++-------- static/css/console.css | 26 ++--- static/icons.svg | 1 + templates/admin/dashboard.html | 28 ++--- templates/auth/login.html | 8 +- templates/landing.html | 92 +++++++++++++--- templates/portal/dashboard.html | 24 +++-- tests/live_parser_samples.json | 19 +++- tests/test_management.py | 23 ++++ tests/test_xiaohongshu_parser.py | 32 ++++++ utils/web_fetcher.py | 10 +- 15 files changed, 383 insertions(+), 121 deletions(-) create mode 100644 tests/test_xiaohongshu_parser.py diff --git a/app.py b/app.py index e47bc9e..0786bb6 100755 --- a/app.py +++ b/app.py @@ -1,6 +1,7 @@ import os import secrets import fcntl +from datetime import timedelta from flask import Flask from src.api.parse import bp as api_bp from src.web.views import bp as web_bp @@ -43,6 +44,7 @@ def create_app(config=None): 'DATABASE_PATH', os.path.join(app.instance_path, 'media_parser.db') ) app.config['MAX_CONTENT_LENGTH'] = 32 * 1024 + app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=30) app.config['JSON_SORT_KEYS'] = False app.config['TRUST_PROXY_HEADERS'] = os.getenv( 'TRUST_PROXY_HEADERS', '' diff --git a/docs/index.md b/docs/index.md index aa95141..98b5cd5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -30,7 +30,7 @@ | 序号 | 平台名称 | 支持媒体类型 | 无水印直链 | 配置门槛 (Cookie 依赖) | 逆向提取模式 | 对应指南 | | :--- | :--- | :--- | :---: | :---: | :--- | :--- | | 1 | **抖音** | 视频 / 图文 / LivePhoto / 音频 / 字幕 / 合集 | ✅ 支持 | 🟢 免配置 | a_bogus 签名 + 动态 ttwid + SSR 容灾兜底 | [查看指南](parsers/douyin.md) | -| 2 | **小红书** | 图文 / LivePhoto / 视频 | ✅ 支持 | ⚠️ 部分需 Cookie | SSR HTML 注入数据提取 | [查看指南](parsers/xiaohongshu.md) | +| 2 | **小红书** | 图文 / LivePhoto / 视频 | ✅ 支持 | 🟢 免配置 | SSR HTML 状态注入 + 双 UA 自适应回退 | [查看指南](parsers/xiaohongshu.md) | | 3 | **快手** | 视频 / 图文 / 音频 | ✅ 支持 | 🟢 免配置 (内置游客凭证) | GraphQL / H5 双端多路由 Fallback | [查看指南](parsers/kuaishou.md) | | 4 | **哔哩哔哩** | 视频 (MP4) / 音频分流 | ✅ 支持 | 🟢 免配置 | 官方 View + PlayURL API | [查看指南](parsers/bilibili.md) | | 5 | **豆包 AI** | AI 视频生成直链 | ✅ 支持 | 🔐 需 `DOUBAO_COOKIE` | Web Session 维持 + 任务轮询 | [查看指南](parsers/doubao.md) | diff --git a/docs/parsers/xiaohongshu.md b/docs/parsers/xiaohongshu.md index a8809ba..c9109a1 100644 --- a/docs/parsers/xiaohongshu.md +++ b/docs/parsers/xiaohongshu.md @@ -12,35 +12,47 @@ * 实况照片 (LivePhoto 动态视频流) * 视频笔记 (H264 Master MP4 流) * 笔记正文与作者信息 + * 笔记封面 (自动从 cover 或首图提取) * **常见链接形态**: - * 短链接:`http://xhslink.com/a/xxxx` + * 短链接:`http://xhslink.com/o/xxxx` 或 `http://xhslink.com/a/xxxx` * 网页发现长链:`https://www.xiaohongshu.com/discovery/item/6a8fbccc000000002a03825a` * 网页探索长链:`https://www.xiaohongshu.com/explore/6a8fbccc000000002a03825a` -* **Cookie 依赖**:公开普通笔记无需 Cookie;部分强受限笔记或高频调用建议在 `.env` 中提供 Cookie。 +* **Cookie 依赖**:🟢 公开普通笔记及移动分享均免配置 Cookie。 --- -## 2. 核心逆向方案 (SSR HTML 状态注入) +## 2. 核心逆向方案 (SSR HTML 状态注入 + 双 UA 自适应回退) -小红书 Web 页面采用 Nuxt / Vue SSR 服务端渲染。页面加载时,首屏所有笔记详情、高清媒体流及作者信息均已经内嵌在 HTML 的 `window.__INITIAL_STATE__` 变量中。 +小红书 Web/H5 页面采用 Nuxt / Vue SSR 服务端渲染。页面加载时,首屏所有笔记详情、高清媒体流及作者信息均已经内嵌在 HTML 的 `window.__INITIAL_STATE__` 变量中。 -### 2.1 请求配置 -* **请求方式**:`GET` 目标长链接 -* **必备请求头**: - * `User-Agent`: 现代 PC 浏览器 UA - * `Referer`: `https://www.xiaohongshu.com/` +### 2.1 请求配置与 URL 参数保留 +* **请求方式**:`GET` 目标解析长链接 +* **关键查询参数保留**:短链 302 重定向后必须保留 `xsec_token`、`xsec_source`、`source`、`xhsshare`、`app_platform` 等核心参数(在 [utils/web_fetcher.py](file:///Users/leo/Projects/media-parser/utils/web_fetcher.py) 的 `UrlParser.extract_video_address` 中维护),缺少来源签名参数会导致服务端返回 404 或拦截。 +* **双 UA 自适应机制**: + * **App 端分享短链**(含 `app_platform=android` 等):优先使用 **Mobile User-Agent** 访问以获取移动端 H5 页面。 + * **PC 桌面端长链**(含 `source=webshare` 等):优先使用 **PC User-Agent** 访问。 + * **自动回退**:若首选 UA 遭遇 302/404 或拦截,解析器自动切换为备用 UA 再次发起请求。 -### 2.2 正则状态提取 +### 2.2 正则状态提取与结构兼容 ```python import re, json -pattern = re.compile(r'window\.__INITIAL_STATE__\s*=\s*(\{.*\})', re.DOTALL) -json_str = BaseParser.parse_html_data(html_content, pattern) +# 1. 正则提取并清洗 :undefined 为 :null +pattern = re.compile(r'window\.__INITIAL_STATE__\s*=\s*(\{.*?\})', re.DOTALL) +match = pattern.search(html_content) or re.search(r'window\.__INITIAL_STATE__\s*=\s*(\{.*\})', html_content, re.DOTALL) -if json_str: +if match: + json_str = re.sub(r':\s*undefined\b', ':null', match.group(1)) full_data = json.loads(json_str) + + # 2. 兼容 PC 桌面端数据结构 first_note_id = full_data.get('note', {}).get('firstNoteId') - note_data = full_data['note']['noteDetailMap'].get(first_note_id, {}).get('note', {}) + if first_note_id: + note_data = full_data['note']['noteDetailMap'].get(first_note_id, {}).get('note', {}) + + # 3. 兼容移动端 H5 数据结构 + if not note_data: + note_data = full_data.get('noteData', {}).get('data', {}).get('noteData', {}) ``` --- @@ -48,23 +60,30 @@ if json_str: ## 3. 字段提取与 LivePhoto 解析 ### 3.1 视频笔记提取 -* 视频直链位于:`note_data.video.media.stream.h264[0].masterUrl` +* 视频直链位于:`note_data.video.media.stream.h264[0].masterUrl` 或 `note_data.video.stream.h264[0].masterUrl`。 * 字符转义处理:将 Unicode 斜杠 `\\u002F` 替换为标准 `/`。 ### 3.2 图文与 LivePhoto 提取 * 遍历 `note_data.imageList`: - * **原图直链**:提取 `urlDefault` 字段。 - * **实况动图 (LivePhoto)**:检查 `image.get('livePhoto') == True`,若为真,则提取 `image.stream.h264[0].masterUrl` 作为实况动图关联的视频流。 + * **原图直链**:按优先级提取 `image.urlDefault` -> `image.url` -> `image.infoList[-1].url`。 + * **实况动图 (LivePhoto)**:检查 `image.get('livePhoto') == True`,若为真,则提取 `image.stream.h264[0].masterUrl` 作为实况动图关联的动态视频流。 + +### 3.3 封面与作者提取 +* **封面**:优先读取 `note_data.cover.urlDefault` / `url`,若为空则以 `imageList[0]` 兜底。 +* **作者**:兼容读取 `user.nickname` / `user.nickName` 及 `user.userId`。 --- ## 4. 常见踩坑与反爬对抗 (Gotchas) -1. **重定向至登录页 (`/login`) 或 404**: - * *原因*:小红书对单一 IP 频次较高的请求,或者未带参数的特定私密笔记会强制要求登录。 - * *解法*:在 [src/api/parse.py](file:///Users/leo/Projects/media-parser/src/api/parse.py) 中针对小红书内置了 3 次自适应重试机制(`_fetch_with_retry`);若持续失败,则友好提示用户配置 Cookie。 -2. **字符转义问题**: - * JSON 字符串中经常包含 JSON 编码的斜杠 `\u002F`,必须统一格式化清洗。 +1. **App 分享短链在 PC UA 下重定向到 `/explore?undertake_note_error=该内容暂时无法查看` (错误码 -510001)**: + * *原因*:小红书服务端对 App 端生成的分享链接(带有 `app_platform=android&xsec_source=app_share` 等参数)进行设备特征校验,PC UA 访问会被判定为非移动环境并重定向到探索页。 + * *解法*:在 [src/parsers/xiaohongshu_parser.py](file:///Users/leo/Projects/media-parser/src/parsers/xiaohongshu_parser.py) 中启用双 UA 自动降级重试机制,对 App 分享链接自动采用移动端 User-Agent 请求,无需 Cookie 即可获取完整 H5 状态树。 +2. **缺少 `xsec_source` 导致 H5 页面返回 404/安全校验**: + * *原因*:在 URL 清洗阶段若仅保留 `xsec_token` 而丢弃了 `xsec_source`,移动端 H5 接口会因缺少渠道来源标识而拒绝渲染笔记数据。 + * *解法*:在 [utils/web_fetcher.py](file:///Users/leo/Projects/media-parser/utils/web_fetcher.py) 的 `UrlParser.extract_video_address` 中完整保留 `xsec_token`, `xsec_source`, `source`, `xhsshare`, `app_platform` 等核心参数。 +3. **字符转义与 `undefined` 序列化**: + * HTML 内嵌的 JS 对象常包含 `:undefined`,直接 `json.loads` 会抛出异常,需统一正则替换为 `:null`。 --- @@ -74,3 +93,7 @@ if json_str: ```bash python tests/manual_verify_parsers.py --platform 小红书 ``` +* **单元测试**: + ```bash + python -m unittest tests/test_xiaohongshu_parser.py + ``` diff --git a/src/auth.py b/src/auth.py index 1e166a1..c82825f 100644 --- a/src/auth.py +++ b/src/auth.py @@ -202,6 +202,7 @@ def login(): else: session.clear() session["user_id"] = user["id"] + session.permanent = request.form.get("remember_me") in {"1", "on", "true"} target = request.args.get("next", "") if not target.startswith("/") or target.startswith("//"): target = url_for("admin.dashboard") if user["role"] == "admin" else url_for("portal.dashboard") diff --git a/src/parsers/xiaohongshu_parser.py b/src/parsers/xiaohongshu_parser.py index 55fdf70..a25b8e5 100644 --- a/src/parsers/xiaohongshu_parser.py +++ b/src/parsers/xiaohongshu_parser.py @@ -3,7 +3,9 @@ import re import json from src.parsers.base_parser import BaseParser from configs.logging_config import get_logger +from configs.general_constants import USER_AGENT_PC, USER_AGENT_M import requests + logger = get_logger(__name__) @@ -11,45 +13,80 @@ logger = get_logger(__name__) class XiaohongshuParser(BaseParser): def __init__(self, real_url): super().__init__(real_url) - self.headers = { - "content-type": "application/json; charset=UTF-8", - 'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", - 'referer': 'https://www.xiaohongshu.com/' - } - - # 获取 HTML 并解析 JSON 状态 - html_content = self.fetch_html_content() - pattern = re.compile(r'window\.__INITIAL_STATE__\s*=\s*(\{.*\})', re.DOTALL) - json_str = BaseParser.parse_html_data(html_content, pattern) - - # 初始化数据容器 self.note_data = {} - try: - if json_str: - full_data = json.loads(json_str) - first_note_id = full_data.get('note', {}).get('firstNoteId') - if first_note_id: - self.note_data = full_data['note']['noteDetailMap'].get(first_note_id, {}).get('note', {}) - except (json.JSONDecodeError, KeyError) as e: - logger.error(f"初始化解析数据失败: {e}") + self.fetch_and_parse() - def fetch_html_content(self): - try: - resp = self.session.get(self.real_url, headers=self.headers, timeout=5) - resp.raise_for_status() - if "xiaohongshu.com/login" in resp.url: - logger.error("小红书解析提示: 请求被重定向到了登录页面。") - elif "xiaohongshu.com/404" in resp.url: - logger.error("小红书解析提示: 遭遇安全拦截或页面未找到(404)。") - self.html_content = resp.text - return self.html_content - except requests.RequestException as e: - logger.error(f"Failed to get the page: {self.real_url}, Error: {e}") + def fetch_and_parse(self): + pc_ua = USER_AGENT_PC[0] if USER_AGENT_PC else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + mobile_ua = USER_AGENT_M[0] if USER_AGENT_M else "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1" + + # 根据链接特征选择优先尝试的 UA + if "app_platform" in self.real_url or "app_share" in self.real_url or "xhslink" in self.real_url: + candidate_uas = [mobile_ua, pc_ua] + else: + candidate_uas = [pc_ua, mobile_ua] + + for ua in candidate_uas: + headers = { + "User-Agent": ua, + "Referer": "https://www.xiaohongshu.com/", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + } + try: + resp = self.session.get(self.real_url, headers=headers, timeout=6) + if resp.status_code != 200: + continue + if "xiaohongshu.com/login" in resp.url or "xiaohongshu.com/404" in resp.url or "undertake_note_error" in resp.url: + continue + + self.html_content = resp.text + note = self._extract_note_from_html(self.html_content) + if note: + self.note_data = note + return + except requests.RequestException as e: + logger.warning(f"小红书请求失败 (UA: {ua[:30]}...): {e}") + continue + except Exception as e: + logger.error(f"解析小红书页面异常: {e}") + continue + + if not self.note_data: + logger.error(f"未能解析出小红书笔记数据: {self.real_url}") + + def _extract_note_from_html(self, html_content): + if not html_content: return None - except Exception as e: - logger.error(f"An unexpected error occurred while fetching {self.real_url}: {e}") + pattern = re.compile(r'window\.__INITIAL_STATE__\s*=\s*(\{.*?\})', re.DOTALL) + match = pattern.search(html_content) + if not match: + pattern = re.compile(r'window\.__INITIAL_STATE__\s*=\s*(\{.*\})', re.DOTALL) + match = pattern.search(html_content) + if not match: return None + json_str = match.group(1) + json_str = re.sub(r':\s*undefined\b', ':null', json_str) + try: + full_data = json.loads(json_str) + except json.JSONDecodeError as e: + logger.error(f"小红书 JSON 解析失败: {e}") + return None + + # 1. PC 端结构: full_data['note']['noteDetailMap'][first_note_id]['note'] + first_note_id = full_data.get('note', {}).get('firstNoteId') + if first_note_id: + note = full_data.get('note', {}).get('noteDetailMap', {}).get(first_note_id, {}).get('note', {}) + if note: + return note + + # 2. 移动端 H5 结构: full_data['noteData']['data']['noteData'] + note = full_data.get('noteData', {}).get('data', {}).get('noteData', {}) + if note: + return note + + # 3. 兜底回退结构 + return full_data.get('noteData', {}).get('note', {}) or full_data.get('note', {}).get('note', {}) or None def get_author_info(self): """ @@ -57,17 +94,30 @@ class XiaohongshuParser(BaseParser): """ user = self.note_data.get('user', {}) return { - 'nickname': user.get('nickname', ''), - 'author_id': user.get('userId', ''), + 'nickname': user.get('nickname') or user.get('nickName', ''), + 'author_id': user.get('userId') or user.get('author_id') or user.get('id', ''), 'avatar': user.get('avatar', '') } def get_real_video_url(self): try: video_info = self.note_data.get('video', {}) - master_url = video_info['media']['stream']['h264'][0].get('masterUrl', '') - return master_url.replace("\\u002F", "/") if master_url else None - except (KeyError, IndexError): + if not isinstance(video_info, dict): + return None + stream = video_info.get('media', {}).get('stream', {}) or video_info.get('stream', {}) + h264_data = stream.get('h264', []) + if h264_data and isinstance(h264_data, list): + master_url = h264_data[0].get('masterUrl', '') + if master_url: + return master_url.replace("\\u002F", "/") + for codec in ('h265', 'av1'): + codec_data = stream.get(codec, []) + if codec_data and isinstance(codec_data, list): + master_url = codec_data[0].get('masterUrl', '') + if master_url: + return master_url.replace("\\u002F", "/") + return None + except (KeyError, IndexError, AttributeError): return None def get_title_content(self): @@ -77,20 +127,55 @@ class XiaohongshuParser(BaseParser): return self.note_data.get('desc', '') or None def get_cover_photo_url(self): + cover = self.note_data.get('cover') + if isinstance(cover, dict): + url = cover.get('urlDefault') or cover.get('url') + if url: + return url.replace("\\u002F", "/") + info_list = cover.get('infoList', []) + if info_list and isinstance(info_list, list): + url = info_list[0].get('url') + if url: + return url.replace("\\u002F", "/") + elif isinstance(cover, str) and cover: + return cover.replace("\\u002F", "/") + + # 兜底从 imageList 中获取首图作为封面 + image_list = self.note_data.get('imageList', []) + if image_list and isinstance(image_list, list): + first_img = image_list[0] + if isinstance(first_img, dict): + url = first_img.get('urlDefault') or first_img.get('url') + if url: + return url.replace("\\u002F", "/") + info_list = first_img.get('infoList', []) + if info_list and isinstance(info_list, list): + url = info_list[0].get('url') + if url: + return url.replace("\\u002F", "/") return None def get_image_list(self): image_url_list = [] image_list = self.note_data.get('imageList', []) + if not isinstance(image_list, list): + return image_url_list + for image in image_list: - url = image.get('urlDefault', '') + if not isinstance(image, dict): + continue + url = image.get('urlDefault') or image.get('url') + if not url: + info_list = image.get('infoList', []) + if info_list and isinstance(info_list, list): + url = info_list[-1].get('url') or info_list[0].get('url', '') if url: img_data = url.replace("\\u002F", "/") - # 检查是否有livePhoto + # 检查是否有 livePhoto if image.get('livePhoto', False): stream = image.get('stream', {}) h264_data = stream.get('h264', []) - if h264_data: + if h264_data and isinstance(h264_data, list): master_url = h264_data[0].get('masterUrl', '') if master_url: img_data = { diff --git a/static/css/console.css b/static/css/console.css index 57f77f3..047329c 100644 --- a/static/css/console.css +++ b/static/css/console.css @@ -1079,16 +1079,16 @@ table input[type="number"] { .pie-wrapper { display: flex; flex-direction: column; - gap: 16px; + gap: 12px; align-items: center; justify-content: center; - padding: 10px 0; + padding: 6px 0; } .pie-svg-box { position: relative; - width: 180px; - height: 180px; + width: 210px; + height: 210px; flex-shrink: 0; } @@ -1099,7 +1099,7 @@ table input[type="number"] { transform: translate(-50%, -50%); text-align: center; pointer-events: none; - width: 120px; + width: 130px; display: flex; flex-direction: column; align-items: center; @@ -1108,7 +1108,7 @@ table input[type="number"] { } .pie-center-lbl { - font-size: 12px; + font-size: 12.5px; color: var(--ink-muted); white-space: nowrap; line-height: 1; @@ -1116,7 +1116,7 @@ table input[type="number"] { } .pie-center-val { - font-size: 21px; + font-size: 24px; font-weight: 800; color: var(--ink); line-height: 1.15; @@ -1145,12 +1145,12 @@ table input[type="number"] { flex-wrap: wrap; justify-content: center; align-items: center; - gap: 8px 16px; - margin-top: 14px; + gap: 5px 10px; + margin-top: 12px; width: 100%; max-height: 220px; overflow-y: auto; - padding: 4px 8px; + padding: 2px 4px; } .pie-pill-grid::-webkit-scrollbar { @@ -1164,13 +1164,13 @@ table input[type="number"] { .pie-pill-item { display: inline-flex; align-items: center; - gap: 6px; - font-size: 12.5px; + gap: 5px; + font-size: 12px; color: var(--ink-secondary); cursor: pointer; background: transparent; border: none; - padding: 3px 6px; + padding: 2px 5px; border-radius: var(--radius-sm); transition: all 0.15s ease; user-select: none; diff --git a/static/icons.svg b/static/icons.svg index 09742dc..0b669e0 100644 --- a/static/icons.svg +++ b/static/icons.svg @@ -20,4 +20,5 @@ + diff --git a/templates/admin/dashboard.html b/templates/admin/dashboard.html index 1996b60..baf1815 100644 --- a/templates/admin/dashboard.html +++ b/templates/admin/dashboard.html @@ -701,31 +701,35 @@ - - - + + + - - - + + + {% for log in logs %} - - - + + + - - - + + {% else %} diff --git a/templates/auth/login.html b/templates/auth/login.html index b165c58..c4a7e3e 100644 --- a/templates/auth/login.html +++ b/templates/auth/login.html @@ -32,7 +32,13 @@ -
+
+ +
+
diff --git a/templates/landing.html b/templates/landing.html index 14031a3..96f76c1 100644 --- a/templates/landing.html +++ b/templates/landing.html @@ -106,20 +106,39 @@

{{ icon('video', 'w-4 h-4 text-slate-500') }} 视频文件预览 (0)

+ {{ icon('link', 'w-3.5 h-3.5') }}新窗口打开 +
-
+
+ - -
-
-
- {{ icon('link', 'w-3.5 h-3.5') }}打开浏览器播放 - + + + + + 点击在新窗口播放 {{ icon('link', 'w-3 h-3') }} + +
时间客户 / Key Prefix平台时间客户 / Key Prefix平台 脱敏 URL状态耗时错误码状态耗时错误码
{{ format_log_time(log['created_at']) }}{{ log['username'] or '在线体验' }} {{ log['key_prefix'] or '' }}{{ log['platform'] or '-' }}{{ format_log_time(log['created_at']) }}{{ log['username'] or '在线体验' }} {{ log['key_prefix'] or '' }}{{ log['platform'] or '-' }} {% if log['input_url'] %} {{ log['input_url'] }} {% else %}-{% endif %} {{ log['status_code'] + {{ log['status_code'] }}{{ log['duration_ms'] }} ms{{ log['error_code'] or '-' }}{{ log['duration_ms'] }} ms + {% if log['error_code'] %} + {{ log['error_code'] }} + {% else %}-{% endif %} +
- - + + - - - + + + {% for log in logs %} - - + + - - - + + {% else %} diff --git a/tests/live_parser_samples.json b/tests/live_parser_samples.json index 6eee3c0..8429f21 100644 --- a/tests/live_parser_samples.json +++ b/tests/live_parser_samples.json @@ -1,7 +1,7 @@ { "schema_version": 2, "description": "50 平台多形态全景自动化回归测试用例库 (涵盖短链/长链/图集/单视频/AI生成等)", - "total_cases": 308, + "total_cases": 310, "cases": [ { "platform": "网易云音乐", @@ -413,6 +413,23 @@ "note": "2026-08-28 已验证", "pattern": "基础标准样例" }, + { + "platform": "小红书", + "url": "http://xhslink.com/o/5EY4JTCKXTr", + "pattern": "App 分享短链与 LivePhoto 实况图文", + "media_types": [ + "images", + "live_media" + ], + "expected_fields": [ + "images", + "live_media", + "cover", + "title", + "author" + ], + "note": "2026-09 已验证: App 端分享短链,含 6 张高清图集与 LivePhoto 实况动图流" + }, { "platform": "视频号", "url": "https://weixin.qq.com/sph/AzGrUgqzFv", diff --git a/tests/test_management.py b/tests/test_management.py index f2544f8..0158c65 100644 --- a/tests/test_management.py +++ b/tests/test_management.py @@ -79,6 +79,29 @@ class ManagementTest(unittest.TestCase): self.assertEqual(response.status_code, 302) self.assertTrue(response.location.endswith("/admin")) + def test_login_remember_me_controls_session_permanence(self): + self.client.post( + "/auth/setup", + data={"csrf_token": self.csrf(), "username": "admin", "password": "password123", "confirm_password": "password123"}, + ) + # Without remember_me + with self.client: + self.client.post( + "/auth/login", + data={"csrf_token": self.csrf(), "username": "admin", "password": "password123"}, + ) + from flask import session + self.assertFalse(session.permanent) + + # With remember_me + with self.client: + self.client.post( + "/auth/login", + data={"csrf_token": self.csrf(), "username": "admin", "password": "password123", "remember_me": "1"}, + ) + from flask import session + self.assertTrue(session.permanent) + def test_setup_rejects_missing_csrf_token(self): response = self.client.post( "/auth/setup", diff --git a/tests/test_xiaohongshu_parser.py b/tests/test_xiaohongshu_parser.py new file mode 100644 index 0000000..de776fa --- /dev/null +++ b/tests/test_xiaohongshu_parser.py @@ -0,0 +1,32 @@ +import unittest +from src.parsers.xiaohongshu_parser import XiaohongshuParser + + +class XiaohongshuParserTest(unittest.TestCase): + def test_extract_note_from_pc_initial_state(self): + html = '' + parser = XiaohongshuParser.__new__(XiaohongshuParser) + note = parser._extract_note_from_html(html) + self.assertIsNotNone(note) + self.assertEqual(note.get("title"), "PC标题") + self.assertEqual(note.get("desc"), "PC描述") + parser.note_data = note + self.assertEqual(parser.get_author_info()["nickname"], "PC用户") + self.assertEqual(parser.get_cover_photo_url(), "http://img1.jpg") + + def test_extract_note_from_mobile_initial_state(self): + html = '' + parser = XiaohongshuParser.__new__(XiaohongshuParser) + note = parser._extract_note_from_html(html) + self.assertIsNotNone(note) + self.assertEqual(note.get("title"), "移动端标题") + parser.note_data = note + self.assertEqual(parser.get_author_info()["nickname"], "移动端用户") + self.assertEqual(parser.get_cover_photo_url(), "http://m_img1.jpg") + images = parser.get_image_list() + self.assertEqual(len(images), 1) + self.assertEqual(images[0]["live_photo_url"], "http://live.mp4") + + +if __name__ == "__main__": + unittest.main() diff --git a/utils/web_fetcher.py b/utils/web_fetcher.py index a1f2eb5..23cb35e 100755 --- a/utils/web_fetcher.py +++ b/utils/web_fetcher.py @@ -172,9 +172,13 @@ class UrlParser: address = f"{address}?id={vid}" elif platform == "小红书": query_params = parse_qs(parsed_url.query) - xsec_token = query_params.get('xsec_token', [None])[0] # 使用 get 方法避免 KeyError - if xsec_token: - address = f"{address}?xsec_token={xsec_token}" + preserved_params = [] + for key in ('xsec_token', 'xsec_source', 'source', 'xhsshare', 'app_platform'): + value = query_params.get(key, [None])[0] + if value is not None: + preserved_params.append((key, value)) + if preserved_params: + address = f"{address}?{urlencode(preserved_params)}" elif platform == "快手": address = address.replace('http://', 'https://') elif platform == "抖音":
时间平台时间平台 脱敏 URL状态耗时错误码状态耗时错误码
{{ format_log_time(log['created_at']) }}{{ log['platform'] or '-' }}{{ format_log_time(log['created_at']) }}{{ log['platform'] or '-' }} {% if log['input_url'] %} {{ log['input_url'] }} {% else %}-{% endif %} {{ log['status_code'] + {{ log['status_code'] }}{{ log['duration_ms'] }} ms{{ log['error_code'] or '-' }}{{ log['duration_ms'] }} ms + {% if log['error_code'] %} + {{ log['error_code'] }} + {% else %}-{% endif %} +