From c668ef2fa38a6187e6cfb85ea7e90db95df3d6d8 Mon Sep 17 00:00:00 2001 From: ucmao <294323976@qq.com> Date: Sat, 10 Oct 2026 14:48:52 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E9=AB=98=E9=97=B2=E9=B1=BC=E5=95=86?= =?UTF-8?q?=E5=93=81=E5=9B=BE=E6=96=87=E6=8F=90=E5=8F=96=E6=8A=97=E9=A3=8E?= =?UTF-8?q?=E6=8E=A7=E7=A8=B3=E5=AE=9A=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/parsers/xianyu.md | 21 +++-- src/parsers/xianyu_parser.py | 158 +++++++++++++++++++++++++---------- tests/test_xianyu_parser.py | 45 ++++++---- 3 files changed, 153 insertions(+), 71 deletions(-) diff --git a/docs/parsers/xianyu.md b/docs/parsers/xianyu.md index 9370a62..97108b6 100644 --- a/docs/parsers/xianyu.md +++ b/docs/parsers/xianyu.md @@ -20,17 +20,16 @@ 1. **短链展开与 itemId 提取**: * 请求淘口令或短链(`m.tb.cn` / `e.tb.cn`),从 HTML 内联跳转变量 `var url = '...'` 或查询参数中提取目标商品 `itemId` / `id` 及初始标价。 -2. **移动端 H5 MTOP 动态令牌签名提取**: - * 采用闲鱼官方 H5 端 MTOP 接口:`mtop.taobao.idle.awesome.detail` (AppKey: `12574478`)。 - * **令牌握手**:先向网关发起初始化请求获取 `_m_h5_tk` Token。 - * **动态签名计算**:使用 `md5(token + "&" + t + "&" + appKey + "&" + data)` 计算合法 `sign`。 - * **媒体与元数据解析**: - * **高清图集**:解析 `itemDO.imageInfos` 提取所有原图直链(由 `img.alicdn.com` 托管,自动清洗缩略裁剪参数,免防盗链且支持 CORS 跨域展示)。 - * **封面**:默认提取相册首图作为主图封面。 - * **标题与文案**:提取 `itemDO.title` 与 `itemDO.desc`。 - * **卖家昵称**:优先提取 `flowData.floating` 浮动组件中的未脱敏昵称,降级提取 `sellerDO.nick`。 - * **视频直链**:若商品包含介绍视频,自动递归提取视频直链。 - * **状态感知**:若宝贝已删除或下架(`FAIL_BIZ_ITEM_DEL_NOT_FOUND`),自动设置 `ITEM_NOT_FOUND` 业务提示。 +2. **双层移动端 H5 MTOP 动态令牌签名提取**: + * **一级高可用接口(抗风控核心)**:`mtop.taobao.idle.item.info` (AppKey: `12574478`)。 + * **免风控特性**:不含个性化追踪负载,完全不受阿里霸下 (Baxia) WAF `RGV587_ERROR` 限制,无需任何登录态与 Cookie。 + * **核心数据提取**:稳定解析 `data.title`(标题)、`data.description`(详细文案)、`data.reservePrice`(价格)、`data.imageInfos`(高清图集与主图,托管于 `img.alicdn.com`,自动清洗裁剪后缀,天然支持 CORS 跨域免防盗链)。 + * **二级扩展接口(非阻塞补充)**:`mtop.taobao.idle.awesome.detail`。 + * 在非阻塞异常捕获下尝试请求,用于补充卖家昵称(`flowData.floating` / `sellerDO.nick`)及视频直链。若遭遇 WAF 拦截则自动跳过,不影响核心图文与价格交付。 + * **令牌握手与动态签名**: + * 向网关发起首轮请求获取 `_m_h5_tk` 临时令牌。 + * 使用 `md5(token + "&" + t + "&" + appKey + "&" + data)` 计算动态 `sign` 请求数据。 + * **状态感知**:若宝贝已删除或下架(`FAIL_BIZ_ITEM_DEL_NOT_FOUND` / `FAIL_BIZ_ITEM_QUERY_FAILED`),自动设置 `ITEM_NOT_FOUND` 业务提示。 --- diff --git a/src/parsers/xianyu_parser.py b/src/parsers/xianyu_parser.py index 95e2a2f..5b0da99 100644 --- a/src/parsers/xianyu_parser.py +++ b/src/parsers/xianyu_parser.py @@ -79,40 +79,54 @@ class XianyuParser(BaseParser): def _fetch_item_detail(self, item_id): """调用闲鱼 H5 MTOP 接口获取商品真实媒体与图文元数据。""" + # 1. 优先调用轻量级、免风控的 item.info 接口获取核心商品详情与原图相册 + self._fetch_via_item_info(item_id) + + # 2. 尝试从 awesome.detail 补充卖家昵称与视频直链(若触发风控静默忽略,不影响主流程) + if not self.author or not self.video_url: + self._try_fetch_awesome_detail(item_id) + + def _fetch_via_item_info(self, item_id): + """调用闲鱼官方公共商品详情接口,提取高清图片、标题、描述与标价。""" app_key = "12574478" - mtop_url = "https://h5api.m.goofish.com/h5/mtop.taobao.idle.awesome.detail/1.0/" + mtop_url = "https://h5api.m.goofish.com/h5/mtop.taobao.idle.item.info/1.0/" mtop_headers = { - "User-Agent": USER_AGENT_M[0], + "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1", "Referer": "https://h5.m.goofish.com/", "Accept": "application/json", } + data_str = json.dumps({"itemId": str(item_id)}, separators=(",", ":")) try: - init_params = { - "jsv": "2.7.3", - "appKey": app_key, - "api": "mtop.taobao.idle.awesome.detail", - "v": "1.0", - "type": "originaljson", - "dataType": "json", - "data": "{}", - } - self.session.get(mtop_url, params=init_params, headers=mtop_headers, timeout=5) + t = str(int(time.time() * 1000)) + self.session.get( + mtop_url, + params={ + "jsv": "2.7.3", + "appKey": app_key, + "t": t, + "sign": "dummy", + "api": "mtop.taobao.idle.item.info", + "v": "1.0", + "type": "originaljson", + "dataType": "json", + "data": data_str, + }, + headers=mtop_headers, + timeout=5, + ) token_cookie = self.session.cookies.get("_m_h5_tk", "") token = token_cookie.split("_")[0] if token_cookie else "" - data_dict = {"itemId": str(item_id)} - data_str = json.dumps(data_dict, separators=(",", ":")) - t = str(int(time.time() * 1000)) - sign_str = f"{token}&{t}&{app_key}&{data_str}" - sign = hashlib.md5(sign_str.encode("utf-8")).hexdigest() + t2 = str(int(time.time() * 1000)) + sign = hashlib.md5(f"{token}&{t2}&{app_key}&{data_str}".encode("utf-8")).hexdigest() req_params = { "jsv": "2.7.3", "appKey": app_key, - "t": t, + "t": t2, "sign": sign, - "api": "mtop.taobao.idle.awesome.detail", + "api": "mtop.taobao.idle.item.info", "v": "1.0", "type": "originaljson", "dataType": "json", @@ -130,30 +144,29 @@ class XianyuParser(BaseParser): ret = result.get("ret", []) ret_code = ret[0] if isinstance(ret, list) and ret else "" - if "FAIL_BIZ_ITEM_DEL_NOT_FOUND" in ret_code: + if "FAIL_BIZ_ITEM_DEL_NOT_FOUND" in ret_code or "FAIL_BIZ_ITEM_QUERY_FAILED" in ret_code: self.terminal_detail = { "detail_msg": "该宝贝不存在或已被卖家删除/下架", "error_code": "ITEM_NOT_FOUND", } return - data = result.get("data", {}) - if not isinstance(data, dict): + if "RGV587_ERROR" in ret_code: + from src.utils.parser_transport import report_cookie_required + report_cookie_required() return - item_do = data.get("itemDO", {}) - if isinstance(item_do, dict): - if item_do.get("title"): - self.title = item_do.get("title") - if item_do.get("desc"): - self.desc = item_do.get("desc") - - sold_price = item_do.get("soldPrice") or item_do.get("defaultPrice") - if sold_price: - self.price = str(sold_price) + data = result.get("data", {}) + if isinstance(data, dict): + if data.get("title"): + self.title = data.get("title") + if data.get("description"): + self.desc = data.get("description") + if data.get("reservePrice"): + self.price = str(data.get("reservePrice")) images = [] - for img_info in item_do.get("imageInfos", []): + for img_info in data.get("imageInfos", []): if isinstance(img_info, dict): img_url = img_info.get("url") if img_url: @@ -168,17 +181,78 @@ class XianyuParser(BaseParser): self.image_list = images self.cover_url = images[0] - seller_nick = self._extract_seller_nick(data) - if seller_nick: - self.author = seller_nick - - video_url = self._extract_video_url(data) - if video_url: - self.video_url = video_url - self.video_list = [video_url] + major_pic = data.get("absoluteMajorPicture") + if major_pic: + if major_pic.startswith("//"): + major_pic = "https:" + major_pic + elif major_pic.startswith("http://"): + major_pic = "https://" + major_pic[7:] + self.cover_url = major_pic except Exception as exc: - logger.warning("Failed to fetch Xianyu item detail via MTOP: %s", exc) + logger.warning("Failed to fetch Xianyu item.info via MTOP: %s", exc) + + def _try_fetch_awesome_detail(self, item_id): + """尝试从 awesome.detail 补充卖家昵称与视频直链(若受风控限制静默忽略)。""" + app_key = "12574478" + mtop_url = "https://h5api.m.goofish.com/h5/mtop.taobao.idle.awesome.detail/1.0/" + mtop_headers = { + "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1", + "Referer": "https://h5.m.goofish.com/", + "Accept": "application/json", + } + data_str = json.dumps({"itemId": str(item_id)}, separators=(",", ":")) + + try: + t = str(int(time.time() * 1000)) + self.session.get( + mtop_url, + params={ + "jsv": "2.7.3", + "appKey": app_key, + "t": t, + "sign": "dummy", + "api": "mtop.taobao.idle.awesome.detail", + "v": "1.0", + "type": "originaljson", + "dataType": "json", + "data": data_str, + }, + headers=mtop_headers, + timeout=5, + ) + + token_cookie = self.session.cookies.get("_m_h5_tk", "") + token = token_cookie.split("_")[0] if token_cookie else "" + t2 = str(int(time.time() * 1000)) + sign = hashlib.md5(f"{token}&{t2}&{app_key}&{data_str}".encode("utf-8")).hexdigest() + + req_params = { + "jsv": "2.7.3", + "appKey": app_key, + "t": t2, + "sign": sign, + "api": "mtop.taobao.idle.awesome.detail", + "v": "1.0", + "type": "originaljson", + "dataType": "json", + "data": data_str, + } + resp = self.session.get(mtop_url, params=req_params, headers=mtop_headers, timeout=5) + result = resp.json() + data = result.get("data", {}) + if isinstance(data, dict): + seller_nick = self._extract_seller_nick(data) + if seller_nick: + self.author = seller_nick + + video_url = self._extract_video_url(data) + if video_url: + self.video_url = video_url + self.video_list = [video_url] + + except Exception: + pass def _extract_seller_nick(self, data): """提取闲鱼卖家昵称,优先未脱敏昵称。""" diff --git a/tests/test_xianyu_parser.py b/tests/test_xianyu_parser.py index 2811a2c..93e7c8b 100644 --- a/tests/test_xianyu_parser.py +++ b/tests/test_xianyu_parser.py @@ -35,20 +35,26 @@ class XianyuParserTest(unittest.TestCase): mtop_init_resp = Mock() mtop_init_resp.raise_for_status.return_value = None - mtop_data_resp = Mock() - mtop_data_resp.raise_for_status.return_value = None - mtop_data_resp.json.return_value = { + mtop_info_resp = Mock() + mtop_info_resp.raise_for_status.return_value = None + mtop_info_resp.json.return_value = { + "ret": ["SUCCESS::接口调用成功"], + "data": { + "title": "测试闲鱼商品标题", + "description": "商品详细描述", + "reservePrice": "99.0", + "imageInfos": [ + {"url": "http://img.alicdn.com/bao/uploaded/i1/test1.png"}, + {"url": "http://img.alicdn.com/bao/uploaded/i2/test2.png"} + ], + }, + } + + mtop_awesome_resp = Mock() + mtop_awesome_resp.raise_for_status.return_value = None + mtop_awesome_resp.json.return_value = { "ret": ["SUCCESS::接口调用成功"], "data": { - "itemDO": { - "title": "测试闲鱼商品标题", - "desc": "商品详细描述", - "soldPrice": "99.0", - "imageInfos": [ - {"url": "http://img.alicdn.com/bao/uploaded/i1/test1.png"}, - {"url": "http://img.alicdn.com/bao/uploaded/i2/test2.png"} - ], - }, "sellerDO": { "nick": "卖家小明", }, @@ -56,11 +62,14 @@ class XianyuParserTest(unittest.TestCase): } def side_effect(url, *args, **kwargs): - if "mtop.taobao.idle.awesome.detail" in url: + if "mtop" in url: params = kwargs.get("params", {}) - if params.get("data") == "{}": + if params.get("sign") == "dummy": return mtop_init_resp - return mtop_data_resp + if "item.info" in url: + return mtop_info_resp + if "awesome.detail" in url: + return mtop_awesome_resp return page_resp url = "https://www.goofish.com/item?itemId=834469366646" @@ -90,14 +99,14 @@ class XianyuParserTest(unittest.TestCase): mtop_data_resp = Mock() mtop_data_resp.raise_for_status.return_value = None mtop_data_resp.json.return_value = { - "ret": ["FAIL_BIZ_ITEM_DEL_NOT_FOUND::您要看的宝贝不存在或已被删除啦!"], + "ret": ["FAIL_BIZ_ITEM_QUERY_FAILED::查询商品信息失败"], "data": {}, } def side_effect(url, *args, **kwargs): - if "mtop.taobao.idle.awesome.detail" in url: + if "mtop" in url: params = kwargs.get("params", {}) - if params.get("data") == "{}": + if params.get("sign") == "dummy": return mtop_init_resp return mtop_data_resp return page_resp