提高闲鱼商品图文提取抗风控稳定性

This commit is contained in:
ucmao
2026-10-10 14:48:52 +08:00
parent f62a6867a4
commit c668ef2fa3
3 changed files with 153 additions and 71 deletions
+10 -11
View File
@@ -20,17 +20,16 @@
1. **短链展开与 itemId 提取**:
* 请求淘口令或短链(`m.tb.cn` / `e.tb.cn`),从 HTML 内联跳转变量 `var url = '...'` 或查询参数中提取目标商品 `itemId` / `id` 及初始标价。
2. **移动端 H5 MTOP 动态令牌签名提取**:
* 采用闲鱼官方 H5 端 MTOP 接口:`mtop.taobao.idle.awesome.detail` (AppKey: `12574478`)。
* **令牌握手**:先向网关发起初始化请求获取 `_m_h5_tk` Token。
* **动态签名计算**:使用 `md5(token + "&" + t + "&" + appKey + "&" + data)` 计算合法 `sign`。
* **媒体与元数据解析**:
* **高清图集**:解析 `itemDO.imageInfos` 提取所有原图直链(由 `img.alicdn.com` 托管,自动清洗缩略裁剪参数,免防盗链且支持 CORS 跨域展示)。
* **封面**:默认提取相册首图作为主图封面。
* **标题与文案**:提取 `itemDO.title` 与 `itemDO.desc`。
* **卖家昵称**:优先提取 `flowData.floating` 浮动组件中的未脱敏昵称,降级提取 `sellerDO.nick`。
* **视频直链**:若商品包含介绍视频,自动递归提取视频直链。
* **状态感知**:若宝贝已删除或下架(`FAIL_BIZ_ITEM_DEL_NOT_FOUND`),自动设置 `ITEM_NOT_FOUND` 业务提示。
2. **双层移动端 H5 MTOP 动态令牌签名提取**:
* **一级高可用接口(抗风控核心)**:`mtop.taobao.idle.item.info` (AppKey: `12574478`)。
* **免风控特性**:不含个性化追踪负载,完全不受阿里霸下 (Baxia) WAF `RGV587_ERROR` 限制,无需任何登录态与 Cookie。
* **核心数据提取**:稳定解析 `data.title`(标题)、`data.description`(详细文案)、`data.reservePrice`(价格)、`data.imageInfos`(高清图集与主图,托管于 `img.alicdn.com`,自动清洗裁剪后缀,天然支持 CORS 跨域免防盗链)。
* **二级扩展接口(非阻塞补充)**:`mtop.taobao.idle.awesome.detail`。
* 在非阻塞异常捕获下尝试请求,用于补充卖家昵称(`flowData.floating` / `sellerDO.nick`)及视频直链。若遭遇 WAF 拦截则自动跳过,不影响核心图文与价格交付。
* **令牌握手与动态签名**:
* 向网关发起首轮请求获取 `_m_h5_tk` 临时令牌。
* 使用 `md5(token + "&" + t + "&" + appKey + "&" + data)` 计算动态 `sign` 请求数据。
* **状态感知**:若宝贝已删除或下架(`FAIL_BIZ_ITEM_DEL_NOT_FOUND` / `FAIL_BIZ_ITEM_QUERY_FAILED`),自动设置 `ITEM_NOT_FOUND` 业务提示。
---
+116 -42
View File
@@ -79,40 +79,54 @@ class XianyuParser(BaseParser):
def _fetch_item_detail(self, item_id):
"""调用闲鱼 H5 MTOP 接口获取商品真实媒体与图文元数据。"""
# 1. 优先调用轻量级、免风控的 item.info 接口获取核心商品详情与原图相册
self._fetch_via_item_info(item_id)
# 2. 尝试从 awesome.detail 补充卖家昵称与视频直链(若触发风控静默忽略,不影响主流程)
if not self.author or not self.video_url:
self._try_fetch_awesome_detail(item_id)
def _fetch_via_item_info(self, item_id):
"""调用闲鱼官方公共商品详情接口,提取高清图片、标题、描述与标价。"""
app_key = "12574478"
mtop_url = "https://h5api.m.goofish.com/h5/mtop.taobao.idle.awesome.detail/1.0/"
mtop_url = "https://h5api.m.goofish.com/h5/mtop.taobao.idle.item.info/1.0/"
mtop_headers = {
"User-Agent": USER_AGENT_M[0],
"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1",
"Referer": "https://h5.m.goofish.com/",
"Accept": "application/json",
}
data_str = json.dumps({"itemId": str(item_id)}, separators=(",", ":"))
try:
init_params = {
"jsv": "2.7.3",
"appKey": app_key,
"api": "mtop.taobao.idle.awesome.detail",
"v": "1.0",
"type": "originaljson",
"dataType": "json",
"data": "{}",
}
self.session.get(mtop_url, params=init_params, headers=mtop_headers, timeout=5)
t = str(int(time.time() * 1000))
self.session.get(
mtop_url,
params={
"jsv": "2.7.3",
"appKey": app_key,
"t": t,
"sign": "dummy",
"api": "mtop.taobao.idle.item.info",
"v": "1.0",
"type": "originaljson",
"dataType": "json",
"data": data_str,
},
headers=mtop_headers,
timeout=5,
)
token_cookie = self.session.cookies.get("_m_h5_tk", "")
token = token_cookie.split("_")[0] if token_cookie else ""
data_dict = {"itemId": str(item_id)}
data_str = json.dumps(data_dict, separators=(",", ":"))
t = str(int(time.time() * 1000))
sign_str = f"{token}&{t}&{app_key}&{data_str}"
sign = hashlib.md5(sign_str.encode("utf-8")).hexdigest()
t2 = str(int(time.time() * 1000))
sign = hashlib.md5(f"{token}&{t2}&{app_key}&{data_str}".encode("utf-8")).hexdigest()
req_params = {
"jsv": "2.7.3",
"appKey": app_key,
"t": t,
"t": t2,
"sign": sign,
"api": "mtop.taobao.idle.awesome.detail",
"api": "mtop.taobao.idle.item.info",
"v": "1.0",
"type": "originaljson",
"dataType": "json",
@@ -130,30 +144,29 @@ class XianyuParser(BaseParser):
ret = result.get("ret", [])
ret_code = ret[0] if isinstance(ret, list) and ret else ""
if "FAIL_BIZ_ITEM_DEL_NOT_FOUND" in ret_code:
if "FAIL_BIZ_ITEM_DEL_NOT_FOUND" in ret_code or "FAIL_BIZ_ITEM_QUERY_FAILED" in ret_code:
self.terminal_detail = {
"detail_msg": "该宝贝不存在或已被卖家删除/下架",
"error_code": "ITEM_NOT_FOUND",
}
return
data = result.get("data", {})
if not isinstance(data, dict):
if "RGV587_ERROR" in ret_code:
from src.utils.parser_transport import report_cookie_required
report_cookie_required()
return
item_do = data.get("itemDO", {})
if isinstance(item_do, dict):
if item_do.get("title"):
self.title = item_do.get("title")
if item_do.get("desc"):
self.desc = item_do.get("desc")
sold_price = item_do.get("soldPrice") or item_do.get("defaultPrice")
if sold_price:
self.price = str(sold_price)
data = result.get("data", {})
if isinstance(data, dict):
if data.get("title"):
self.title = data.get("title")
if data.get("description"):
self.desc = data.get("description")
if data.get("reservePrice"):
self.price = str(data.get("reservePrice"))
images = []
for img_info in item_do.get("imageInfos", []):
for img_info in data.get("imageInfos", []):
if isinstance(img_info, dict):
img_url = img_info.get("url")
if img_url:
@@ -168,17 +181,78 @@ class XianyuParser(BaseParser):
self.image_list = images
self.cover_url = images[0]
seller_nick = self._extract_seller_nick(data)
if seller_nick:
self.author = seller_nick
video_url = self._extract_video_url(data)
if video_url:
self.video_url = video_url
self.video_list = [video_url]
major_pic = data.get("absoluteMajorPicture")
if major_pic:
if major_pic.startswith("//"):
major_pic = "https:" + major_pic
elif major_pic.startswith("http://"):
major_pic = "https://" + major_pic[7:]
self.cover_url = major_pic
except Exception as exc:
logger.warning("Failed to fetch Xianyu item detail via MTOP: %s", exc)
logger.warning("Failed to fetch Xianyu item.info via MTOP: %s", exc)
def _try_fetch_awesome_detail(self, item_id):
"""尝试从 awesome.detail 补充卖家昵称与视频直链(若受风控限制静默忽略)。"""
app_key = "12574478"
mtop_url = "https://h5api.m.goofish.com/h5/mtop.taobao.idle.awesome.detail/1.0/"
mtop_headers = {
"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1",
"Referer": "https://h5.m.goofish.com/",
"Accept": "application/json",
}
data_str = json.dumps({"itemId": str(item_id)}, separators=(",", ":"))
try:
t = str(int(time.time() * 1000))
self.session.get(
mtop_url,
params={
"jsv": "2.7.3",
"appKey": app_key,
"t": t,
"sign": "dummy",
"api": "mtop.taobao.idle.awesome.detail",
"v": "1.0",
"type": "originaljson",
"dataType": "json",
"data": data_str,
},
headers=mtop_headers,
timeout=5,
)
token_cookie = self.session.cookies.get("_m_h5_tk", "")
token = token_cookie.split("_")[0] if token_cookie else ""
t2 = str(int(time.time() * 1000))
sign = hashlib.md5(f"{token}&{t2}&{app_key}&{data_str}".encode("utf-8")).hexdigest()
req_params = {
"jsv": "2.7.3",
"appKey": app_key,
"t": t2,
"sign": sign,
"api": "mtop.taobao.idle.awesome.detail",
"v": "1.0",
"type": "originaljson",
"dataType": "json",
"data": data_str,
}
resp = self.session.get(mtop_url, params=req_params, headers=mtop_headers, timeout=5)
result = resp.json()
data = result.get("data", {})
if isinstance(data, dict):
seller_nick = self._extract_seller_nick(data)
if seller_nick:
self.author = seller_nick
video_url = self._extract_video_url(data)
if video_url:
self.video_url = video_url
self.video_list = [video_url]
except Exception:
pass
def _extract_seller_nick(self, data):
"""提取闲鱼卖家昵称,优先未脱敏昵称。"""
+27 -18
View File
@@ -35,20 +35,26 @@ class XianyuParserTest(unittest.TestCase):
mtop_init_resp = Mock()
mtop_init_resp.raise_for_status.return_value = None
mtop_data_resp = Mock()
mtop_data_resp.raise_for_status.return_value = None
mtop_data_resp.json.return_value = {
mtop_info_resp = Mock()
mtop_info_resp.raise_for_status.return_value = None
mtop_info_resp.json.return_value = {
"ret": ["SUCCESS::接口调用成功"],
"data": {
"title": "测试闲鱼商品标题",
"description": "商品详细描述",
"reservePrice": "99.0",
"imageInfos": [
{"url": "http://img.alicdn.com/bao/uploaded/i1/test1.png"},
{"url": "http://img.alicdn.com/bao/uploaded/i2/test2.png"}
],
},
}
mtop_awesome_resp = Mock()
mtop_awesome_resp.raise_for_status.return_value = None
mtop_awesome_resp.json.return_value = {
"ret": ["SUCCESS::接口调用成功"],
"data": {
"itemDO": {
"title": "测试闲鱼商品标题",
"desc": "商品详细描述",
"soldPrice": "99.0",
"imageInfos": [
{"url": "http://img.alicdn.com/bao/uploaded/i1/test1.png"},
{"url": "http://img.alicdn.com/bao/uploaded/i2/test2.png"}
],
},
"sellerDO": {
"nick": "卖家小明",
},
@@ -56,11 +62,14 @@ class XianyuParserTest(unittest.TestCase):
}
def side_effect(url, *args, **kwargs):
if "mtop.taobao.idle.awesome.detail" in url:
if "mtop" in url:
params = kwargs.get("params", {})
if params.get("data") == "{}":
if params.get("sign") == "dummy":
return mtop_init_resp
return mtop_data_resp
if "item.info" in url:
return mtop_info_resp
if "awesome.detail" in url:
return mtop_awesome_resp
return page_resp
url = "https://www.goofish.com/item?itemId=834469366646"
@@ -90,14 +99,14 @@ class XianyuParserTest(unittest.TestCase):
mtop_data_resp = Mock()
mtop_data_resp.raise_for_status.return_value = None
mtop_data_resp.json.return_value = {
"ret": ["FAIL_BIZ_ITEM_DEL_NOT_FOUND::您要看的宝贝不存在或已被删除啦!"],
"ret": ["FAIL_BIZ_ITEM_QUERY_FAILED::查询商品信息失败"],
"data": {},
}
def side_effect(url, *args, **kwargs):
if "mtop.taobao.idle.awesome.detail" in url:
if "mtop" in url:
params = kwargs.get("params", {})
if params.get("data") == "{}":
if params.get("sign") == "dummy":
return mtop_init_resp
return mtop_data_resp
return page_resp