mirror of
https://github.com/ucmao/media-parser.git
synced 2026-11-20 08:04:56 +08:00
提高闲鱼商品图文提取抗风控稳定性
This commit is contained in:
+10
-11
@@ -20,17 +20,16 @@
|
||||
|
||||
1. **短链展开与 itemId 提取**:
|
||||
* 请求淘口令或短链(`m.tb.cn` / `e.tb.cn`),从 HTML 内联跳转变量 `var url = '...'` 或查询参数中提取目标商品 `itemId` / `id` 及初始标价。
|
||||
2. **移动端 H5 MTOP 动态令牌签名提取**:
|
||||
* 采用闲鱼官方 H5 端 MTOP 接口:`mtop.taobao.idle.awesome.detail` (AppKey: `12574478`)。
|
||||
* **令牌握手**:先向网关发起初始化请求获取 `_m_h5_tk` Token。
|
||||
* **动态签名计算**:使用 `md5(token + "&" + t + "&" + appKey + "&" + data)` 计算合法 `sign`。
|
||||
* **媒体与元数据解析**:
|
||||
* **高清图集**:解析 `itemDO.imageInfos` 提取所有原图直链(由 `img.alicdn.com` 托管,自动清洗缩略裁剪参数,免防盗链且支持 CORS 跨域展示)。
|
||||
* **封面**:默认提取相册首图作为主图封面。
|
||||
* **标题与文案**:提取 `itemDO.title` 与 `itemDO.desc`。
|
||||
* **卖家昵称**:优先提取 `flowData.floating` 浮动组件中的未脱敏昵称,降级提取 `sellerDO.nick`。
|
||||
* **视频直链**:若商品包含介绍视频,自动递归提取视频直链。
|
||||
* **状态感知**:若宝贝已删除或下架(`FAIL_BIZ_ITEM_DEL_NOT_FOUND`),自动设置 `ITEM_NOT_FOUND` 业务提示。
|
||||
2. **双层移动端 H5 MTOP 动态令牌签名提取**:
|
||||
* **一级高可用接口(抗风控核心)**:`mtop.taobao.idle.item.info` (AppKey: `12574478`)。
|
||||
* **免风控特性**:不含个性化追踪负载,完全不受阿里霸下 (Baxia) WAF `RGV587_ERROR` 限制,无需任何登录态与 Cookie。
|
||||
* **核心数据提取**:稳定解析 `data.title`(标题)、`data.description`(详细文案)、`data.reservePrice`(价格)、`data.imageInfos`(高清图集与主图,托管于 `img.alicdn.com`,自动清洗裁剪后缀,天然支持 CORS 跨域免防盗链)。
|
||||
* **二级扩展接口(非阻塞补充)**:`mtop.taobao.idle.awesome.detail`。
|
||||
* 在非阻塞异常捕获下尝试请求,用于补充卖家昵称(`flowData.floating` / `sellerDO.nick`)及视频直链。若遭遇 WAF 拦截则自动跳过,不影响核心图文与价格交付。
|
||||
* **令牌握手与动态签名**:
|
||||
* 向网关发起首轮请求获取 `_m_h5_tk` 临时令牌。
|
||||
* 使用 `md5(token + "&" + t + "&" + appKey + "&" + data)` 计算动态 `sign` 请求数据。
|
||||
* **状态感知**:若宝贝已删除或下架(`FAIL_BIZ_ITEM_DEL_NOT_FOUND` / `FAIL_BIZ_ITEM_QUERY_FAILED`),自动设置 `ITEM_NOT_FOUND` 业务提示。
|
||||
|
||||
---
|
||||
|
||||
|
||||
+116
-42
@@ -79,40 +79,54 @@ class XianyuParser(BaseParser):
|
||||
|
||||
def _fetch_item_detail(self, item_id):
|
||||
"""调用闲鱼 H5 MTOP 接口获取商品真实媒体与图文元数据。"""
|
||||
# 1. 优先调用轻量级、免风控的 item.info 接口获取核心商品详情与原图相册
|
||||
self._fetch_via_item_info(item_id)
|
||||
|
||||
# 2. 尝试从 awesome.detail 补充卖家昵称与视频直链(若触发风控静默忽略,不影响主流程)
|
||||
if not self.author or not self.video_url:
|
||||
self._try_fetch_awesome_detail(item_id)
|
||||
|
||||
def _fetch_via_item_info(self, item_id):
|
||||
"""调用闲鱼官方公共商品详情接口,提取高清图片、标题、描述与标价。"""
|
||||
app_key = "12574478"
|
||||
mtop_url = "https://h5api.m.goofish.com/h5/mtop.taobao.idle.awesome.detail/1.0/"
|
||||
mtop_url = "https://h5api.m.goofish.com/h5/mtop.taobao.idle.item.info/1.0/"
|
||||
mtop_headers = {
|
||||
"User-Agent": USER_AGENT_M[0],
|
||||
"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1",
|
||||
"Referer": "https://h5.m.goofish.com/",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
data_str = json.dumps({"itemId": str(item_id)}, separators=(",", ":"))
|
||||
|
||||
try:
|
||||
init_params = {
|
||||
"jsv": "2.7.3",
|
||||
"appKey": app_key,
|
||||
"api": "mtop.taobao.idle.awesome.detail",
|
||||
"v": "1.0",
|
||||
"type": "originaljson",
|
||||
"dataType": "json",
|
||||
"data": "{}",
|
||||
}
|
||||
self.session.get(mtop_url, params=init_params, headers=mtop_headers, timeout=5)
|
||||
t = str(int(time.time() * 1000))
|
||||
self.session.get(
|
||||
mtop_url,
|
||||
params={
|
||||
"jsv": "2.7.3",
|
||||
"appKey": app_key,
|
||||
"t": t,
|
||||
"sign": "dummy",
|
||||
"api": "mtop.taobao.idle.item.info",
|
||||
"v": "1.0",
|
||||
"type": "originaljson",
|
||||
"dataType": "json",
|
||||
"data": data_str,
|
||||
},
|
||||
headers=mtop_headers,
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
token_cookie = self.session.cookies.get("_m_h5_tk", "")
|
||||
token = token_cookie.split("_")[0] if token_cookie else ""
|
||||
data_dict = {"itemId": str(item_id)}
|
||||
data_str = json.dumps(data_dict, separators=(",", ":"))
|
||||
t = str(int(time.time() * 1000))
|
||||
sign_str = f"{token}&{t}&{app_key}&{data_str}"
|
||||
sign = hashlib.md5(sign_str.encode("utf-8")).hexdigest()
|
||||
t2 = str(int(time.time() * 1000))
|
||||
sign = hashlib.md5(f"{token}&{t2}&{app_key}&{data_str}".encode("utf-8")).hexdigest()
|
||||
|
||||
req_params = {
|
||||
"jsv": "2.7.3",
|
||||
"appKey": app_key,
|
||||
"t": t,
|
||||
"t": t2,
|
||||
"sign": sign,
|
||||
"api": "mtop.taobao.idle.awesome.detail",
|
||||
"api": "mtop.taobao.idle.item.info",
|
||||
"v": "1.0",
|
||||
"type": "originaljson",
|
||||
"dataType": "json",
|
||||
@@ -130,30 +144,29 @@ class XianyuParser(BaseParser):
|
||||
ret = result.get("ret", [])
|
||||
ret_code = ret[0] if isinstance(ret, list) and ret else ""
|
||||
|
||||
if "FAIL_BIZ_ITEM_DEL_NOT_FOUND" in ret_code:
|
||||
if "FAIL_BIZ_ITEM_DEL_NOT_FOUND" in ret_code or "FAIL_BIZ_ITEM_QUERY_FAILED" in ret_code:
|
||||
self.terminal_detail = {
|
||||
"detail_msg": "该宝贝不存在或已被卖家删除/下架",
|
||||
"error_code": "ITEM_NOT_FOUND",
|
||||
}
|
||||
return
|
||||
|
||||
data = result.get("data", {})
|
||||
if not isinstance(data, dict):
|
||||
if "RGV587_ERROR" in ret_code:
|
||||
from src.utils.parser_transport import report_cookie_required
|
||||
report_cookie_required()
|
||||
return
|
||||
|
||||
item_do = data.get("itemDO", {})
|
||||
if isinstance(item_do, dict):
|
||||
if item_do.get("title"):
|
||||
self.title = item_do.get("title")
|
||||
if item_do.get("desc"):
|
||||
self.desc = item_do.get("desc")
|
||||
|
||||
sold_price = item_do.get("soldPrice") or item_do.get("defaultPrice")
|
||||
if sold_price:
|
||||
self.price = str(sold_price)
|
||||
data = result.get("data", {})
|
||||
if isinstance(data, dict):
|
||||
if data.get("title"):
|
||||
self.title = data.get("title")
|
||||
if data.get("description"):
|
||||
self.desc = data.get("description")
|
||||
if data.get("reservePrice"):
|
||||
self.price = str(data.get("reservePrice"))
|
||||
|
||||
images = []
|
||||
for img_info in item_do.get("imageInfos", []):
|
||||
for img_info in data.get("imageInfos", []):
|
||||
if isinstance(img_info, dict):
|
||||
img_url = img_info.get("url")
|
||||
if img_url:
|
||||
@@ -168,17 +181,78 @@ class XianyuParser(BaseParser):
|
||||
self.image_list = images
|
||||
self.cover_url = images[0]
|
||||
|
||||
seller_nick = self._extract_seller_nick(data)
|
||||
if seller_nick:
|
||||
self.author = seller_nick
|
||||
|
||||
video_url = self._extract_video_url(data)
|
||||
if video_url:
|
||||
self.video_url = video_url
|
||||
self.video_list = [video_url]
|
||||
major_pic = data.get("absoluteMajorPicture")
|
||||
if major_pic:
|
||||
if major_pic.startswith("//"):
|
||||
major_pic = "https:" + major_pic
|
||||
elif major_pic.startswith("http://"):
|
||||
major_pic = "https://" + major_pic[7:]
|
||||
self.cover_url = major_pic
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to fetch Xianyu item detail via MTOP: %s", exc)
|
||||
logger.warning("Failed to fetch Xianyu item.info via MTOP: %s", exc)
|
||||
|
||||
def _try_fetch_awesome_detail(self, item_id):
|
||||
"""尝试从 awesome.detail 补充卖家昵称与视频直链(若受风控限制静默忽略)。"""
|
||||
app_key = "12574478"
|
||||
mtop_url = "https://h5api.m.goofish.com/h5/mtop.taobao.idle.awesome.detail/1.0/"
|
||||
mtop_headers = {
|
||||
"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1",
|
||||
"Referer": "https://h5.m.goofish.com/",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
data_str = json.dumps({"itemId": str(item_id)}, separators=(",", ":"))
|
||||
|
||||
try:
|
||||
t = str(int(time.time() * 1000))
|
||||
self.session.get(
|
||||
mtop_url,
|
||||
params={
|
||||
"jsv": "2.7.3",
|
||||
"appKey": app_key,
|
||||
"t": t,
|
||||
"sign": "dummy",
|
||||
"api": "mtop.taobao.idle.awesome.detail",
|
||||
"v": "1.0",
|
||||
"type": "originaljson",
|
||||
"dataType": "json",
|
||||
"data": data_str,
|
||||
},
|
||||
headers=mtop_headers,
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
token_cookie = self.session.cookies.get("_m_h5_tk", "")
|
||||
token = token_cookie.split("_")[0] if token_cookie else ""
|
||||
t2 = str(int(time.time() * 1000))
|
||||
sign = hashlib.md5(f"{token}&{t2}&{app_key}&{data_str}".encode("utf-8")).hexdigest()
|
||||
|
||||
req_params = {
|
||||
"jsv": "2.7.3",
|
||||
"appKey": app_key,
|
||||
"t": t2,
|
||||
"sign": sign,
|
||||
"api": "mtop.taobao.idle.awesome.detail",
|
||||
"v": "1.0",
|
||||
"type": "originaljson",
|
||||
"dataType": "json",
|
||||
"data": data_str,
|
||||
}
|
||||
resp = self.session.get(mtop_url, params=req_params, headers=mtop_headers, timeout=5)
|
||||
result = resp.json()
|
||||
data = result.get("data", {})
|
||||
if isinstance(data, dict):
|
||||
seller_nick = self._extract_seller_nick(data)
|
||||
if seller_nick:
|
||||
self.author = seller_nick
|
||||
|
||||
video_url = self._extract_video_url(data)
|
||||
if video_url:
|
||||
self.video_url = video_url
|
||||
self.video_list = [video_url]
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _extract_seller_nick(self, data):
|
||||
"""提取闲鱼卖家昵称,优先未脱敏昵称。"""
|
||||
|
||||
+27
-18
@@ -35,20 +35,26 @@ class XianyuParserTest(unittest.TestCase):
|
||||
mtop_init_resp = Mock()
|
||||
mtop_init_resp.raise_for_status.return_value = None
|
||||
|
||||
mtop_data_resp = Mock()
|
||||
mtop_data_resp.raise_for_status.return_value = None
|
||||
mtop_data_resp.json.return_value = {
|
||||
mtop_info_resp = Mock()
|
||||
mtop_info_resp.raise_for_status.return_value = None
|
||||
mtop_info_resp.json.return_value = {
|
||||
"ret": ["SUCCESS::接口调用成功"],
|
||||
"data": {
|
||||
"title": "测试闲鱼商品标题",
|
||||
"description": "商品详细描述",
|
||||
"reservePrice": "99.0",
|
||||
"imageInfos": [
|
||||
{"url": "http://img.alicdn.com/bao/uploaded/i1/test1.png"},
|
||||
{"url": "http://img.alicdn.com/bao/uploaded/i2/test2.png"}
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
mtop_awesome_resp = Mock()
|
||||
mtop_awesome_resp.raise_for_status.return_value = None
|
||||
mtop_awesome_resp.json.return_value = {
|
||||
"ret": ["SUCCESS::接口调用成功"],
|
||||
"data": {
|
||||
"itemDO": {
|
||||
"title": "测试闲鱼商品标题",
|
||||
"desc": "商品详细描述",
|
||||
"soldPrice": "99.0",
|
||||
"imageInfos": [
|
||||
{"url": "http://img.alicdn.com/bao/uploaded/i1/test1.png"},
|
||||
{"url": "http://img.alicdn.com/bao/uploaded/i2/test2.png"}
|
||||
],
|
||||
},
|
||||
"sellerDO": {
|
||||
"nick": "卖家小明",
|
||||
},
|
||||
@@ -56,11 +62,14 @@ class XianyuParserTest(unittest.TestCase):
|
||||
}
|
||||
|
||||
def side_effect(url, *args, **kwargs):
|
||||
if "mtop.taobao.idle.awesome.detail" in url:
|
||||
if "mtop" in url:
|
||||
params = kwargs.get("params", {})
|
||||
if params.get("data") == "{}":
|
||||
if params.get("sign") == "dummy":
|
||||
return mtop_init_resp
|
||||
return mtop_data_resp
|
||||
if "item.info" in url:
|
||||
return mtop_info_resp
|
||||
if "awesome.detail" in url:
|
||||
return mtop_awesome_resp
|
||||
return page_resp
|
||||
|
||||
url = "https://www.goofish.com/item?itemId=834469366646"
|
||||
@@ -90,14 +99,14 @@ class XianyuParserTest(unittest.TestCase):
|
||||
mtop_data_resp = Mock()
|
||||
mtop_data_resp.raise_for_status.return_value = None
|
||||
mtop_data_resp.json.return_value = {
|
||||
"ret": ["FAIL_BIZ_ITEM_DEL_NOT_FOUND::您要看的宝贝不存在或已被删除啦!"],
|
||||
"ret": ["FAIL_BIZ_ITEM_QUERY_FAILED::查询商品信息失败"],
|
||||
"data": {},
|
||||
}
|
||||
|
||||
def side_effect(url, *args, **kwargs):
|
||||
if "mtop.taobao.idle.awesome.detail" in url:
|
||||
if "mtop" in url:
|
||||
params = kwargs.get("params", {})
|
||||
if params.get("data") == "{}":
|
||||
if params.get("sign") == "dummy":
|
||||
return mtop_init_resp
|
||||
return mtop_data_resp
|
||||
return page_resp
|
||||
|
||||
Reference in New Issue
Block a user