兼容闲鱼itemId链接并优化短链口令数字后缀清洗

This commit is contained in:
ucmao
2026-09-16 16:50:36 +08:00
parent 1da33baebb
commit 2e1f56a3f9
5 changed files with 55 additions and 5 deletions
+1
View File
@@ -10,6 +10,7 @@
* **支持媒体类型**:商品高清大图 (JPEG/PNG) / 宝贝标题与描述
* **常见链接形态**:
* 口令/短链:`https://e.tb.cn/h.87fj9SNrqHW8kfC?tk=y1E5Tb4wkGd`
* 闲鱼 Web/H5 链接:`https://www.goofish.com/item?itemId=834469366646`、`https://h5.m.goofish.com/item?id=...`
* 商品长链:`https://2.taobao.com/item.htm?id=123456789`
* **Cookie 依赖**:无需 Cookie。
+11 -1
View File
@@ -47,7 +47,17 @@ class XianyuParser(BaseParser):
if not self.item_id:
parsed_real = urlparse(self.real_url)
params_real = parse_qs(parsed_real.query)
self.item_id = (params_real.get("id") or [None])[0]
self.item_id = (
params_real.get("itemId")
or params_real.get("id")
or params_real.get("item_id")
or [None]
)[0]
if not self.price:
self.price = (params_real.get("price") or [None])[0]
if not self.target_url and self.item_id:
self.target_url = f"https://www.goofish.com/item?itemId={self.item_id}"
if self.target_url:
self.image_list = [self.target_url]
+9
View File
@@ -11,6 +11,14 @@ class UrlParserTest(unittest.TestCase):
text = "复制打开应用 https://v.douyin.com/abc123/ 查看作品"
self.assertEqual(UrlParser.get_url(text), "https://v.douyin.com/abc123/")
def test_extracts_and_cleans_douyin_passcode_numeric_suffix(self):
text = "7.46 复制打开抖音 https://v.douyin.com/TkRst--_MmU/3.05"
self.assertEqual(UrlParser.get_url(text), "https://v.douyin.com/TkRst--_MmU/")
def test_extracts_first_url_when_glued_together(self):
text = "https://weixin.qq.com/sph/AUIdAnYCUMhttps://weixin.qq.com/sph/AH81IDXvsU"
self.assertEqual(UrlParser.get_url(text), "https://weixin.qq.com/sph/AUIdAnYCUM")
def test_get_url_handles_non_string_values(self):
for value in (None, 1, {}, []):
with self.subTest(value=value):
@@ -18,6 +26,7 @@ class UrlParserTest(unittest.TestCase):
def test_preserves_only_platform_specific_query_parameters(self):
cases = [
("https://www.goofish.com/item?itemId=834469366646&noise=x", "https://www.goofish.com/item?itemId=834469366646"),
("https://haokan.baidu.com/v?vid=11&noise=x", "https://haokan.baidu.com/v?vid=11"),
("https://isee.weishi.qq.com/ws/app-pages/share/index.html?id=22&noise=x", "https://isee.weishi.qq.com/ws/app-pages/share/index.html?id=22"),
("https://www.xiaohongshu.com/explore/33?xsec_token=token&noise=x", "https://www.xiaohongshu.com/explore/33?xsec_token=token"),
+12 -1
View File
@@ -27,7 +27,18 @@ class XianyuParserTest(unittest.TestCase):
with patch("requests.Session.get", return_value=page_resp):
parser = XianyuParser(url)
self.assertEqual(parser.get_title_content(), "闲鱼商品 (商品ID: 967494320598, 标价: ¥12.2)")
def test_parses_goofish_item_id_url(self):
page_resp = Mock()
page_resp.raise_for_status.return_value = None
page_resp.text = "<html><body>闲鱼商品页</body></html>"
url = "https://www.goofish.com/item?itemId=834469366646"
with patch("requests.Session.get", return_value=page_resp):
parser = XianyuParser(url)
self.assertEqual(parser.get_title_content(), "闲鱼商品 (商品ID: 834469366646)")
self.assertEqual(parser.get_image_list(), ["https://www.goofish.com/item?itemId=834469366646"])
self.assertEqual(parser.get_cover_photo_url(), "https://www.goofish.com/item?itemId=834469366646")
def test_invalid_url_handles_gracefully(self):
parser = XianyuParser("https://e.tb.cn/invalid")
+22 -3
View File
@@ -103,10 +103,17 @@ class UrlParser:
return None
url_pattern = re.compile(r'https?:\/\/(?:www\.|[-a-zA-Z0-9.@:%_+~#=]{1,256}\.[a-zA-Z0-9()]{1,24})\b(?:[-a-zA-Z0-9()@:%_+.~#?&//=]*?)(?=(?:https?:\/\/|$|\s))')
match = url_pattern.search(text)
if match:
return match.group()
else:
if not match:
return None
raw_url = match.group()
# 清洗抖音/快手等短链误拼接的口令数字后缀(如 /3.05、/7.46)
douyin_match = re.match(r'^(https?:\/\/v\.douyin\.com\/[a-zA-Z0-9_-]+)\/\d+(?:\.\d+)?\/?$', raw_url)
if douyin_match:
return douyin_match.group(1) + '/'
kuaishou_match = re.match(r'^(https?:\/\/v\.kuaishou\.com\/[a-zA-Z0-9_-]+)\/\d+(?:\.\d+)?\/?$', raw_url)
if kuaishou_match:
return kuaishou_match.group(1) + '/'
return raw_url
@staticmethod
def get_domain(url):
@@ -182,6 +189,15 @@ class UrlParser:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "快手":
address = address.replace('http://', 'https://')
elif platform == "闲鱼":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ('id', 'itemId', 'item_id', 'price', 'tk'):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "酷狗音乐":
query_params = parse_qs(parsed_url.query)
preserved_params = []
@@ -497,6 +513,9 @@ class UrlParser:
params_hash = query_params.get('hash', [None])[0]
if params_hash:
return params_hash
params_item_id = query_params.get('itemId', [None])[0] or query_params.get('item_id', [None])[0]
if params_item_id:
return params_item_id
params_id = query_params.get('id', [None])[0]
if params_id:
return params_id