mirror of
https://github.com/ucmao/media-parser.git
synced 2026-11-20 08:04:56 +08:00
兼容闲鱼itemId链接并优化短链口令数字后缀清洗
This commit is contained in:
@@ -10,6 +10,7 @@
|
||||
* **支持媒体类型**:商品高清大图 (JPEG/PNG) / 宝贝标题与描述
|
||||
* **常见链接形态**:
|
||||
* 口令/短链:`https://e.tb.cn/h.87fj9SNrqHW8kfC?tk=y1E5Tb4wkGd`
|
||||
* 闲鱼 Web/H5 链接:`https://www.goofish.com/item?itemId=834469366646`、`https://h5.m.goofish.com/item?id=...`
|
||||
* 商品长链:`https://2.taobao.com/item.htm?id=123456789`
|
||||
* **Cookie 依赖**:无需 Cookie。
|
||||
|
||||
|
||||
@@ -47,7 +47,17 @@ class XianyuParser(BaseParser):
|
||||
if not self.item_id:
|
||||
parsed_real = urlparse(self.real_url)
|
||||
params_real = parse_qs(parsed_real.query)
|
||||
self.item_id = (params_real.get("id") or [None])[0]
|
||||
self.item_id = (
|
||||
params_real.get("itemId")
|
||||
or params_real.get("id")
|
||||
or params_real.get("item_id")
|
||||
or [None]
|
||||
)[0]
|
||||
if not self.price:
|
||||
self.price = (params_real.get("price") or [None])[0]
|
||||
|
||||
if not self.target_url and self.item_id:
|
||||
self.target_url = f"https://www.goofish.com/item?itemId={self.item_id}"
|
||||
|
||||
if self.target_url:
|
||||
self.image_list = [self.target_url]
|
||||
|
||||
@@ -11,6 +11,14 @@ class UrlParserTest(unittest.TestCase):
|
||||
text = "复制打开应用 https://v.douyin.com/abc123/ 查看作品"
|
||||
self.assertEqual(UrlParser.get_url(text), "https://v.douyin.com/abc123/")
|
||||
|
||||
def test_extracts_and_cleans_douyin_passcode_numeric_suffix(self):
|
||||
text = "7.46 复制打开抖音 https://v.douyin.com/TkRst--_MmU/3.05"
|
||||
self.assertEqual(UrlParser.get_url(text), "https://v.douyin.com/TkRst--_MmU/")
|
||||
|
||||
def test_extracts_first_url_when_glued_together(self):
|
||||
text = "https://weixin.qq.com/sph/AUIdAnYCUMhttps://weixin.qq.com/sph/AH81IDXvsU"
|
||||
self.assertEqual(UrlParser.get_url(text), "https://weixin.qq.com/sph/AUIdAnYCUM")
|
||||
|
||||
def test_get_url_handles_non_string_values(self):
|
||||
for value in (None, 1, {}, []):
|
||||
with self.subTest(value=value):
|
||||
@@ -18,6 +26,7 @@ class UrlParserTest(unittest.TestCase):
|
||||
|
||||
def test_preserves_only_platform_specific_query_parameters(self):
|
||||
cases = [
|
||||
("https://www.goofish.com/item?itemId=834469366646&noise=x", "https://www.goofish.com/item?itemId=834469366646"),
|
||||
("https://haokan.baidu.com/v?vid=11&noise=x", "https://haokan.baidu.com/v?vid=11"),
|
||||
("https://isee.weishi.qq.com/ws/app-pages/share/index.html?id=22&noise=x", "https://isee.weishi.qq.com/ws/app-pages/share/index.html?id=22"),
|
||||
("https://www.xiaohongshu.com/explore/33?xsec_token=token&noise=x", "https://www.xiaohongshu.com/explore/33?xsec_token=token"),
|
||||
|
||||
@@ -27,7 +27,18 @@ class XianyuParserTest(unittest.TestCase):
|
||||
with patch("requests.Session.get", return_value=page_resp):
|
||||
parser = XianyuParser(url)
|
||||
|
||||
self.assertEqual(parser.get_title_content(), "闲鱼商品 (商品ID: 967494320598, 标价: ¥12.2)")
|
||||
def test_parses_goofish_item_id_url(self):
|
||||
page_resp = Mock()
|
||||
page_resp.raise_for_status.return_value = None
|
||||
page_resp.text = "<html><body>闲鱼商品页</body></html>"
|
||||
|
||||
url = "https://www.goofish.com/item?itemId=834469366646"
|
||||
with patch("requests.Session.get", return_value=page_resp):
|
||||
parser = XianyuParser(url)
|
||||
|
||||
self.assertEqual(parser.get_title_content(), "闲鱼商品 (商品ID: 834469366646)")
|
||||
self.assertEqual(parser.get_image_list(), ["https://www.goofish.com/item?itemId=834469366646"])
|
||||
self.assertEqual(parser.get_cover_photo_url(), "https://www.goofish.com/item?itemId=834469366646")
|
||||
|
||||
def test_invalid_url_handles_gracefully(self):
|
||||
parser = XianyuParser("https://e.tb.cn/invalid")
|
||||
|
||||
+22
-3
@@ -103,10 +103,17 @@ class UrlParser:
|
||||
return None
|
||||
url_pattern = re.compile(r'https?:\/\/(?:www\.|[-a-zA-Z0-9.@:%_+~#=]{1,256}\.[a-zA-Z0-9()]{1,24})\b(?:[-a-zA-Z0-9()@:%_+.~#?&//=]*?)(?=(?:https?:\/\/|$|\s))')
|
||||
match = url_pattern.search(text)
|
||||
if match:
|
||||
return match.group()
|
||||
else:
|
||||
if not match:
|
||||
return None
|
||||
raw_url = match.group()
|
||||
# 清洗抖音/快手等短链误拼接的口令数字后缀(如 /3.05、/7.46)
|
||||
douyin_match = re.match(r'^(https?:\/\/v\.douyin\.com\/[a-zA-Z0-9_-]+)\/\d+(?:\.\d+)?\/?$', raw_url)
|
||||
if douyin_match:
|
||||
return douyin_match.group(1) + '/'
|
||||
kuaishou_match = re.match(r'^(https?:\/\/v\.kuaishou\.com\/[a-zA-Z0-9_-]+)\/\d+(?:\.\d+)?\/?$', raw_url)
|
||||
if kuaishou_match:
|
||||
return kuaishou_match.group(1) + '/'
|
||||
return raw_url
|
||||
|
||||
@staticmethod
|
||||
def get_domain(url):
|
||||
@@ -182,6 +189,15 @@ class UrlParser:
|
||||
address = f"{address}?{urlencode(preserved_params)}"
|
||||
elif platform == "快手":
|
||||
address = address.replace('http://', 'https://')
|
||||
elif platform == "闲鱼":
|
||||
query_params = parse_qs(parsed_url.query)
|
||||
preserved_params = []
|
||||
for key in ('id', 'itemId', 'item_id', 'price', 'tk'):
|
||||
value = query_params.get(key, [None])[0]
|
||||
if value is not None:
|
||||
preserved_params.append((key, value))
|
||||
if preserved_params:
|
||||
address = f"{address}?{urlencode(preserved_params)}"
|
||||
elif platform == "酷狗音乐":
|
||||
query_params = parse_qs(parsed_url.query)
|
||||
preserved_params = []
|
||||
@@ -497,6 +513,9 @@ class UrlParser:
|
||||
params_hash = query_params.get('hash', [None])[0]
|
||||
if params_hash:
|
||||
return params_hash
|
||||
params_item_id = query_params.get('itemId', [None])[0] or query_params.get('item_id', [None])[0]
|
||||
if params_item_id:
|
||||
return params_item_id
|
||||
params_id = query_params.get('id', [None])[0]
|
||||
if params_id:
|
||||
return params_id
|
||||
|
||||
Reference in New Issue
Block a user