Files
media-parser/utils/web_fetcher.py
T

621 lines
29 KiB
Python
Executable File

import re
import requests
import random
from src.utils.parser_transport import ParserSession, current_attempt
from urllib.parse import urljoin, urlparse, parse_qs, urlencode
from configs.logging_config import get_logger
from configs.general_constants import USER_AGENT_PC, DOMAIN_TO_NAME
logger = get_logger(__name__)
class WebFetcher:
headers = {
"content-type": "application/json; charset=UTF-8",
"User-Agent": random.choice(USER_AGENT_PC)
}
@staticmethod
def _is_allowed_target(url):
"""仅允许向已支持的平台域名发起 HTTP(S) 请求。"""
if not isinstance(url, str) or not url.strip():
return False
try:
parsed = urlparse(url)
if parsed.scheme not in {"http", "https"}:
return False
if not parsed.hostname or parsed.username is not None or parsed.password is not None:
return False
if parsed.port not in {None, 80, 443}:
return False
except ValueError:
return False
return UrlParser.get_platform(url) is not None
@staticmethod
def fetch_redirect_url(url, max_redirects=5):
if not isinstance(url, str) or not url.strip() or max_redirects < 1:
return None
if not WebFetcher._is_allowed_target(url):
logger.warning("Blocked outbound request to unsafe target: %s", UrlParser.get_domain(url))
return None
# 知乎公开 API 可直接通过内容 ID 解析,访问网页常触发 403 风控。
# 绿洲分享页会对通用桌面请求头返回 500,由解析器使用移动端请求头抓取。
# 微博博文可通过 API 直接根据 ID 解析,直接访问网页端常触发访客系统重定向。
domain = UrlParser.get_domain(url)
if domain not in {"t.cn", "b23.tv", "xhslink.cn", "xhslink.com", "hy.fan", "dw4.co", "yspapp.cn", "mapp.api.weibo.cn", "s.izuiyou.com", "adshare.toutiao.com"}:
if UrlParser.get_platform(url) in {"知乎", "绿洲", "新片场", "夸克AI", "通义千问", "微博", "小云雀AI", "哔哩哔哩", "快影", "微信公众号", "海螺AI", "网易LOFTER", "得物", "央视"}:
return UrlParser.extract_video_address(url)
try:
current_url = url
for _ in range(max_redirects):
# 发送请求,禁止重定向
if current_attempt():
with ParserSession() as session:
resp = session.get(current_url, headers=WebFetcher.headers, allow_redirects=False, timeout=5)
else:
resp = requests.get(current_url, headers=WebFetcher.headers, allow_redirects=False, timeout=5)
resp.raise_for_status()
redirect_url = resp.headers.get("location")
if not redirect_url and resp.status_code == 200 and isinstance(getattr(resp, "text", None), str):
meta_match = re.search(r'<meta[^>]*http-equiv=["\']?refresh["\']?[^>]*content=["\']?[^"\']*URL=[\'"]?([^\'" >]+)', resp.text, re.I)
if meta_match:
redirect_url = meta_match.group(1).strip("'\"")
if redirect_url:
redirect_url = urljoin(current_url, redirect_url)
# 如果重定向到了登录页、404拦截页、验证码校验页或错误页,不要更新 url,直接中断以保留原始有效 URL
if any(path in redirect_url for path in ["/login", "/404", "/captcha", "/verify", "/error", "/visitor"]):
break
if not WebFetcher._is_allowed_target(redirect_url):
logger.warning(
"Blocked redirect to unsafe target: %s",
UrlParser.get_domain(redirect_url),
)
return None
current_url = redirect_url
if UrlParser.get_platform(current_url):
break
else:
break
else:
return None
if not UrlParser.get_platform(current_url):
return None
return UrlParser.extract_video_address(current_url)
except requests.RequestException as e:
logger.error(f"Failed to get the page: {e}")
return None
except Exception as e:
logger.error(f"An error occurred: {e}")
return None
class UrlParser:
@staticmethod
def convert_to_https(url):
if not url:
return None
if url.startswith('http://'):
return 'https://' + url[7:]
return url
@staticmethod
def get_url(text):
if not isinstance(text, str):
return None
url_pattern = re.compile(r'https?:\/\/(?:www\.|[-a-zA-Z0-9.@:%_+~#=]{1,256}\.[a-zA-Z0-9()]{1,24})\b(?:[-a-zA-Z0-9()@:%_+.~#?&//=]*?)(?=(?:https?:\/\/|$|\s))')
match = url_pattern.search(text)
if not match:
return None
raw_url = match.group()
# 清洗抖音/快手等短链误拼接的口令数字后缀(如 /3.05、/7.46)
douyin_match = re.match(r'^(https?:\/\/v\.douyin\.com\/[a-zA-Z0-9_-]+)\/\d+(?:\.\d+)?\/?$', raw_url)
if douyin_match:
return douyin_match.group(1) + '/'
kuaishou_match = re.match(r'^(https?:\/\/v\.kuaishou\.com\/[a-zA-Z0-9_-]+)\/\d+(?:\.\d+)?\/?$', raw_url)
if kuaishou_match:
return kuaishou_match.group(1) + '/'
return raw_url
@staticmethod
def get_domain(url):
if not isinstance(url, str):
return ''
parsed_url = urlparse(url)
return (parsed_url.hostname or '').lower().rstrip('.')
@staticmethod
def get_platform(url):
"""识别链接所属平台,并兼容快手生成的随机移动端子域名。"""
domain = UrlParser.get_domain(url)
path = urlparse(url).path
if domain in {"www.iesdouyin.com", "iesdouyin.com"} and path.startswith("/xg/"):
return "西瓜视频"
platform = DOMAIN_TO_NAME.get(domain)
if platform:
return platform
if domain.endswith('.m.chenzhongtech.com'):
return '快手'
if domain == 'lofter.com' or domain.endswith('.lofter.com'):
return '网易LOFTER'
if domain.endswith('.dewu.com') or domain.endswith('.poizon.com'):
return '得物'
if domain == 'butterflyai.cn' or domain.endswith('.butterflyai.cn'):
return '星绘AI'
if domain in {'cctv.com', 'cctv.cn'} or domain.endswith('.cctv.com') or domain.endswith('.cctv.cn'):
return '央视'
if domain in {'yangshipin.cn', 'yspapp.cn'} or domain.endswith('.yangshipin.cn') or domain.endswith('.yspapp.cn'):
return '央视频'
return None
@staticmethod
def extract_video_address(url):
if not isinstance(url, str) or not url:
return None
parsed_url = urlparse(url)
domain = UrlParser.get_domain(url)
platform = UrlParser.get_platform(url)
address = f"{parsed_url.scheme}://{domain}{parsed_url.path}"
if address.endswith('/'):
address = address[:-1]
if platform == '好看视频':
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ('vid', 'id', 'context', 'pd', 'src', 'nid', 'sign', 'word', 'oword', 'atn', 'frsrcid'):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
from urllib.parse import quote
address = f"{address}?{urlencode(preserved_params, quote_via=quote)}"
elif platform == "微视":
query_params = parse_qs(parsed_url.query)
vid = query_params.get('id', [None])[0] # 使用 get 方法避免 KeyError
if vid:
address = f"{address}?id={vid}"
elif platform == "小红书":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ('xsec_token', 'xsec_source', 'source', 'xhsshare', 'app_platform'):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "快手":
address = address.replace('http://', 'https://')
elif platform == "闲鱼":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ('id', 'itemId', 'item_id', 'price', 'tk'):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "酷狗音乐":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ('hash', 'sruserid', 'kgsscty1', 'chain'):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "抖音":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ('modal_id', 'ep_id', 'album_id', 'episode_id', 'playlet_id', 'series_id'):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "全民K歌":
query_params = parse_qs(parsed_url.query)
s = query_params.get('s', [None])[0]
if s:
address = f"{address}?s={s}"
elif platform == "最右":
query_params = parse_qs(parsed_url.query)
pid = query_params.get('pid', [None])[0]
if pid:
address = f"{address}?pid={pid}"
elif platform == "豆包":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ('share_id', 'source_type', 'video_id', 'share_scene', 'vid'):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "即梦AI":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in (
'item_id',
'id',
'published_item_id',
'search_keyword',
'share_token',
'share_sec_uid',
'share_sec_did',
'share_campaign_key',
'share_category',
'share_from',
'ct',
'ab_vid',
'scene',
'reward_type',
'is_self_create_apperance',
'author_id',
'template_type_id',
'workDetailType',
'itemType',
'act_key',
):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "夸克AI":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("shareId", "share_id", "authorId", "author_id", "channel_from", "biz_id", "qwcontainer", "url", "env"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "可灵AI":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("creative_id", "work_id", "creative_type"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "微博":
query_params = parse_qs(parsed_url.query)
if value := query_params.get("fid", [None])[0]:
address = f"{address}?{urlencode({'fid': value})}"
elif platform == "Soul":
fragment = parsed_url.fragment
if "?" in fragment:
fragment_path, fragment_query = fragment.split("?", 1)
query_params = parse_qs(fragment_query)
preserved_params = []
for key in ("postIdEcpt", "sign", "signVersion"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}#{fragment_path}?{urlencode(preserved_params)}"
elif platform == "汽水音乐":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("track_id", "ugc_video_id"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "QQ音乐":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("vid", "songid", "songId", "songmid", "songMid", "id", "__"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "网易云音乐":
query_params = parse_qs(parsed_url.query)
fragment = parsed_url.fragment.lstrip("/")
if "?" in fragment:
fragment_path, fragment_query = fragment.split("?", 1)
if fragment_path in {"song", "mv", "event"}:
address = f"{parsed_url.scheme}://{domain}/{fragment_path}"
query_params = parse_qs(fragment_query)
preserved_params = []
for key in ("id", "uid", "userid", "type", "eventId", "songId"):
value = query_params.get(key, [None])[0]
if value is not None and value != "null":
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "酷狗音乐":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("hash", "chain", "sruserid", "kgsscty1", "chl"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "松果时刻":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("sharer_id", "author_id", "channel", "version", "style_id", "share_id", "project_id", "story_type", "story_id", "item_type", "item_id"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "腾讯频道":
query_params = parse_qs(parsed_url.query)
if value := query_params.get("b", [None])[0]:
address = f"{address}?{urlencode({'b': value})}"
elif platform == "剪映":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("template_id", "item_type"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "快影":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("id", "templateId", "template_id", "userId", "app_name", "page_name"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "微信公众号":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in (
"__biz", "mid", "idx", "sn", "chksm", "t", "vid", "vidsn",
"voice_id", "item_show_type", "scene", "subscene", "clicktime",
"enterid", "sharer_shareinfo", "sharer_shareinfo_first"
):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params, safe='=/')}"
elif platform == "视频号":
query_params = parse_qs(parsed_url.query)
short_uri = query_params.get('id', [None])[0]
if short_uri:
address = f"{address}?{urlencode({'id': short_uri})}"
elif platform == "绿洲":
query_params = parse_qs(parsed_url.query)
if value := query_params.get("sid", [None])[0]:
address = f"{address}?{urlencode({'sid': value})}"
elif platform == "通义千问":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("shareId", "share_id", "authorId", "author_id", "bizId", "biz_id", "enter_from", "fp_from", "channel_from", "image_index", "qwcontainer", "env"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "腾讯元宝":
query_params = parse_qs(parsed_url.query)
user_id = query_params.get("userId", [None])[0]
if user_id:
address = f"{address}?{urlencode({'userId': user_id})}"
elif platform == "闲鱼":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("tk", "id", "price", "shareurl", "short_name", "sp_tk"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "拼多多":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in (
"goods_id",
"feed_id",
"review_id",
"ps",
"_oak_share_url",
"_t_timestamp",
"share_uin",
"refer_share_uin",
"refer_share_id",
"channel",
"page_from",
):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform in ("红果短剧", "番茄小说", "红果漫剧", "小云雀AI"):
if parsed_url.query:
address = f"{address}?{parsed_url.query}"
elif platform == "得物":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("trendId", "shareId", "source", "shareType", "shareChannel", "isScreenShot"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "网易LOFTER":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("incantation", "permalink", "postId", "blogId"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "星绘AI":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("share_code", "share_id", "entity_type", "is_mine", "resource_count", "share_token"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "央视":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("item_id", "articleId", "pid", "guid"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
elif platform == "央视频":
query_params = parse_qs(parsed_url.query)
preserved_params = []
for key in ("vid", "cid", "type", "serverFrom", "articleid", "articleId", "id"):
value = query_params.get(key, [None])[0]
if value is not None:
preserved_params.append((key, value))
if preserved_params:
address = f"{address}?{urlencode(preserved_params)}"
return address
@staticmethod
def get_video_id(url):
try:
parsed_url = urlparse(url)
query_params = parse_qs(parsed_url.query)
# 尝试从查询参数中获取视频ID
match_vid = re.search(r'(?:["%22]|%22)?(?:vid|video_id|content_id|material_id)(?:["%22]|%22)?\s*(?:[:=]|%3A)\s*(?:["%22]|%22)?(\d{15,22})', url)
if match_vid:
return match_vid.group(1)
params_trend_id = query_params.get('trendId', [None])[0]
if params_trend_id:
return params_trend_id
params_share_id = query_params.get('shareId', [None])[0]
if params_share_id:
return params_share_id
params_vid = query_params.get('vid', [None])[0]
if params_vid:
return params_vid
params_songid = (
query_params.get('songid', [None])[0]
or query_params.get('songId', [None])[0]
or query_params.get('songmid', [None])[0]
or query_params.get('songMid', [None])[0]
)
if params_songid:
return params_songid
params_hash = query_params.get('hash', [None])[0]
if params_hash:
return params_hash
params_item_id = query_params.get('itemId', [None])[0] or query_params.get('item_id', [None])[0]
if params_item_id:
return params_item_id
params_id = query_params.get('id', [None])[0]
if params_id:
return params_id
params_modal_id = query_params.get('modal_id', [None])[0]
if params_modal_id:
return params_modal_id
params_ep_id = query_params.get('ep_id', [None])[0]
if params_ep_id:
return params_ep_id
params_episode_id = query_params.get('episode_id', [None])[0]
if params_episode_id:
return params_episode_id
params_album_id = query_params.get('album_id', [None])[0]
if params_album_id:
return params_album_id
params_playlet_id = query_params.get('playlet_id', [None])[0]
if params_playlet_id:
return params_playlet_id
params_series_id = query_params.get('series_id', [None])[0]
if params_series_id:
return params_series_id
params_v = query_params.get('v', [None])[0]
if params_v:
return params_v
params_s = query_params.get('s', [None])[0]
if params_s:
return params_s
params_pid = query_params.get('pid', [None])[0]
if params_pid:
return params_pid
params_video_id = query_params.get('video_id', [None])[0]
if params_video_id:
return params_video_id
params_creative_id = query_params.get('creative_id', [None])[0]
if params_creative_id:
return params_creative_id
params_work_id = query_params.get('work_id', [None])[0]
if params_work_id:
return params_work_id
fragment_query = parse_qs(parsed_url.fragment.split('?', 1)[1]) if '?' in parsed_url.fragment else {}
params_post_id = fragment_query.get('postIdEcpt', [None])[0]
if params_post_id:
return params_post_id
params_track_id = query_params.get('track_id', [None])[0]
if params_track_id:
return params_track_id
params_ugc_video_id = query_params.get('ugc_video_id', [None])[0]
if params_ugc_video_id:
return params_ugc_video_id
params_template_id = query_params.get('template_id', [None])[0]
if params_template_id:
return params_template_id
params_feed_id = query_params.get('feed_id', [None])[0]
if params_feed_id:
return params_feed_id
params_goods_id = query_params.get('goods_id', [None])[0]
if params_goods_id:
return params_goods_id
params_review_id = query_params.get('review_id', [None])[0]
if params_review_id:
return params_review_id
params_ps = query_params.get('ps', [None])[0]
if params_ps:
return params_ps
params_share_code = query_params.get('share_code', [None])[0]
if params_share_code:
return params_share_code
params_item_id = query_params.get('item_id', [None])[0]
if params_item_id:
return params_item_id
params_article_id = query_params.get('articleId', [None])[0] or query_params.get('articleid', [None])[0]
if params_article_id:
return params_article_id
params_guid = query_params.get('guid', [None])[0]
if params_guid:
return params_guid
# 尝试从URL路径中获取视频ID
path_segments = parsed_url.path.strip('/').split('/')
if path_segments:
video_id = path_segments[-1]
if video_id.endswith('.html'):
video_id = video_id[:-5]
elif video_id.endswith('.shtml'):
video_id = video_id[:-6]
desktop_mv = re.fullmatch(r'mv_([0-9a-fA-F]{32})', video_id)
if desktop_mv:
return desktop_mv.group(1)
return video_id
logger.warning(f'Unable to retrieve video ID from URL: {url}')
return None
except Exception as e:
logger.error(f"An error occurred while extracting video ID: {e}")
return None