mirror of
https://github.com/ucmao/media-parser.git
synced 2026-11-20 08:04:56 +08:00
移除 TikTok 和 YouTube 平台支持
This commit is contained in:
@@ -52,7 +52,6 @@
|
||||
| **皮皮虾** | ✓ | ✓ | ✓ | ✓ | | | | **全民小视频** | ✓ | ✓ | ✓ | ✓ | | | |
|
||||
| **全民K歌** | | ✓ | ✓ | ✓ | | | | **六间房** | | ✓ | ✓ | ✓ | | | |
|
||||
| **新片场** | | ✓ | ✓ | ✓ | | | | **最右** | ✓ | ✓ | | ✓ | | | |
|
||||
| **TikTok** | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | **YouTube** | ✓ | ✓ | ✓ | ✓ | | ✓ | |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -18,17 +18,11 @@
|
||||
"h5.pipigx.com": "皮皮搞笑",
|
||||
"m.acfun.cn": "AcFun",
|
||||
"www.acfun.cn": "AcFun",
|
||||
"www.tiktok.com": "TikTok",
|
||||
"vt.tiktok.com": "TikTok",
|
||||
"vm.tiktok.com": "TikTok",
|
||||
"weibo.com": "微博",
|
||||
"m.weibo.cn": "微博",
|
||||
"www.ixigua.com": "西瓜视频",
|
||||
"v.ixigua.com": "西瓜视频",
|
||||
"www.youtube.com": "YouTube",
|
||||
"youtu.be": "YouTube",
|
||||
"m.youtube.com": "YouTube",
|
||||
"www.zhihu.com": "知乎",
|
||||
"www.zhihu.com": "知乎",
|
||||
"zhuanlan.zhihu.com": "知乎",
|
||||
"doupai.cc": "逗拍",
|
||||
"v2.doupai.cc": "逗拍",
|
||||
|
||||
@@ -7,10 +7,8 @@ from src.parsers.weishi_parser import WeishiParser
|
||||
from src.parsers.lishipin_parser import LishipinParser
|
||||
from src.parsers.pipigaoxiao_parser import PipigaoxiaoParser
|
||||
from src.parsers.acfun_parser import AcfunParser
|
||||
from src.parsers.tiktok_parser import TiktokParser
|
||||
from src.parsers.weibo_parser import WeiboParser
|
||||
from src.parsers.xigua_parser import XiguaParser
|
||||
from src.parsers.youtube_parser import YoutubeParser
|
||||
from src.parsers.zhihu_parser import ZhihuParser
|
||||
from src.parsers.doupai_parser import DoupaiParser
|
||||
from src.parsers.huya_parser import HuyaParser
|
||||
@@ -36,10 +34,8 @@ class ParserFactory:
|
||||
"梨视频": LishipinParser,
|
||||
"皮皮搞笑": PipigaoxiaoParser,
|
||||
"AcFun": AcfunParser,
|
||||
"TikTok": TiktokParser,
|
||||
"微博": WeiboParser,
|
||||
"西瓜视频": XiguaParser,
|
||||
"YouTube": YoutubeParser,
|
||||
"知乎": ZhihuParser,
|
||||
"逗拍": DoupaiParser,
|
||||
"虎牙": HuyaParser,
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
import re
|
||||
import json
|
||||
import random
|
||||
from src.parsers.base_parser import BaseParser
|
||||
from configs.general_constants import USER_AGENT_PC, USER_AGENT_M
|
||||
from configs.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
class TiktokParser(BaseParser):
|
||||
def __init__(self, real_url):
|
||||
super().__init__(real_url)
|
||||
self.headers = {
|
||||
'User-Agent': random.choice(USER_AGENT_PC),
|
||||
}
|
||||
self.video_id = self._extract_id()
|
||||
self.post_data = self._fetch_post_data()
|
||||
|
||||
def _extract_id(self):
|
||||
if not self.real_url:
|
||||
return None
|
||||
|
||||
# Match URL patterns like:
|
||||
# https://www.tiktok.com/@username/video/7106594312292453675
|
||||
# https://m.tiktok.com/v/7106594312292453675.html
|
||||
# https://vt.tiktok.com/ZSdQXXXXX/ -> redirect
|
||||
|
||||
match = re.search(r'/video/(\d+)', self.real_url)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
match = re.search(r'/v/(\d+)', self.real_url)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
return None
|
||||
|
||||
def _fetch_post_data(self):
|
||||
if not self.video_id:
|
||||
logger.error("TiktokParser: Could not extract video ID.")
|
||||
return {}
|
||||
|
||||
# Using tikwm API as primary, since TikTok's own web download addresses
|
||||
# are protected by rigid browser fingerprints and Akamai WAF
|
||||
url = "https://www.tikwm.com/api/"
|
||||
params = {
|
||||
'url': self.real_url,
|
||||
'count': 12,
|
||||
'cursor': 0,
|
||||
'web': 1,
|
||||
'hd': 1
|
||||
}
|
||||
try:
|
||||
resp = self.session.get(url, params=params, headers=self.headers, timeout=10)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
if data.get('code') == 0:
|
||||
return data.get('data', {})
|
||||
except Exception as e:
|
||||
logger.warning(f"Tiktok API fetch failed: {e}")
|
||||
|
||||
return {}
|
||||
|
||||
def get_real_video_url(self):
|
||||
try:
|
||||
# prefer hdplay then play
|
||||
play_addr = self.post_data.get('hdplay') or self.post_data.get('play')
|
||||
if play_addr:
|
||||
if play_addr.startswith('/'):
|
||||
return f"https://www.tikwm.com{play_addr}"
|
||||
return play_addr
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to extract real video url: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def get_title_content(self):
|
||||
return self.post_data.get('title', '')
|
||||
|
||||
def get_cover_photo_url(self):
|
||||
try:
|
||||
cover = self.post_data.get('cover')
|
||||
if cover:
|
||||
if cover.startswith('/'):
|
||||
return f"https://www.tikwm.com{cover}"
|
||||
return cover
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
def get_image_list(self):
|
||||
images = []
|
||||
try:
|
||||
# TikWM supports photo mode as well in the "images" field array
|
||||
if 'images' in self.post_data and isinstance(self.post_data['images'], list):
|
||||
for img in self.post_data['images']:
|
||||
images.append(img)
|
||||
except:
|
||||
pass
|
||||
return images
|
||||
|
||||
def get_audio_url(self):
|
||||
try:
|
||||
music_url = self.post_data.get('play', '') if self.post_data.get('music') is None else self.post_data.get('music')
|
||||
# Fallback to music_info if music is empty string
|
||||
if not music_url:
|
||||
music_url = self.post_data.get('music_info', {}).get('play')
|
||||
|
||||
# Note: actual tikwm API typically uses 'music' for the background music track
|
||||
if not music_url:
|
||||
music_url = self.post_data.get('music')
|
||||
|
||||
if music_url:
|
||||
if music_url.startswith('/'):
|
||||
return f"https://www.tikwm.com{music_url}"
|
||||
return music_url
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to extract audio url: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def get_author_info(self):
|
||||
try:
|
||||
author = self.post_data.get('author', {})
|
||||
if author:
|
||||
uid = author.get('unique_id', '') or author.get('id', '')
|
||||
nickname = author.get('nickname', '')
|
||||
avatar = author.get('avatar', '')
|
||||
if avatar.startswith('/'):
|
||||
avatar = f"https://www.tikwm.com{avatar}"
|
||||
|
||||
return {
|
||||
"nickname": nickname,
|
||||
"author_id": str(uid),
|
||||
"avatar": avatar
|
||||
}
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
real_url = 'https://www.tiktok.com/@tiktok/video/7106594312292453675'
|
||||
dl = TiktokParser(real_url)
|
||||
print("-" * 30)
|
||||
print(f"作者信息:{dl.get_author_info()}")
|
||||
print(f"标题内容:{dl.get_title_content()[:30]}...")
|
||||
print(f"封面图片:{dl.get_cover_photo_url()}")
|
||||
print(f"视频链接:{dl.get_real_video_url()}")
|
||||
print(f"音频链接:{dl.get_audio_url()}")
|
||||
print(f"图片列表:{dl.get_image_list()}")
|
||||
print("-" * 30)
|
||||
@@ -1,115 +0,0 @@
|
||||
import logging
|
||||
import re
|
||||
import json
|
||||
from src.parsers.base_parser import BaseParser
|
||||
from configs.logging_config import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
class YoutubeParser(BaseParser):
|
||||
def __init__(self, real_url):
|
||||
super().__init__(real_url)
|
||||
self.headers = {
|
||||
'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
}
|
||||
self.post_data = self._fetch_post_data()
|
||||
|
||||
def _fetch_post_data(self):
|
||||
try:
|
||||
resp = self.session.get(self.real_url, headers=self.headers, timeout=10)
|
||||
if resp.status_code == 200:
|
||||
match = re.search(r'ytInitialPlayerResponse\s*=\s*({.+?});', resp.text)
|
||||
if match:
|
||||
player_res = json.loads(match.group(1))
|
||||
return player_res
|
||||
except Exception as e:
|
||||
logger.warning(f"Youtube raw fetch failed: {e}")
|
||||
|
||||
return {}
|
||||
|
||||
|
||||
def get_real_video_url(self):
|
||||
try:
|
||||
streamingData = self.post_data.get('streamingData', {})
|
||||
formats = streamingData.get('formats', [])
|
||||
|
||||
# YouTube format with Audio and Video multiplexed
|
||||
for fmt in formats:
|
||||
# Prioritize 720p or 1080p multiplexed
|
||||
url = fmt.get('url')
|
||||
if url:
|
||||
return url
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Youtube URL extraction error: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def get_audio_url(self):
|
||||
try:
|
||||
streamingData = self.post_data.get('streamingData', {})
|
||||
adaptiveFormats = streamingData.get('adaptiveFormats', [])
|
||||
|
||||
# Prioritize m4a (audio/mp4) or webm audio
|
||||
for fmt in adaptiveFormats:
|
||||
mime = fmt.get('mimeType', '')
|
||||
if 'audio/' in mime:
|
||||
url = fmt.get('url')
|
||||
if url:
|
||||
return url
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Youtube audio URL extraction error: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def get_title_content(self):
|
||||
details = self.post_data.get('videoDetails', {})
|
||||
title = details.get('title', '')
|
||||
desc = details.get('shortDescription', '')
|
||||
if desc:
|
||||
return f"{title}\n{desc[:200]}"
|
||||
return title
|
||||
|
||||
def get_cover_photo_url(self):
|
||||
try:
|
||||
thumbnails = self.post_data.get('videoDetails', {}).get('thumbnail', {}).get('thumbnails', [])
|
||||
if thumbnails:
|
||||
# get max resolution thumbnail
|
||||
return thumbnails[-1].get('url')
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
def get_image_list(self):
|
||||
return []
|
||||
|
||||
def get_author_info(self):
|
||||
details = self.post_data.get('videoDetails', {})
|
||||
author_name = details.get('author', '')
|
||||
channel_id = details.get('channelId', '')
|
||||
|
||||
avatar = None
|
||||
try:
|
||||
micro = self.post_data.get('microformat', {}).get('playerMicroformatRenderer', {})
|
||||
avatar = micro.get('ownerProfileUrl', '')
|
||||
except:
|
||||
pass
|
||||
|
||||
return {
|
||||
"nickname": author_name,
|
||||
"author_id": channel_id,
|
||||
"avatar": avatar
|
||||
}
|
||||
|
||||
if __name__ == '__main__':
|
||||
real_url = 'https://www.youtube.com/watch?v=kffacxfA7G4'
|
||||
dl = YoutubeParser(real_url)
|
||||
print("-" * 30)
|
||||
print(f"作者信息:{dl.get_author_info()}")
|
||||
print(f"标题内容:{dl.get_title_content()[:50]}...")
|
||||
print(f"封面图片:{dl.get_cover_photo_url()}")
|
||||
print(f"视频链接:{dl.get_real_video_url()}")
|
||||
print(f"音频链接:{dl.get_audio_url()} ")
|
||||
print("-" * 30)
|
||||
@@ -131,7 +131,7 @@
|
||||
<div class="max-w-4xl mx-auto">
|
||||
<div class="text-center mb-12">
|
||||
<h2 class="text-3xl font-black text-slate-800 mb-4">在线体验去水印</h2>
|
||||
<p class="text-slate-500">支持抖音、快手、小红书、B站、微博、知乎、西瓜视频、TikTok、YouTube、AcFun、微视、好看视频、皮皮搞笑、梨视频等主流平台视频或图文链接
|
||||
<p class="text-slate-500">支持抖音、快手、小红书、B站、微博、知乎、西瓜视频、AcFun、微视、好看视频、皮皮搞笑、梨视频等主流平台视频或图文链接
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@@ -278,7 +278,7 @@
|
||||
</div>
|
||||
<h3 class="text-xl font-bold text-slate-800 mb-3">多平台支持</h3>
|
||||
<p class="text-slate-500 leading-relaxed">
|
||||
全面兼容抖音、快手、小红书、B站、微博、知乎、西瓜视频、豆包、即梦 AI、TikTok、YouTube、AcFun、微视、好看视频、皮皮搞笑、梨视频等主流媒体平台,覆盖全网素材。</p>
|
||||
全面兼容抖音、快手、小红书、B站、微博、知乎、西瓜视频、豆包、即梦 AI、AcFun、微视、好看视频、皮皮搞笑、梨视频等主流媒体平台,覆盖全网素材。</p>
|
||||
</div>
|
||||
<div
|
||||
class="p-8 bg-slate-50 rounded-3xl border border-slate-100 hover:border-indigo-200 transition-all group">
|
||||
|
||||
@@ -22,7 +22,6 @@ class UrlParserTest(unittest.TestCase):
|
||||
("https://isee.weishi.qq.com/ws/app-pages/share/index.html?id=22&noise=x", "https://isee.weishi.qq.com/ws/app-pages/share/index.html?id=22"),
|
||||
("https://www.xiaohongshu.com/explore/33?xsec_token=token&noise=x", "https://www.xiaohongshu.com/explore/33?xsec_token=token"),
|
||||
("https://www.douyin.com/?modal_id=44&noise=x", "https://www.douyin.com?modal_id=44"),
|
||||
("https://www.youtube.com/watch?v=55&feature=share", "https://www.youtube.com/watch?v=55"),
|
||||
("https://kg.qq.com/node/play?s=66&noise=x", "https://kg.qq.com/node/play?s=66"),
|
||||
("https://izuiyou.com/post/detail?pid=77&noise=x", "https://izuiyou.com/post/detail?pid=77"),
|
||||
]
|
||||
@@ -32,7 +31,6 @@ class UrlParserTest(unittest.TestCase):
|
||||
|
||||
def test_get_video_id_supports_query_path_and_html_suffix(self):
|
||||
cases = [
|
||||
("https://www.youtube.com/watch?v=query-id", "query-id"),
|
||||
("https://www.doubao.com/video-sharing?video_id=video-id", "video-id"),
|
||||
("https://www.bilibili.com/video/BV123", "BV123"),
|
||||
("https://www.pearvideo.com/video_123.html", "video_123"),
|
||||
@@ -62,7 +60,6 @@ class UrlParserTest(unittest.TestCase):
|
||||
with self.subTest(url=url):
|
||||
self.assertIsNone(UrlParser.get_platform(url))
|
||||
|
||||
|
||||
class WebFetcherTest(unittest.TestCase):
|
||||
@staticmethod
|
||||
def response(location=None, status_code=200):
|
||||
|
||||
@@ -121,11 +121,6 @@ class UrlParser:
|
||||
modal_id = query_params.get('modal_id', [None])[0]
|
||||
if modal_id:
|
||||
address = f"{address}?modal_id={modal_id}"
|
||||
elif platform == "YouTube":
|
||||
query_params = parse_qs(parsed_url.query)
|
||||
v = query_params.get('v', [None])[0]
|
||||
if v:
|
||||
address = f"{address}?v={v}"
|
||||
elif platform == "全民K歌":
|
||||
query_params = parse_qs(parsed_url.query)
|
||||
s = query_params.get('s', [None])[0]
|
||||
@@ -207,10 +202,8 @@ class UrlParser:
|
||||
'快手': 'https://www.kuaishou.com/short-video/',
|
||||
'梨视频': 'https://www.pearvideo.com/',
|
||||
'AcFun': 'https://www.acfun.cn/v/',
|
||||
'TikTok': 'https://www.tiktok.com/@/video/',
|
||||
'微博': 'https://m.weibo.cn/status/',
|
||||
'西瓜视频': 'https://www.ixigua.com/',
|
||||
'YouTube': 'https://www.youtube.com/watch?v=',
|
||||
'知乎': 'https://www.zhihu.com/question/',
|
||||
'逗拍': 'https://v2.doupai.cc/topic/',
|
||||
'虎牙': 'https://v.huya.com/play/',
|
||||
|
||||
Reference in New Issue
Block a user