mirror of
https://github.com/ucmao/media-parser.git
synced 2026-11-20 08:04:56 +08:00
468 lines
20 KiB
Python
468 lines
20 KiB
Python
"""代理窗口、跨进程协调、失败重试与请求预算的确定性回归。"""
|
|
|
|
import json
|
|
import multiprocessing
|
|
import os
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from unittest.mock import Mock, patch
|
|
|
|
import requests
|
|
|
|
from app import create_app
|
|
from src.api.parse import _execute_parse
|
|
from src.utils.parser_transport import (
|
|
Attempt, ParseFailure, ParserSession, ProxyAccessFailure, attempt_context, current_attempt,
|
|
)
|
|
from src.utils.proxy_manager import ProxyAddress, ProxyManager
|
|
|
|
|
|
def activate_worker(database):
|
|
manager = ProxyManager(database, 'http://provider.invalid')
|
|
manager.activate('快手')
|
|
|
|
|
|
def replenish_worker(database, api_url, event):
|
|
manager = ProxyManager(database, api_url)
|
|
event.wait(10)
|
|
manager.replenish(time.monotonic() + 5)
|
|
|
|
|
|
class Clock:
|
|
def __init__(self):
|
|
self.now = 1000.0
|
|
|
|
def time(self):
|
|
return self.now
|
|
|
|
def sleep(self, seconds):
|
|
self.now += seconds
|
|
|
|
|
|
def payload(ip='114.106.134.52', remain=49):
|
|
return {'code': 200, 'data': {'count': 1, 'surplus_quantity': 0, 'proxy_list': [
|
|
{'ip': ip, 'port': 35152, 'ip_remain': remain, 'city_code': '341700', 'city_name': '安徽|池州'}
|
|
]}}
|
|
|
|
|
|
class ManagerTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.database = os.path.join(self.temp.name, 'proxy.db')
|
|
self.manager = ProxyManager(self.database, 'http://provider.invalid')
|
|
self.clock = Clock()
|
|
self.patches = [patch('time.time', self.clock.time), patch('time.monotonic', self.clock.time)]
|
|
for item in self.patches:
|
|
item.start()
|
|
self.addCleanup(item.stop)
|
|
|
|
def supply(self, ip='114.106.134.52', remain=49):
|
|
session = Mock()
|
|
session.get.return_value.json.return_value = payload(ip, remain)
|
|
with patch('src.utils.proxy_manager.requests.Session') as factory:
|
|
factory.return_value.__enter__.return_value = session
|
|
called = self.manager.replenish(self.clock.now + 5)
|
|
return called, session
|
|
|
|
def test_json2_remaining_and_zero_surplus(self):
|
|
proxy = self.manager.decode(payload(), 998)
|
|
self.assertEqual(proxy.address, '114.106.134.52:35152')
|
|
self.assertEqual(proxy.expires_at, 1047)
|
|
self.assertEqual(proxy.url, 'http://114.106.134.52:35152')
|
|
for remain in (None, 0, 9, 'bad', float('nan'), float('inf')):
|
|
with self.subTest(remain=remain), self.assertRaises((ValueError, TypeError)):
|
|
self.manager.decode(payload(remain=remain), 1000)
|
|
|
|
def test_window_does_not_extend_and_other_platform_is_direct(self):
|
|
self.manager.activate('快手')
|
|
self.clock.now = 1100
|
|
self.manager.activate('快手')
|
|
with self.manager.connection() as db:
|
|
self.assertEqual(db.execute('SELECT until_at FROM proxy_platform_windows').fetchone()[0], 1900)
|
|
self.assertTrue(self.manager.active('快手'))
|
|
self.assertFalse(self.manager.active('抖音'))
|
|
self.clock.now = 1900
|
|
self.assertFalse(self.manager.active('快手'))
|
|
self.manager.activate('快手')
|
|
self.assertTrue(self.manager.active('快手'))
|
|
|
|
def test_prefetch_retirement_and_inflight_validity(self):
|
|
self.manager.activate('快手')
|
|
called, session = self.supply()
|
|
self.assertTrue(called)
|
|
self.assertFalse(session.trust_env)
|
|
first = self.manager.available()
|
|
self.clock.now += 1
|
|
self.supply('114.106.134.53')
|
|
self.assertFalse(self.supply('114.106.134.54')[0])
|
|
self.clock.now = 1030
|
|
self.assertTrue(self.supply('114.106.134.54')[0])
|
|
self.assertNotEqual(self.manager.available().address, first.address)
|
|
self.assertTrue(self.manager.valid(first))
|
|
self.clock.now = 1049
|
|
self.assertFalse(self.manager.valid(first))
|
|
|
|
def test_invalid_address_shared_and_duplicate_does_not_revive(self):
|
|
self.supply()
|
|
proxy = self.manager.available()
|
|
other = ProxyManager(self.database, self.manager.api_url)
|
|
other.invalidate(proxy, '访问失败')
|
|
self.assertFalse(self.manager.valid(proxy))
|
|
self.assertIsNone(self.manager.available())
|
|
self.clock.now += 1
|
|
self.supply()
|
|
self.assertIsNone(self.manager.available())
|
|
|
|
def test_maintenance_idle_and_inflight_use(self):
|
|
with patch.object(self.manager, 'replenish') as replenish:
|
|
self.manager.maintain_once()
|
|
replenish.assert_not_called()
|
|
self.manager.activate('快手')
|
|
self.manager.maintain_once()
|
|
self.assertEqual(replenish.call_count, 1)
|
|
self.clock.now = 1899
|
|
token = self.manager.begin_use(90)
|
|
self.clock.now = 1901
|
|
self.manager.maintain_once()
|
|
self.assertEqual(replenish.call_count, 2)
|
|
self.manager.end_use(token)
|
|
self.manager.maintain_once()
|
|
self.assertEqual(replenish.call_count, 2)
|
|
|
|
def test_lease_takeover_and_extraction_frequency(self):
|
|
with self.manager.connection(write=True) as db:
|
|
db.execute("UPDATE proxy_coordination SET owner='dead', lease_until=1008, next_fetch=1001")
|
|
self.assertFalse(self.supply()[0])
|
|
self.clock.now = 1008
|
|
self.assertTrue(self.supply()[0])
|
|
self.assertFalse(self.supply('114.106.134.53')[0])
|
|
self.clock.now += 1
|
|
self.assertTrue(self.supply('114.106.134.53')[0])
|
|
|
|
def test_supplier_failure_backoff(self):
|
|
session = Mock()
|
|
session.get.side_effect = requests.Timeout()
|
|
with patch('src.utils.proxy_manager.requests.Session') as factory:
|
|
factory.return_value.__enter__.return_value = session
|
|
for delay in (1, 2, 4, 8, 15, 15):
|
|
self.assertTrue(self.manager.replenish(self.clock.now + 5))
|
|
with self.manager.connection() as db:
|
|
next_fetch = db.execute('SELECT next_fetch FROM proxy_coordination').fetchone()[0]
|
|
self.assertEqual(next_fetch, self.clock.now + delay)
|
|
self.assertFalse(self.manager.replenish(self.clock.now + 5))
|
|
self.clock.now = next_fetch
|
|
|
|
|
|
class ProcessCoordinationTest(unittest.TestCase):
|
|
def test_two_processes_share_window(self):
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
database = os.path.join(temp, 'shared.db')
|
|
manager = ProxyManager(database, 'http://provider.invalid')
|
|
context = multiprocessing.get_context('spawn')
|
|
processes = [context.Process(target=activate_worker, args=(database,)) for _ in range(2)]
|
|
for process in processes:
|
|
process.start()
|
|
for process in processes:
|
|
process.join(15)
|
|
self.assertEqual(process.exitcode, 0)
|
|
self.assertTrue(manager.active('快手'))
|
|
with manager.connection() as db:
|
|
self.assertEqual(db.execute('SELECT count(*) FROM proxy_platform_windows').fetchone()[0], 1)
|
|
|
|
def test_two_processes_only_extract_once(self):
|
|
calls = []
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
calls.append(time.monotonic())
|
|
time.sleep(0.2)
|
|
data = json.dumps(payload()).encode()
|
|
self.send_response(200)
|
|
self.send_header('Content-Length', str(len(data)))
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
with ThreadingHTTPServer(('127.0.0.1', 0), Handler) as server:
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
try:
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
database = os.path.join(temp, 'shared.db')
|
|
api_url = f'http://127.0.0.1:{server.server_port}'
|
|
ProxyManager(database, api_url)
|
|
context = multiprocessing.get_context('spawn')
|
|
event = context.Event()
|
|
processes = [context.Process(target=replenish_worker, args=(database, api_url, event)) for _ in range(2)]
|
|
for process in processes:
|
|
process.start()
|
|
event.set()
|
|
for process in processes:
|
|
process.join(15)
|
|
self.assertEqual(process.exitcode, 0)
|
|
self.assertEqual(len(calls), 1)
|
|
finally:
|
|
server.shutdown()
|
|
thread.join()
|
|
|
|
|
|
def fake_parser(kind='success'):
|
|
parser = Mock()
|
|
parser.terminal_error = None
|
|
parser.no_media_in_content = False
|
|
parser.cookie_required = kind == 'cookie'
|
|
parser.get_title_content.return_value = '测试标题'
|
|
parser.get_description.return_value = None
|
|
parser.get_real_video_url.return_value = 'https://cdn.example/video.mp4' if kind == 'success' else None
|
|
parser.get_video_list.return_value = []
|
|
parser.get_image_list.return_value = []
|
|
parser.get_cover_photo_url.return_value = None
|
|
parser.get_author_info.return_value = None
|
|
parser.get_audio_url.return_value = None
|
|
parser.get_subtitles.return_value = None
|
|
if kind == 'cookie':
|
|
parser.terminal_error = {'error_code': 'KUAISHOU_COOKIE_REQUIRED', 'detail_msg': '需要 Cookie'}
|
|
if kind == 'deleted':
|
|
parser.terminal_error = {'detail_msg': '内容已删除'}
|
|
if kind == 'text':
|
|
parser.no_media_in_content = True
|
|
return parser
|
|
|
|
|
|
class RetryIntegrationTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.app = create_app({'TESTING': True, 'API_ONLY': True, 'SECRET_KEY': 'test',
|
|
'DATABASE': os.path.join(self.temp.name, 'test.db'),
|
|
'DYNAMIC_PROXY_API_URL': 'http://provider.invalid'})
|
|
self.manager = self.app.extensions['proxy_manager']
|
|
self.client = self.app.test_client()
|
|
self.url = 'https://v.kuaishou.com/test'
|
|
self.clock = Clock()
|
|
for target, value in [('time.time', self.clock.time), ('time.monotonic', self.clock.time),
|
|
('src.api.parse.time.sleep', self.clock.sleep)]:
|
|
item = patch(target, value)
|
|
item.start()
|
|
self.addCleanup(item.stop)
|
|
item = patch('src.api.parse.WebFetcher.fetch_redirect_url', return_value=self.url)
|
|
self.redirect = item.start()
|
|
self.addCleanup(item.stop)
|
|
self.sessions = []
|
|
|
|
def supply(self):
|
|
session = Mock()
|
|
index = len(self.sessions) + 1
|
|
session.get.return_value.json.return_value = payload(f'114.106.134.{index}', 300)
|
|
self.sessions.append(session)
|
|
return session
|
|
|
|
def parse(self, parsers):
|
|
with patch('src.api.parse.ParserFactory.create_parser', side_effect=parsers) as factory:
|
|
with patch('src.utils.proxy_manager.requests.Session') as sessions:
|
|
sessions.return_value.__enter__.side_effect = self.supply
|
|
response = self.client.get('/api/v1/parse', query_string={'url': self.url})
|
|
return response, factory
|
|
|
|
def test_direct_success_does_not_fetch_proxy(self):
|
|
response, factory = self.parse([fake_parser()])
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(len(self.sessions), 0)
|
|
self.assertFalse(self.manager.active('快手'))
|
|
self.assertEqual(factory.call_count, 1)
|
|
|
|
def test_cookie_retry_recreates_parser_and_redirect(self):
|
|
seen = []
|
|
|
|
def build(*args):
|
|
seen.append(current_attempt().proxy)
|
|
return fake_parser('cookie' if len(seen) == 1 else 'success')
|
|
|
|
with patch('src.api.parse.platform_access', return_value=None) as access:
|
|
response, factory = self.parse(build)
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIsNone(seen[0])
|
|
self.assertIsNotNone(seen[1])
|
|
self.assertTrue(self.manager.active('快手'))
|
|
self.assertEqual(self.redirect.call_count, 2)
|
|
self.assertEqual(factory.call_count, 2)
|
|
access.assert_called_once_with('快手')
|
|
|
|
def test_three_cookie_failures_keep_cookie_error(self):
|
|
response, factory = self.parse([fake_parser('cookie') for _ in range(4)])
|
|
self.assertEqual(response.status_code, 400)
|
|
self.assertEqual(response.json['error_code'], 'KUAISHOU_COOKIE_REQUIRED')
|
|
self.assertEqual(factory.call_count, 4)
|
|
self.assertEqual(len(self.sessions), 3)
|
|
self.assertIsNone(self.manager.available())
|
|
|
|
def test_three_network_failures_exhaust(self):
|
|
self.manager.activate('快手')
|
|
|
|
def fail(*args):
|
|
current_attempt().fail_proxy('网络异常')
|
|
|
|
response, factory = self.parse(fail)
|
|
self.assertEqual(response.status_code, 502)
|
|
self.assertEqual(response.json['error_code'], 'PROXY_RETRY_EXHAUSTED')
|
|
self.assertEqual(factory.call_count, 3)
|
|
|
|
def test_swallowed_network_failure_still_rotates(self):
|
|
self.manager.activate('快手')
|
|
|
|
def build(*args):
|
|
if len(self.sessions) == 1:
|
|
try:
|
|
current_attempt().fail_proxy('网络异常')
|
|
except requests.RequestException:
|
|
pass
|
|
return fake_parser('empty')
|
|
return fake_parser()
|
|
|
|
response, _ = self.parse(build)
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(len(self.sessions), 2)
|
|
|
|
def test_content_and_plain_empty_do_not_discard_proxy(self):
|
|
self.manager.activate('快手')
|
|
for kind in ('deleted', 'text', 'empty'):
|
|
with self.subTest(kind=kind):
|
|
response, _ = self.parse([fake_parser(kind)])
|
|
self.assertEqual(response.status_code, 400)
|
|
self.assertIsNotNone(self.manager.available())
|
|
self.assertEqual(len(self.sessions), 1)
|
|
|
|
def test_expired_window_restores_direct(self):
|
|
self.manager.activate('快手')
|
|
self.clock.now += 900
|
|
response, _ = self.parse([fake_parser()])
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(len(self.sessions), 0)
|
|
|
|
def test_supplier_failure_has_three_call_limit(self):
|
|
self.manager.activate('快手')
|
|
with patch('src.utils.proxy_manager.requests.Session') as sessions:
|
|
sessions.return_value.__enter__.return_value.get.side_effect = requests.Timeout()
|
|
response = self.client.get('/api/v1/parse', query_string={'url': self.url})
|
|
self.assertEqual(sessions.return_value.__enter__.return_value.get.call_count, 3)
|
|
self.assertEqual(response.status_code, 503)
|
|
self.assertEqual(response.json['error_code'], 'PROXY_UNAVAILABLE')
|
|
self.assertTrue(self.manager.active('快手'))
|
|
|
|
def test_budget_stops_even_if_parser_returns_success(self):
|
|
def build(*args):
|
|
self.clock.now += 91
|
|
return fake_parser()
|
|
|
|
response, _ = self.parse(build)
|
|
self.assertEqual(response.status_code, 504)
|
|
self.assertEqual(response.json['error_code'], 'PARSE_TIMEOUT')
|
|
|
|
def test_credit_and_request_log_happen_once(self):
|
|
with self.app.test_request_context('/api/v1/parse'):
|
|
with patch('src.api.parse.reserve_user_credit', return_value=True) as reserve, \
|
|
patch('src.api.parse.refund_user_credit') as refund, \
|
|
patch('src.api.parse.record_request') as record, \
|
|
patch('src.api.parse.ParserFactory.create_parser', side_effect=[fake_parser('cookie'), fake_parser()]), \
|
|
patch('src.utils.proxy_manager.requests.Session') as sessions:
|
|
sessions.return_value.__enter__.side_effect = self.supply
|
|
_, status = _execute_parse(self.url, {'user_id': 123})
|
|
self.assertEqual(status, 200)
|
|
reserve.assert_called_once_with(123)
|
|
refund.assert_not_called()
|
|
record.assert_called_once()
|
|
|
|
def test_failure_refunds_credit_and_logs_only_once(self):
|
|
with self.app.test_request_context('/api/v1/parse'):
|
|
with patch('src.api.parse.reserve_user_credit', return_value=True) as reserve, \
|
|
patch('src.api.parse.refund_user_credit') as refund, \
|
|
patch('src.api.parse.record_request') as record, \
|
|
patch('src.api.parse.ParserFactory.create_parser', side_effect=[fake_parser('cookie') for _ in range(4)]), \
|
|
patch('src.utils.proxy_manager.requests.Session') as sessions:
|
|
sessions.return_value.__enter__.side_effect = self.supply
|
|
_, status = _execute_parse(self.url, {'user_id': 123})
|
|
self.assertEqual(status, 400)
|
|
reserve.assert_called_once_with(123)
|
|
refund.assert_called_once_with(123)
|
|
record.assert_called_once()
|
|
|
|
def test_second_request_uses_proxy_from_first_network_step(self):
|
|
self.manager.activate('快手')
|
|
seen = []
|
|
|
|
def redirect(url):
|
|
seen.append(current_attempt().proxy)
|
|
return url
|
|
|
|
self.redirect.side_effect = redirect
|
|
for _ in range(2):
|
|
response, _ = self.parse([fake_parser()])
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertTrue(all(seen))
|
|
self.assertEqual(seen[0].address, seen[1].address)
|
|
self.assertEqual(len(self.sessions), 1)
|
|
|
|
def test_other_platform_stays_direct(self):
|
|
self.manager.activate('快手')
|
|
self.url = 'https://www.douyin.com/video/123'
|
|
self.redirect.return_value = self.url
|
|
response, _ = self.parse([fake_parser()])
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(len(self.sessions), 0)
|
|
|
|
def test_xhs_cookie_detail_keeps_cookie_code(self):
|
|
self.url = 'https://www.xiaohongshu.com/explore/123'
|
|
self.redirect.return_value = self.url
|
|
parsers = []
|
|
for _ in range(4):
|
|
parser = fake_parser('empty')
|
|
parser.terminal_error = {'detail_msg': '小红书 Cookie 凭据可能已失效或访问受限,请在后台更新'}
|
|
parsers.append(parser)
|
|
response, _ = self.parse(parsers)
|
|
self.assertEqual(response.status_code, 400)
|
|
self.assertEqual(response.json['error_code'], 'XIAOHONGSHU_COOKIE_REQUIRED')
|
|
self.assertEqual(len(self.sessions), 3)
|
|
|
|
def test_cookie_challenge_in_constructor_rotates_immediately(self):
|
|
from src.utils.parser_transport import report_cookie_required
|
|
self.manager.activate('快手')
|
|
|
|
def build(*args):
|
|
report_cookie_required()
|
|
self.fail('代理 Cookie 拦截后不应继续解析')
|
|
|
|
response, factory = self.parse(build)
|
|
self.assertEqual(response.status_code, 400)
|
|
self.assertEqual(response.json['error_code'], 'KUAISHOU_COOKIE_REQUIRED')
|
|
self.assertEqual(factory.call_count, 3)
|
|
|
|
def test_universal_decoders(self):
|
|
start = time.time()
|
|
# 1. 巨量 IP (Juliang)
|
|
p1 = ProxyManager.decode({'code': 200, 'data': {'proxy_list': [{'ip': '1.2.3.4', 'port': 8080, 'ip_remain': 120}]}}, start)
|
|
self.assertEqual(p1.address, '1.2.3.4:8080')
|
|
self.assertAlmostEqual(p1.expires_at - start, 120, delta=1)
|
|
|
|
# 2. 芝麻代理 (Zhima)
|
|
p2 = ProxyManager.decode({'code': 0, 'data': [{'ip': '5.6.7.8', 'port': 9090, 'expire_time': '2026-09-30 20:00:00'}]}, start)
|
|
self.assertEqual(p2.address, '5.6.7.8:9090')
|
|
|
|
# 3. 青果网络 (Qingguo)
|
|
p3 = ProxyManager.decode({'code': 'SUCCESS', 'data': [{'ip': '9.10.11.12', 'port': 7070, 'ttl': 300}]}, start)
|
|
self.assertEqual(p3.address, '9.10.11.12:7070')
|
|
|
|
# 4. Data5U / 无忧代理
|
|
p4 = ProxyManager.decode({'success': True, 'data': [{'ip': '13.14.15.16', 'port': 6060}]}, start)
|
|
self.assertEqual(p4.address, '13.14.15.16:6060')
|
|
|
|
# 5. 纯文本 TXT
|
|
p5 = ProxyManager.decode("17.18.19.20:5050\n1.1.1.1:80", start)
|
|
self.assertEqual(p5.address, '17.18.19.20:5050')
|
|
|