DenozordecandClaude Opus 5.5
e3af778068
tools/audit_tracks.py и reports/tracks_audit.md — аудит всех дорожек техданных по data/raw с планом. Пункт 1 плана: harvest_core.codec_line_rows добавляет студии из строк «Аудио 2: AC3 … (MVO, LostFilm)», «Аудио Rus: … | TVShows» (studio_source=codec), которых нет среди строк релиза; тот же шаг в tools/relabel_studios.py (повторный запуск ничего не добавляет). labels.csv: +6 428 строк в 2 862 релизах (kinozal 3 444, nnm 1 609, rutracker 1 375), со студией 29 331 -> 35 759. tests/e2e_segmenter.py: модель через E2E_MODEL. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2229 lines
106 KiB
Python
2229 lines
106 KiB
Python
#!/usr/bin/env python
|
||
"""
|
||
dataset.py — один скрипт: собрать датасет «раздача -> студия озвучки» с
|
||
kinozal.me и rutracker.org, разобрать метки и слить в один CSV.
|
||
|
||
Только стандартная библиотека: Chrome запускается как процесс, общение с ним
|
||
идёт по CDP (DevTools Protocol) через WebSocket, который реализован здесь же
|
||
на сокетах. Ни Node.js, ни pip-пакеты не нужны.
|
||
|
||
Команды
|
||
-------
|
||
python dataset.py chrome # запустить Chrome (если не запущен)
|
||
python dataset.py harvest kinozal --ids 2155882 --limit 5
|
||
python dataset.py harvest kinozal --browse https://kinozal.me/top.php --limit 20
|
||
python dataset.py harvest rutracker --forums 313,252 --pages 2 --limit 100
|
||
python dataset.py parse # сырые страницы -> строки
|
||
python dataset.py merge # строки -> out/dataset/labels.csv
|
||
python dataset.py all --rutracker-forums 313 --pages 1 --limit 20
|
||
python dataset.py stats
|
||
|
||
Chrome НЕ закрывается: профиль out/chrome-profile хранит логин и cookie, а
|
||
повторный запуск не провоцирует Cloudflare. Первый раз войдите в аккаунт
|
||
rutracker вручную в открывшемся окне — этого достаточно для скачивания
|
||
торрентов и обхода лимитов гостя.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import base64
|
||
import csv
|
||
import json
|
||
import os
|
||
import random
|
||
import re
|
||
import socket
|
||
import struct
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
_REPO = Path(__file__).resolve().parents[2]
|
||
if str(_REPO) not in sys.path:
|
||
sys.path.insert(0, str(_REPO))
|
||
from src.audio18 import audio_counts, audio_lines # noqa: E402
|
||
from src.html_extract import RE_LINE_PREFIX, RE_TYPE_HINT, _split_entries # noqa: E402
|
||
from src.trackinfo import audio_lines_all, lang_of_text, pick_for_row # noqa: E402
|
||
|
||
ROOT = Path(__file__).resolve().parent
|
||
OUT = Path(os.environ.get("DUB_HARVEST_OUT") or ROOT / "out")
|
||
RAW_DIR = OUT / "raw"
|
||
DATASET = OUT / "dataset"
|
||
CHROME_PORT = int(os.environ.get("DUB_CHROME_PORT") or 9222)
|
||
CHROME_PROFILE = Path(os.environ.get("DUB_CHROME_PROFILE") or OUT / "chrome-profile")
|
||
CHROME_PATHS = [
|
||
r"C:\Program Files\Google\Chrome\Application\chrome.exe",
|
||
r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe",
|
||
]
|
||
UA_HINT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36"
|
||
|
||
|
||
def log(msg: str) -> None:
|
||
print(msg, flush=True)
|
||
|
||
|
||
# =====================================================================
|
||
# Состояние обхода: у каждого источника свой файл, чтобы два
|
||
# параллельных процесса (kinozal + rutracker) не затирали прогресс
|
||
# =====================================================================
|
||
STATE_FILES = {
|
||
"rutracker": OUT / "state_rutracker.json",
|
||
"kinozal": OUT / "state_kinozal.json",
|
||
"nnm": OUT / "state_nnm.json",
|
||
}
|
||
LEGACY_STATE = OUT / "state.json"
|
||
|
||
|
||
def _migrate_legacy_state() -> None:
|
||
if not LEGACY_STATE.exists():
|
||
return
|
||
try:
|
||
old = json.loads(LEGACY_STATE.read_text("utf-8"))
|
||
except (OSError, json.JSONDecodeError):
|
||
return
|
||
for src, path in STATE_FILES.items():
|
||
part = old.get(src)
|
||
if part and not path.exists():
|
||
path.write_text(json.dumps({"version": 1, src: part},
|
||
ensure_ascii=False, indent=2), "utf-8")
|
||
LEGACY_STATE.replace(LEGACY_STATE.with_suffix(".migrated.json"))
|
||
|
||
|
||
def load_state(source: str) -> dict:
|
||
_migrate_legacy_state()
|
||
path = STATE_FILES[source]
|
||
if path.exists():
|
||
try:
|
||
return json.loads(path.read_text("utf-8"))
|
||
except json.JSONDecodeError:
|
||
log(f" {path.name} повреждён — начинаю этот источник заново")
|
||
base = {"version": 1}
|
||
return base
|
||
|
||
|
||
def save_state(source: str, state: dict) -> None:
|
||
OUT.mkdir(parents=True, exist_ok=True)
|
||
path = STATE_FILES[source]
|
||
tmp = path.with_suffix(".tmp")
|
||
tmp.write_text(json.dumps(state, ensure_ascii=False, indent=2), "utf-8")
|
||
tmp.replace(path) # атомарно: не порвётся при падении
|
||
|
||
|
||
def stamp() -> str:
|
||
return time.strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
|
||
# =====================================================================
|
||
# WebSocket (RFC 6455) — минимальный клиент для CDP
|
||
# =====================================================================
|
||
class WebSocket:
|
||
def __init__(self, url: str, timeout: float = 30.0):
|
||
m = re.match(r"ws://([^/:]+):(\d+)(/.*)", url)
|
||
if not m:
|
||
raise ValueError(f"плохой ws url: {url}")
|
||
host, port, path = m.group(1), int(m.group(2)), m.group(3)
|
||
self.sock = socket.create_connection((host, port), timeout=timeout)
|
||
self.sock.settimeout(timeout)
|
||
key = base64.b64encode(os.urandom(16)).decode()
|
||
req = (
|
||
f"GET {path} HTTP/1.1\r\n"
|
||
f"Host: {host}:{port}\r\n"
|
||
"Upgrade: websocket\r\n"
|
||
"Connection: Upgrade\r\n"
|
||
f"Sec-WebSocket-Key: {key}\r\n"
|
||
"Sec-WebSocket-Version: 13\r\n\r\n"
|
||
)
|
||
self.sock.sendall(req.encode())
|
||
self._buf = b""
|
||
while b"\r\n\r\n" not in self._buf:
|
||
chunk = self.sock.recv(4096)
|
||
if not chunk:
|
||
raise ConnectionError("handshake закрыт")
|
||
self._buf += chunk
|
||
head, self._buf = self._buf.split(b"\r\n\r\n", 1)
|
||
if b"101" not in head.split(b"\r\n")[0]:
|
||
raise ConnectionError(f"handshake failed: {head[:120]!r}")
|
||
|
||
# -- низкий уровень -------------------------------------------------
|
||
def _read(self, n: int) -> bytes:
|
||
while len(self._buf) < n:
|
||
chunk = self.sock.recv(65536)
|
||
if not chunk:
|
||
raise ConnectionError("соединение закрыто")
|
||
self._buf += chunk
|
||
data, self._buf = self._buf[:n], self._buf[n:]
|
||
return data
|
||
|
||
def send(self, text: str) -> None:
|
||
payload = text.encode()
|
||
header = bytearray([0x81])
|
||
n = len(payload)
|
||
if n < 126:
|
||
header.append(0x80 | n)
|
||
elif n < 65536:
|
||
header.append(0x80 | 126)
|
||
header += struct.pack(">H", n)
|
||
else:
|
||
header.append(0x80 | 127)
|
||
header += struct.pack(">Q", n)
|
||
mask = os.urandom(4)
|
||
header += mask
|
||
masked = bytes(b ^ mask[i % 4] for i, b in enumerate(payload))
|
||
self.sock.sendall(bytes(header) + masked)
|
||
|
||
def recv(self) -> str:
|
||
fragments = b""
|
||
while True:
|
||
b1, b2 = self._read(2)
|
||
fin, opcode = b1 & 0x80, b1 & 0x0F
|
||
masked, ln = b2 & 0x80, b2 & 0x7F
|
||
if ln == 126:
|
||
ln = struct.unpack(">H", self._read(2))[0]
|
||
elif ln == 127:
|
||
ln = struct.unpack(">Q", self._read(8))[0]
|
||
mask = self._read(4) if masked else None
|
||
data = self._read(ln)
|
||
if mask:
|
||
data = bytes(b ^ mask[i % 4] for i, b in enumerate(data))
|
||
if opcode == 0x9: # ping
|
||
self.sock.sendall(b"\x8a\x00")
|
||
continue
|
||
if opcode == 0x8: # close
|
||
raise ConnectionError("сервер закрыл соединение")
|
||
fragments += data
|
||
if fin:
|
||
return fragments.decode("utf-8", "replace")
|
||
|
||
def close(self) -> None:
|
||
try:
|
||
self.sock.close()
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
class CDP:
|
||
"""Клиент DevTools Protocol поверх WebSocket."""
|
||
|
||
def __init__(self, ws_url: str, timeout: float = 60.0):
|
||
self.ws_url = ws_url
|
||
self.ws = WebSocket(ws_url, timeout=timeout)
|
||
self.next_id = 0
|
||
self.timeout = timeout
|
||
|
||
def reconnect(self) -> None:
|
||
"""Новое соединение с той же вкладкой (после обрыва или принудительного закрытия)."""
|
||
try:
|
||
self.ws.close()
|
||
except OSError:
|
||
pass
|
||
self.ws = WebSocket(self.ws_url, timeout=self.timeout)
|
||
self.next_id = 0
|
||
self.send("Page.enable")
|
||
self.send("Runtime.enable")
|
||
|
||
def abort(self) -> None:
|
||
"""Оборвать зависший вызов из другого потока: recv() в рабочем потоке упадёт с ошибкой."""
|
||
try:
|
||
self.ws.sock.shutdown(socket.SHUT_RDWR)
|
||
except OSError:
|
||
pass
|
||
try:
|
||
self.ws.close()
|
||
except OSError:
|
||
pass
|
||
|
||
def send(self, method: str, params: dict | None = None) -> dict:
|
||
self.next_id += 1
|
||
mid = self.next_id
|
||
self.ws.send(json.dumps({"id": mid, "method": method, "params": params or {}}))
|
||
deadline = time.time() + self.timeout
|
||
while time.time() < deadline:
|
||
msg = json.loads(self.ws.recv())
|
||
if msg.get("id") != mid:
|
||
continue # событие, не наш ответ
|
||
if "error" in msg:
|
||
raise RuntimeError(f"{method}: {msg['error']}")
|
||
return msg.get("result", {})
|
||
raise TimeoutError(f"{method}: нет ответа за {self.timeout}s")
|
||
|
||
def eval(self, expression: str, await_promise: bool = True) -> Any:
|
||
res = self.send("Runtime.evaluate", {
|
||
"expression": expression,
|
||
"returnByValue": True,
|
||
"awaitPromise": await_promise,
|
||
})
|
||
if res.get("exceptionDetails"):
|
||
desc = res["exceptionDetails"].get("exception", {}).get("description", "eval error")
|
||
raise RuntimeError(desc)
|
||
return res.get("result", {}).get("value")
|
||
|
||
def close(self) -> None:
|
||
self.ws.close()
|
||
|
||
|
||
# =====================================================================
|
||
# Chrome / вкладки
|
||
# =====================================================================
|
||
def http_json(path: str, method: str = "GET", timeout: float = 5.0) -> Any:
|
||
req = urllib.request.Request(f"http://127.0.0.1:{CHROME_PORT}{path}", method=method)
|
||
req.add_header("User-Agent", UA_HINT)
|
||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||
return json.loads(r.read().decode("utf-8", "replace"))
|
||
|
||
|
||
def chrome_alive() -> bool:
|
||
try:
|
||
http_json("/json/version", timeout=1.5)
|
||
return True
|
||
except (urllib.error.URLError, OSError, json.JSONDecodeError):
|
||
return False
|
||
|
||
|
||
def find_chrome() -> str:
|
||
for p in CHROME_PATHS:
|
||
if Path(p).exists():
|
||
return p
|
||
raise FileNotFoundError("Google Chrome не найден в стандартных путях")
|
||
|
||
|
||
def chrome_start(url: str = "https://rutracker.org/forum/index.php") -> bool:
|
||
"""Запустить Chrome, если он ещё не запущен. Процесс остаётся жить."""
|
||
if chrome_alive():
|
||
return True
|
||
CHROME_PROFILE.mkdir(parents=True, exist_ok=True)
|
||
exe = find_chrome()
|
||
flags = 0
|
||
if os.name == "nt":
|
||
flags = subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
|
||
subprocess.Popen(
|
||
[exe,
|
||
f"--remote-debugging-port={CHROME_PORT}",
|
||
f"--user-data-dir={CHROME_PROFILE}",
|
||
"--no-first-run", "--no-default-browser-check",
|
||
"--disable-blink-features=AutomationControlled",
|
||
"--test-type", # скрывает плашку «неподдерживаемый флаг командной строки»
|
||
"--lang=ru-RU", "--window-size=1400,1100", url],
|
||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||
creationflags=flags, close_fds=True,
|
||
)
|
||
for _ in range(80):
|
||
if chrome_alive():
|
||
return True
|
||
time.sleep(0.25)
|
||
return False
|
||
|
||
|
||
def page_target(prefer: str = "") -> dict:
|
||
"""Вкладка под конкретный сайт: своя у каждого процесса."""
|
||
pages = [t for t in http_json("/json/list") if t.get("type") == "page"]
|
||
if prefer:
|
||
for p in pages:
|
||
if prefer in (p.get("url") or ""):
|
||
return p
|
||
url = "https://rutracker.org/forum/index.php" if "rutracker" in prefer else "https://kinozal.me/"
|
||
return http_json(f"/json/new?{urllib.parse.quote(url, safe=':/?=&')}", method="PUT")
|
||
if not pages:
|
||
return http_json("/json/new?about:blank", method="PUT")
|
||
return pages[0]
|
||
|
||
|
||
def connect(prefer: str = "") -> tuple[CDP, dict]:
|
||
if not chrome_start():
|
||
raise RuntimeError("не удалось запустить Chrome")
|
||
page = page_target(prefer)
|
||
cdp = CDP(page["webSocketDebuggerUrl"])
|
||
cdp.send("Page.enable")
|
||
cdp.send("Runtime.enable")
|
||
return cdp, page
|
||
|
||
|
||
CF_MARKERS = ("just a moment", "один момент", "проверки безопасности",
|
||
"checking your browser", "выполнение проверки безопасности")
|
||
|
||
|
||
# тексты страниц удалённой/несуществующей раздачи (kinozal, rutracker, nnm)
|
||
DEAD_MARKERS = ("нет раздачи с таким id", "раздача не найдена", "тема не найдена",
|
||
"такой темы не существует", "тема удалена", "topic does not exist")
|
||
|
||
|
||
# страница списка без тем (за последней страницей раздела): это конец списка, не ошибка
|
||
EMPTY_LIST_MARKERS = ("подходящих тем или сообщений не найдено", "тем не найдено")
|
||
|
||
|
||
# страницы ошибок сервера / Cloudflare (503, 502, 504, 52x, 1015, 1020): лечатся повтором
|
||
RE_ERROR_TITLE = re.compile(
|
||
r"(?i)^\s*(?:error\s*|ошибка\s*)?(?:50[0-4]|52\d)\b|service (?:temporarily )?unavailable|bad gateway"
|
||
r"|gateway time-?out|attention required|temporarily unavailable|too many requests")
|
||
RE_ERROR_HEAD = re.compile(
|
||
r"(?i)(?:error|ошибка)\s*(?:code)?\s*:?\s*(?:50[0-4]|52\d|1015|1020)\b|service (?:temporarily )?unavailable"
|
||
r"|bad gateway|gateway time-?out|web server is (?:down|returning an unknown error)"
|
||
r"|has banned you temporarily|you are being rate limited")
|
||
|
||
RETRY_DELAYS = (3.0, 8.0, 20.0) # пауза перед 1-м, 2-м и 3-м повтором
|
||
RETRY_HOOK = None # callable(url, attempt, retries, reason) — для интерфейса
|
||
|
||
|
||
def _goto_once(cdp: CDP, url: str, min_len: int = 600, timeout: float = 45.0) -> dict:
|
||
"""Один заход: ждём контент, ловим пустой список, удалённую раздачу и страницу ошибки сервера."""
|
||
cdp.send("Page.navigate", {"url": url})
|
||
t0 = time.time()
|
||
deadline = t0 + timeout
|
||
last = {"title": "", "len": 0}
|
||
still_since = None # с какого момента длина текста не меняется
|
||
while time.time() < deadline:
|
||
time.sleep(0.4)
|
||
try:
|
||
state = json.loads(cdp.eval(
|
||
"JSON.stringify({t:document.title, r:document.readyState,"
|
||
" len:document.body?document.body.innerText.length:0})"
|
||
))
|
||
except RuntimeError:
|
||
continue
|
||
prev_len = last["len"]
|
||
last = {"title": state.get("t", ""), "len": state.get("len", 0)}
|
||
if state.get("r") == "complete" and last["len"] == prev_len:
|
||
still_since = still_since or time.time()
|
||
else:
|
||
still_since = None
|
||
# короткая, но загруженная и неменяющаяся страница (закрытая тема, «перенесено»):
|
||
# не ждём таймаут, решает вызывающий (has_body); страницу проверки Cloudflare не берём
|
||
if (still_since and time.time() - still_since > 4 and last["len"] < min_len
|
||
and not any(m in (last["title"] or "").lower() for m in CF_MARKERS)):
|
||
return {"ok": True, "short": True, **last, "url": url}
|
||
if state.get("r") == "complete" and RE_ERROR_TITLE.search(last["title"] or "") and last["len"] < 3000:
|
||
return {"ok": False, "server_error": True, "reason": (last["title"] or "ошибка")[:40],
|
||
"len": last["len"], "title": last["title"], "url": url}
|
||
if state.get("r") == "complete" and last["len"] < min_len:
|
||
# короткая страница: удалённая раздача — не ждём таймаут
|
||
head = (cdp.eval("document.body ? document.body.innerText.slice(0, 800).toLowerCase() : ''")
|
||
or "")
|
||
if RE_ERROR_HEAD.search(head):
|
||
m = RE_ERROR_HEAD.search(head)
|
||
return {"ok": False, "server_error": True, "reason": m.group(0)[:40].strip(),
|
||
"len": last["len"], "title": last["title"], "url": url}
|
||
if any(m in head for m in EMPTY_LIST_MARKERS):
|
||
return {"ok": True, "empty": True, "title": last["title"],
|
||
"len": last["len"], "url": url}
|
||
if any(m in head for m in DEAD_MARKERS):
|
||
return {"ok": False, "dead": True, "title": "нет раздачи: " + last["title"],
|
||
"len": last["len"], "url": url}
|
||
# тяжёлые темы (десятки сообщений, картинки, реклама) годами остаются в «interactive»:
|
||
# текст уже есть, ждать «complete» незачем
|
||
ready = state.get("r") == "complete" or (
|
||
state.get("r") == "interactive" and time.time() - t0 > 5)
|
||
if not ready or last["len"] < min_len:
|
||
continue
|
||
title = (last["title"] or "").lower()
|
||
if any(m in title for m in CF_MARKERS):
|
||
continue
|
||
return {"ok": True, **last, "url": url}
|
||
return {"ok": False, "timeout": True, "reason": "таймаут", **last, "url": url}
|
||
|
||
|
||
def goto(cdp: CDP, url: str, min_len: int = 600, timeout: float = 45.0, retries: int = 3) -> dict:
|
||
"""Перейти по адресу и дождаться контента; при ошибке сервера/Cloudflare повторить.
|
||
|
||
Страница ошибки (503, 502, 504, 52x, 1015…) обновляется до `retries` раз с нарастающей
|
||
паузой (3, 8, 20 с) — на практике обновление её лечит. Таймаут повторяется один раз.
|
||
Удалённая раздача и пустой список повтора не требуют.
|
||
"""
|
||
res = _goto_once(cdp, url, min_len, timeout)
|
||
for attempt in range(1, retries + 1):
|
||
if res.get("ok") or res.get("dead") or not (res.get("server_error") or res.get("timeout")):
|
||
return res
|
||
if res.get("timeout") and attempt > 1:
|
||
break
|
||
if RETRY_HOOK:
|
||
RETRY_HOOK(url, attempt, retries, res.get("reason", ""))
|
||
time.sleep(RETRY_DELAYS[min(attempt - 1, len(RETRY_DELAYS) - 1)])
|
||
res = _goto_once(cdp, url, min_len, timeout)
|
||
return res
|
||
|
||
|
||
# =====================================================================
|
||
# Сбор: kinozal
|
||
# =====================================================================
|
||
KZ_BASE = "https://kinozal.me"
|
||
|
||
|
||
def kz_page_url(base: str, page: int) -> str:
|
||
"""kinozal нумерует страницы с нуля: browse.php?...&page=N"""
|
||
clean = re.sub(r"([?&])page=\d+", "", base).rstrip("?&")
|
||
sep = "&" if "?" in clean else "?"
|
||
return f"{clean}{sep}page={page}"
|
||
|
||
|
||
def kz_browse_ids(cdp: CDP, url: str) -> tuple[bool, list[str]]:
|
||
"""(открылась ли страница, id раздач). Пустая страница != ошибка."""
|
||
nav = goto(cdp, url, min_len=800)
|
||
if not nav["ok"]:
|
||
log(f" kinozal: страница списка не открылась ({nav['title']})")
|
||
return False, []
|
||
# только ссылки на раздачи: «…/userdetails.php?id=» (профили пользователей) содержит
|
||
# подстроку «details.php?id=», но раздачей не является
|
||
ids = cdp.eval(
|
||
"JSON.stringify([...new Set([...document.querySelectorAll('a[href*=\"details.php?id=\"]')]"
|
||
".map(a => (a.getAttribute('href')||'').match(/(?:^|\\/)details\\.php\\?id=(\\d+)/))"
|
||
".filter(Boolean).map(m => m[1]))])"
|
||
)
|
||
return True, json.loads(ids or "[]")
|
||
|
||
|
||
KZ_ITEMS_JS = r"""JSON.stringify((() => {
|
||
const out = {}, order = [];
|
||
for (const a of document.querySelectorAll('a[href*="details.php?id="]')) {
|
||
const m = (a.getAttribute('href') || '').match(/(?:^|\/)details\.php\?id=(\d+)/);
|
||
if (!m) continue;
|
||
const t = (a.innerText || '').replace(/\s+/g, ' ').trim();
|
||
if (!(m[1] in out)) { out[m[1]] = ''; order.push(m[1]); }
|
||
if (t.length > out[m[1]].length) out[m[1]] = t;
|
||
}
|
||
return order.map(id => ({ id, text: out[id].slice(0, 140) }));
|
||
})())"""
|
||
|
||
|
||
def kz_browse_items(cdp: CDP, url: str) -> tuple[bool, list[dict]]:
|
||
"""Как kz_browse_ids, но с заголовками тем: [{id, text}]."""
|
||
nav = goto(cdp, url, min_len=800)
|
||
if not nav["ok"]:
|
||
log(f" kinozal: страница списка не открылась ({nav['title']})")
|
||
return False, []
|
||
return True, json.loads(cdp.eval(KZ_ITEMS_JS) or "[]")
|
||
|
||
|
||
DEPTH_JS = r"""(() => {
|
||
let last = 0;
|
||
for (const a of document.querySelectorAll('a[href]')) {
|
||
const h = a.getAttribute('href') || '';
|
||
let m = h.match(/viewforum\.php\?[^#]*[?&]?start=(\d+)/);
|
||
if (m) last = Math.max(last, Math.floor(+m[1] / 50));
|
||
if (h.startsWith('?') || /browse\.php/.test(h)) { // kinozal: ссылки вида «?c=21&page=99»
|
||
m = h.match(/[?&]page=(\d+)/);
|
||
if (m) last = Math.max(last, +m[1]);
|
||
}
|
||
}
|
||
return last + 1;
|
||
})()"""
|
||
|
||
|
||
def list_depth(cdp: CDP) -> int:
|
||
"""Сколько страниц в разделе — по ссылкам пагинации открытой страницы списка."""
|
||
try:
|
||
return max(1, int(cdp.eval(DEPTH_JS) or 1))
|
||
except (TypeError, ValueError):
|
||
return 1
|
||
|
||
|
||
KZ_TABS_JS = "document.getElementById('tabs') ? document.getElementById('tabs').innerText : ''"
|
||
|
||
|
||
def kz_read_tab(cdp: CDP, release_id: str, tab: int, timeout: float = 6.0) -> str:
|
||
"""Переключить вкладку и дождаться её содержимого (вместо фиксированной паузы 1.2 с).
|
||
|
||
Ждём, пока текст изменится и перестанет меняться; если вкладка уже была показана
|
||
(текст тот же), хватает 1.5 с.
|
||
"""
|
||
before = cdp.eval(KZ_TABS_JS) or ""
|
||
cdp.eval(f"try {{ showtab({release_id}, {tab}); }} catch (e) {{}} true")
|
||
start, prev = time.time(), None
|
||
while time.time() - start < timeout:
|
||
time.sleep(0.25)
|
||
text = cdp.eval(KZ_TABS_JS) or ""
|
||
if text and text != before and text == prev:
|
||
return text # сменился и стабилен
|
||
if text and text == before and time.time() - start >= 1.5:
|
||
return text # та же вкладка уже была открыта
|
||
prev = text
|
||
return cdp.eval(KZ_TABS_JS) or ""
|
||
|
||
|
||
def kz_release(cdp: CDP, release_id: str) -> dict:
|
||
url = f"{KZ_BASE}/details.php?id={release_id}"
|
||
nav = goto(cdp, url, min_len=800)
|
||
rec: dict = {"id": release_id, "url": url, "source": "kinozal",
|
||
"ok": nav["ok"], "page_title": nav["title"]}
|
||
if not nav["ok"]:
|
||
rec["error"] = f"не открылась: {nav['title']}"
|
||
return rec
|
||
rec["h1"] = cdp.eval("(document.querySelector('h1')||{}).innerText||''") or ""
|
||
rec["release_tab"] = kz_read_tab(cdp, release_id, 0)
|
||
rec["tech_tab"] = kz_read_tab(cdp, release_id, 100)
|
||
rec["page_text"] = cdp.eval("document.body.innerText") or ""
|
||
return rec
|
||
|
||
|
||
def harvest_kinozal(ids: list[str], browse: str | None, limit: int, delay: float,
|
||
pages: int = 1, from_page: int | None = None,
|
||
reset: bool = False) -> int:
|
||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||
raw_file = RAW_DIR / "kinozal.jsonl"
|
||
seen = _seen_ids(raw_file)
|
||
state = load_state("kinozal")
|
||
bstate = state.setdefault("kinozal", {}).setdefault("browse", {})
|
||
if reset:
|
||
bstate.clear()
|
||
save_state("kinozal", state)
|
||
|
||
cdp, _ = connect("kinozal.me")
|
||
done = 0
|
||
try:
|
||
for rid in dict.fromkeys(ids):
|
||
if rid in seen or done >= limit:
|
||
continue
|
||
try:
|
||
rec = kz_release(cdp, rid)
|
||
except Exception as exc: # noqa: BLE001
|
||
rec = {"id": rid, "source": "kinozal", "ok": False, "error": str(exc)}
|
||
_store(raw_file, rec)
|
||
seen.add(rid)
|
||
done += 1
|
||
mark = "+" if rec.get("release_tab") else "-"
|
||
log(f" {mark} {rid} {(rec.get('page_title') or '')[:78]}")
|
||
time.sleep(delay + random.random() * 1.2)
|
||
|
||
if browse:
|
||
st = bstate.get(browse, {"next_page": 0, "releases": 0})
|
||
start = int(st.get("next_page", 0)) if from_page is None else from_page
|
||
log(f"kinozal: {browse} — продолжаю со страницы {start + 1} "
|
||
f"(собрано раздач: {st.get('releases', 0)})")
|
||
for k in range(pages):
|
||
if done >= limit:
|
||
break
|
||
page = start + k
|
||
ok, page_ids = kz_browse_ids(cdp, kz_page_url(browse, page))
|
||
if not ok:
|
||
log(f" стр.{page + 1}: не открылась — попробую в следующий раз")
|
||
break
|
||
if not page_ids:
|
||
bstate[browse] = {**st, "finished": True, "updated": stamp()}
|
||
save_state("kinozal", state)
|
||
log(f" стр.{page + 1}: пусто — список закончился")
|
||
break
|
||
fresh = [i for i in page_ids if i not in seen]
|
||
log(f" стр.{page + 1}: {len(page_ids)} раздач, новых {len(fresh)}")
|
||
|
||
completed = True
|
||
for rid in fresh:
|
||
if done >= limit:
|
||
completed = False
|
||
break
|
||
try:
|
||
rec = kz_release(cdp, rid)
|
||
except Exception as exc: # noqa: BLE001
|
||
rec = {"id": rid, "source": "kinozal", "ok": False,
|
||
"error": str(exc)}
|
||
_store(raw_file, rec)
|
||
seen.add(rid)
|
||
done += 1
|
||
mark = "+" if rec.get("release_tab") else "-"
|
||
log(f" {mark} {rid} {(rec.get('page_title') or '')[:70]}")
|
||
time.sleep(delay + random.random() * 1.2)
|
||
|
||
bstate[browse] = {
|
||
"next_page": page if not completed else page + 1,
|
||
"releases": st.get("releases", 0) + (len(fresh) if completed else 0),
|
||
"updated": stamp(),
|
||
}
|
||
save_state("kinozal", state)
|
||
st = bstate[browse]
|
||
if not completed:
|
||
log(f" лимит {limit} исчерпан, следующая страница {page + 1}")
|
||
break
|
||
log(f"kinozal: добавлено {done} записей -> {raw_file}")
|
||
return done
|
||
finally:
|
||
cdp.close()
|
||
|
||
|
||
# =====================================================================
|
||
# Сбор: rutracker
|
||
# =====================================================================
|
||
RT_BASE = "https://rutracker.org/forum/"
|
||
RT_SKIP = {"1045", "6661913", "6452965", "2234744", "2694168", "2968462", "2276119"}
|
||
# служебные темы разделов: правила, критерии, заявки — не раздачи
|
||
RT_SERVICE_RE = re.compile(
|
||
r"^\s*(==|правила|критерии|вниманию|внимание|обсуждени|заявки|помощь|faq|список|"
|
||
r"профильные|бездоказательн|создание новых|восстановление оформления|"
|
||
r"премьеры и продленки|что такое|запрет на рекламу|просьба к юзерам|"
|
||
r"осторожно|об использовании|акция|изменения|итоги|конкурс|опрос)",
|
||
re.I)
|
||
|
||
|
||
def is_service_topic(title: str) -> bool:
|
||
return bool(RT_SERVICE_RE.match(title or ""))
|
||
|
||
RT_FORUM_JS = """JSON.stringify((() => {
|
||
const SKIP = new Set(%s);
|
||
const out = [], seen = new Set();
|
||
const rows = document.querySelectorAll('table.forumline tr').length;
|
||
for (const tr of document.querySelectorAll('tr[id^="tr-"]')) {
|
||
let best = null, bestLen = 0;
|
||
for (const a of tr.querySelectorAll('a[href*="viewtopic.php?t="]')) {
|
||
const t = (a.innerText || '').replace(/\\s+/g, ' ').trim();
|
||
if (t.length > bestLen) { bestLen = t.length; best = a; }
|
||
}
|
||
if (!best) continue;
|
||
const m = (best.getAttribute('href') || '').match(/viewtopic\\.php\\?t=(\\d+)/);
|
||
if (!m || SKIP.has(m[1]) || seen.has(m[1])) continue;
|
||
const text = (best.innerText || '').replace(/\\s+/g, ' ').trim();
|
||
if (text.length <= 25) continue;
|
||
seen.add(m[1]);
|
||
out.push({ id: m[1], text: text.slice(0, 140) });
|
||
}
|
||
return { rows, items: out };
|
||
})())"""
|
||
|
||
RT_TOPIC_JS = """JSON.stringify((() => {
|
||
const pick = (sels) => { for (const s of sels) { const el = document.querySelector(s);
|
||
if (el && el.innerText.trim().length > 30) return el; } return null; };
|
||
const post = pick(['.post_body', 'div[id^="post-text-"]', '.postbody']);
|
||
const title = pick(['h1.maintitle', '#topic-title', 'a.maintitle', 'h1']);
|
||
return { title: title ? title.innerText.trim() : document.title,
|
||
post: post ? post.innerText : '',
|
||
hasTorrent: !!document.querySelector('a[href*="dl.php?t="]') };
|
||
})())"""
|
||
|
||
|
||
def rt_forum_page(cdp: CDP, forum: str, page: int) -> tuple[bool, list[dict]]:
|
||
"""(открылась ли страница, темы). Пустая страница != ошибка загрузки."""
|
||
url = f"{RT_BASE}viewforum.php?f={forum}&start={page * 50}"
|
||
for attempt in (1, 2):
|
||
nav = goto(cdp, url, min_len=1200)
|
||
if nav.get("empty"):
|
||
return True, []
|
||
if not nav["ok"]:
|
||
log(f" f={forum} стр.{page + 1}: не открылась ({nav['title']})")
|
||
else:
|
||
data = json.loads(cdp.eval(RT_FORUM_JS % json.dumps(sorted(RT_SKIP))) or "{}")
|
||
if data.get("rows"):
|
||
items = data.get("items", [])
|
||
releases = [it for it in items if not is_service_topic(it["text"])]
|
||
skipped = len(items) - len(releases)
|
||
if skipped:
|
||
log(f" f={forum} стр.{page + 1}: отброшено служебных тем: {skipped}")
|
||
return True, releases
|
||
log(f" f={forum} стр.{page + 1}: список не отрисовался (0 строк)")
|
||
if attempt == 1:
|
||
time.sleep(3)
|
||
return False, []
|
||
|
||
|
||
def rt_topic(cdp: CDP, topic_id: str) -> dict:
|
||
url = f"{RT_BASE}viewtopic.php?t={topic_id}"
|
||
nav = goto(cdp, url, min_len=900)
|
||
rec: dict = {"id": topic_id, "url": url, "source": "rutracker",
|
||
"ok": nav["ok"], "page_title": nav["title"]}
|
||
if not nav["ok"]:
|
||
rec["error"] = f"не открылась: {nav['title']}"
|
||
return rec
|
||
data = json.loads(cdp.eval(RT_TOPIC_JS) or "{}")
|
||
rec.update(data)
|
||
return rec
|
||
|
||
|
||
def _seen_ids(raw_file: Path) -> set[str]:
|
||
if not raw_file.exists():
|
||
return set()
|
||
ids = set()
|
||
for line in raw_file.read_text("utf-8").splitlines():
|
||
if line.strip():
|
||
try:
|
||
ids.add(str(json.loads(line)["id"]))
|
||
except (json.JSONDecodeError, KeyError):
|
||
pass
|
||
return ids
|
||
|
||
|
||
def _store(raw_file: Path, rec: dict) -> None:
|
||
with raw_file.open("a", encoding="utf-8") as f:
|
||
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||
|
||
|
||
def harvest_rutracker(forums: list[str], topics: list[str], pages: int,
|
||
limit: int, delay: float, from_page: int | None = None,
|
||
reset: bool = False) -> int:
|
||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||
raw_file = RAW_DIR / "rutracker.jsonl"
|
||
seen = _seen_ids(raw_file)
|
||
state = load_state("rutracker")
|
||
fstate = state.setdefault("rutracker", {}).setdefault("forums", {})
|
||
if reset:
|
||
fstate.clear()
|
||
save_state("rutracker", state)
|
||
|
||
cdp, _ = connect("rutracker.org")
|
||
done = 0
|
||
try:
|
||
# явно заданные темы — вне очереди
|
||
for tid in dict.fromkeys(topics):
|
||
if tid in seen or done >= limit:
|
||
continue
|
||
try:
|
||
rec = rt_topic(cdp, tid)
|
||
except Exception as exc: # noqa: BLE001
|
||
rec = {"id": tid, "source": "rutracker", "ok": False, "error": str(exc)}
|
||
_store(raw_file, rec)
|
||
seen.add(tid)
|
||
done += 1
|
||
mark = "+" if re.search(r"Перевод|Озвучк", rec.get("post") or "", re.I) else "-"
|
||
log(f" {mark} {tid} {(rec.get('title') or '')[:78]}")
|
||
time.sleep(delay + random.random() * 0.8)
|
||
|
||
for forum in forums:
|
||
st = fstate.get(forum, {"next_page": 0, "topics": 0})
|
||
start = 0 if from_page is None else from_page
|
||
if from_page is None:
|
||
start = int(st.get("next_page", 0))
|
||
log(f"раздел {forum}: продолжаю со страницы {start + 1} "
|
||
f"(уже собрано тем: {st.get('topics', 0)})")
|
||
|
||
for k in range(pages):
|
||
if done >= limit:
|
||
break
|
||
page = start + k
|
||
ok, items = rt_forum_page(cdp, forum, page)
|
||
if not ok:
|
||
log(f" стр.{page + 1}: не открылась — попробую в следующий раз")
|
||
break
|
||
if not items:
|
||
fstate[forum] = {**st, "finished": True, "updated": stamp()}
|
||
save_state("rutracker", state)
|
||
log(f" стр.{page + 1}: тем нет — раздел пройден до конца")
|
||
break
|
||
fresh = [it for it in items if it["id"] not in seen]
|
||
log(f" стр.{page + 1}: {len(items)} тем, новых {len(fresh)}")
|
||
|
||
completed = True
|
||
for it in fresh:
|
||
if done >= limit:
|
||
completed = False
|
||
break
|
||
try:
|
||
rec = rt_topic(cdp, it["id"])
|
||
except Exception as exc: # noqa: BLE001
|
||
rec = {"id": it["id"], "source": "rutracker", "ok": False,
|
||
"error": str(exc)}
|
||
_store(raw_file, rec)
|
||
seen.add(it["id"])
|
||
done += 1
|
||
mark = "+" if re.search(r"Перевод|Озвучк", rec.get("post") or "", re.I) else "-"
|
||
log(f" {mark} {it['id']} {(rec.get('title') or '')[:70]}")
|
||
time.sleep(delay + random.random() * 0.8)
|
||
|
||
fstate[forum] = {
|
||
"next_page": page if not completed else page + 1,
|
||
"topics": st.get("topics", 0) + (len(fresh) if completed else 0),
|
||
"updated": stamp(),
|
||
}
|
||
save_state("rutracker", state) # прогресс после каждой страницы
|
||
st = fstate[forum]
|
||
if not completed:
|
||
log(f" лимит {limit} исчерпан, следующая страница {page + 1} — докачается потом")
|
||
break
|
||
log(f"rutracker: добавлено {done} тем -> {raw_file}")
|
||
return done
|
||
finally:
|
||
cdp.close()
|
||
|
||
|
||
# =====================================================================
|
||
# Сбор: nnmclub (nnm-club)
|
||
# =====================================================================
|
||
NNM_BASE = "https://nnmclub.to/forum/"
|
||
|
||
NNM_FORUM_JS = """JSON.stringify((() => {
|
||
const out = [], seen = new Set();
|
||
const rows = document.querySelectorAll('table.forumline tr').length;
|
||
for (const tr of document.querySelectorAll('table.forumline tr')) {
|
||
const a = tr.querySelector('a[href*="viewtopic.php?t="]');
|
||
if (!a) continue;
|
||
const m = (a.getAttribute('href') || '').match(/t=(\\d+)/);
|
||
if (!m || seen.has(m[1])) continue;
|
||
const text = (a.innerText || '').replace(/\\s+/g, ' ').trim();
|
||
if (text.length < 12) continue;
|
||
seen.add(m[1]);
|
||
out.push({ id: m[1], text: text.slice(0, 160) });
|
||
}
|
||
return { rows, items: out };
|
||
})())"""
|
||
|
||
NNM_TOPIC_JS = """JSON.stringify((() => {
|
||
const pick = (sels) => { for (const s of sels) { const el = document.querySelector(s);
|
||
if (el && el.innerText.trim().length > 30) return el; } return null; };
|
||
// табличная вёрстка: innerText склеивает строки, поэтому <br> и блоки
|
||
// превращаем в переводы строк вручную
|
||
const toText = (el) => {
|
||
const c = el.cloneNode(true);
|
||
c.querySelectorAll('br').forEach(b => b.replaceWith(document.createTextNode('\\n')));
|
||
c.querySelectorAll('p,div,tr,li,h1,h2,h3').forEach(n => n.append(document.createTextNode('\\n')));
|
||
return c.innerText;
|
||
};
|
||
const post = pick(['.postbody', '.post_body']);
|
||
const title = pick(['h1', '.maintitle', '#topic-title']);
|
||
return { title: title ? title.innerText.trim() : document.title,
|
||
post: post ? toText(post) : '' };
|
||
})())"""
|
||
|
||
|
||
def nnm_forum_page(cdp: CDP, forum: str, page: int) -> tuple[bool, list[dict]]:
|
||
url = f"{NNM_BASE}viewforum.php?f={forum}&start={page * 50}"
|
||
for attempt in (1, 2):
|
||
nav = goto(cdp, url, min_len=1500)
|
||
if nav.get("empty"):
|
||
return True, []
|
||
if not nav["ok"]:
|
||
log(f" nnm f={forum} стр.{page + 1}: не открылась ({nav['title']})")
|
||
else:
|
||
data = json.loads(cdp.eval(NNM_FORUM_JS) or "{}")
|
||
if data.get("rows"):
|
||
items = data.get("items", [])
|
||
releases = [it for it in items if not is_service_topic(it["text"])]
|
||
skipped = len(items) - len(releases)
|
||
if skipped:
|
||
log(f" nnm f={forum} стр.{page + 1}: отброшено служебных тем: {skipped}")
|
||
return True, releases
|
||
log(f" nnm f={forum} стр.{page + 1}: список не отрисовался (0 строк)")
|
||
if attempt == 1:
|
||
time.sleep(3)
|
||
return False, []
|
||
|
||
|
||
def nnm_topic(cdp: CDP, topic_id: str) -> dict:
|
||
url = f"{NNM_BASE}viewtopic.php?t={topic_id}"
|
||
nav = goto(cdp, url, min_len=1200)
|
||
rec: dict = {"id": topic_id, "url": url, "source": "nnm",
|
||
"ok": nav["ok"], "page_title": nav["title"]}
|
||
if not nav["ok"]:
|
||
rec["error"] = f"не открылась: {nav['title']}"
|
||
return rec
|
||
rec.update(json.loads(cdp.eval(NNM_TOPIC_JS) or "{}"))
|
||
return rec
|
||
|
||
|
||
def harvest_nnm(forums: list[str], topics: list[str], pages: int, limit: int,
|
||
delay: float, from_page: int | None = None,
|
||
reset: bool = False) -> int:
|
||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||
raw_file = RAW_DIR / "nnm.jsonl"
|
||
seen = _seen_ids(raw_file)
|
||
state = load_state("nnm")
|
||
fstate = state.setdefault("nnm", {}).setdefault("forums", {})
|
||
if reset:
|
||
fstate.clear()
|
||
save_state("nnm", state)
|
||
|
||
cdp, _ = connect("nnmclub.to")
|
||
done = 0
|
||
try:
|
||
for tid in dict.fromkeys(topics):
|
||
if tid in seen or done >= limit:
|
||
continue
|
||
try:
|
||
rec = nnm_topic(cdp, tid)
|
||
except Exception as exc: # noqa: BLE001
|
||
rec = {"id": tid, "source": "nnm", "ok": False, "error": str(exc)}
|
||
_store(raw_file, rec)
|
||
seen.add(tid)
|
||
done += 1
|
||
mark = "+" if re.search(r"Перевод|Аудио\s*\d", rec.get("post") or "", re.I) else "-"
|
||
log(f" {mark} {tid} {(rec.get('title') or '')[:78]}")
|
||
time.sleep(delay + random.random() * 0.8)
|
||
|
||
for forum in forums:
|
||
st = fstate.get(forum, {"next_page": 0, "topics": 0})
|
||
start = int(st.get("next_page", 0)) if from_page is None else from_page
|
||
log(f"раздел nnm {forum}: продолжаю со страницы {start + 1} "
|
||
f"(уже собрано тем: {st.get('topics', 0)})")
|
||
|
||
for k in range(pages):
|
||
if done >= limit:
|
||
break
|
||
page = start + k
|
||
ok, items = nnm_forum_page(cdp, forum, page)
|
||
if not ok:
|
||
log(f" стр.{page + 1}: не открылась — попробую в следующий раз")
|
||
break
|
||
if not items:
|
||
fstate[forum] = {**st, "finished": True, "updated": stamp()}
|
||
save_state("nnm", state)
|
||
log(f" стр.{page + 1}: тем нет — раздел пройден до конца")
|
||
break
|
||
fresh = [it for it in items if it["id"] not in seen]
|
||
log(f" стр.{page + 1}: {len(items)} тем, новых {len(fresh)}")
|
||
|
||
completed = True
|
||
for it in fresh:
|
||
if done >= limit:
|
||
completed = False
|
||
break
|
||
try:
|
||
rec = nnm_topic(cdp, it["id"])
|
||
except Exception as exc: # noqa: BLE001
|
||
rec = {"id": it["id"], "source": "nnm", "ok": False,
|
||
"error": str(exc)}
|
||
_store(raw_file, rec)
|
||
seen.add(it["id"])
|
||
done += 1
|
||
mark = "+" if re.search(r"Перевод|Аудио\s*\d", rec.get("post") or "", re.I) else "-"
|
||
log(f" {mark} {it['id']} {(rec.get('title') or '')[:70]}")
|
||
time.sleep(delay + random.random() * 0.8)
|
||
|
||
fstate[forum] = {
|
||
"next_page": page if not completed else page + 1,
|
||
"topics": st.get("topics", 0) + (len(fresh) if completed else 0),
|
||
"updated": stamp(),
|
||
}
|
||
save_state("nnm", state)
|
||
st = fstate[forum]
|
||
if not completed:
|
||
log(f" лимит {limit} исчерпан, следующая страница {page + 1}")
|
||
break
|
||
log(f"nnm: добавлено {done} тем -> {raw_file}")
|
||
return done
|
||
finally:
|
||
cdp.close()
|
||
|
||
|
||
# =====================================================================
|
||
# Разбор: общие нормализаторы
|
||
# =====================================================================
|
||
CYR_TO_LAT = str.maketrans({"А": "A", "В": "B", "С": "C", "Е": "E", "Н": "H",
|
||
"К": "K", "М": "M", "О": "O", "Р": "P", "Т": "T", "Х": "X"})
|
||
|
||
STUDIO_ALIASES = {
|
||
"hdrezka": "HDrezka", "hd rezka": "HDrezka", "hdrezka studio": "HDrezka",
|
||
"red head sound": "Red Head Sound", "read head sound": "Red Head Sound",
|
||
"redhead sound": "Red Head Sound",
|
||
"tvshows": "TVShows", "tv shows": "TVShows",
|
||
"lostfilm": "LostFilm", "lost film": "LostFilm",
|
||
"кубик в кубе": "Кубик в Кубе", "кубик": "Кубик в Кубе",
|
||
"jaskier": "Jaskier", "newcomers": "NewComers", "new comers": "NewComers",
|
||
"movie dubbing": "Movie Dubbing",
|
||
"videofilm int": "Videofilm Int", "videofilm international": "Videofilm Int",
|
||
"мосфильм-мастер": "Мосфильм-Мастер", "мосфильм мастер": "Мосфильм-Мастер",
|
||
"moviedalen": "MovieDalen", "movie dalen": "MovieDalen",
|
||
"paramount comedy": "Paramount Comedy",
|
||
"winmedia": "WinMedia", "viruse project": "ViruseProject",
|
||
"viruseproject": "ViruseProject", "ultradoxstudio": "Ultradox",
|
||
"ultradox studio": "Ultradox",
|
||
"а. гаврилов": "Андрей Гаврилов", "гаврилов": "Андрей Гаврилов",
|
||
"ю. сербин": "Юрий Сербин", "сербин": "Юрий Сербин",
|
||
"дмитрий есарев": "Дмитрий Есарев", "есарев": "Дмитрий Есарев",
|
||
"а. киреев": "Андрей Киреев", "киреев": "Андрей Киреев",
|
||
"володарский": "Леонид Володарский", "леонид володарский": "Леонид Володарский",
|
||
"живов": "Юрий Живов", "ю. живов": "Юрий Живов",
|
||
"яроцкий": "Михаил Яроцкий", "м. яроцкий": "Михаил Яроцкий",
|
||
"нтв +": "НТВ+",
|
||
}
|
||
STUDIO_PLACEHOLDERS = {"official", "официальный", "official dub", "офиц.",
|
||
"не требуется", "ru", "rus", "нет", "ориг.", "оригинал"}
|
||
SOURCE_LIKE = {"itunes", "blu-ray", "blu ray", "bluray", "netflix", "amazon",
|
||
"dvd", "hdtv", "web-dl", "webdl", "webrip", "bd", "bdrip",
|
||
"uhd", "original", "оригинал", "bd cee", "dvd r5", "r5"}
|
||
STUDIO_NOISE = re.compile(
|
||
r"\b(закадров\w*|многоголос\w*|двухголос\w*|одноголос\w*|"
|
||
r"профессиональн\w*|любительск\w*|авторск\w*|дублированн\w*|"
|
||
r"неофициальн\w*|официальн\w*|полный|перевод|роли озвучивали)\b", re.I)
|
||
# «дубляж» как тип, а не часть имени студии («Дубляжная», «Продубляж», «Дубляж-Студия»)
|
||
RE_DUB_TYPE = r"дублир|(?<![а-яё])дубляж(?![а-яё])"
|
||
ROLE_PATTERNS = [(RE_DUB_TYPE, "дубляж"), (r"многоголос", "многоголосый"),
|
||
(r"двухголос", "двухголосый"), (r"одноголос", "одноголосый"),
|
||
(r"субтитр", "субтитры"), (r"оригинал", "оригинал")]
|
||
|
||
# 18+ у kinozal/rutracker/nnm стоит в строке дорожки; отдельная строка
|
||
# «Возраст: 18+» — это рейтинг фильма, её флагом дорожки считать нельзя
|
||
RE_18 = re.compile(r"\b18\s*\+")
|
||
|
||
# «АVO, Ю. Сербин»: аббревиатура перевода перед именем студии
|
||
LEAD_TAG_TOKEN = re.compile(r"^[A-Za-zА-Яа-я]{2,4}\b[\s:.,|\-–—]*")
|
||
TAG_NAMES = {"AVO", "DVO", "MVO", "DUB", "VO", "ЛО", "ЛД", "ЛМ", "ДБ", "ПМ", "ПД", "АП"}
|
||
|
||
|
||
def _is_lead_tag(text: str) -> bool:
|
||
m = re.match(r"^([A-Za-zА-Яа-я]{2,4})\b", text or "")
|
||
return bool(m and m.group(1).translate(CYR_TO_LAT).upper() in TAG_NAMES)
|
||
|
||
|
||
def has_18(text: str) -> bool:
|
||
return bool(RE_18.search(text or ""))
|
||
|
||
|
||
def mark_18(note: str, age: bool) -> str:
|
||
if age and "18+" not in (note or ""):
|
||
return (note + " 18+").strip()
|
||
return (note or "").strip()
|
||
|
||
|
||
def clean_studio(raw: str) -> tuple[str, str]:
|
||
note = ""
|
||
s = (raw or "").strip().strip("|").strip()
|
||
s = re.sub(r"\[[^\]]*\]", " ", s)
|
||
s = re.sub(r"\([^)]*\)", " ", s)
|
||
if re.search(r"\b18\s*\+", s):
|
||
note = "18+"
|
||
s = re.sub(r"\b18\s*\+", " ", s)
|
||
s = STUDIO_NOISE.sub(" ", s)
|
||
s = re.sub(r"[«»\"']", "", s)
|
||
s = re.sub(r"\s*\([^)]*$", "", s) # незакрытая скобка в названии студии
|
||
s = re.sub(r"\s*\|\s*(RU|RUS|Rus|UKR|ENG)\b.*$", " ", s, flags=re.I)
|
||
s = s.replace("|", " ")
|
||
s = re.sub(r"\s+", " ", s).strip(" |.-–—:,")
|
||
s = LEAD_TAG_TOKEN.sub("", s) if _is_lead_tag(s) else s
|
||
s = s.strip("()[]{},;: ").strip() # остатки скобок от «(MVO, X) (18+)»
|
||
s = re.sub(r"\s+Studio$", "", s, flags=re.I)
|
||
if not s.strip(" ,.-–—|"):
|
||
s = ""
|
||
return s, note
|
||
|
||
|
||
def canonical_studio(s: str) -> tuple[str, str]:
|
||
key = re.sub(r"\s+", " ", (s or "").strip().lower())
|
||
if key in STUDIO_PLACEHOLDERS:
|
||
return "", "официальный дубляж"
|
||
return STUDIO_ALIASES.get(key, (s or "").strip()), ""
|
||
|
||
|
||
def translation_type(text: str) -> tuple[str, str]:
|
||
t = (text or "").lower()
|
||
kind = "неизвестно"
|
||
for pat, name in ROLE_PATTERNS:
|
||
if re.search(pat, t):
|
||
kind = name
|
||
break
|
||
if kind == "дубляж":
|
||
if re.search(r"любительск|псевдо", t): # любительский дубляж — отдельный класс, не «проф.»
|
||
return "люб. дубляж", "люб."
|
||
return "проф. дубляж", "проф."
|
||
if "профессиональн" in t:
|
||
pref = "проф."
|
||
elif "любительск" in t:
|
||
pref = "люб."
|
||
elif "авторск" in t:
|
||
pref = "авт."
|
||
else:
|
||
pref = ""
|
||
if "авторск" in t and kind == "одноголосый":
|
||
pref = "авт."
|
||
return f"{pref} {kind}".strip(), pref
|
||
|
||
|
||
# ---- kinozal -------------------------------------------------------
|
||
KZ_RE_TRANSLATION = re.compile(
|
||
r"^\s*(?:Перевод|Озвучиван\w*|Озвучк\w*)\s*(?:#\s*\d+|\d+)?\s*:\s*(?P<body>.+)$", re.I)
|
||
KZ_RE_AUDIO_X = re.compile(
|
||
r"^\s*[АA]удио(?:\s*#?\s*\d+)?\s*x\s*(?P<lang>[A-ZА-Я/]\w*)\s*:\s*(?P<body>.+)$", re.I)
|
||
KZ_RE_AUDIO = re.compile(r"^\s*[АA]удио\s*:\s*(?P<body>.+)$", re.I)
|
||
KZ_RE_SUBS = re.compile(r"^\s*[СC]убтитры\s*:\s*(?P<body>.+)$", re.I)
|
||
KZ_RE_ACTORS = re.compile(r"Роли озвучивали\s*:\s*(?P<names>[^\]]+)", re.I)
|
||
RE_CODEC = re.compile(r"\b(E-?AC-?3|AC-?3|DTS-?HD|DTS|AAC|FLAC|MP3|OPUS|PCM|TrueHD)\b", re.I)
|
||
RE_LANG = re.compile(r"\b(Russian|Русский|English|Английский|Ukrainian|Украинский)\b", re.I)
|
||
|
||
|
||
def kz_split_title(title: str) -> dict:
|
||
parts = [p.strip() for p in (title or "").split(" / ")]
|
||
year, shorthand = "", ""
|
||
for i, p in enumerate(parts):
|
||
if re.fullmatch(r"(19|20)\d{2}(\s*-\s*(19|20)\d{2})?", p):
|
||
year = p
|
||
shorthand = parts[i + 1] if i + 1 < len(parts) else ""
|
||
break
|
||
return {"ru_title": parts[0] if parts else "", "year": year, "shorthand": shorthand}
|
||
|
||
|
||
def kz_audio(lang: str, body: str) -> dict:
|
||
tag, note = "", ""
|
||
m = re.search(r"\|([^|]*)\|", body)
|
||
if m:
|
||
tag, n1 = clean_studio(m.group(1))
|
||
tag, n2 = canonical_studio(tag)
|
||
note = " ".join(x for x in (n1, n2) if x)
|
||
body = body[:m.start()] + " " + body[m.end():]
|
||
flat = body.translate(CYR_TO_LAT)
|
||
codec = ""
|
||
cm = RE_CODEC.search(flat)
|
||
if cm:
|
||
codec = cm.group(1).upper().replace(" ", "")
|
||
codec = {"EAC3": "E-AC3", "AC3": "AC3"}.get(codec.replace("-", ""), codec)
|
||
ch = ""
|
||
cm2 = re.search(r"(\d+)\s*ch\b", flat, re.I)
|
||
if cm2:
|
||
ch = cm2.group(1)
|
||
khz = ""
|
||
km = re.search(r"(\d+(?:\.\d+)?)\s*(?:kHz|кГц)", body, re.I)
|
||
if km:
|
||
khz = str(int(float(km.group(1))))
|
||
kbps = ""
|
||
bm = re.search(r"(\d+)\s*(?:kbps|кбит\s*/?\s*с?)", body, re.I)
|
||
if bm:
|
||
kbps = bm.group(1)
|
||
return {"lang": lang.upper(), "codec": codec, "channels": ch, "khz": khz,
|
||
"kbps": kbps, "tag": tag, "note": note,
|
||
"age_18": int(has_18(body) or "18+" in note)}
|
||
|
||
|
||
def _kz_parse_core(rec: dict) -> tuple[list[dict], dict]:
|
||
release_text = (rec.get("release_tab") or "").strip()
|
||
tech_text = (rec.get("tech_tab") or "").strip()
|
||
|
||
def has_transl(t: str) -> bool:
|
||
return any(KZ_RE_TRANSLATION.match(l) for l in t.splitlines())
|
||
|
||
if has_transl(release_text):
|
||
source, studio_source = release_text, "release"
|
||
else:
|
||
source, studio_source = tech_text, "tech"
|
||
|
||
translations: list[dict] = []
|
||
audios: list[dict] = []
|
||
subs = ""
|
||
# переводы берём только из приоритетной вкладки, иначе «Техданные»
|
||
# добавляют вторую, обезличенную строку «Перевод: Дублированный»
|
||
for line in source.splitlines():
|
||
line = line.strip()
|
||
m = KZ_RE_TRANSLATION.match(line)
|
||
if m:
|
||
body = m.group("body").strip()
|
||
actors = ""
|
||
am = KZ_RE_ACTORS.search(body)
|
||
if am:
|
||
actors = am.group("names").strip(" .")
|
||
role, candidates = "", []
|
||
pm = re.search(r"\(([^)]*)\)", body)
|
||
if pm:
|
||
role = pm.group(1)
|
||
candidates = [body[pm.end():].strip(" |.-–—"), role,
|
||
body[:pm.start()].strip(" |.-–—")]
|
||
else:
|
||
candidates = [body]
|
||
studio, note = "", ""
|
||
for cand in candidates:
|
||
s1, n1 = clean_studio(cand)
|
||
s1, n2 = canonical_studio(s1)
|
||
if s1:
|
||
studio, note = s1, " ".join(x for x in (n1, n2) if x)
|
||
break
|
||
kind, pro = translation_type(body)
|
||
if re.search(r"не требуется", body, re.I):
|
||
kind = "оригинал"
|
||
translations.append({"studio": studio, "type": kind, "pro": pro,
|
||
"role": role, "actors": actors, "note": note,
|
||
"line": line})
|
||
continue
|
||
|
||
# аудиодорожки и субтитры могут быть в любой из вкладок
|
||
for line in (source + "\n" + tech_text).splitlines():
|
||
line = line.strip()
|
||
m = KZ_RE_AUDIO_X.match(line)
|
||
if m:
|
||
audios.append(kz_audio(m.group("lang"), m.group("body")))
|
||
continue
|
||
m = KZ_RE_AUDIO.match(line)
|
||
if m and not audios:
|
||
body = m.group("body")
|
||
lang = "RUS"
|
||
lm = RE_LANG.search(body)
|
||
if lm:
|
||
lang = {"russian": "RUS", "русский": "RUS", "english": "ENG",
|
||
"английский": "ENG", "ukrainian": "UKR",
|
||
"украинский": "UKR"}[lm.group(1).lower()]
|
||
body = body[:lm.start()] + body[lm.end():]
|
||
audios.append(kz_audio(lang, body.strip(" :,()")))
|
||
continue
|
||
m = KZ_RE_SUBS.match(line)
|
||
if m and not subs:
|
||
subs = m.group("body").strip()
|
||
|
||
info = kz_split_title((rec.get("page_title") or "").split(" :: ")[0])
|
||
is_series = bool(re.search(r"сезон|серии", rec.get("page_title") or "", re.I))
|
||
ru_audio = [a for a in audios if a["lang"].startswith("RU")]
|
||
|
||
rows, used = [], set()
|
||
for t in translations:
|
||
if t["type"] in ("оригинал", "субтитры"):
|
||
continue
|
||
match = None
|
||
for j, a in enumerate(audios): # по имени студии — среди всех языков (украинский дубляж и т.п.)
|
||
if j in used:
|
||
continue
|
||
tag, st = a["tag"].lower(), t["studio"].lower()
|
||
if tag and st and (tag == st or tag.startswith(st) or st.startswith(tag)):
|
||
match = (j, a)
|
||
break
|
||
if match is None:
|
||
for j, a in enumerate(audios): # без имени — только безымянная русская дорожка
|
||
if j not in used and not a["tag"] and a["lang"].startswith("RU"):
|
||
match = (j, a)
|
||
break
|
||
if match:
|
||
used.add(match[0])
|
||
a = match[1] if match else {}
|
||
age_18 = bool(a.get("age_18") or has_18(t["line"]))
|
||
note = " ".join(x for x in (t["note"], a.get("note", "")) if x).strip()
|
||
rows.append({
|
||
"source": "kinozal", "release_id": rec["id"], "ru_title": info["ru_title"],
|
||
"year": info["year"], "quality": (tech_text.split("\n")[0] or "").replace("Качество:", "").strip(),
|
||
"is_series": is_series, "studio": t["studio"] or "(не указана)",
|
||
"translation_type": t["type"], "professional": t["pro"],
|
||
"role_raw": t["role"], "actors": t["actors"],
|
||
"note": mark_18(note, age_18), "age_18": int(age_18),
|
||
"codec": a.get("codec", ""), "channels": a.get("channels", ""),
|
||
"khz": a.get("khz", ""), "kbps": a.get("kbps", ""),
|
||
"subtitles": subs, "studio_source": studio_source, "raw": t["line"],
|
||
"lang": (a.get("lang") or "RUS") if match else "",
|
||
})
|
||
rel = {"source": "kinozal", "release_id": rec["id"], "ru_title": info["ru_title"],
|
||
"year": info["year"], "quality": rows[0]["quality"] if rows else "",
|
||
"is_series": is_series, "n_translations": len(rows),
|
||
"ok": bool(rows),
|
||
"studios": " | ".join(sorted({r["studio"] for r in rows}))}
|
||
return rows, rel
|
||
|
||
|
||
# ---- rutracker -----------------------------------------------------
|
||
RT_RE_FIELD = re.compile(
|
||
r"^\s*(?:Перевод|Озвучка|Озвучивание|Озвучено|Звук)\s*(?:#\s*\d+|\d+)?\s*:\s*(?P<body>.+)$", re.I)
|
||
RT_RE_SUBS = re.compile(r"^\s*Субтитры\s*:\s*(?P<body>.+)$", re.I)
|
||
RT_RE_BRACKET = re.compile(r"\[(?P<body>[^\]]*)\]")
|
||
RT_RE_SEP = re.compile(r"\s*(?:\+|;|/)\s*(?=Проф|Люб|Авт|Дубл|Многоголос|Двухголос|Одноголос)", re.I)
|
||
|
||
|
||
def rt_split_entries(body: str) -> list[str]:
|
||
text = body.strip()
|
||
numbered = re.split(r"\s(?=\d\.\s)", text)
|
||
parts = [re.sub(r"^\d\.\s*", "", p) for p in numbered] if len(numbered) > 1 else RT_RE_SEP.split(text)
|
||
return [p.strip(" .") for p in parts if p and p.strip(" .")]
|
||
|
||
|
||
def rt_parse_entry(text: str) -> dict:
|
||
role, rest = "", text
|
||
pm = re.search(r"\(([^)]*)\)", text)
|
||
if pm:
|
||
role = pm.group(1)
|
||
rest = text[pm.end():].strip(" |.-–—:") or text[:pm.start()].strip(" |.-–—:")
|
||
studio, note = clean_studio(rest)
|
||
studio, extra = canonical_studio(studio)
|
||
note = " ".join(x for x in (note, extra) if x).strip()
|
||
age_18 = has_18(text)
|
||
note = mark_18(note, age_18)
|
||
if studio.lower() in SOURCE_LIKE:
|
||
note = (note + f" источник: {studio}").strip()
|
||
studio = ""
|
||
kind, pro = translation_type(text)
|
||
return {"role": role, "studio": studio, "type": kind, "pro": pro,
|
||
"note": note, "age_18": int(age_18), "raw": text}
|
||
|
||
|
||
def rt_parse_title(title: str) -> dict:
|
||
ru = (title or "").split(" / ")[0].strip()
|
||
year, quality = "", ""
|
||
m = RT_RE_BRACKET.search(title or "")
|
||
if m:
|
||
inside = [p.strip() for p in m.group("body").split(",")]
|
||
for p in inside:
|
||
if re.fullmatch(r"(19|20)\d{2}", p):
|
||
year = p
|
||
quality = inside[-1] if inside else ""
|
||
is_series = bool(re.search(r"Сезон\s*:|Серии\s*:\s*\d", title or "", re.I))
|
||
return {"ru_title": ru, "year": year, "quality": quality, "is_series": is_series}
|
||
|
||
|
||
def audio18_rows(rows: list[dict], post: str) -> list[dict]:
|
||
"""Строки «Аудио … |Студия| 18+» дополняют строки «Перевод».
|
||
|
||
У студии может быть две аудиодорожки (обычная и 18+) при одной строке
|
||
«Перевод»: тогда добавляется отдельная строка 18+. Кодек, каналы, частота
|
||
и битрейт берутся из соответствующей строки «Аудио».
|
||
"""
|
||
studios = sorted({r["studio"] for r in rows if r["studio"]})
|
||
lines = audio_lines(post, studios)
|
||
extra = []
|
||
for studio in studios:
|
||
mine = [r for r in rows if r["studio"] == studio]
|
||
auds = [a for a in lines if a["studio"] == studio]
|
||
plain = [a for a in auds if not a["adult"]]
|
||
adult = [a for a in auds if a["adult"]]
|
||
for r, a in zip(mine, plain or adult):
|
||
for k in ("codec", "channels", "khz", "kbps"):
|
||
r[k] = r[k] or a[k]
|
||
if not adult or any(int(r["age_18"] or 0) for r in mine):
|
||
continue
|
||
if plain:
|
||
row = dict(mine[0])
|
||
for k in ("codec", "channels", "khz", "kbps"):
|
||
row[k] = adult[0][k]
|
||
row["age_18"], row["note"] = 1, mark_18(row["note"], True)
|
||
row["raw"] = f"{row['raw']} 18+"
|
||
extra.append(row)
|
||
else:
|
||
for r in mine:
|
||
r["age_18"], r["note"] = 1, mark_18(r["note"], True)
|
||
return rows + extra
|
||
|
||
|
||
def _rt_parse_core(rec: dict) -> tuple[list[dict], dict]:
|
||
post = rec.get("post") or ""
|
||
info = rt_parse_title(rec.get("page_title") or rec.get("title") or "")
|
||
entries, subs = [], ""
|
||
for line in post.splitlines():
|
||
m = RT_RE_FIELD.match(line)
|
||
if m:
|
||
for part in rt_split_entries(m.group("body")):
|
||
e = rt_parse_entry(part)
|
||
if e["type"] in ("оригинал", "субтитры"):
|
||
continue
|
||
if not e["studio"] and e["type"] == "неизвестно":
|
||
continue
|
||
entries.append(e)
|
||
continue
|
||
m = RT_RE_SUBS.match(line)
|
||
if m and not subs:
|
||
subs = m.group("body").strip()
|
||
rows = [{
|
||
"source": "rutracker", "release_id": rec["id"], "ru_title": info["ru_title"],
|
||
"year": info["year"], "quality": info["quality"], "is_series": info["is_series"],
|
||
"studio": e["studio"] or "(не указана)", "translation_type": e["type"],
|
||
"professional": e["pro"], "role_raw": e["role"], "actors": "",
|
||
"note": e["note"], "age_18": e.get("age_18", 0),
|
||
"codec": "", "channels": "", "khz": "", "kbps": "",
|
||
"subtitles": subs, "studio_source": "release", "raw": e["raw"],
|
||
} for e in entries]
|
||
rows = audio18_rows(rows, post)
|
||
rel = {"source": "rutracker", "release_id": rec["id"], "ru_title": info["ru_title"],
|
||
"year": info["year"], "quality": info["quality"], "is_series": info["is_series"],
|
||
"n_translations": len(rows), "ok": bool(rows),
|
||
"studios": " | ".join(sorted({r["studio"] for r in rows}))}
|
||
return rows, rel
|
||
|
||
|
||
# ---- nnmclub -------------------------------------------------------
|
||
NNM_RE_AUDIO = re.compile(r"^\s*Аудио\s*(?:#?\s*\d+)?\s*:\s*(?P<body>.+)$", re.I)
|
||
NNM_RE_PEREVOD = re.compile(r"^\s*Перевод\s*:\s*(?P<body>.+)$", re.I)
|
||
NNM_LANG_RE = re.compile(r"(Русский|Английский|Украинский)", re.I)
|
||
NNM_LANG_MAP = {"русский": "RUS", "английский": "ENG", "украинский": "UKR"}
|
||
NNM_TAG_RE = re.compile(r"\b(DUB|MVO|DVO|AVO|VO|ДБ|ПМ|ПД|АП|ЛМ|ЛД|ЛО)\b", re.I)
|
||
NNM_RE_CODEC = re.compile(r"\b(Opus|AC-?3|E-?AC-?3|AAC|DTS-?HD|DTS|FLAC|MP3|TrueHD)\b", re.I)
|
||
NNM_RE_KBP = re.compile(r"(\d+)\s*(?:Kbps|Кбит)", re.I)
|
||
NNM_RE_CH = re.compile(r"(\d+)\s*ch\b", re.I)
|
||
NNM_RE_LAYOUT = re.compile(r"\b(\d)\.(\d)\b")
|
||
NNM_RE_YEAR = re.compile(r"\((\d{4})")
|
||
NNM_RE_QUAL = re.compile(r"\b(BDRemux|BDRip|WEB-?DL|WEBRip|HDRip|DVDRip|SATRip|UHD)\b|(\d{3,4}p)", re.I)
|
||
|
||
# порядок важен: AVO/DVO/MVO содержат «VO» внутри
|
||
NNM_TAG_TYPES = [
|
||
("DUB", "проф. дубляж", "проф."),
|
||
("ДБ", "проф. дубляж", "проф."),
|
||
("MVO", "проф. многоголосый", "проф."),
|
||
("ПМ", "проф. многоголосый", "проф."),
|
||
("DVO", "проф. двухголосый", "проф."),
|
||
("ПД", "проф. двухголосый", "проф."),
|
||
("AVO", "авт. одноголосый", "авт."),
|
||
("АП", "авт. одноголосый", "авт."),
|
||
("ЛМ", "люб. многоголосый", "люб."),
|
||
("ЛД", "люб. двухголосый", "люб."),
|
||
("ЛО", "люб. одноголосый", "люб."),
|
||
("VO", "одноголосый", ""),
|
||
]
|
||
|
||
|
||
def nnm_tag_type(tag: str) -> tuple[str, str]:
|
||
up = (tag or "").upper().replace(".", "")
|
||
for needle, label, pro in NNM_TAG_TYPES:
|
||
if needle in up:
|
||
return label, pro
|
||
return "", ""
|
||
|
||
|
||
NNM_LEAD_TAG = re.compile(r"^(AVO|DVO|MVO|DUB|VO|ЛО|ЛД|ЛМ|ДБ|ПМ|ПД|АП)\b[\s:.,|\-–—]*", re.I)
|
||
NNM_TAG_NAMES = {needle for needle, _, _ in NNM_TAG_TYPES}
|
||
NNM_LEAD_TOKEN = re.compile(r"^([A-Za-zА-Яа-я]{2,4})\b[\s:.,|\-–—]*")
|
||
|
||
|
||
def nnm_studio(name: str) -> tuple[str, str]:
|
||
"""Имя из скобок может начинаться с аббревиатуры: 'AVO Юрий Немахов'."""
|
||
cleaned, note = clean_studio(name)
|
||
# в скобках иногда не имя, а техпараметры: 'DTS / 5.1 / 48 kHz / 1536 kbps'
|
||
if re.search(r"kHz|кГц|kbps|Кбит|\b\d\.\d\b|\b\d+/\d+\b", cleaned, re.I):
|
||
return "", note
|
||
cleaned = NNM_LEAD_TAG.sub("", cleaned)
|
||
# 'АVO, Ю. Сербин' — кириллическая «А» в латинской аббревиатуре
|
||
m = NNM_LEAD_TOKEN.match(cleaned)
|
||
if m and m.group(1).translate(CYR_TO_LAT).upper() in NNM_TAG_NAMES:
|
||
cleaned = cleaned[m.end():]
|
||
cleaned = cleaned.strip("()[]{},;: ").strip() # остатки скобок от «(MVO, X) (18+)»
|
||
cleaned, extra = canonical_studio(cleaned)
|
||
return cleaned, " ".join(x for x in (note, extra) if x).strip()
|
||
|
||
|
||
# слова, которые в «Перевод: <тип> - <студия>» не студия: тип, кратность, язык
|
||
NNM_PART_TYPEISH = re.compile(
|
||
r"(?i)голос|дубл|закадр|авторск|любител|профессион|русск|оригинал|субтитр|лицензи"
|
||
r"|^\W*[xх]?\d+[xх]?\W*$")
|
||
|
||
|
||
def nnm_part_studio(part: str) -> str:
|
||
"""Студия из части строки «Перевод:» nnm: «Многоголосый профессиональный - Jaskier»,
|
||
«Дублированный (Невафильм)», «Авторский (Живов» -> имя; «Дублированный (Лицензия)»,
|
||
«Одноголосый x2», «Двуголосый профессиональный» -> ""."""
|
||
if len(part) > 90 or re.search(r"(?i)премьер|\bдата\b", part):
|
||
return "" # склеилось соседнее поле описания
|
||
m = re.search(r"\s[-–—:]\s*(.+)$", part)
|
||
cands = ([m.group(1)] if m else []) + [c for c in re.findall(r"\(([^()]*)\)?", part) if c]
|
||
for cand in cands:
|
||
studio, _ = clean_studio(cand)
|
||
studio, _ = canonical_studio(studio)
|
||
if len(studio) >= 2 and not NNM_PART_TYPEISH.search(studio) \
|
||
and studio.lower() not in SOURCE_LIKE:
|
||
return studio
|
||
return ""
|
||
|
||
|
||
def _nnm_parse_core(rec: dict) -> tuple[list[dict], dict]:
|
||
post = rec.get("post") or ""
|
||
title = rec.get("page_title") or rec.get("title") or ""
|
||
ru_title = title.split(" / ")[0].strip()
|
||
ym = NNM_RE_YEAR.search(title)
|
||
year = ym.group(1) if ym else ""
|
||
qm = NNM_RE_QUAL.search(title)
|
||
quality = (qm.group(1) or qm.group(2)) if qm else ""
|
||
is_series = bool(re.search(r"сезон|серии|\(S\d+|сериал", title, re.I))
|
||
|
||
rows: list[dict] = []
|
||
perevod_types: list[str] = []
|
||
|
||
for line in post.splitlines():
|
||
line = line.strip()
|
||
m = NNM_RE_AUDIO.match(line)
|
||
if m:
|
||
body = m.group("body").strip()
|
||
lm = NNM_LANG_RE.search(body)
|
||
if not lm or NNM_LANG_MAP.get(lm.group(1).lower()) != "RUS":
|
||
continue # английскую/украинскую дорожку пропускаем
|
||
rest = body[lm.end():]
|
||
tm = NNM_TAG_RE.search(rest)
|
||
tag = tm.group(1).upper() if tm else ""
|
||
if not tag: # формат «... - дубляж» без аббревиатуры
|
||
low = rest.lower()
|
||
if "дубляж" in low:
|
||
tag = "DUB"
|
||
elif "многоголос" in low:
|
||
tag = "MVO"
|
||
elif "двухголос" in low:
|
||
tag = "DVO"
|
||
elif "авторск" in low:
|
||
tag = "AVO"
|
||
elif "одноголос" in low:
|
||
tag = "VO"
|
||
name_raw = ""
|
||
pm = re.search(r"\((.*)\)", rest)
|
||
if pm:
|
||
name_raw = pm.group(1).strip()
|
||
studio, note = nnm_studio(name_raw)
|
||
age_18 = has_18(rest)
|
||
note = mark_18(note, age_18)
|
||
ttype, pro = nnm_tag_type(tag)
|
||
if not ttype:
|
||
ttype, pro = "неизвестно", ""
|
||
codec_m = NNM_RE_CODEC.search(body)
|
||
kb_m = NNM_RE_KBP.search(body)
|
||
ch_m = NNM_RE_CH.search(body)
|
||
channels = ch_m.group(1) if ch_m else ""
|
||
if not channels: # формат «6.0» / «5.1»
|
||
lay = NNM_RE_LAYOUT.search(body)
|
||
if lay:
|
||
channels = str(int(lay.group(1)) + (1 if lay.group(2) != "0" else 0))
|
||
rows.append({
|
||
"source": "nnm", "release_id": rec["id"], "ru_title": ru_title,
|
||
"year": year, "quality": quality, "is_series": is_series,
|
||
"studio": studio or "(не указана)", "translation_type": ttype,
|
||
"professional": pro, "role_raw": tag, "actors": name_raw,
|
||
"note": note,
|
||
"age_18": int(age_18),
|
||
"codec": (codec_m.group(1).upper() if codec_m else ""),
|
||
"channels": channels,
|
||
"khz": "",
|
||
"kbps": (kb_m.group(1) if kb_m else ""),
|
||
"subtitles": "", "studio_source": "release", "raw": line,
|
||
})
|
||
continue
|
||
|
||
m = NNM_RE_PEREVOD.match(line)
|
||
if m:
|
||
for part in re.split(r"[,/]", m.group("body")):
|
||
part = part.strip()
|
||
if not part:
|
||
continue
|
||
ttype, pro = translation_type(part)
|
||
if ttype in ("неизвестно", "оригинал", "субтитры"):
|
||
continue
|
||
perevod_types.append(part)
|
||
# студия — если названа в части («Дублированный (Невафильм)»); без неё тип всё равно ценен
|
||
rows.append({
|
||
"source": "nnm", "release_id": rec["id"], "ru_title": ru_title,
|
||
"year": year, "quality": quality, "is_series": is_series,
|
||
"studio": nnm_part_studio(part) or "(не указана)", "translation_type": ttype,
|
||
"professional": pro, "role_raw": part, "actors": "",
|
||
"note": mark_18("", has_18(line)), "age_18": int(has_18(line)),
|
||
"codec": "", "channels": "", "khz": "", "kbps": "",
|
||
"subtitles": "", "studio_source": "tech", "raw": line,
|
||
})
|
||
|
||
# есть построчные «Аудио N» с именами — строки «Перевод:» (tech) не нужны; иначе
|
||
# оставляем tech-строки: типы и студии, названные в «Перевод:»
|
||
if any(r["studio"] != "(не указана)" and r["studio_source"] == "release" for r in rows):
|
||
rows = [r for r in rows if r["studio_source"] == "release" and r["studio"] != "(не указана)"]
|
||
|
||
# дорожка без имени и без типа («- Русский») ничего не даёт датасету
|
||
rows = [r for r in rows
|
||
if not (r["studio"] == "(не указана)" and r["translation_type"] == "неизвестно")]
|
||
|
||
# одна и та же строка «Перевод:» может встретиться дважды
|
||
seen_keys, uniq = set(), []
|
||
for r in rows:
|
||
key = (r["studio"].lower(), r["translation_type"], r["kbps"], r["codec"])
|
||
if key in seen_keys:
|
||
continue
|
||
seen_keys.add(key)
|
||
uniq.append(r)
|
||
rows = uniq
|
||
|
||
rel = {"source": "nnm", "release_id": rec["id"], "ru_title": ru_title,
|
||
"year": year, "quality": quality, "is_series": is_series,
|
||
"n_translations": len(rows), "ok": bool(rows),
|
||
"studios": " | ".join(sorted({r["studio"] for r in rows}))}
|
||
return rows, rel
|
||
|
||
|
||
# =====================================================================
|
||
# Стадии: parse / merge / stats
|
||
# =====================================================================
|
||
RAW_SOURCES = [
|
||
("kinozal", RAW_DIR / "kinozal.jsonl", OUT / "harvest" / "raw.jsonl"),
|
||
("rutracker", RAW_DIR / "rutracker.jsonl", OUT / "rutracker" / "raw.jsonl"),
|
||
("nnm", RAW_DIR / "nnm.jsonl", OUT / "nnm" / "raw.jsonl"),
|
||
]
|
||
# ---- язык и строка «Аудио» для меток (общий разбор src/trackinfo.py) ---------
|
||
def enrich_rows(rows: list[dict], text: str) -> list[dict]:
|
||
"""К строкам разметки: lang и audio_raw из подходящей строки «Аудио» (с тем же 18+)."""
|
||
lines = audio_lines_all(text)
|
||
for r in rows:
|
||
info = pick_for_row(r.get("raw") or "", bool(int(r.get("age_18") or 0)), lines)
|
||
r["lang"], r["audio_raw"] = r.get("lang") or "", ""
|
||
if info:
|
||
r["audio_raw"] = info["raw"]
|
||
r["lang"] = info["lang"] or lang_of_text(r.get("raw") or "")
|
||
for key in ("codec", "channels", "kbps"):
|
||
r[key] = r.get(key) or info[key]
|
||
else:
|
||
r["lang"] = r["lang"] or lang_of_text(r.get("raw") or "")
|
||
return rows
|
||
|
||
|
||
def kz_audio_only_rows(rec: dict, text: str) -> list[dict]:
|
||
"""kinozal: «Аудио 1: AC3, 6 ch, 448 Кбит/с - Русский дубляж (Студия)» без строк «Перевод»."""
|
||
title = rec.get("page_title") or rec.get("h1") or ""
|
||
info = kz_split_title(title.split(" :: ")[0])
|
||
is_series = bool(re.search(r"сезон|серии", title, re.I))
|
||
rows = []
|
||
for a in audio_lines_all(text):
|
||
if not a["names"] or a["original"] or a["commentary"]:
|
||
continue
|
||
descr = re.split(r"(?i)(?:кбит/с|kbps)\s*[-–—]\s*", a["raw"], maxsplit=1)[-1]
|
||
kind, pro = translation_type(descr)
|
||
if kind == "неизвестно" or kind.endswith("неизвестно"):
|
||
continue
|
||
studio, note = clean_studio(a["names"][0])
|
||
studio, extra = canonical_studio(studio)
|
||
rows.append({
|
||
"source": "kinozal", "release_id": rec["id"], "ru_title": info["ru_title"],
|
||
"year": info["year"], "quality": "", "is_series": is_series,
|
||
"studio": studio or "(не указана)", "translation_type": kind, "professional": pro,
|
||
"role_raw": descr, "actors": "",
|
||
"note": mark_18(" ".join(x for x in (note, extra) if x).strip(), a["adult"]),
|
||
"age_18": int(a["adult"]), "codec": a["codec"], "channels": a["channels"], "khz": "",
|
||
"kbps": a["kbps"], "subtitles": "", "studio_source": "audio",
|
||
"raw": descr, "lang": a["lang"], "audio_raw": a["raw"]})
|
||
return rows
|
||
|
||
|
||
def split_multi_rows(rows: list[dict]) -> list[dict]:
|
||
"""Строка с несколькими дорожками («Проф. многоголосый, авторский (одноголосый)») -> по строке на дорожку.
|
||
|
||
Режет теми же правилами, что сервис при разборе страницы (src.html_extract._split_entries), чтобы
|
||
обучение и инференс видели одинаковые фрагменты. Одну дорожку («Многоголосый, профессиональный -
|
||
LostFilm») не трогает. nnm tech-строки уже по одной на дорожку (role_raw) — тоже не трогает.
|
||
"""
|
||
out: list[dict] = []
|
||
for r in rows:
|
||
if r.get("studio_source") == "tech" and (r.get("role_raw") or "").strip():
|
||
out.append(r)
|
||
continue
|
||
body = RE_LINE_PREFIX.sub("", (r.get("raw") or "").strip())
|
||
parts = [p for p in _split_entries(re.sub(r"\s+", " ", body)) if RE_TYPE_HINT.search(p)]
|
||
if len(parts) < 2:
|
||
out.append(r)
|
||
continue
|
||
base_studio = (r.get("studio") or "").strip()
|
||
for part in parts:
|
||
e = rt_parse_entry(part)
|
||
if e["type"] in ("оригинал", "субтитры"):
|
||
continue
|
||
new = dict(r)
|
||
keep = base_studio not in ("", "(не указана)") and base_studio.casefold() in part.casefold()
|
||
new.update(raw=part, translation_type=e["type"], professional=e["pro"],
|
||
role_raw=e["role"], studio=base_studio if keep else (e["studio"] or "(не указана)"),
|
||
age_18=int(has_18(part) or (keep and int(r.get("age_18") or 0))))
|
||
new["note"] = mark_18(e["note"] or "", bool(new["age_18"]))
|
||
out.append(new)
|
||
return out
|
||
|
||
|
||
def audio_only_rows(rec: dict, text: str, rows: list[dict]) -> list[dict]:
|
||
"""Строки «Аудио … |MVO, Студия|» с языком, которых нет среди строк разметки (украинский дубляж и т.п).
|
||
|
||
Берутся только дорожки с блоком |…| (тип, студия) и определимым типом; оригинал и комментарии —
|
||
не обучающие строки. Имена из скобок («(L,C,R,l,r)» — раскладка каналов) сюда не попадают.
|
||
"""
|
||
def key(x: str) -> str:
|
||
return re.sub(r"[\W_]+", "", (x or "").casefold())
|
||
|
||
have = key(" ".join(((r.get("studio") or "") + " " + (r.get("raw") or "")) for r in rows))
|
||
out = []
|
||
for a in audio_lines_all(text):
|
||
if a["original"] or a["commentary"]:
|
||
continue
|
||
m = re.search(r"\|([^|]+)\|", a["raw"])
|
||
if not m:
|
||
continue
|
||
chunk = m.group(1)
|
||
tokens = [t.strip() for t in re.split(r"\s*[,;/]\s*", chunk) if t.strip()]
|
||
kind = pro = ""
|
||
for t in tokens:
|
||
if t.upper().replace(".", "") in NNM_TAG_NAMES: # короткий тег: DUB, MVO, ПМ …
|
||
kind, pro = nnm_tag_type(t)
|
||
break
|
||
k, pr = translation_type(t)
|
||
if not k.endswith("неизвестно"):
|
||
kind, pro = k, pr
|
||
break
|
||
if not kind:
|
||
continue
|
||
studio = ""
|
||
for t in tokens:
|
||
if t.upper().replace(".", "") in NNM_TAG_NAMES or re.search(r"(?i)голос|дубл|оригинал|original", t):
|
||
continue
|
||
st, note = clean_studio(t)
|
||
st, extra = canonical_studio(st)
|
||
if len(st) >= 3:
|
||
studio = st
|
||
break
|
||
if not studio or key(studio) in have or re.match(r"(?i)(?:blu-?ray|bd|dvd|web)\b", studio): # источник, а не студия
|
||
continue
|
||
base = rows[0] if rows else {}
|
||
out.append({
|
||
"source": base.get("source", ""), "release_id": rec.get("id", ""),
|
||
"ru_title": base.get("ru_title", ""), "year": base.get("year", ""),
|
||
"quality": base.get("quality", ""), "is_series": base.get("is_series", False),
|
||
"studio": studio, "translation_type": kind, "professional": pro,
|
||
"role_raw": chunk.strip(), "actors": "",
|
||
"note": mark_18(" ".join(x for x in (note, extra) if x).strip(), a["adult"]),
|
||
"age_18": int(a["adult"]), "codec": a["codec"], "channels": a["channels"], "khz": "",
|
||
"kbps": a["kbps"], "subtitles": "", "studio_source": "audio",
|
||
"raw": a["raw"], "lang": a["lang"], "audio_raw": a["raw"]})
|
||
have += key(studio)
|
||
return out
|
||
|
||
|
||
def kz_title_rows(rec: dict, text: str, rows: list[dict]) -> list[dict]:
|
||
"""kinozal: студии из заголовка раздачи «… / ПМ (LostFilm, HDrezka Studio), ДБ (X) / …».
|
||
|
||
На kinozal студии часто названы только в заголовке и в строках «Аудио Rus: … | LostFilm»
|
||
(без типа), а «Перевод:» во «Техданных» — без студий. Строка-вход — тот же синтетический
|
||
сегмент, что строит сервис из заголовка (src.release.title_segments); строка «Аудио» с этой
|
||
студией (если есть) — в audio_raw (позитив для сегментатора).
|
||
"""
|
||
from src.release import RE_TITLE_TRACK, TITLE_TYPES
|
||
|
||
def key(x: str) -> str:
|
||
return re.sub(r"[\W_]+", "", (x or "").casefold())
|
||
|
||
title = (rec.get("h1") or rec.get("page_title") or "").split(" :: ")[0]
|
||
have = key(" ".join(((r.get("studio") or "") + " " + (r.get("raw") or "")) for r in rows))
|
||
lines = [ln for ln in audio_lines_all(text) if not ln["original"] and not ln["commentary"]]
|
||
base = rows[0] if rows else {}
|
||
info = kz_split_title(title)
|
||
out = []
|
||
for abbr, names in RE_TITLE_TRACK.findall(title):
|
||
for name in re.split(r"\s*[,;]\s*", names.strip()):
|
||
studio, note = clean_studio(name)
|
||
studio, extra = canonical_studio(studio)
|
||
k = key(studio)
|
||
if len(k) < 2 or k in have or studio.lower() in SOURCE_LIKE:
|
||
continue
|
||
kind, pro = translation_type(TITLE_TYPES[abbr])
|
||
a = next((ln for ln in lines if k in key(ln["raw"])), None)
|
||
out.append({
|
||
"source": "kinozal", "release_id": rec.get("id", ""),
|
||
"ru_title": base.get("ru_title") or info["ru_title"],
|
||
"year": base.get("year") or info["year"], "quality": base.get("quality", ""),
|
||
"is_series": base.get("is_series", bool(re.search(r"сезон|серии", title, re.I))),
|
||
"studio": studio, "translation_type": kind, "professional": pro,
|
||
"role_raw": TITLE_TYPES[abbr], "actors": "",
|
||
"note": mark_18(" ".join(x for x in (note, extra) if x).strip(), bool(a and a["adult"])),
|
||
"age_18": int(bool(a and a["adult"])),
|
||
"codec": a["codec"] if a else "", "channels": a["channels"] if a else "", "khz": "",
|
||
"kbps": a["kbps"] if a else "", "subtitles": "", "studio_source": "title",
|
||
"raw": f"{TITLE_TYPES[abbr]} | {name.strip()}",
|
||
"lang": (a["lang"] or "RUS") if a else "RUS", "audio_raw": a["raw"] if a else ""})
|
||
have += k
|
||
return out
|
||
|
||
|
||
# кодек-строка одной дорожки: «Аудио 2: …», «Аудио Rus: …», «Аудио pic RUS: …», «Звук #1: …»,
|
||
# «• №3: Русский (АС3 …)», «Russian (Dolby AC3, 6 ch) …»
|
||
RE_CODEC_LINE = re.compile(
|
||
r"(?i)^\s*[•#№*\-\s]*(?:(?:аудио\w*|audio|звук\w*)\b[^:\n]{0,24}:|№\s*\d+\s*:|(?:russian|русск\w*|украин\w*)\b)")
|
||
RE_CODEC_PARAM = re.compile(r"(?i)\b(?:e-?ac-?3|ac-?3|ас3|dts|aac|truehd|flac|mp3|opus|l?pcm|mp2)\b|\d\s*(?:kbps|кбит|kb/s)")
|
||
RE_LANG_WORD = re.compile(
|
||
r"(?i)русск|russian|\brus\b|украин|ukrain|\bukr\b|україн|англ|english|\beng\b|япон|japan|казах|kazakh"
|
||
r"|француз|french|немецк|german|итальян|испан|корей|китайск")
|
||
# не студия: язык, тех. параметры, тип, источник, пометки
|
||
CODEC_NOT_STUDIO = re.compile(
|
||
r"(?i)^(?:rus|ukr|eng|ru|ua|int|ext|pic|dub|mvo|dvo|avo|vo|лицензи\w*|ненормативн\w*.*|.*лексик\w*"
|
||
r"|присутствует.*|внешн\w*|в составе.*|серии.*|эп\w*\.?\s*\d.*|[\d\s.,:/\-]+|.*\b(?:kbps|ch|khz|кбит)\b.*"
|
||
r"|.*(?:русск|английск|украинск|russian|english|ukrainian|оригинал|original)\w*.*"
|
||
r"|.*(?:голос|дублир|закадр|перевод).*|.*(?<![а-яё])дубляж(?![а-яё]).*|звук.*"
|
||
r"|.*(?:blu-?ray|itunes|netflix|amazon|\bbd\b|\bdvd\b|web-?dl).*)$")
|
||
|
||
|
||
# имя из скобок кодек-строки, которого нет в словаре: «(L,C,R,l,r)», «(АС3 …)», «(1-8 серии)», «(поздний)»
|
||
CODEC_NAME_NOISE = re.compile(
|
||
r"(?i)^(?:[lrc]|sl|sr|ls|rs|lfe)\b|\b(?:[lrc]|lfe)\s*[,;]|^(?:cee|eur|vhs|cbr|vbr|lfe|stereo|custom"
|
||
r"|line|production|россия|сек|[aа]{1,2}[сc]\w?|мр\w?|front|side)\b|\bmono\b|dolby|поздн|ранн|отдельн|тифло|комментар"
|
||
r"|сезон|сери[ия]|эпизод|файл|контейнер|bits?\b|sound\s*track|sdh|forced|вставк|сэмпл|sample|неизвестн|unknown")
|
||
|
||
|
||
def _lang_key(word: str) -> str:
|
||
w = word.lower()
|
||
return "RUS" if w.startswith(("русс", "russ", "rus")) else "UKR" if w.startswith(("укр", "ukr")) else w[:4]
|
||
|
||
|
||
def _known_studios() -> re.Pattern | None:
|
||
"""Словарь студий из data/labels.csv (≥ 5 строк): имена, которые kinozal пишет без скобок
|
||
(«Аудио Rus: … | LostFilm», «… - Русский, LostFilm», «… MVO DniproFilm / HDRezka Studio»)."""
|
||
global _KNOWN_RX
|
||
if _KNOWN_RX is None:
|
||
path = _REPO / "data" / "labels.csv"
|
||
cnt: Counter = Counter()
|
||
if path.exists():
|
||
with path.open(encoding="utf-8-sig", newline="") as f:
|
||
for r in csv.DictReader(f):
|
||
if r.get("studio_source") == "codec": # словарь — без своих же строк: повторный прогон стабилен
|
||
continue
|
||
st = (r.get("studio") or "").strip()
|
||
if len(st) >= 3 and st != "(не указана)" and not CODEC_NOT_STUDIO.match(st) \
|
||
and st.lower() not in SOURCE_LIKE and not CODEC_NAME_NOISE.search(st):
|
||
cnt[st] += 1
|
||
names = sorted((s for s, c in cnt.items() if c >= 5), key=len, reverse=True)
|
||
_KNOWN_RX = re.compile(r"(?i)(?<![\w])(" + "|".join(map(re.escape, names)) + r")(?![\w])") \
|
||
if names else re.compile(r"(?!x)x")
|
||
return _KNOWN_RX
|
||
|
||
|
||
_KNOWN_RX: re.Pattern | None = None
|
||
|
||
|
||
def codec_line_rows(rec: dict, text: str, rows: list[dict], known: re.Pattern | None = None) -> list[dict]:
|
||
"""Студии из кодек-строк русских/украинских дорожек, которых нет среди строк разметки.
|
||
|
||
kinozal: «Перевод: Профессиональный (многоголосый закадровый)» без студий + «Аудио 2: AC3 …
|
||
(MVO, LostFilm)», «Аудио Rus: … | TVShows»; nnm/rutracker — те же кодек-строки рядом с
|
||
«Перевод N:» со списком, где этой студии нет. Строка-вход — сегмент кодек-строки, как его
|
||
режет сервис (`src.release._entries`); тип — из самой строки (MVO, «дубляж»), иначе
|
||
единственный тип релиза, иначе «неизвестно». Сводка из нескольких языков
|
||
(«Аудио: Русский (AC3…), украинский (AC3…)»), оригинал и комментарии — пропускаются.
|
||
"""
|
||
from src.release import _entries, same_studio
|
||
from src.trackinfo import parse_audio_line
|
||
|
||
known = known if known is not None else _known_studios()
|
||
have = [r["studio"] for r in rows if r.get("studio") and r["studio"] != "(не указана)"]
|
||
types = {r["translation_type"] for r in rows} - {"неизвестно", ""}
|
||
base = rows[0] if rows else {}
|
||
out: list[dict] = []
|
||
for line in (text or "").splitlines():
|
||
line = re.sub(r"\s+", " ", line).strip()
|
||
if len(line) > 300 or not RE_CODEC_LINE.match(line) or not RE_CODEC_PARAM.search(line):
|
||
continue
|
||
if len({_lang_key(m.group(0)) for m in RE_LANG_WORD.finditer(line)}) > 1:
|
||
continue
|
||
info = parse_audio_line(re.sub(r"(?i)^\s*[^:]{0,30}:\s*", "", line, count=1))
|
||
if info["original"] or info["commentary"] or info["lang"] not in ("RUS", "UKR", ""):
|
||
continue
|
||
if not info["lang"] and not re.search(r"(?i)\brus\b|\bukr\b", line):
|
||
continue
|
||
tag = NNM_TAG_RE.search(line) or NNM_TAG_RE.search(line.translate(CYR_TO_LAT)) # «АVO» кириллицей
|
||
kind, pro = nnm_tag_type(tag.group(1)) if tag else ("", "")
|
||
if not kind:
|
||
kind, pro = translation_type(line)
|
||
if kind.endswith("неизвестно"):
|
||
kind, pro = "", ""
|
||
if not kind:
|
||
kind = next(iter(types)) if len(types) == 1 else "неизвестно"
|
||
pro = kind.split()[0] if kind.split()[0] in ("проф.", "люб.", "авт.") else ""
|
||
cands = [(m.group(1), True) for m in known.finditer(line)] + [(n, False) for n in info["names"]]
|
||
for name, in_vocab in cands:
|
||
studio, note = clean_studio(name)
|
||
studio, extra = canonical_studio(studio)
|
||
if len(studio) < 3 or studio.lower() in SOURCE_LIKE or CODEC_NOT_STUDIO.match(studio):
|
||
continue
|
||
if CODEC_NAME_NOISE.search(studio) or not in_vocab and re.search(r"\d", studio):
|
||
continue
|
||
if any(same_studio(studio, x) for x in have):
|
||
continue
|
||
entries = _entries(line)
|
||
seg = next((e for e in entries if name.lower() in e.lower()), entries[0] if entries else line)
|
||
out.append({
|
||
"source": base.get("source") or rec.get("source", ""), "release_id": rec.get("id", ""),
|
||
"ru_title": base.get("ru_title", ""), "year": base.get("year", ""),
|
||
"quality": base.get("quality", ""), "is_series": base.get("is_series", False),
|
||
"studio": studio, "translation_type": kind, "professional": pro,
|
||
"role_raw": "", "actors": "",
|
||
"note": mark_18(" ".join(x for x in (note, extra) if x).strip(), info["adult"]),
|
||
"age_18": int(info["adult"]), "codec": info["codec"], "channels": info["channels"],
|
||
"khz": "", "kbps": info["kbps"], "subtitles": "", "studio_source": "codec",
|
||
"raw": seg, "lang": info["lang"] or ("UKR" if re.search(r"(?i)\bukr\b", line) else "RUS"),
|
||
"audio_raw": line})
|
||
have.append(studio)
|
||
return out
|
||
|
||
|
||
def _with_audio(core, texts):
|
||
"""Обёртка парсера: lang/audio_raw у строк; число переводов в rel — по итоговым строкам."""
|
||
def parse(rec: dict) -> tuple[list[dict], dict]:
|
||
rows, rel = core(rec)
|
||
text = texts(rec)
|
||
if rows:
|
||
rows = enrich_rows(split_multi_rows(rows), text)
|
||
rows += audio_only_rows(rec, text, rows)
|
||
elif core is _kz_parse_core:
|
||
rows = kz_audio_only_rows(rec, text)
|
||
if core is _kz_parse_core:
|
||
rows += kz_title_rows(rec, text, rows)
|
||
if rows:
|
||
rows += codec_line_rows(rec, text, rows)
|
||
rel["n_translations"], rel["ok"] = len(rows), True
|
||
rel["studios"] = " | ".join(sorted({r["studio"] for r in rows}))
|
||
return rows, rel
|
||
parse.__name__ = core.__name__.lstrip("_").replace("_core", "")
|
||
return parse
|
||
|
||
|
||
kz_parse = _with_audio(_kz_parse_core, lambda r: "\n".join(
|
||
x for x in ((r.get("release_tab") or ""), (r.get("tech_tab") or "")) if x))
|
||
rt_parse = _with_audio(_rt_parse_core, lambda r: r.get("post") or "")
|
||
nnm_parse = _with_audio(_nnm_parse_core, lambda r: r.get("post") or "")
|
||
|
||
|
||
FIELDS = ["source", "release_id", "ru_title", "year", "quality", "is_series",
|
||
"studio", "translation_type", "professional", "role_raw", "actors",
|
||
"note", "age_18", "codec", "channels", "khz", "kbps", "subtitles",
|
||
"studio_source", "raw", "lang", "audio_raw"]
|
||
|
||
|
||
def read_jsonl(paths: list[Path]) -> list[dict]:
|
||
rows: dict[str, dict] = {}
|
||
for p in paths:
|
||
if not p.exists():
|
||
continue
|
||
for line in p.read_text("utf-8").splitlines():
|
||
if not line.strip():
|
||
continue
|
||
try:
|
||
rec = json.loads(line)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
# одна тема может лежать и в старом, и в новом файле —
|
||
# оставляем последнюю версию
|
||
rows[str(rec.get("id"))] = rec
|
||
return list(rows.values())
|
||
|
||
|
||
def cmd_parse(_: argparse.Namespace) -> int:
|
||
RAW_DIR.mkdir(parents=True, exist_ok=True)
|
||
DATASET.mkdir(parents=True, exist_ok=True)
|
||
total_rows = 0
|
||
for source, new_raw, legacy in RAW_SOURCES:
|
||
records = read_jsonl([new_raw, legacy])
|
||
if not records:
|
||
continue
|
||
parser = {"kinozal": kz_parse, "rutracker": rt_parse, "nnm": nnm_parse}[source]
|
||
rows, rels = [], []
|
||
for rec in records:
|
||
if not rec.get("ok"):
|
||
continue
|
||
try:
|
||
r, rel = parser(rec)
|
||
except Exception as exc: # noqa: BLE001
|
||
log(f" {source} {rec.get('id')}: parse error {exc}")
|
||
continue
|
||
rows.extend(r)
|
||
rels.append(rel)
|
||
(DATASET / f"{source}_entries.csv").write_text("", "utf-8")
|
||
with (DATASET / f"{source}_entries.csv").open("w", encoding="utf-8-sig", newline="") as f:
|
||
w = csv.DictWriter(f, fieldnames=FIELDS)
|
||
w.writeheader()
|
||
w.writerows(rows)
|
||
with (DATASET / f"{source}_releases.csv").open("w", encoding="utf-8-sig", newline="") as f:
|
||
w = csv.DictWriter(f, fieldnames=["source", "release_id", "ru_title", "year",
|
||
"quality", "is_series", "n_translations",
|
||
"studios", "ok"])
|
||
w.writeheader()
|
||
w.writerows(rels)
|
||
total_rows += len(rows)
|
||
log(f"{source:9s}: тем {len(records):4d}, с переводами {sum(1 for r in rels if r['ok']):4d}, "
|
||
f"записей {len(rows):4d}")
|
||
log(f"итого строк: {total_rows}")
|
||
return 0
|
||
|
||
|
||
def cmd_merge(_: argparse.Namespace) -> int:
|
||
rows, seen = [], set()
|
||
for source, *_ in RAW_SOURCES:
|
||
path = DATASET / f"{source}_entries.csv"
|
||
if not path.exists():
|
||
continue
|
||
with path.open(encoding="utf-8-sig", newline="") as f:
|
||
for r in csv.DictReader(f):
|
||
row = {k: (r.get(k) or "").strip() for k in FIELDS}
|
||
key = (row["source"], row["release_id"], row["studio"].lower(),
|
||
row["translation_type"], row["raw"][:120].lower())
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
rows.append(row)
|
||
|
||
DATASET.mkdir(parents=True, exist_ok=True)
|
||
out = DATASET / "labels.csv"
|
||
with out.open("w", encoding="utf-8-sig", newline="") as f:
|
||
w = csv.DictWriter(f, fieldnames=FIELDS)
|
||
w.writeheader()
|
||
w.writerows(rows)
|
||
|
||
stats = dataset_stats(rows)
|
||
(DATASET / "stats.json").write_text(json.dumps(stats, ensure_ascii=False, indent=2), "utf-8")
|
||
print_stats(stats)
|
||
log(f"\n-> {out}")
|
||
return 0
|
||
|
||
|
||
def dataset_stats(rows: list[dict]) -> dict:
|
||
studios = Counter(r["studio"] for r in rows if r["studio"] != "(не указана)")
|
||
adult = [r for r in rows if str(r.get("age_18", "")).strip() in ("1", "True")]
|
||
return {
|
||
"rows": len(rows),
|
||
"releases": len({(r["source"], r["release_id"]) for r in rows}),
|
||
"films": sum(1 for r in rows if str(r["is_series"]).lower() not in ("true", "1")),
|
||
"series": sum(1 for r in rows if str(r["is_series"]).lower() in ("true", "1")),
|
||
"with_studio": len(rows) - sum(1 for r in rows if r["studio"] == "(не указана)"),
|
||
"rows_18plus": len(adult),
|
||
"18plus_by_studio": dict(Counter(r["studio"] for r in adult).most_common(10)),
|
||
"unique_studios": len(studios),
|
||
"by_studio": dict(studios.most_common(50)),
|
||
"by_type": dict(Counter(r["translation_type"] for r in rows).most_common()),
|
||
"by_source": dict(Counter(r["source"] for r in rows)),
|
||
}
|
||
|
||
|
||
def print_stats(stats: dict) -> None:
|
||
log(f"строк : {stats['rows']} {stats['by_source']}")
|
||
log(f"раздач : {stats['releases']} (фильмы {stats['films']} / сериалы {stats['series']})")
|
||
log(f"со студией : {stats['with_studio']} ({100 * stats['with_studio'] / max(stats['rows'], 1):.0f}%)")
|
||
log(f"студий : {stats['unique_studios']}")
|
||
if "rows_18plus" in stats:
|
||
log(f"дорожек 18+ : {stats['rows_18plus']} {stats.get('18plus_by_studio', {})}")
|
||
log("\nтоп-15 студий:")
|
||
for s, n in list(stats["by_studio"].items())[:15]:
|
||
log(f" {n:4d} {s}")
|
||
log("\nтипы переводов:")
|
||
for t, n in list(stats["by_type"].items())[:8]:
|
||
log(f" {n:4d} {t}")
|
||
|
||
|
||
def cmd_status(_: argparse.Namespace) -> int:
|
||
_migrate_legacy_state()
|
||
if not any(p.exists() for p in STATE_FILES.values()):
|
||
log("прогресса пока нет: файлы out/state_*.json появятся после первого harvest")
|
||
return 0
|
||
|
||
forums = load_state("rutracker").get("rutracker", {}).get("forums", {})
|
||
log("rutracker:")
|
||
if not forums:
|
||
log(" — ещё ничего не собрано")
|
||
for f, st in sorted(forums.items(), key=lambda x: str(x[0])):
|
||
done = " [раздел пройден]" if st.get("finished") else ""
|
||
log(f" f={f:<5} следующая страница {int(st.get('next_page', 0)) + 1:>3}"
|
||
f" новых тем {st.get('topics', 0):>4} {st.get('updated', '')}{done}")
|
||
|
||
browse = load_state("kinozal").get("kinozal", {}).get("browse", {})
|
||
log("\nkinozal:")
|
||
if not browse:
|
||
log(" — ещё ничего не собрано")
|
||
for url, st in browse.items():
|
||
done = " [список закончился]" if st.get("finished") else ""
|
||
log(f" {url}")
|
||
log(f" следующая страница {int(st.get('next_page', 0)) + 1}"
|
||
f" раздач {st.get('releases', 0)} {st.get('updated', '')}{done}")
|
||
|
||
nnm_forums = load_state("nnm").get("nnm", {}).get("forums", {})
|
||
log("\nnnmclub:")
|
||
if not nnm_forums:
|
||
log(" — ещё ничего не собрано")
|
||
for f, st in sorted(nnm_forums.items(), key=lambda x: str(x[0])):
|
||
done = " [раздел пройден]" if st.get("finished") else ""
|
||
log(f" f={f:<5} следующая страница {int(st.get('next_page', 0)) + 1:>3}"
|
||
f" новых тем {st.get('topics', 0):>4} {st.get('updated', '')}{done}")
|
||
|
||
log("\nсостояние: " + ", ".join(p.name for p in STATE_FILES.values()))
|
||
return 0
|
||
|
||
|
||
def cmd_stats(args: argparse.Namespace) -> int:
|
||
path = DATASET / "labels.csv"
|
||
if not path.exists():
|
||
log("нет out/dataset/labels.csv — сначала: python dataset.py merge")
|
||
return 1
|
||
with path.open(encoding="utf-8-sig", newline="") as f:
|
||
rows = list(csv.DictReader(f))
|
||
print_stats(dataset_stats(rows))
|
||
if getattr(args, "adult", False):
|
||
adult = [r for r in rows if r.get("age_18") == "1"]
|
||
log(f"\nдорожки 18+ ({len(adult)}):")
|
||
for r in adult:
|
||
log(f" {r['source']:<9} | {r['studio']:<20} | {r['translation_type']:<18} | "
|
||
f"{r['ru_title'][:40]} [{r['release_id']}]")
|
||
return 0
|
||
|
||
|
||
# =====================================================================
|
||
# CLI
|
||
# =====================================================================
|
||
def main() -> int:
|
||
try:
|
||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||
except (AttributeError, ValueError):
|
||
pass
|
||
|
||
ap = argparse.ArgumentParser(description="Сборщик датасета студий озвучки")
|
||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||
|
||
p_chrome = sub.add_parser("chrome", help="запустить/проверить Chrome")
|
||
p_chrome.add_argument("--url", default="https://rutracker.org/forum/index.php")
|
||
|
||
p_h = sub.add_parser("harvest", help="собрать страницы")
|
||
p_h.add_argument("site", choices=["kinozal", "rutracker", "nnm"])
|
||
p_h.add_argument("--ids", default="")
|
||
p_h.add_argument("--topics", default="", help="id тем rutracker (алиас --ids)")
|
||
p_h.add_argument("--browse", default=None, help="URL списка kinozal")
|
||
p_h.add_argument("--forums", default="", help="id разделов rutracker: 313,252")
|
||
p_h.add_argument("--pages", type=int, default=1,
|
||
help="сколько страниц пройти за этот запуск (с сохранённой)")
|
||
p_h.add_argument("--from-page", type=int, default=None,
|
||
help="принудительно начать с этой страницы (1 = первая)")
|
||
p_h.add_argument("--reset", action="store_true",
|
||
help="забыть сохранённый прогресс и начать с первой страницы")
|
||
p_h.add_argument("--limit", type=int, default=50)
|
||
p_h.add_argument("--delay", type=float, default=1.6)
|
||
p_h.add_argument("--no-build", action="store_true",
|
||
help="не пересобирать labels.csv после сбора")
|
||
|
||
sub.add_parser("parse", help="разобрать сырые страницы в строки")
|
||
sub.add_parser("merge", help="слить источники в out/dataset/labels.csv")
|
||
sub.add_parser("stats", help="показать статистику датасета").add_argument(
|
||
"--adult", action="store_true", help="вывести все дорожки с 18+")
|
||
sub.add_parser("status", help="где остановился сбор (out/state.json)")
|
||
|
||
p_all = sub.add_parser("all", help="Chrome + сбор + разбор + слияние")
|
||
p_all.add_argument("--kinozal-ids", default="")
|
||
p_all.add_argument("--kinozal-browse", default=None)
|
||
p_all.add_argument("--kinozal-limit", type=int, default=0)
|
||
p_all.add_argument("--rutracker-forums", default="313,252")
|
||
p_all.add_argument("--rutracker-topics", default="")
|
||
p_all.add_argument("--nnm-forums", default="")
|
||
p_all.add_argument("--nnm-topics", default="")
|
||
p_all.add_argument("--pages", type=int, default=1)
|
||
p_all.add_argument("--from-page", type=int, default=None)
|
||
p_all.add_argument("--reset", action="store_true")
|
||
p_all.add_argument("--limit", type=int, default=50)
|
||
p_all.add_argument("--delay", type=float, default=1.6)
|
||
|
||
args = ap.parse_args()
|
||
|
||
if args.cmd == "chrome":
|
||
if chrome_start(args.url):
|
||
pages = [p for p in http_json("/json/list") if p.get("type") == "page"]
|
||
log(f"Chrome работает на :{CHROME_PORT}, профиль {CHROME_PROFILE}")
|
||
for p in pages:
|
||
log(f" вкладка: {p.get('title', '')} {p.get('url', '')}")
|
||
return 0
|
||
log("не удалось запустить Chrome")
|
||
return 1
|
||
|
||
if args.cmd == "harvest":
|
||
ids = [x for x in re.split(r"[,\s]+", args.ids or args.topics) if x]
|
||
from_page = None if args.from_page is None else max(0, args.from_page - 1)
|
||
if args.site == "kinozal":
|
||
harvest_kinozal(ids, args.browse, args.limit, args.delay,
|
||
args.pages, from_page, args.reset)
|
||
else:
|
||
forums = [x for x in re.split(r"[,\s]+", args.forums) if x]
|
||
collector = harvest_nnm if args.site == "nnm" else harvest_rutracker
|
||
collector(forums, ids, args.pages, args.limit, args.delay,
|
||
from_page, args.reset)
|
||
if not args.no_build:
|
||
log("\nпересобираю датасет...")
|
||
cmd_parse(args)
|
||
cmd_merge(args)
|
||
return 0
|
||
|
||
if args.cmd == "parse":
|
||
return cmd_parse(args)
|
||
if args.cmd == "merge":
|
||
return cmd_merge(args)
|
||
if args.cmd == "stats":
|
||
return cmd_stats(args)
|
||
if args.cmd == "status":
|
||
return cmd_status(args)
|
||
|
||
if args.cmd == "all":
|
||
if not chrome_start():
|
||
log("не удалось запустить Chrome")
|
||
return 1
|
||
ids = [x for x in re.split(r"[,\s]+", args.kinozal_ids) if x]
|
||
from_page = None if args.from_page is None else max(0, args.from_page - 1)
|
||
if args.kinozal_browse or ids:
|
||
harvest_kinozal(ids, args.kinozal_browse, args.kinozal_limit or args.limit,
|
||
args.delay, args.pages, from_page, args.reset)
|
||
forums = [x for x in re.split(r"[,\s]+", args.rutracker_forums) if x]
|
||
topics = [x for x in re.split(r"[,\s]+", args.rutracker_topics) if x]
|
||
harvest_rutracker(forums, topics, args.pages, args.limit, args.delay,
|
||
from_page, args.reset)
|
||
nnm_forums = [x for x in re.split(r"[,\s]+", args.nnm_forums) if x]
|
||
nnm_topics = [x for x in re.split(r"[,\s]+", args.nnm_topics) if x]
|
||
if nnm_forums or nnm_topics:
|
||
harvest_nnm(nnm_forums, nnm_topics, args.pages, args.limit, args.delay,
|
||
from_page, args.reset)
|
||
cmd_parse(args)
|
||
return cmd_merge(args)
|
||
|
||
return 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|