DenozordecandClaude Sonnet 5.5
efc9a4f56a
docker / build (push) Failing after 39s
- Docker: Dockerfile (CPU), compose с FlareSolverr, настройка через DUB_* переменные, токен доступа к /predict, tools/pack_models.py, workflow Gitea Actions, docs/docker.md - Модель грузится офлайн (config.json рядом с весами); общий код вынесен в src/spans.py, сервису не нужны sklearn/matplotlib/pandas - Обучение: профили tiny2/base/large/modern, запуски в models/runs, реестр и честное сравнение на одной тестовой выборке (src/runs.py), пункт меню «Модели» - Датасет: язык и аудио-параметры дорожек, 18+ по строкам «Аудио», ручные метки по ссылкам, переиндексация сохранённого HTML, параллельная сборка, сбалансированная выборка - Тексты страниц (data/pages) больше не хранятся в git Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
315 lines
16 KiB
Python
315 lines
16 KiB
Python
"""Метаданные релиза со страницы трекера (фильм / сериал / мультфильм / аниме).
|
||
|
||
extract(site, rec, section_name="") -> dict # одна строка data/meta.csv
|
||
|
||
Вход — запись сборщика (`data/raw/<трекер>/<id>.json` или старый harvest):
|
||
rutracker / nnm: поля `title` и `post`; kinozal: `h1`, `release_tab`, `tech_tab`.
|
||
Всё берётся из сохранённого текста, поэтому парсер можно улучшать и пересобирать
|
||
таблицу без повторных заходов на трекер.
|
||
|
||
Дорожки (`audio_tracks`, JSON): язык, кодек, число каналов, битрейт, тип
|
||
(DUB/MVO/DVO/AVO/VO/ORIG), студия, диапазон серий, 18+.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
|
||
# Поднимайте при правке извлечения: кэш строк meta.csv сбросится и пересоберётся один раз.
|
||
META_VERSION = 2
|
||
|
||
FIELDS = [
|
||
"source", "release_id", "title", "ru_title", "orig_title", "year", "kind", "is_series",
|
||
"season", "episodes", "episodes_total", "section",
|
||
"quality", "resolution", "video_codec", "hdr", "bit_depth", "fps", "container",
|
||
"size_gb", "duration_min", "duration_per_episode",
|
||
"countries", "genres", "directors", "cast", "age_limit", "mpaa", "release_group", "source_media",
|
||
"audio_langs", "orig_audio", "n_audio", "audio_tracks",
|
||
"sub_langs", "sub_forced", "sub_format",
|
||
]
|
||
|
||
LANGS = [("русск", "RUS"), ("russian", "RUS"), ("рус", "RUS"), ("rus", "RUS"),
|
||
("англий", "ENG"), ("english", "ENG"), ("eng", "ENG"),
|
||
("украин", "UKR"), ("ukrainian", "UKR"), ("ukr", "UKR"),
|
||
("япон", "JPN"), ("japanese", "JPN"), ("jpn", "JPN"),
|
||
("корей", "KOR"), ("korean", "KOR"), ("француз", "FRA"), ("french", "FRA"),
|
||
("немец", "DEU"), ("german", "DEU"), ("испан", "SPA"), ("spanish", "SPA"),
|
||
("итальян", "ITA"), ("italian", "ITA"), ("китайск", "CHI"), ("chinese", "CHI"),
|
||
("польск", "POL"), ("polish", "POL"), ("белорус", "BEL"), ("чешск", "CZE")]
|
||
|
||
CODEC_AUDIO = re.compile(
|
||
r"(?i)E-?AC-?3(?:\s*JOC)?|AC-?3|DTS-HD\s*MA|DTS-HD|DTS:X|DTS|TrueHD|FLAC|AAC|Opus|MP3|L?PCM")
|
||
TAG = re.compile(r"(?i)\b(DUB|MVO|DVO|AVO|VO|Original)\b")
|
||
TAG_WORDS = [("дублир", "DUB"), ("дубляж", "DUB"), ("многоголос", "MVO"), ("двухголос", "DVO"),
|
||
("одноголос", "AVO"), ("оригинал", "ORIG")]
|
||
RE_BITRATE = re.compile(r"(?i)(\d+(?:[.,]\d+)?)\s*(?:k|к)(?:bps|бит)")
|
||
RE_CHANNELS = re.compile(r"(?i)(\d+(?:\.\d)?)\s*(?:ch\b|канал)")
|
||
RE_LAYOUT = re.compile(r"(\d)/(\d)(?:\.\d)?")
|
||
RE_EPISODES_IN_TRACK = re.compile(r"(?i)сери[яию]\s*([\d\s,\-–]+)")
|
||
|
||
KEYS = {
|
||
"year": ("Год выпуска", "Год"),
|
||
"country": ("Страна", "Производство", "Выпущено"),
|
||
"genre": ("Жанр",),
|
||
"director": ("Режиссер", "Режиссёр"),
|
||
"cast": ("В ролях", "Актеры", "Актёры"),
|
||
"duration": ("Продолжительность",),
|
||
"quality": ("Качество видео", "Тип релиза", "Качество"),
|
||
"container": ("Контейнер",),
|
||
"size": ("Размер",),
|
||
"age": ("Возраст",),
|
||
"mpaa": ("Рейтинг MPAA",),
|
||
"group": ("Релиз от", "Релиз группы", "Релизер"),
|
||
"source_media": ("Источник", "Исходник"),
|
||
"orig_audio": ("Оригинальная аудиодорожка",),
|
||
"voice_langs": ("Язык озвучки",),
|
||
"subs": ("Субтитры",),
|
||
"sub_format": ("Формат субтитров",),
|
||
"video": ("Видео",),
|
||
}
|
||
|
||
|
||
def lang_of(word: str) -> str | None:
|
||
w = (word or "").strip().lower()
|
||
for stem, code in LANGS:
|
||
if w.startswith(stem):
|
||
return code
|
||
return None
|
||
|
||
|
||
def langs_in(text: str) -> list[str]:
|
||
out: list[str] = []
|
||
for part in re.split(r"[,;/+|]|\bи\b", text or ""):
|
||
code = lang_of(re.sub(r"[^A-Za-zА-Яа-яЁё]+", " ", part).strip().split(" ")[0] if part.strip() else "")
|
||
if code and code not in out:
|
||
out.append(code)
|
||
return out
|
||
|
||
|
||
def field(text: str, key: str) -> str:
|
||
for label in KEYS[key]:
|
||
m = re.search(rf"(?im)(?:^|\s){re.escape(label)}\s*:[ \t]*([^\n]+)", text)
|
||
if m and m.group(1).strip():
|
||
return m.group(1).strip()
|
||
return ""
|
||
|
||
|
||
def _num_range(s: str) -> str:
|
||
return re.sub(r"\s+", "", s).replace("–", "-")
|
||
|
||
|
||
def parse_title(title: str) -> dict:
|
||
t = re.sub(r"\s+", " ", title or "").strip()
|
||
parts = [p.strip() for p in t.split(" / ")]
|
||
ru = parts[0] if parts else ""
|
||
orig = ""
|
||
if len(parts) > 1 and not re.match(r"^(?:19|20)\d\d\b", parts[1]):
|
||
orig = re.split(r"\s*[\(\[]", parts[1])[0].strip()
|
||
year = re.search(r"\b(19\d\d|20[0-2]\d)\b", t)
|
||
season = re.search(r"(?i)сезон[ы]?\s*(\d+(?:\s*[-–]\s*\d+)?)", t) or re.search(r"\bS(\d{1,2})\b", t)
|
||
ep = re.search(r"(?i)сери[ияй]\s*(\d+(?:\s*[-–]\s*\d+)?)(?:\s*из\s*(\d+))?", t)
|
||
return {"ru_title": ru, "orig_title": orig, "year": year.group(1) if year else "",
|
||
"season": _num_range(season.group(1)) if season else "",
|
||
"episodes": _num_range(ep.group(1)) if ep else "",
|
||
"episodes_total": ep.group(2) if ep and ep.group(2) else ""}
|
||
|
||
|
||
def parse_video(line: str, extra: str = "") -> dict:
|
||
s = f"{line} {extra}"
|
||
codec = ""
|
||
if re.search(r"(?i)HEVC|H\.?265|x265", s):
|
||
codec = "HEVC"
|
||
elif re.search(r"(?i)AVC|H\.?264|x264", s):
|
||
codec = "AVC"
|
||
elif re.search(r"(?i)MPEG-?2", s):
|
||
codec = "MPEG2"
|
||
elif re.search(r"(?i)VC-?1", s):
|
||
codec = "VC1"
|
||
elif re.search(r"(?i)\bAV1\b", s):
|
||
codec = "AV1"
|
||
elif re.search(r"(?i)xvid|divx", s):
|
||
codec = "XviD/DivX"
|
||
res = ""
|
||
m = re.search(r"(\d{3,4})\s*[x×]\s*(\d{3,4})", s)
|
||
if m:
|
||
w = int(m.group(1))
|
||
res = "2160p" if w >= 3500 else "1080p" if w >= 1800 else "720p" if w >= 1200 else "480p"
|
||
else:
|
||
m = re.search(r"(?i)\b(2160|1440|1080|720|576|480)\s*p\b", s)
|
||
res = f"{m.group(1)}p" if m else ("2160p" if re.search(r"(?i)\b4K\b|UHD", s) else "")
|
||
hdr = []
|
||
for pat, name in ((r"HDR10\+", "HDR10+"), (r"HDR10(?!\+)", "HDR10"), (r"Dolby\s*Vision|\bDV\b", "DV"),
|
||
(r"(?<![A-Za-z0-9])HDR(?!10)", "HDR")):
|
||
if re.search(pat, s):
|
||
hdr.append(name)
|
||
bd = re.search(r"(?i)(\d{1,2})\s*-?\s*bit", s)
|
||
fps = re.search(r"(?i)(\d{2}(?:\.\d+)?)\s*(?:fps|кадр)", s)
|
||
return {"video_codec": codec, "resolution": res, "hdr": "+".join(dict.fromkeys(hdr)),
|
||
"bit_depth": bd.group(1) if bd else "", "fps": fps.group(1) if fps else ""}
|
||
|
||
|
||
_HOMOGLYPH = str.maketrans("АСЕ", "ACE") # «АС3» кириллицей на rutracker
|
||
|
||
|
||
def _track(body: str, lang: str | None = None) -> dict:
|
||
codec = CODEC_AUDIO.search(body.translate(_HOMOGLYPH))
|
||
ch = RE_CHANNELS.search(body)
|
||
channels = ""
|
||
if ch:
|
||
channels = ch.group(1).split(".")[0] if "." in ch.group(1) and ch.group(1).endswith(".0") else ch.group(1)
|
||
else:
|
||
lay = RE_LAYOUT.search(body)
|
||
if lay:
|
||
channels = str(int(lay.group(1)) + int(lay.group(2)) + (1 if re.search(r"LFE", body) else 0))
|
||
br = RE_BITRATE.search(body)
|
||
note = re.split(r"\s\|\s|\s-\s|\|", body, maxsplit=1)
|
||
note = note[1] if len(note) > 1 else ""
|
||
tag_m = TAG.search(note) or TAG.search(body)
|
||
tag = tag_m.group(1).upper() if tag_m else ""
|
||
if not tag:
|
||
low = note.lower()
|
||
tag = next((v for k, v in TAG_WORDS if k in low), "")
|
||
tag = "ORIG" if tag == "ORIGINAL" else tag
|
||
if lang is None:
|
||
lead = re.match(r"\s*([A-Za-z]{2,3})\s*:", body) # «RUS: AC3, …»
|
||
lang = lang_of(lead.group(1)) if lead else None
|
||
if lang is None:
|
||
for w in re.findall(r"[A-Za-zА-Яа-яЁё]+", note):
|
||
lang = lang_of(w)
|
||
if lang:
|
||
break
|
||
if lang is None and tag in ("DUB", "MVO", "DVO", "AVO", "VO"):
|
||
lang = "RUS"
|
||
studio = ""
|
||
m = re.search(r"\(([^()]*)\)", note)
|
||
if m:
|
||
studio = TAG.sub("", m.group(1)).strip(" ,;")
|
||
else: # rutracker: «… 192 kbps |HDrezka| 18+»
|
||
piped = re.search(r"\|\s*([^|]+?)\s*\|", body)
|
||
if piped and not re.fullmatch(r"(?i)original|оригинал|18\s*\+", piped.group(1)):
|
||
studio = TAG.sub("", piped.group(1)).strip(" ,;")
|
||
try:
|
||
from src.episodes import format_ranges, parse_episodes
|
||
episodes = format_ranges(parse_episodes(note)["ranges"])
|
||
except ImportError: # без репозитория рядом
|
||
eps = RE_EPISODES_IN_TRACK.search(note)
|
||
episodes = _num_range(eps.group(1)).strip(",") if eps else ""
|
||
return {"lang": lang or "", "codec": re.sub(r"\s+", " ", codec.group(0)).upper() if codec else "",
|
||
"channels": channels, "bitrate_kbps": int(float(br.group(1).replace(",", "."))) if br else "",
|
||
"tag": tag, "studio": studio, "episodes": episodes,
|
||
"is_18plus": "18+" in body}
|
||
|
||
|
||
def parse_audio(text: str) -> list[dict]:
|
||
tracks: list[dict] = []
|
||
# «Аудио: …», «Аудио 2: …» и rutracker-вид «Аудио RUS: AC3, 2 ch … |Студия| 18+»
|
||
for m in re.finditer(r"(?im)^\s*Аудио(?:\s*0*(\d*)\s*:[ \t]*|[^\S\n]+(?=[A-Za-z]{2,3}\s*:))([^\n]+)", text):
|
||
body = m.group(2).strip()
|
||
groups = re.findall(r"\(([^()]*(?:ch|канал)[^()]*)\)", body, flags=re.I)
|
||
if groups and not m.group(1): # kinozal: «Русский (AC3, 2 ch, …), (…), английский (…)»
|
||
lang = None
|
||
for g in re.finditer(r"(?:([A-Za-zА-Яа-яЁё]+)\s*)?\(([^()]*(?:ch|канал)[^()]*)\)", body, flags=re.I):
|
||
lang = lang_of(g.group(1)) or lang
|
||
tracks.append(_track(g.group(2), lang))
|
||
elif body:
|
||
tracks.append(_track(body))
|
||
return tracks
|
||
|
||
|
||
RE_TRANSLATION = re.compile(r"(?im)^\s*(?:Перевод|Озвучка|Озвучивание)\s*\d*\s*:[^\S\n]*([^\n]+)")
|
||
|
||
|
||
def translation_tags(text: str) -> dict[str, str]:
|
||
"""{студия: DUB/MVO/DVO/AVO} из строк «Перевод: Профессиональный (многоголосый) Студия»."""
|
||
out: dict[str, str] = {}
|
||
for m in RE_TRANSLATION.finditer(text):
|
||
body = m.group(1)
|
||
low = body.lower()
|
||
tag = next((v for k, v in TAG_WORDS if k in low), "")
|
||
studio = re.sub(r"\[.*?\]|\(.*?\)|18\s*\+", " ", body).strip(" ,;|-")
|
||
studio = studio.split("|")[-1].strip()
|
||
studio = re.sub(r"(?i)^(?:(?:профессиональн|любительск|авторск|дублирован|многоголос|двухголос|"
|
||
r"одноголос|закадров)\w*[\s,]*)+", "", studio).strip()
|
||
if tag and studio:
|
||
out.setdefault(_studio_canon(studio).casefold(), tag)
|
||
return out
|
||
|
||
|
||
def kind_of(section: str, title: str, genres: str, is_series: bool) -> str:
|
||
blob = f"{section} {genres} {title}".lower()
|
||
if re.search(r"аниме|anime|онгоинг", blob):
|
||
return "anime"
|
||
if re.search(r"мульт|анимац|animation", blob):
|
||
return "cartoon"
|
||
return "series" if is_series else "film"
|
||
|
||
|
||
def _studio_canon(s: str) -> str:
|
||
try:
|
||
from src.studio_norm import canonical
|
||
return canonical(s) or s
|
||
except Exception: # noqa: BLE001
|
||
return s
|
||
|
||
|
||
def extract(site: str, rec: dict, section_name: str = "", page_text: str = "") -> dict:
|
||
"""page_text — текст полной страницы (если сохранён HTML): добавляет размер раздачи."""
|
||
if site == "kinozal":
|
||
title = rec.get("h1") or rec.get("page_title") or ""
|
||
# жанр, страна, режиссёр — в теле страницы; вкладки идут первыми, чтобы
|
||
# их поля (качество, аудио) имели приоритет над списками «подобных раздач»
|
||
body = (rec.get("page_text") or "")[:8000]
|
||
text = "\n".join(x for x in (title, rec.get("release_tab"), rec.get("tech_tab"), body) if x)
|
||
else:
|
||
title = rec.get("title") or rec.get("page_title") or ""
|
||
text = f"{title}\n{rec.get('post') or ''}"
|
||
tp = parse_title(title)
|
||
quality = field(text, "quality")
|
||
video_line = field(text, "video")
|
||
vid = parse_video(video_line, f"{title} {quality}")
|
||
tracks = parse_audio(text)
|
||
for t in tracks:
|
||
if t["studio"]:
|
||
t["studio"] = _studio_canon(t["studio"])
|
||
by_studio = translation_tags(text) # тип озвучки из строк «Перевод»
|
||
for t in tracks:
|
||
if not t["tag"] and t["studio"]:
|
||
t["tag"] = by_studio.get(t["studio"].casefold(), "")
|
||
orig_audio = langs_in(field(text, "orig_audio"))
|
||
audio_langs = list(dict.fromkeys(
|
||
[t["lang"] for t in tracks if t["lang"]] + langs_in(field(text, "voice_langs")) + orig_audio))
|
||
subs = field(text, "subs")
|
||
size = re.search(r"(?i)([\d.,]+)\s*(ТБ|ГБ|GB|МБ|MB)", field(text, "size"))
|
||
if not size and page_text:
|
||
size = re.search(r"(?i)(?:Размер|Size)\s*[:\s]\s*([\d.,]+)\s*(ТБ|ГБ|GB|МБ|MB)", page_text)
|
||
size_gb = ""
|
||
if size:
|
||
v = float(size.group(1).replace(",", "."))
|
||
size_gb = round(v * {"ТБ": 1024, "ГБ": 1, "GB": 1, "МБ": 1 / 1024, "MB": 1 / 1024}[size.group(2).upper()], 2)
|
||
dur_line = field(text, "duration")
|
||
dur = re.search(r"(\d+):(\d\d):(\d\d)", dur_line)
|
||
genres = field(text, "genre")
|
||
is_series = bool(tp["season"] or tp["episodes"] or re.search(r"(?i)сериал", section_name)
|
||
or re.search(r"(?i)сериал", title))
|
||
age = re.search(r"(\d+)\s*\+", field(text, "age"))
|
||
cast = [c.strip() for c in re.split(r"[,;]", field(text, "cast")) if c.strip()][:8]
|
||
return {
|
||
"source": site, "release_id": str(rec.get("id", "")), "title": re.sub(r"\s+", " ", title).strip(),
|
||
"ru_title": tp["ru_title"], "orig_title": tp["orig_title"],
|
||
"year": tp["year"] or (re.search(r"\d{4}", field(text, "year")) or [""])[0],
|
||
"kind": kind_of(section_name, title, genres, is_series), "is_series": is_series,
|
||
"season": tp["season"], "episodes": tp["episodes"], "episodes_total": tp["episodes_total"],
|
||
"section": section_name or rec.get("section_ref", ""),
|
||
"quality": quality, **vid, "container": field(text, "container"),
|
||
"size_gb": size_gb, "duration_min": int(dur.group(1)) * 60 + int(dur.group(2)) if dur else "",
|
||
"duration_per_episode": bool(dur and re.search(r"(?i)сери", dur_line)),
|
||
"countries": field(text, "country"), "genres": genres, "directors": field(text, "director"),
|
||
"cast": "; ".join(cast), "age_limit": age.group(1) if age else "", "mpaa": field(text, "mpaa"),
|
||
"release_group": field(text, "group"), "source_media": field(text, "source_media"),
|
||
"audio_langs": "+".join(audio_langs), "orig_audio": "+".join(orig_audio), "n_audio": len(tracks),
|
||
"audio_tracks": json.dumps(tracks, ensure_ascii=False) if tracks else "",
|
||
"sub_langs": "+".join(langs_in(subs)), "sub_forced": bool(re.search(r"(?i)forced|форсир", subs)),
|
||
"sub_format": field(text, "sub_format"),
|
||
}
|