"""Метаданные релиза со страницы трекера (фильм / сериал / мультфильм / аниме). extract(site, rec, section_name="") -> dict # одна строка data/meta.csv Вход — запись сборщика (`data/raw/<трекер>/.json` или старый harvest): rutracker / nnm: поля `title` и `post`; kinozal: `h1`, `release_tab`, `tech_tab`. Всё берётся из сохранённого текста, поэтому парсер можно улучшать и пересобирать таблицу без повторных заходов на трекер. Дорожки (`audio_tracks`, JSON): язык, кодек, число каналов, битрейт, тип (DUB/MVO/DVO/AVO/VO/ORIG), студия, диапазон серий, 18+. """ from __future__ import annotations import json import re # Поднимайте при правке извлечения: кэш строк meta.csv сбросится и пересоберётся один раз. META_VERSION = 2 FIELDS = [ "source", "release_id", "title", "ru_title", "orig_title", "year", "kind", "is_series", "season", "episodes", "episodes_total", "section", "quality", "resolution", "video_codec", "hdr", "bit_depth", "fps", "container", "size_gb", "duration_min", "duration_per_episode", "countries", "genres", "directors", "cast", "age_limit", "mpaa", "release_group", "source_media", "audio_langs", "orig_audio", "n_audio", "audio_tracks", "sub_langs", "sub_forced", "sub_format", ] LANGS = [("русск", "RUS"), ("russian", "RUS"), ("рус", "RUS"), ("rus", "RUS"), ("англий", "ENG"), ("english", "ENG"), ("eng", "ENG"), ("украин", "UKR"), ("ukrainian", "UKR"), ("ukr", "UKR"), ("япон", "JPN"), ("japanese", "JPN"), ("jpn", "JPN"), ("корей", "KOR"), ("korean", "KOR"), ("француз", "FRA"), ("french", "FRA"), ("немец", "DEU"), ("german", "DEU"), ("испан", "SPA"), ("spanish", "SPA"), ("итальян", "ITA"), ("italian", "ITA"), ("китайск", "CHI"), ("chinese", "CHI"), ("польск", "POL"), ("polish", "POL"), ("белорус", "BEL"), ("чешск", "CZE")] CODEC_AUDIO = re.compile( r"(?i)E-?AC-?3(?:\s*JOC)?|AC-?3|DTS-HD\s*MA|DTS-HD|DTS:X|DTS|TrueHD|FLAC|AAC|Opus|MP3|L?PCM") TAG = re.compile(r"(?i)\b(DUB|MVO|DVO|AVO|VO|Original)\b") TAG_WORDS = [("дублир", "DUB"), ("дубляж", "DUB"), ("многоголос", "MVO"), ("двухголос", "DVO"), ("одноголос", "AVO"), ("оригинал", "ORIG")] RE_BITRATE = re.compile(r"(?i)(\d+(?:[.,]\d+)?)\s*(?:k|к)(?:bps|бит)") RE_CHANNELS = re.compile(r"(?i)(\d+(?:\.\d)?)\s*(?:ch\b|канал)") RE_LAYOUT = re.compile(r"(\d)/(\d)(?:\.\d)?") RE_EPISODES_IN_TRACK = re.compile(r"(?i)сери[яию]\s*([\d\s,\-–]+)") KEYS = { "year": ("Год выпуска", "Год"), "country": ("Страна", "Производство", "Выпущено"), "genre": ("Жанр",), "director": ("Режиссер", "Режиссёр"), "cast": ("В ролях", "Актеры", "Актёры"), "duration": ("Продолжительность",), "quality": ("Качество видео", "Тип релиза", "Качество"), "container": ("Контейнер",), "size": ("Размер",), "age": ("Возраст",), "mpaa": ("Рейтинг MPAA",), "group": ("Релиз от", "Релиз группы", "Релизер"), "source_media": ("Источник", "Исходник"), "orig_audio": ("Оригинальная аудиодорожка",), "voice_langs": ("Язык озвучки",), "subs": ("Субтитры",), "sub_format": ("Формат субтитров",), "video": ("Видео",), } def lang_of(word: str) -> str | None: w = (word or "").strip().lower() for stem, code in LANGS: if w.startswith(stem): return code return None def langs_in(text: str) -> list[str]: out: list[str] = [] for part in re.split(r"[,;/+|]|\bи\b", text or ""): code = lang_of(re.sub(r"[^A-Za-zА-Яа-яЁё]+", " ", part).strip().split(" ")[0] if part.strip() else "") if code and code not in out: out.append(code) return out def field(text: str, key: str) -> str: for label in KEYS[key]: m = re.search(rf"(?im)(?:^|\s){re.escape(label)}\s*:[ \t]*([^\n]+)", text) if m and m.group(1).strip(): return m.group(1).strip() return "" def _num_range(s: str) -> str: return re.sub(r"\s+", "", s).replace("–", "-") def parse_title(title: str) -> dict: t = re.sub(r"\s+", " ", title or "").strip() parts = [p.strip() for p in t.split(" / ")] ru = parts[0] if parts else "" orig = "" if len(parts) > 1 and not re.match(r"^(?:19|20)\d\d\b", parts[1]): orig = re.split(r"\s*[\(\[]", parts[1])[0].strip() year = re.search(r"\b(19\d\d|20[0-2]\d)\b", t) season = re.search(r"(?i)сезон[ы]?\s*(\d+(?:\s*[-–]\s*\d+)?)", t) or re.search(r"\bS(\d{1,2})\b", t) ep = re.search(r"(?i)сери[ияй]\s*(\d+(?:\s*[-–]\s*\d+)?)(?:\s*из\s*(\d+))?", t) return {"ru_title": ru, "orig_title": orig, "year": year.group(1) if year else "", "season": _num_range(season.group(1)) if season else "", "episodes": _num_range(ep.group(1)) if ep else "", "episodes_total": ep.group(2) if ep and ep.group(2) else ""} def parse_video(line: str, extra: str = "") -> dict: s = f"{line} {extra}" codec = "" if re.search(r"(?i)HEVC|H\.?265|x265", s): codec = "HEVC" elif re.search(r"(?i)AVC|H\.?264|x264", s): codec = "AVC" elif re.search(r"(?i)MPEG-?2", s): codec = "MPEG2" elif re.search(r"(?i)VC-?1", s): codec = "VC1" elif re.search(r"(?i)\bAV1\b", s): codec = "AV1" elif re.search(r"(?i)xvid|divx", s): codec = "XviD/DivX" res = "" m = re.search(r"(\d{3,4})\s*[x×]\s*(\d{3,4})", s) if m: w = int(m.group(1)) res = "2160p" if w >= 3500 else "1080p" if w >= 1800 else "720p" if w >= 1200 else "480p" else: m = re.search(r"(?i)\b(2160|1440|1080|720|576|480)\s*p\b", s) res = f"{m.group(1)}p" if m else ("2160p" if re.search(r"(?i)\b4K\b|UHD", s) else "") hdr = [] for pat, name in ((r"HDR10\+", "HDR10+"), (r"HDR10(?!\+)", "HDR10"), (r"Dolby\s*Vision|\bDV\b", "DV"), (r"(? dict: codec = CODEC_AUDIO.search(body.translate(_HOMOGLYPH)) ch = RE_CHANNELS.search(body) channels = "" if ch: channels = ch.group(1).split(".")[0] if "." in ch.group(1) and ch.group(1).endswith(".0") else ch.group(1) else: lay = RE_LAYOUT.search(body) if lay: channels = str(int(lay.group(1)) + int(lay.group(2)) + (1 if re.search(r"LFE", body) else 0)) br = RE_BITRATE.search(body) note = re.split(r"\s\|\s|\s-\s|\|", body, maxsplit=1) note = note[1] if len(note) > 1 else "" tag_m = TAG.search(note) or TAG.search(body) tag = tag_m.group(1).upper() if tag_m else "" if not tag: low = note.lower() tag = next((v for k, v in TAG_WORDS if k in low), "") tag = "ORIG" if tag == "ORIGINAL" else tag if lang is None: lead = re.match(r"\s*([A-Za-z]{2,3})\s*:", body) # «RUS: AC3, …» lang = lang_of(lead.group(1)) if lead else None if lang is None: for w in re.findall(r"[A-Za-zА-Яа-яЁё]+", note): lang = lang_of(w) if lang: break if lang is None and tag in ("DUB", "MVO", "DVO", "AVO", "VO"): lang = "RUS" studio = "" m = re.search(r"\(([^()]*)\)", note) if m: studio = TAG.sub("", m.group(1)).strip(" ,;") else: # rutracker: «… 192 kbps |HDrezka| 18+» piped = re.search(r"\|\s*([^|]+?)\s*\|", body) if piped and not re.fullmatch(r"(?i)original|оригинал|18\s*\+", piped.group(1)): studio = TAG.sub("", piped.group(1)).strip(" ,;") try: from src.episodes import format_ranges, parse_episodes episodes = format_ranges(parse_episodes(note)["ranges"]) except ImportError: # без репозитория рядом eps = RE_EPISODES_IN_TRACK.search(note) episodes = _num_range(eps.group(1)).strip(",") if eps else "" return {"lang": lang or "", "codec": re.sub(r"\s+", " ", codec.group(0)).upper() if codec else "", "channels": channels, "bitrate_kbps": int(float(br.group(1).replace(",", "."))) if br else "", "tag": tag, "studio": studio, "episodes": episodes, "is_18plus": "18+" in body} def parse_audio(text: str) -> list[dict]: tracks: list[dict] = [] # «Аудио: …», «Аудио 2: …» и rutracker-вид «Аудио RUS: AC3, 2 ch … |Студия| 18+» for m in re.finditer(r"(?im)^\s*Аудио(?:\s*0*(\d*)\s*:[ \t]*|[^\S\n]+(?=[A-Za-z]{2,3}\s*:))([^\n]+)", text): body = m.group(2).strip() groups = re.findall(r"\(([^()]*(?:ch|канал)[^()]*)\)", body, flags=re.I) if groups and not m.group(1): # kinozal: «Русский (AC3, 2 ch, …), (…), английский (…)» lang = None for g in re.finditer(r"(?:([A-Za-zА-Яа-яЁё]+)\s*)?\(([^()]*(?:ch|канал)[^()]*)\)", body, flags=re.I): lang = lang_of(g.group(1)) or lang tracks.append(_track(g.group(2), lang)) elif body: tracks.append(_track(body)) return tracks RE_TRANSLATION = re.compile(r"(?im)^\s*(?:Перевод|Озвучка|Озвучивание)\s*\d*\s*:[^\S\n]*([^\n]+)") def translation_tags(text: str) -> dict[str, str]: """{студия: DUB/MVO/DVO/AVO} из строк «Перевод: Профессиональный (многоголосый) Студия».""" out: dict[str, str] = {} for m in RE_TRANSLATION.finditer(text): body = m.group(1) low = body.lower() tag = next((v for k, v in TAG_WORDS if k in low), "") studio = re.sub(r"\[.*?\]|\(.*?\)|18\s*\+", " ", body).strip(" ,;|-") studio = studio.split("|")[-1].strip() studio = re.sub(r"(?i)^(?:(?:профессиональн|любительск|авторск|дублирован|многоголос|двухголос|" r"одноголос|закадров)\w*[\s,]*)+", "", studio).strip() if tag and studio: out.setdefault(_studio_canon(studio).casefold(), tag) return out def kind_of(section: str, title: str, genres: str, is_series: bool) -> str: blob = f"{section} {genres} {title}".lower() if re.search(r"аниме|anime|онгоинг", blob): return "anime" if re.search(r"мульт|анимац|animation", blob): return "cartoon" return "series" if is_series else "film" def _studio_canon(s: str) -> str: try: from src.studio_norm import canonical return canonical(s) or s except Exception: # noqa: BLE001 return s def extract(site: str, rec: dict, section_name: str = "", page_text: str = "") -> dict: """page_text — текст полной страницы (если сохранён HTML): добавляет размер раздачи.""" if site == "kinozal": title = rec.get("h1") or rec.get("page_title") or "" # жанр, страна, режиссёр — в теле страницы; вкладки идут первыми, чтобы # их поля (качество, аудио) имели приоритет над списками «подобных раздач» body = (rec.get("page_text") or "")[:8000] text = "\n".join(x for x in (title, rec.get("release_tab"), rec.get("tech_tab"), body) if x) else: title = rec.get("title") or rec.get("page_title") or "" text = f"{title}\n{rec.get('post') or ''}" tp = parse_title(title) quality = field(text, "quality") video_line = field(text, "video") vid = parse_video(video_line, f"{title} {quality}") tracks = parse_audio(text) for t in tracks: if t["studio"]: t["studio"] = _studio_canon(t["studio"]) by_studio = translation_tags(text) # тип озвучки из строк «Перевод» for t in tracks: if not t["tag"] and t["studio"]: t["tag"] = by_studio.get(t["studio"].casefold(), "") orig_audio = langs_in(field(text, "orig_audio")) audio_langs = list(dict.fromkeys( [t["lang"] for t in tracks if t["lang"]] + langs_in(field(text, "voice_langs")) + orig_audio)) subs = field(text, "subs") size = re.search(r"(?i)([\d.,]+)\s*(ТБ|ГБ|GB|МБ|MB)", field(text, "size")) if not size and page_text: size = re.search(r"(?i)(?:Размер|Size)\s*[:\s]\s*([\d.,]+)\s*(ТБ|ГБ|GB|МБ|MB)", page_text) size_gb = "" if size: v = float(size.group(1).replace(",", ".")) size_gb = round(v * {"ТБ": 1024, "ГБ": 1, "GB": 1, "МБ": 1 / 1024, "MB": 1 / 1024}[size.group(2).upper()], 2) dur_line = field(text, "duration") dur = re.search(r"(\d+):(\d\d):(\d\d)", dur_line) genres = field(text, "genre") is_series = bool(tp["season"] or tp["episodes"] or re.search(r"(?i)сериал", section_name) or re.search(r"(?i)сериал", title)) age = re.search(r"(\d+)\s*\+", field(text, "age")) cast = [c.strip() for c in re.split(r"[,;]", field(text, "cast")) if c.strip()][:8] return { "source": site, "release_id": str(rec.get("id", "")), "title": re.sub(r"\s+", " ", title).strip(), "ru_title": tp["ru_title"], "orig_title": tp["orig_title"], "year": tp["year"] or (re.search(r"\d{4}", field(text, "year")) or [""])[0], "kind": kind_of(section_name, title, genres, is_series), "is_series": is_series, "season": tp["season"], "episodes": tp["episodes"], "episodes_total": tp["episodes_total"], "section": section_name or rec.get("section_ref", ""), "quality": quality, **vid, "container": field(text, "container"), "size_gb": size_gb, "duration_min": int(dur.group(1)) * 60 + int(dur.group(2)) if dur else "", "duration_per_episode": bool(dur and re.search(r"(?i)сери", dur_line)), "countries": field(text, "country"), "genres": genres, "directors": field(text, "director"), "cast": "; ".join(cast), "age_limit": age.group(1) if age else "", "mpaa": field(text, "mpaa"), "release_group": field(text, "group"), "source_media": field(text, "source_media"), "audio_langs": "+".join(audio_langs), "orig_audio": "+".join(orig_audio), "n_audio": len(tracks), "audio_tracks": json.dumps(tracks, ensure_ascii=False) if tracks else "", "sub_langs": "+".join(langs_in(subs)), "sub_forced": bool(re.search(r"(?i)forced|форсир", subs)), "sub_format": field(text, "sub_format"), }