DenozordecandClaude Sonnet 5.5
efc9a4f56a
docker / build (push) Failing after 39s
- Docker: Dockerfile (CPU), compose с FlareSolverr, настройка через DUB_* переменные, токен доступа к /predict, tools/pack_models.py, workflow Gitea Actions, docs/docker.md - Модель грузится офлайн (config.json рядом с весами); общий код вынесен в src/spans.py, сервису не нужны sklearn/matplotlib/pandas - Обучение: профили tiny2/base/large/modern, запуски в models/runs, реестр и честное сравнение на одной тестовой выборке (src/runs.py), пункт меню «Модели» - Датасет: язык и аудио-параметры дорожек, 18+ по строкам «Аудио», ручные метки по ссылкам, переиндексация сохранённого HTML, параллельная сборка, сбалансированная выборка - Тексты страниц (data/pages) больше не хранятся в git Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
270 lines
11 KiB
Python
270 lines
11 KiB
Python
"""Интерфейс консоли для tools/collect.py: живая панель сбора, статистика, меню (rich)."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import time
|
||
from collections import Counter, deque
|
||
from dataclasses import dataclass, field
|
||
|
||
from rich import box
|
||
from rich.console import Console, Group
|
||
from rich.live import Live
|
||
from rich.panel import Panel
|
||
from rich.progress import (BarColumn, MofNCompleteColumn, Progress, ProgressColumn,
|
||
SpinnerColumn, TaskProgressColumn, TextColumn, TimeElapsedColumn,
|
||
TimeRemainingColumn)
|
||
from rich.prompt import Prompt
|
||
from rich.table import Table
|
||
from rich.text import Text
|
||
|
||
console = Console()
|
||
|
||
SITE_STYLE = {"rutracker": "cyan", "nnm": "magenta", "kinozal": "yellow"}
|
||
STATE_STYLE = {"ожидание": "dim", "готово": "green", "ошибка": "red", "остановлено": "yellow"}
|
||
|
||
|
||
def fmt_time(sec: float | None) -> str:
|
||
if sec is None or sec != sec:
|
||
return "—"
|
||
sec = int(sec)
|
||
h, rest = divmod(sec, 3600)
|
||
m, s = divmod(rest, 60)
|
||
return f"{h}:{m:02d}:{s:02d}" if h else f"{m:02d}:{s:02d}"
|
||
|
||
|
||
def bar(frac: float, width: int = 16, color: str = "green") -> Text:
|
||
frac = max(0.0, min(1.0, frac))
|
||
full = int(round(frac * width))
|
||
t = Text()
|
||
t.append("█" * full, style=color)
|
||
t.append("░" * (width - full), style="grey35")
|
||
return t
|
||
|
||
|
||
@dataclass
|
||
class Stats:
|
||
"""Счётчики одного трекера в текущем запуске сбора."""
|
||
|
||
site: str
|
||
limit: int = 0
|
||
state: str = "ожидание"
|
||
section: str = ""
|
||
saved: int = 0
|
||
failed: int = 0
|
||
skipped: int = 0 # удалённые раздачи («Нет раздачи с таким ID»)
|
||
started: float = 0.0
|
||
finished: float = 0.0
|
||
last: str = ""
|
||
errors: deque = field(default_factory=lambda: deque(maxlen=3))
|
||
stamps: deque = field(default_factory=lambda: deque(maxlen=12))
|
||
|
||
def begin(self, limit: int) -> None:
|
||
self.limit, self.saved, self.failed, self.skipped = limit, 0, 0, 0
|
||
self.started, self.finished = time.time(), 0.0
|
||
self.stamps.clear()
|
||
self.errors.clear()
|
||
|
||
def ok(self, title: str) -> None:
|
||
self.saved += 1
|
||
self.last = title
|
||
self.stamps.append(time.time())
|
||
|
||
def fail(self, msg: str) -> None:
|
||
self.failed += 1
|
||
self.errors.append(msg)
|
||
|
||
def done(self, state: str = "готово") -> None:
|
||
self.finished = time.time()
|
||
self.state = state
|
||
|
||
@property
|
||
def elapsed(self) -> float:
|
||
if not self.started:
|
||
return 0.0
|
||
return (self.finished or time.time()) - self.started
|
||
|
||
@property
|
||
def speed(self) -> float:
|
||
"""Страниц в минуту: по последним сохранениям, пока их мало — за всё время."""
|
||
if len(self.stamps) >= 4 and not self.finished:
|
||
span = self.stamps[-1] - self.stamps[0]
|
||
if span > 0:
|
||
return (len(self.stamps) - 1) / span * 60
|
||
return self.saved / self.elapsed * 60 if self.elapsed > 0 and self.saved else 0.0
|
||
|
||
@property
|
||
def eta(self) -> float | None:
|
||
if self.finished or self.speed <= 0 or self.limit <= self.saved:
|
||
return None
|
||
return (self.limit - self.saved) / self.speed * 60
|
||
|
||
|
||
STATS: dict[str, Stats] = {}
|
||
|
||
|
||
def get_stats(site: str) -> Stats:
|
||
return STATS.setdefault(site, Stats(site))
|
||
|
||
|
||
def dashboard(sites: list[str]) -> Panel:
|
||
"""Живая таблица по трекерам: состояние, прогресс, скорость, ETA, последняя раздача."""
|
||
t = Table(box=box.SIMPLE_HEAD, expand=True, pad_edge=False)
|
||
t.add_column("Трекер", style="bold", no_wrap=True)
|
||
t.add_column("Состояние", no_wrap=True)
|
||
t.add_column("Прогресс", no_wrap=True)
|
||
t.add_column("Сохр.", justify="right")
|
||
t.add_column("Ош.", justify="right")
|
||
t.add_column("Удал.", justify="right")
|
||
t.add_column("стр/мин", justify="right")
|
||
t.add_column("Время", justify="right")
|
||
t.add_column("ETA", justify="right")
|
||
t.add_column("Раздел / последняя раздача", overflow="ellipsis", no_wrap=True, ratio=1)
|
||
total_saved = total_failed = 0
|
||
speed_sum = 0.0
|
||
for s in sites:
|
||
st = get_stats(s)
|
||
total_saved += st.saved
|
||
total_failed += st.failed
|
||
speed_sum += st.speed if not st.finished else 0
|
||
frac = st.saved / st.limit if st.limit else 0
|
||
prog = Text.assemble(bar(frac, 14, SITE_STYLE.get(s, "green")),
|
||
f" {st.saved}/{st.limit or '?'}")
|
||
state = Text(st.state, style=STATE_STYLE.get(st.state, "white"))
|
||
info = Text(st.section or "", style="dim")
|
||
if st.last:
|
||
info.append(" ▸ " + st.last[:70], style="white")
|
||
t.add_row(Text(s, style=SITE_STYLE.get(s, "white")), state, prog, str(st.saved),
|
||
Text(str(st.failed), style="red" if st.failed else "dim"),
|
||
Text(str(st.skipped), style="yellow" if st.skipped else "dim"),
|
||
f"{st.speed:4.1f}" if st.speed else "—", fmt_time(st.elapsed) if st.started else "—",
|
||
fmt_time(st.eta), info)
|
||
foot = Text.assemble(("Итого сохранено: ", "dim"), (str(total_saved), "bold green"),
|
||
(" ошибок: ", "dim"), (str(total_failed), "red" if total_failed else "dim"),
|
||
(" общая скорость: ", "dim"), (f"{speed_sum:.1f} стр/мин", "bold"),
|
||
(" Ctrl+C — остановить после текущей раздачи", "dim italic"))
|
||
errs = [Text(f" ! {s}: {e}", style="red") for s in sites for e in list(get_stats(s).errors)[-1:]]
|
||
return Panel(Group(t, foot, *errs), title="[bold]Сбор страниц[/bold]", border_style="blue")
|
||
|
||
|
||
def live(sites: list[str]) -> Live:
|
||
return Live(dashboard(sites), console=console, refresh_per_second=4, transient=False,
|
||
get_renderable=lambda: dashboard(sites))
|
||
|
||
|
||
def distribution(title: str, counter: Counter, width: int = 18, top: int = 8) -> Table:
|
||
t = Table(box=box.SIMPLE, title=title, title_justify="left", show_header=False, pad_edge=False)
|
||
t.add_column(justify="left")
|
||
t.add_column(justify="right")
|
||
t.add_column()
|
||
items = counter.most_common(top)
|
||
mx = max((n for _, n in items), default=1)
|
||
for k, n in items:
|
||
t.add_row(str(k) or "—", str(n), bar(n / mx, width, "cyan"))
|
||
return t
|
||
|
||
|
||
def coverage_table(rows: list[dict]) -> Table:
|
||
"""rows: {site, labeled, pages, postonly, raw, meta, missing}."""
|
||
t = Table(box=box.ROUNDED, title="Покрытие датасета", title_justify="left")
|
||
for col, just in (("Трекер", "left"), ("Размечено", "right"), ("Страниц", "right"),
|
||
("Полных", "right"), ("Урезан.", "right"),
|
||
("Raw", "right"), ("Meta", "right"), ("Без стр.", "right")):
|
||
t.add_column(col, justify=just, no_wrap=True, min_width=len(col))
|
||
for r in rows:
|
||
full = r["pages"] - r["postonly"]
|
||
t.add_row(Text(r["site"], style=SITE_STYLE.get(r["site"], "white")), str(r["labeled"]),
|
||
str(r["pages"]), str(full), str(r["postonly"]), str(r["raw"]), str(r["meta"]),
|
||
Text(str(r["missing"]), style="red" if r["missing"] else "green"))
|
||
return t
|
||
|
||
|
||
def _short_ref(ref: str) -> str:
|
||
if ref.startswith("http"):
|
||
import re
|
||
m = re.search(r"c=(\d+)&v=(\d+)&d=(\d+)", ref)
|
||
return f"c={m.group(1)} v={m.group(2)} d={m.group(3)}" if m else ref[-24:]
|
||
return f"f={ref}"
|
||
|
||
|
||
def sections_table(site: str, entries: list[dict], rows: int = 14) -> Panel:
|
||
"""Итог поиска разделов: что взято, что пропущено, сколько тем на первой странице."""
|
||
kept = [e for e in entries if e.get("keep")]
|
||
skipped = [e for e in entries if not e.get("keep")]
|
||
t = Table(box=box.SIMPLE_HEAD, expand=True, pad_edge=False)
|
||
t.add_column("", width=2)
|
||
t.add_column("Раздел / список", overflow="ellipsis", no_wrap=True, ratio=1)
|
||
t.add_column("ID", style="dim", no_wrap=True)
|
||
t.add_column("Тем", justify="right")
|
||
for e in (kept[:rows] + skipped[:3]):
|
||
ok = e.get("keep")
|
||
mark = Text("✓", style="green") if e.get("status") == "ok" else (
|
||
Text("?", style="yellow") if ok else Text("✗", style="red"))
|
||
topics = "—" if e.get("topics") is None else str(e["topics"])
|
||
t.add_row(mark, e["name"], _short_ref(e["ref"]), topics)
|
||
hidden = len(kept) - min(len(kept), rows)
|
||
foot = f"взято {len(kept)}" + (f", пропущено {len(skipped)} (мало тем)" if skipped else "")
|
||
if hidden:
|
||
foot += f" [dim]… и ещё {hidden} в списке[/]"
|
||
unsure = sum(1 for e in kept if e.get("status") == "?")
|
||
if unsure:
|
||
foot += f" [yellow]? — не удалось проверить: {unsure}[/]"
|
||
return Panel(Group(t, Text.from_markup(foot)), title=f"[bold]{site}[/] · найденные разделы",
|
||
border_style=SITE_STYLE.get(site, "blue"))
|
||
|
||
|
||
def banner(lines: list[str]) -> Panel:
|
||
body = Text("\n".join(lines))
|
||
return Panel(body, title="[bold]DUB-AI · датасет и модель[/bold]", border_style="green",
|
||
padding=(0, 2))
|
||
|
||
|
||
def menu_table(items: list[tuple[str, str]]) -> Table:
|
||
t = Table(box=None, show_header=False, pad_edge=False)
|
||
t.add_column(style="bold cyan", justify="right")
|
||
t.add_column()
|
||
for key, label in items:
|
||
t.add_row(key, label)
|
||
return t
|
||
|
||
|
||
def clear() -> None:
|
||
"""Очистить терминал (если вывод не в консоль — ничего не делает)."""
|
||
if console.is_terminal:
|
||
console.clear()
|
||
|
||
|
||
def pause(text: str = "Enter — вернуться в меню") -> None:
|
||
try:
|
||
console.input(f"\n[dim]{text}…[/dim]")
|
||
except (EOFError, KeyboardInterrupt):
|
||
pass
|
||
|
||
|
||
def ask(prompt: str, default: str = "") -> str:
|
||
return Prompt.ask(f"[bold]{prompt}[/bold]", default=default or None, console=console,
|
||
show_default=bool(default)) or ""
|
||
|
||
|
||
class _Rate(ProgressColumn):
|
||
"""Скорость: элементов в секунду (по окну rich)."""
|
||
|
||
def render(self, task) -> Text:
|
||
return Text(f"{task.speed:.1f}/с" if task.speed else "—/с", style="cyan")
|
||
|
||
|
||
def progress() -> Progress:
|
||
"""Полоса прогресса: что делаем, N/M, %, скорость, прошло, осталось."""
|
||
return Progress(SpinnerColumn(), TextColumn("[bold]{task.description:<26}"), BarColumn(bar_width=26),
|
||
MofNCompleteColumn(), TaskProgressColumn(), _Rate(),
|
||
TextColumn("прошло"), TimeElapsedColumn(), TextColumn("осталось"),
|
||
TimeRemainingColumn(), console=console, transient=False)
|
||
|
||
|
||
def summary_table(title: str, columns: list[str], rows: list[list]) -> Table:
|
||
t = Table(box=box.SIMPLE_HEAD, title=title, title_justify="left", pad_edge=False)
|
||
for i, c in enumerate(columns):
|
||
t.add_column(c, justify="left" if i == 0 else "right")
|
||
for r in rows:
|
||
t.add_row(*[str(x) for x in r])
|
||
return t
|