"""Интерфейс консоли для tools/collect.py: живая панель сбора, статистика, меню (rich).""" from __future__ import annotations import time from collections import Counter, deque from dataclasses import dataclass, field from rich import box from rich.console import Console, Group from rich.live import Live from rich.panel import Panel from rich.progress import (BarColumn, MofNCompleteColumn, Progress, ProgressColumn, SpinnerColumn, TaskProgressColumn, TextColumn, TimeElapsedColumn, TimeRemainingColumn) from rich.prompt import Prompt from rich.table import Table from rich.text import Text console = Console() SITE_STYLE = {"rutracker": "cyan", "nnm": "magenta", "kinozal": "yellow"} STATE_STYLE = {"ожидание": "dim", "готово": "green", "ошибка": "red", "остановлено": "yellow"} def fmt_time(sec: float | None) -> str: if sec is None or sec != sec: return "—" sec = int(sec) h, rest = divmod(sec, 3600) m, s = divmod(rest, 60) return f"{h}:{m:02d}:{s:02d}" if h else f"{m:02d}:{s:02d}" def bar(frac: float, width: int = 16, color: str = "green") -> Text: frac = max(0.0, min(1.0, frac)) full = int(round(frac * width)) t = Text() t.append("█" * full, style=color) t.append("░" * (width - full), style="grey35") return t @dataclass class Stats: """Счётчики одного трекера в текущем запуске сбора.""" site: str limit: int = 0 state: str = "ожидание" section: str = "" saved: int = 0 failed: int = 0 skipped: int = 0 # удалённые раздачи («Нет раздачи с таким ID») started: float = 0.0 finished: float = 0.0 last: str = "" errors: deque = field(default_factory=lambda: deque(maxlen=3)) stamps: deque = field(default_factory=lambda: deque(maxlen=12)) def begin(self, limit: int) -> None: self.limit, self.saved, self.failed, self.skipped = limit, 0, 0, 0 self.started, self.finished = time.time(), 0.0 self.stamps.clear() self.errors.clear() def ok(self, title: str) -> None: self.saved += 1 self.last = title self.stamps.append(time.time()) def fail(self, msg: str) -> None: self.failed += 1 self.errors.append(msg) def done(self, state: str = "готово") -> None: self.finished = time.time() self.state = state @property def elapsed(self) -> float: if not self.started: return 0.0 return (self.finished or time.time()) - self.started @property def speed(self) -> float: """Страниц в минуту: по последним сохранениям, пока их мало — за всё время.""" if len(self.stamps) >= 4 and not self.finished: span = self.stamps[-1] - self.stamps[0] if span > 0: return (len(self.stamps) - 1) / span * 60 return self.saved / self.elapsed * 60 if self.elapsed > 0 and self.saved else 0.0 @property def eta(self) -> float | None: if self.finished or self.speed <= 0 or self.limit <= self.saved: return None return (self.limit - self.saved) / self.speed * 60 STATS: dict[str, Stats] = {} def get_stats(site: str) -> Stats: return STATS.setdefault(site, Stats(site)) def dashboard(sites: list[str]) -> Panel: """Живая таблица по трекерам: состояние, прогресс, скорость, ETA, последняя раздача.""" t = Table(box=box.SIMPLE_HEAD, expand=True, pad_edge=False) t.add_column("Трекер", style="bold", no_wrap=True) t.add_column("Состояние", no_wrap=True) t.add_column("Прогресс", no_wrap=True) t.add_column("Сохр.", justify="right") t.add_column("Ош.", justify="right") t.add_column("Удал.", justify="right") t.add_column("стр/мин", justify="right") t.add_column("Время", justify="right") t.add_column("ETA", justify="right") t.add_column("Раздел / последняя раздача", overflow="ellipsis", no_wrap=True, ratio=1) total_saved = total_failed = 0 speed_sum = 0.0 for s in sites: st = get_stats(s) total_saved += st.saved total_failed += st.failed speed_sum += st.speed if not st.finished else 0 frac = st.saved / st.limit if st.limit else 0 prog = Text.assemble(bar(frac, 14, SITE_STYLE.get(s, "green")), f" {st.saved}/{st.limit or '?'}") state = Text(st.state, style=STATE_STYLE.get(st.state, "white")) info = Text(st.section or "", style="dim") if st.last: info.append(" ▸ " + st.last[:70], style="white") t.add_row(Text(s, style=SITE_STYLE.get(s, "white")), state, prog, str(st.saved), Text(str(st.failed), style="red" if st.failed else "dim"), Text(str(st.skipped), style="yellow" if st.skipped else "dim"), f"{st.speed:4.1f}" if st.speed else "—", fmt_time(st.elapsed) if st.started else "—", fmt_time(st.eta), info) foot = Text.assemble(("Итого сохранено: ", "dim"), (str(total_saved), "bold green"), (" ошибок: ", "dim"), (str(total_failed), "red" if total_failed else "dim"), (" общая скорость: ", "dim"), (f"{speed_sum:.1f} стр/мин", "bold"), (" Ctrl+C — остановить после текущей раздачи", "dim italic")) errs = [Text(f" ! {s}: {e}", style="red") for s in sites for e in list(get_stats(s).errors)[-1:]] return Panel(Group(t, foot, *errs), title="[bold]Сбор страниц[/bold]", border_style="blue") def live(sites: list[str]) -> Live: return Live(dashboard(sites), console=console, refresh_per_second=4, transient=False, get_renderable=lambda: dashboard(sites)) def distribution(title: str, counter: Counter, width: int = 18, top: int = 8) -> Table: t = Table(box=box.SIMPLE, title=title, title_justify="left", show_header=False, pad_edge=False) t.add_column(justify="left") t.add_column(justify="right") t.add_column() items = counter.most_common(top) mx = max((n for _, n in items), default=1) for k, n in items: t.add_row(str(k) or "—", str(n), bar(n / mx, width, "cyan")) return t def coverage_table(rows: list[dict]) -> Table: """rows: {site, labeled, pages, postonly, raw, meta, missing}.""" t = Table(box=box.ROUNDED, title="Покрытие датасета", title_justify="left") for col, just in (("Трекер", "left"), ("Размечено", "right"), ("Страниц", "right"), ("Полных", "right"), ("Урезан.", "right"), ("Raw", "right"), ("Meta", "right"), ("Без стр.", "right")): t.add_column(col, justify=just, no_wrap=True, min_width=len(col)) for r in rows: full = r["pages"] - r["postonly"] t.add_row(Text(r["site"], style=SITE_STYLE.get(r["site"], "white")), str(r["labeled"]), str(r["pages"]), str(full), str(r["postonly"]), str(r["raw"]), str(r["meta"]), Text(str(r["missing"]), style="red" if r["missing"] else "green")) return t def _short_ref(ref: str) -> str: if ref.startswith("http"): import re m = re.search(r"c=(\d+)&v=(\d+)&d=(\d+)", ref) return f"c={m.group(1)} v={m.group(2)} d={m.group(3)}" if m else ref[-24:] return f"f={ref}" def sections_table(site: str, entries: list[dict], rows: int = 14) -> Panel: """Итог поиска разделов: что взято, что пропущено, сколько тем на первой странице.""" kept = [e for e in entries if e.get("keep")] skipped = [e for e in entries if not e.get("keep")] t = Table(box=box.SIMPLE_HEAD, expand=True, pad_edge=False) t.add_column("", width=2) t.add_column("Раздел / список", overflow="ellipsis", no_wrap=True, ratio=1) t.add_column("ID", style="dim", no_wrap=True) t.add_column("Тем", justify="right") for e in (kept[:rows] + skipped[:3]): ok = e.get("keep") mark = Text("✓", style="green") if e.get("status") == "ok" else ( Text("?", style="yellow") if ok else Text("✗", style="red")) topics = "—" if e.get("topics") is None else str(e["topics"]) t.add_row(mark, e["name"], _short_ref(e["ref"]), topics) hidden = len(kept) - min(len(kept), rows) foot = f"взято {len(kept)}" + (f", пропущено {len(skipped)} (мало тем)" if skipped else "") if hidden: foot += f" [dim]… и ещё {hidden} в списке[/]" unsure = sum(1 for e in kept if e.get("status") == "?") if unsure: foot += f" [yellow]? — не удалось проверить: {unsure}[/]" return Panel(Group(t, Text.from_markup(foot)), title=f"[bold]{site}[/] · найденные разделы", border_style=SITE_STYLE.get(site, "blue")) def banner(lines: list[str]) -> Panel: body = Text("\n".join(lines)) return Panel(body, title="[bold]grabber-ai · датасет и модель[/bold]", border_style="green", padding=(0, 2)) def menu_table(items: list[tuple[str, str]]) -> Table: t = Table(box=None, show_header=False, pad_edge=False) t.add_column(style="bold cyan", justify="right") t.add_column() for key, label in items: t.add_row(key, label) return t def clear() -> None: """Очистить терминал (если вывод не в консоль — ничего не делает).""" if console.is_terminal: console.clear() def pause(text: str = "Enter — вернуться в меню") -> None: try: console.input(f"\n[dim]{text}…[/dim]") except (EOFError, KeyboardInterrupt): pass def ask(prompt: str, default: str = "") -> str: return Prompt.ask(f"[bold]{prompt}[/bold]", default=default or None, console=console, show_default=bool(default)) or "" class _Rate(ProgressColumn): """Скорость: элементов в секунду (по окну rich).""" def render(self, task) -> Text: return Text(f"{task.speed:.1f}/с" if task.speed else "—/с", style="cyan") def progress() -> Progress: """Полоса прогресса: что делаем, N/M, %, скорость, прошло, осталось.""" return Progress(SpinnerColumn(), TextColumn("[bold]{task.description:<26}"), BarColumn(bar_width=26), MofNCompleteColumn(), TaskProgressColumn(), _Rate(), TextColumn("прошло"), TimeElapsedColumn(), TextColumn("осталось"), TimeRemainingColumn(), console=console, transient=False) def summary_table(title: str, columns: list[str], rows: list[list]) -> Table: t = Table(box=box.SIMPLE_HEAD, title=title, title_justify="left", pad_edge=False) for i, c in enumerate(columns): t.add_column(c, justify="left" if i == 0 else "right") for r in rows: t.add_row(*[str(x) for x in r]) return t