From 666a266b2f69db127b76d60508c849141ce91cc6 Mon Sep 17 00:00:00 2001 From: Denozordec Date: Mon, 2 Nov 2026 11:44:18 +0700 Subject: [PATCH] =?UTF-8?q?=D0=A1=D0=B5=D0=B3=D0=BC=D0=B5=D0=BD=D1=82?= =?UTF-8?q?=D0=B0=D1=82=D0=BE=D1=80:=20truncation=20longest=5Ffirst=20?= =?UTF-8?q?=E2=80=94=20=D0=BD=D0=B5=20=D0=BF=D0=B0=D0=B4=D0=B0=D1=82=D1=8C?= =?UTF-8?q?=20=D0=BD=D0=B0=20=D1=81=D1=82=D1=80=D0=BE=D0=BA=D0=B5=20=D0=B4?= =?UTF-8?q?=D0=BB=D0=B8=D0=BD=D0=BD=D0=B5=D0=B5=20max=5Flength?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit С only_second токенизатор бросал исключение, если сама строка (до 220 символов) не помещалась в MAX_LEN и обрезать было нечего, кроме контекста. Co-Authored-By: Claude Opus 5.5 --- src/segmenter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/segmenter.py b/src/segmenter.py index e54faf5..713f19b 100644 --- a/src/segmenter.py +++ b/src/segmenter.py @@ -138,7 +138,7 @@ class LineSegmenter: out = [] for s in range(0, len(pairs), 128): chunk = pairs[s:s + 128] - enc = self.tok([a for a, _ in chunk], [b for _, b in chunk], truncation="only_second", + enc = self.tok([a for a, _ in chunk], [b for _, b in chunk], truncation="longest_first", max_length=MAX_LEN, padding=True, return_tensors="pt").to(self.device) out.append(torch.softmax(self.model(**enc).logits, -1)[:, 1].cpu().numpy()) return np.concatenate(out) if out else np.zeros(0) @@ -166,7 +166,7 @@ def train(out_dir: str = "models/segmenter", epochs: int = 4, batch: int = 32, l sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps, pct_start=0.1) def enc(b): - return tok([x[0] for x in b], [x[1] for x in b], truncation="only_second", + return tok([x[0] for x in b], [x[1] for x in b], truncation="longest_first", max_length=MAX_LEN, padding=True, return_tensors="pt").to(device) def evaluate(data):