Сегментатор: truncation longest_first — не падать на строке длиннее max_length

С only_second токенизатор бросал исключение, если сама строка (до 220
символов) не помещалась в MAX_LEN и обрезать было нечего, кроме контекста.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/segmenter.py

@@ -138,7 +138,7 @@ class LineSegmenter:
         out = []
         for s in range(0, len(pairs), 128):
             chunk = pairs[s:s + 128]
-            enc = self.tok([a for a, _ in chunk], [b for _, b in chunk], truncation="only_second",
+            enc = self.tok([a for a, _ in chunk], [b for _, b in chunk], truncation="longest_first",
                            max_length=MAX_LEN, padding=True, return_tensors="pt").to(self.device)
             out.append(torch.softmax(self.model(**enc).logits, -1)[:, 1].cpu().numpy())
         return np.concatenate(out) if out else np.zeros(0)
@@ -166,7 +166,7 @@ def train(out_dir: str = "models/segmenter", epochs: int = 4, batch: int = 32, l
     sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps, pct_start=0.1)
 
     def enc(b):
-        return tok([x[0] for x in b], [x[1] for x in b], truncation="only_second",
+        return tok([x[0] for x in b], [x[1] for x in b], truncation="longest_first",
                    max_length=MAX_LEN, padding=True, return_tensors="pt").to(device)
 
     def evaluate(data):