diff --git a/src/segmenter.py b/src/segmenter.py index e54faf5..713f19b 100644 --- a/src/segmenter.py +++ b/src/segmenter.py @@ -138,7 +138,7 @@ class LineSegmenter: out = [] for s in range(0, len(pairs), 128): chunk = pairs[s:s + 128] - enc = self.tok([a for a, _ in chunk], [b for _, b in chunk], truncation="only_second", + enc = self.tok([a for a, _ in chunk], [b for _, b in chunk], truncation="longest_first", max_length=MAX_LEN, padding=True, return_tensors="pt").to(self.device) out.append(torch.softmax(self.model(**enc).logits, -1)[:, 1].cpu().numpy()) return np.concatenate(out) if out else np.zeros(0) @@ -166,7 +166,7 @@ def train(out_dir: str = "models/segmenter", epochs: int = 4, batch: int = 32, l sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps, pct_start=0.1) def enc(b): - return tok([x[0] for x in b], [x[1] for x in b], truncation="only_second", + return tok([x[0] for x in b], [x[1] for x in b], truncation="longest_first", max_length=MAX_LEN, padding=True, return_tensors="pt").to(device) def evaluate(data):