diff --git a/embedlab.py b/embedlab.py index 1f43bb0..29b6f36 100755 --- a/embedlab.py +++ b/embedlab.py @@ -138,6 +138,10 @@ def show_tokens(text, preview=50): tokenizer = _loaded[current][0] ids = tokenizer(text, truncation=False)["input_ids"] # includes special tokens toks = tokenizer.convert_ids_to_tokens(ids) + if type(getattr(tokenizer.backend_tokenizer, "decoder", None)).__name__ == "ByteLevel": + # byte-level BPE (Qwen) stores raw bytes as odd characters; decode each token to readable text. + # A token holding only part of a multi-byte character can't be decoded, so keep its raw form. + toks = [d if d and "\ufffd" not in d else t for t, d in zip(toks, map(tokenizer.decode, ([i] for i in ids)))] shown = ", ".join(f"{t}({i})" for t, i in zip(toks[:preview], ids[:preview])) more = f", ... (+{len(ids) - preview} more)" if len(ids) > preview else "" print(f"[{current}] {len(ids)} tokens: {shown}{more}")