From 2f59bddba0ff36c9a2c6f0a1153ba8e40fc47e0a Mon Sep 17 00:00:00 2001 From: Guofang Date: Tue, 10 Nov 2026 22:19:56 +0800 Subject: [PATCH] Show readable tokens for byte-level BPE tokenizers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qwen tokens were printed as raw byte characters (e.g. 你好 as ä½łå¥½). Decode each token for byte-level tokenizers, keeping the raw form for tokens that hold only part of a multi-byte character. Co-Authored-By: Claude Sonnet 5.5 --- embedlab.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/embedlab.py b/embedlab.py index 1f43bb0..29b6f36 100755 --- a/embedlab.py +++ b/embedlab.py @@ -138,6 +138,10 @@ def show_tokens(text, preview=50): tokenizer = _loaded[current][0] ids = tokenizer(text, truncation=False)["input_ids"] # includes special tokens toks = tokenizer.convert_ids_to_tokens(ids) + if type(getattr(tokenizer.backend_tokenizer, "decoder", None)).__name__ == "ByteLevel": + # byte-level BPE (Qwen) stores raw bytes as odd characters; decode each token to readable text. + # A token holding only part of a multi-byte character can't be decoded, so keep its raw form. + toks = [d if d and "\ufffd" not in d else t for t, d in zip(toks, map(tokenizer.decode, ([i] for i in ids)))] shown = ", ".join(f"{t}({i})" for t, i in zip(toks[:preview], ids[:preview])) more = f", ... (+{len(ids) - preview} more)" if len(ids) > preview else "" print(f"[{current}] {len(ids)} tokens: {shown}{more}")