Show readable tokens for byte-level BPE tokenizers

Qwen tokens were printed as raw byte characters (e.g. 你好 as ä½łå¥½).
Decode each token for byte-level tokenizers, keeping the raw form for
tokens that hold only part of a multi-byte character.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

embedlab.py

@@ -138,6 +138,10 @@ def show_tokens(text, preview=50):
     tokenizer = _loaded[current][0]
     ids = tokenizer(text, truncation=False)["input_ids"]  # includes special tokens
     toks = tokenizer.convert_ids_to_tokens(ids)
+    if type(getattr(tokenizer.backend_tokenizer, "decoder", None)).__name__ == "ByteLevel":
+        # byte-level BPE (Qwen) stores raw bytes as odd characters; decode each token to readable text.
+        # A token holding only part of a multi-byte character can't be decoded, so keep its raw form.
+        toks = [d if d and "\ufffd" not in d else t for t, d in zip(toks, map(tokenizer.decode, ([i] for i in ids)))]
     shown = ", ".join(f"{t}({i})" for t, i in zip(toks[:preview], ids[:preview]))
     more = f", ... (+{len(ids) - preview} more)" if len(ids) > preview else ""
     print(f"[{current}] {len(ids)} tokens: {shown}{more}")