\documentclass[11pt]{article}

\usepackage{geometry}

\usepackage{mlmodern}

\usepackage{enumitem}
\setlist[itemize]{align=parleft,left=0pt..30pt}

\parindent=0pt
\parskip=\medskipamount

\usepackage[hidelinks]{hyperref}

\protected\def\link{\futurelet\tempa\linkA}
\def\linkA{\ifx[\tempa \expandafter\linkB \else \expandafter\linkC \fi}
\long\def\linkB[#1]#2{\href{#2}{\ttul{#1}}}
\long\def\linkC#1{\href{#1}{\ttul{#1}}}
\protected\long\def\email#1{\href{mailto:#1}{\ttul{#1}}}

\protected\long\def\ttul#1{%
	\leavevmode
	\setbox0=\hbox{\ttfamily #1}%
	\lower2.25pt\rlap{\leaders\hrule height.625px\hskip\wd0}%
	\pdfliteral{q 1 G 2 Tr 1.7 w 1 j 1 J}%
	\rlap{\copy0}%
	\pdfliteral{Q}%
	\box0
}

\input visualtoks

\title{The \texttt{visualtoks} Package, version 1.1}
\author{plante}

\begin{document}

\maketitle

\noindent
In The \TeX book, Knuth demonstrates the concept of tokens with the following example:
\begin{quote}
	\medskip
	For example, if the normal conventions of plain \TeX\ are in force, the text
	`\verb|{\hskip 36 pt}|' is converted into a list of eight tokens:
	\[
		\hbox{\visualtokssep=1.25em \visualtoks{{\hskip 36 pt}}}
	\]
	The subscripts here are the category codes, as listed earlier: 1 for ``beginning of group,''
	12 for ``other character,'' and so on. The {\ttfamily \visualtoksB{hskip}} doesn't
	get a subscript, because it represents a control sequence token instead of a character token.
	Notice that the space after \verb|\hskip| does not get into the token list, because it
	follows a control word. \hfill (p.\ 38){\parfillskip=0pt \par}
	\medskip
\end{quote}
The same style of token display is used several times in the \TeX book.
It would be useful to be able to generate the display automatically
for an arbitrary list of tokens, for pedagogical or debugging purposes.
This package provides the \verb|\visualtoks| command which does exactly that.

\subsection*{Usage}

Usage: \verb|\visualtoks{|$\,\langle\textit{token list}\rangle\,$\verb|}|.

This package may be loaded by \verb|\input{visualtoks}| (plain \TeX\ and other formats) or \verb|\usepackage{visualtoks}| (\LaTeX). % The $\varepsilon$-\TeX\ extensions are required for the \verb|\detokenize| primitive.

The horizontal separation between displayed tokens may be configured by the dimen register \verb|\visualtokssep|. The default value is 1em.

$\langle\textit{token list}\rangle$ must be balanced with respect to explicit braces, and must not contain the token \verb|\visualtoks@cycle@nil|.
It is assumed that \verb|{| and \verb|}| are the only characters with category codes 1 (beginning of group) and 2 (end of group) respectively.

An \emph{anomalous} control sequence is one that differs in shape from the control sequence with the same name constructed by \verb|\csname|. Anomalous control sequences are marked with a star next to their box.

\subsection*{Samples}

\begin{itemize}

\item \verb|\visualtoks{\def \macro#1{abc #1\egroup}}| gives
\[ \hbox{\visualtoks{\def \macro #1{abc #1\egroup}}}. \]

\item \verb|\visualtoks{$$\halign{&##\hfil\crcr}$$\par}| gives
\[ \hbox{\visualtoks{$$\halign{&##\hfil\crcr}$$\par}}. \]

\item Unbalanced \verb|\if...| tokens: \\
	\verb|\visualtoks{\ifnum\iffalse{\fi`} = 0\else}| gives
\[ \hbox{\visualtoks{\ifnum\iffalse{\fi`} = 0\else}}. \]

\makeatletter
\item To demonstrate how \TeX\ tokenizes consecutive spaces: \\
	\verb|\makeatletter |\verb*|\edef\temp{{    }{\@spaces}}| \\
	\verb|\expandafter\visualtoks\expandafter{\temp}| gives
\[ \hbox{\edef\temp{{    }{\@spaces}} \expandafter\visualtoks\expandafter{\temp}}. \]

\item To demonstrate the \verb|\lowercase| technique: \\
	\verb|\begingroup| \\
	\verb|    \lccode`&=`$ \lccode`#=`$ \lccode`^=`$ \lccode`_=`$| \\
	\verb|    \lccode`|\verb*| |\verb|=`$ \lccode`A=`$ \lccode`?=`$ \lccode`~=`$| \\
	\verb|\lowercase{\endgroup\def\temp{$&##^_|\verb*| |\verb|A?~}}| \\
	\verb|\expandafter\visualtoks\expandafter{\temp}| gives
\begingroup
	\lccode`&=`$ \lccode`#=`$ \lccode`^=`$ \lccode`_=`$
	\lccode` =`$ \lccode`A=`$ \lccode`?=`$ \lccode`~=`$
\lowercase{\endgroup\def\temp{$&##^_ A?~}}
\[ \hbox{\expandafter\visualtoks\expandafter{\temp}}. \]

\item To show anomalous tokens: \\
\verb|\font\tenrm=cmr10 \tenrm| \\
\verb|\expandafter\visualtoks\expandafter{\the\font \tenrm}| gives

\[ \hbox{\font\tenrm=cmr10 \tenrm
	\expandafter\visualtoks\expandafter{\the\font \tenrm}}. \]

% \setbox0=\vbox{\halign{#\crcr \global\futurelet\temp\relax \crcr}}
% \expandafter\visualtoks\expandafter{\temp \endtemplate}

\end{itemize}

\subsection*{License}

This package is copyright \copyright\ 2025 plante, and
released under the \LaTeX\ Project Public License (\textsc{lppl}) 1.3c.

\subsection*{Repository}

The upstream repository of this package may be found at
	\[ \link{https://github.com/plante3/visualtoks/tree/main}. \]

\end{document}