What Is Tokenization?

The tokenization pipeline

How BPE Tokenization Works

Interactive Tokenizer Playground

Tokenization in the npm Ecosystem

Code Examples

1"color:#C77DFF">import Tiktoken "color:#C77DFF">from "color:#6BCB77">'tiktoken';
2
3"color:#555;font-style:italic">// cl100k_base is used by GPT-3.5-turbo and GPT-4
4"color:#C77DFF">const enc = Tiktoken.getEncoding("color:#6BCB77">'cl100k_base');
5
6/**
7 * Returns the "color:#4D96FF">number of tokens in a "color:#4D96FF">string.
8 * Useful for estimating API cost before sending a request.
9 */
10"color:#C77DFF">function countTokens(text: "color:#4D96FF">string): "color:#4D96FF">number {
11  "color:#C77DFF">const tokens: Uint32Array = enc.encode(text);
12  "color:#C77DFF">return tokens.length;
13}
14
15"color:#C77DFF">const sample = "color:#6BCB77">'Tokenization is the first step in every LLM pipeline.';
16console.log(`"${sample}"`);
17console.log(`Token count: ${countTokens(sample)}`); "color:#555;font-style:italic">// → 11
18"color:#555;font-style:italic">// Approx GPT-4 cost at $0.03/1K input tokens: $0.00033

1"color:#C77DFF">import Anthropic "color:#C77DFF">from "color:#6BCB77">'@anthropic-ai/sdk';
2
3"color:#C77DFF">const client = "color:#C77DFF">new Anthropic(); "color:#555;font-style:italic">// uses ANTHROPIC_API_KEY env var
4
5"color:#C77DFF">async "color:#C77DFF">function streamWithTokenCount(prompt: "color:#4D96FF">string): Promise<"color:#4D96FF">void> {
6  "color:#555;font-style:italic">// Count tokens BEFORE sending — avoids surprises on large prompts
7  "color:#C77DFF">const { input_tokens } = "color:#C77DFF">await client.messages.countTokens({
8    model: "color:#6BCB77">'claude-opus-4-5',
9    messages: [{ role: "color:#6BCB77">'user', content: prompt }],
10  });
11  console.log(`Input tokens: ${input_tokens}`);
12
13  "color:#555;font-style:italic">// Stream the response
14  "color:#C77DFF">const stream = "color:#C77DFF">await client.messages.stream({
15    model: "color:#6BCB77">'claude-opus-4-5',
16    max_tokens: 256,
17    messages: [{ role: "color:#6BCB77">'user', content: prompt }],
18  });
19
20  for "color:#C77DFF">await ("color:#C77DFF">const chunk of stream.text_stream) {
21    process.stdout.write(chunk);
22  }
23
24  "color:#C77DFF">const finalMsg = "color:#C77DFF">await stream.finalMessage();
25  console.log(`\nOutput tokens: ${finalMsg.usage.output_tokens}`);
26}
27
28streamWithTokenCount("color:#6BCB77">'Explain tokenization in one paragraph.');

1"color:#C77DFF">import { AutoTokenizer } "color:#C77DFF">from "color:#6BCB77">'@huggingface/transformers';
2
3"color:#555;font-style:italic">// Load the tokenizer for any model on the HuggingFace Hub
4"color:#C77DFF">const tokenizer = "color:#C77DFF">await AutoTokenizer.from_pretrained(
5  "color:#6BCB77">'Xenova/gpt2' "color:#555;font-style:italic">// ~500 KB WASM download
6);
7
8"color:#C77DFF">const text = "color:#6BCB77">'Tokenization splits text into manageable pieces.';
9
10"color:#555;font-style:italic">// encode() returns an object with input_ids, attention_mask, etc.
11"color:#C77DFF">const encoded = tokenizer(text, {
12  padding: false,
13  truncation: true,
14  max_length: 128,
15});
16
17"color:#555;font-style:italic">// Typed Int32Array of token IDs
18"color:#C77DFF">const ids: Int32Array = encoded.input_ids.data;
19console.log("color:#6BCB77">'Token IDs:', Array."color:#C77DFF">from(ids));
20
21"color:#555;font-style:italic">// Decode back to a "color:#4D96FF">string (round-trip check)
22"color:#C77DFF">const decoded: "color:#4D96FF">string = tokenizer.decode(ids, {
23  skip_special_tokens: true,
24});
25console.log("color:#6BCB77">'Decoded:', decoded);
26
27"color:#555;font-style:italic">// Convert each ID to its "color:#4D96FF">string representation
28"color:#C77DFF">const tokenStrings: "color:#4D96FF">string[] = tokenizer.convert_ids_to_tokens(
29  Array."color:#C77DFF">from(ids)
30);
31console.log("color:#6BCB77">'Token strings:', tokenStrings);

Context Windows & Token Cost

1"color:#C77DFF">import Tiktoken "color:#C77DFF">from "color:#6BCB77">'tiktoken';
2
3"color:#C77DFF">const enc = Tiktoken.getEncoding("color:#6BCB77">'cl100k_base');
4
5/**
6 * Split a long document into chunks that fit within a token budget.
7 * Uses sentence boundaries to avoid cutting mid-sentence.
8 */
9"color:#C77DFF">function chunkByTokens(
10  document: "color:#4D96FF">string,
11  maxTokensPerChunk: "color:#4D96FF">number = 512,
12  overlapTokens: "color:#4D96FF">number = 64
13): "color:#4D96FF">string[] {
14  "color:#C77DFF">const sentences = document.match(/[^.!?]+[.!?]+/g) ?? [document];
15  "color:#C77DFF">const chunks: "color:#4D96FF">string[] = [];
16  "color:#C77DFF">let current = "color:#6BCB77">'';
17  "color:#C77DFF">let currentTokens = 0;
18
19  for ("color:#C77DFF">const sentence of sentences) {
20    "color:#C77DFF">const sentenceTokens = enc.encode(sentence).length;
21
22    if (currentTokens + sentenceTokens > maxTokensPerChunk && current) {
23      chunks.push(current.trim());
24      "color:#555;font-style:italic">// Keep the last "color:#6BCB77">'overlapTokens' worth of text for continuity
25      "color:#C77DFF">const words = current.split("color:#6BCB77">' ');
26      current = words.slice(-overlapTokens / 4 | 0).join("color:#6BCB77">' ') + "color:#6BCB77">' ';
27      currentTokens = enc.encode(current).length;
28    }
29
30    current += sentence;
31    currentTokens += sentenceTokens;
32  }
33
34  if (current.trim()) chunks.push(current.trim());
35  "color:#C77DFF">return chunks;
36}