Upload folder using huggingface_hub

Tokenizer/BPE/special_tokens_map.json

@@ -0,0 +1,53 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<user>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<assistant>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<system>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Tokenizer/BPE/spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:debb3ad91c0745bb304129ee9a0332a33c2bb1fffe7313d573608e5014ab69bb
+size 747364

Tokenizer/BPE/test_tokenizer.py

@@ -0,0 +1,51 @@
+from transformers import AutoTokenizer
+tok = AutoTokenizer.from_pretrained("./Tokenizer/BPE")
+
+
+text1 = "Hello world! <user> write code </s>"
+text2 = "myHTTPRequestHandler is calling process_payment_v2"
+text3 = "methylphenidate hydrochloride dopamine reuptake modulation"
+text4 = "hello 🔥🔥🔥💀💀"
+text5 = "https://github.com/Avinash-MiniLLM?tab=repos"
+
+
+print(text1)
+print(text2)
+print(text3)
+print(text4)
+print(text5)
+
+print(tok.tokenize(text1))
+print(tok.tokenize(text2))
+print(tok.tokenize(text3))
+print(tok.tokenize(text4))
+print(tok.tokenize(text5))
+
+
+ids1 = tok.encode(text1)
+ids2 = tok.encode(text2)
+ids3 = tok.encode(text3)
+ids4 = tok.encode(text4)
+ids5 = tok.encode(text5)
+
+print(ids1)
+print(tok.decode(ids1))
+print(tok.decode(ids1, skip_special_tokens=True))
+
+print(ids2)
+print(tok.decode(ids2))
+print(tok.decode(ids2, skip_special_tokens=True))
+
+print(ids3)
+print(tok.decode(ids3))
+print(tok.decode(ids3, skip_special_tokens=True))
+
+ids4 = tok.encode(text4)
+print(ids4)
+print(tok.decode(ids4))
+print(tok.decode(ids4, skip_special_tokens=True))
+
+ids5 = tok.encode(text5)
+print(ids5)
+print(tok.decode(ids5))
+print(tok.decode(ids5, skip_special_tokens=True))

Tokenizer/BPE/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:debb3ad91c0745bb304129ee9a0332a33c2bb1fffe7313d573608e5014ab69bb
+size 747364

Tokenizer/BPE/tokenizer_config.json

@@ -0,0 +1,80 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<user>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<assistant>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<system>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<user>",
+    "<assistant>",
+    "<system>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

Tokenizer/README.md

@@ -0,0 +1,160 @@
+# Tokenizer Module
+
+This module handles all tokenization tasks for the Mini-LLM project, converting raw text into numerical tokens that the model can process.
+
+## Overview
+
+The tokenizer uses **SentencePiece** with **Byte Pair Encoding (BPE)** to create a 32,000 token vocabulary. BPE is the same algorithm used by GPT-3, GPT-4, and LLaMA models.
+
+## Directory Structure
+
+```
+Tokenizer/
+├── BPE/                      # BPE tokenizer artifacts
+│   ├── spm.model            # Trained SentencePiece model
+│   ├── spm.vocab            # Vocabulary file
+│   ├── tokenizer.json       # HuggingFace format
+│   ├── tokenizer_config.json
+│   └── special_tokens_map.json
+├── Unigram/                 # Unigram tokenizer (baseline)
+│   └── ...
+├── train_spm_bpe.py         # Train BPE tokenizer
+├── train_spm_unigram.py     # Train Unigram tokenizer
+└── convert_to_hf.py         # Convert to HuggingFace format
+```
+
+## How It Works
+
+### 1. Training the Tokenizer
+
+**Script**: `train_spm_bpe.py`
+
+```python
+import sentencepiece as spm
+
+spm.SentencePieceTrainer.Train(
+    input="data/raw/merged_text/corpus.txt",
+    model_prefix="Tokenizer/BPE/spm",
+    vocab_size=32000,
+    model_type="bpe",
+    byte_fallback=True,  # Handles emojis, special chars
+    character_coverage=1.0,
+    user_defined_symbols=["<user>", "<assistant>", "<system>"]
+)
+```
+
+**What happens:**
+1. Reads raw text corpus
+2. Learns byte-pair merges (e.g., "th" + "e" → "the")
+3. Builds 32,000 most frequent tokens
+4. Saves model to `spm.model`
+
+### 2. Example: Tokenization Process
+
+**Input Text:**
+```
+"Hello world! <user> write code </s>"
+```
+
+**Tokenization Steps:**
+
+```
+┌─────────────────────────────────────────┐
+│ 1. Text Input                           │
+│    "Hello world! <user> write code"     │
+└─────────────────────────────────────────┘
+              ↓
+┌─────────────────────────────────────────┐
+│ 2. BPE Segmentation                     │
+│    ['H', 'ello', '▁world', '!',         │
+│     '▁', '<user>', '▁write', '▁code']   │
+└─────────────────────────────────────────┘
+              ↓
+┌─────────────────────────────────────────┐
+│ 3. Token IDs                            │
+│    [334, 3855, 288, 267, 2959,          │
+│     354, 267, 12397]                    │
+└─────────────────────────────────────────┘
+```
+
+**Key Features:**
+- `▁` represents space (SentencePiece convention)
+- Special tokens like `<user>` are preserved
+- Byte fallback handles emojis: 🔥 → `<0xF0><0x9F><0x94><0xA5>`
+
+### 3. Converting to HuggingFace Format
+
+**Script**: `convert_to_hf.py`
+
+```python
+from transformers import LlamaTokenizerFast
+
+tokenizer = LlamaTokenizerFast(vocab_file="Tokenizer/BPE/spm.model")
+tokenizer.add_special_tokens({
+    'bos_token': '<s>',
+    'eos_token': '</s>',
+    'unk_token': '<unk>',
+    'pad_token': '<pad>'
+})
+tokenizer.save_pretrained("Tokenizer/BPE")
+```
+
+This creates `tokenizer.json` and config files compatible with HuggingFace Transformers.
+
+## Usage
+
+### Load Tokenizer
+
+```python
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("Tokenizer/BPE")
+```
+
+### Encode Text
+
+```python
+text = "Hello world!"
+ids = tokenizer.encode(text)
+# Output: [1, 334, 3855, 288, 267, 2]
+#         [<s>, H, ello, ▁world, !, </s>]
+```
+
+### Decode IDs
+
+```python
+decoded = tokenizer.decode(ids)
+# Output: "<s> Hello world! </s>"
+
+decoded = tokenizer.decode(ids, skip_special_tokens=True)
+# Output: "Hello world!"
+```
+
+## BPE vs Unigram
+
+| Feature | BPE | Unigram |
+|---------|-----|---------|
+| **Algorithm** | Merge frequent pairs | Probabilistic segmentation |
+| **Emoji Handling** | ✅ Byte fallback | ❌ Creates `<unk>` |
+| **URL Handling** | ✅ Clean splits | ⚠️ Unstable |
+| **Used By** | GPT-3, GPT-4, LLaMA | BERT, T5 |
+| **Recommendation** | ✅ **Primary** | Baseline only |
+
+## Vocabulary Statistics
+
+- **Total Tokens**: 32,000
+- **Special Tokens**: 4 (`<s>`, `</s>`, `<unk>`, `<pad>`)
+- **User-Defined**: 3 (`<user>`, `<assistant>`, `<system>`)
+- **Coverage**: 100% (byte fallback ensures no `<unk>`)
+
+## Performance
+
+- **Compression Ratio**: ~3.5 bytes/token (English text)
+- **Tokenization Speed**: ~1M tokens/second
+- **Vocab Usage**: ~70% of tokens used in typical corpus
+
+## References
+
+- [SentencePiece Documentation](https://github.com/google/sentencepiece)
+- [BPE Paper (Sennrich et al., 2016)](https://arxiv.org/abs/1508.07909)
+- [Tokenizer Comparison Report](../tokenizer_report.md)

Tokenizer/Unigram/special_tokens_map.json

@@ -0,0 +1,53 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<user>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<assistant>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<system>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Tokenizer/Unigram/spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:247ddb0f3561179d04614a59d7eb594da59ad881575a6d3860f859be9b709508
+size 768238

Tokenizer/Unigram/test_tokenizer.py

@@ -0,0 +1,51 @@
+from transformers import AutoTokenizer
+tok = AutoTokenizer.from_pretrained("./Tokenizer/Unigram")
+
+
+text1 = "Hello world! <user> write code </s>"
+text2 = "myHTTPRequestHandler is calling process_payment_v2"
+text3 = "methylphenidate hydrochloride dopamine reuptake modulation"
+text4 = "hello 🔥🔥🔥💀💀"
+text5 = "https://github.com/Avinash-MiniLLM?tab=repos"
+
+
+print(text1)
+print(text2)
+print(text3)
+print(text4)
+print(text5)
+
+print(tok.tokenize(text1))
+print(tok.tokenize(text2))
+print(tok.tokenize(text3))
+print(tok.tokenize(text4))
+print(tok.tokenize(text5))
+
+
+ids1 = tok.encode(text1)
+ids2 = tok.encode(text2)
+ids3 = tok.encode(text3)
+ids4 = tok.encode(text4)
+ids5 = tok.encode(text5)
+
+print(ids1)
+print(tok.decode(ids1))
+print(tok.decode(ids1, skip_special_tokens=True))
+
+print(ids2)
+print(tok.decode(ids2))
+print(tok.decode(ids2, skip_special_tokens=True))
+
+print(ids3)
+print(tok.decode(ids3))
+print(tok.decode(ids3, skip_special_tokens=True))
+
+ids4 = tok.encode(text4)
+print(ids4)
+print(tok.decode(ids4))
+print(tok.decode(ids4, skip_special_tokens=True))
+
+ids5 = tok.encode(text5)
+print(ids5)
+print(tok.decode(ids5))
+print(tok.decode(ids5, skip_special_tokens=True))

Tokenizer/Unigram/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:247ddb0f3561179d04614a59d7eb594da59ad881575a6d3860f859be9b709508
+size 768238

Tokenizer/Unigram/tokenizer_config.json

@@ -0,0 +1,80 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<user>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<assistant>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<system>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<user>",
+    "<assistant>",
+    "<system>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

Tokenizer/convert_to_hf.py

@@ -0,0 +1,18 @@
+from transformers import LlamaTokenizerFast
+
+# Load the raw spm model
+tokenizer = LlamaTokenizerFast(vocab_file="/home/aviinashh/projects/Mini-LLM/Tokenizer/BPE/spm.model")
+
+# Add your special tokens manually to the HF config part
+tokenizer.add_special_tokens({
+    "bos_token": "<s>",
+    "eos_token": "</s>",
+    "unk_token": "<unk>",
+    "pad_token": "<pad>",
+    "additional_special_tokens": ["<user>", "<assistant>", "<system>"]
+})
+
+# Save the json version
+tokenizer.save_pretrained("Tokenizer/")
+
+print("Converted to tokenizer.json successfully!")

Tokenizer/test_tokenizer.py

@@ -0,0 +1,9 @@
+from transformers import AutoTokenizer
+tok = AutoTokenizer.from_pretrained(".")
+print(tok.tokenize("Hello world! <user> write code </s>"))
+
+text = "Hello world! <user> write code </s>"
+ids = tok.encode(text)
+print(ids)
+print(tok.decode(ids))
+print(tok.decode(ids, skip_special_tokens=True))

Tokenizer/train_spm_bpe.py

@@ -0,0 +1,18 @@
+import sentencepiece as spm
+
+spm.SentencePieceTrainer.Train(
+    input="/home/aviinashh/projects/Mini-LLM/data/raw/merged_text/corpus.txt",
+    model_prefix="/home/aviinashh/projects/Mini-LLM/Tokenizer/BPE/spm",
+    vocab_size=32000,
+    model_type="bpe",
+    byte_fallback=True,
+    character_coverage=1.0,
+    unk_id=0,
+    bos_id=1,
+    eos_id=2,
+    pad_id=3,
+    user_defined_symbols=["<user>", "<assistant>", "<system>"],
+)
+
+print("Tokenizer trained!")
+# Model and vocab will be saved as spm.model and spm.vocab in the specified path

Tokenizer/train_spm_unigram.py

@@ -0,0 +1,17 @@
+import sentencepiece as spm
+
+spm.SentencePieceTrainer.Train(
+    input="/home/aviinashh/projects/Mini-LLM/data/raw/merged_text/corpus.txt",
+    model_prefix="/home/aviinashh/projects/Mini-LLM/Tokenizer/spm",
+    vocab_size=32000,
+    model_type="unigram",
+    character_coverage=1.0,
+    unk_id=0,
+    bos_id=1,
+    eos_id=2,
+    pad_id=3,
+    user_defined_symbols=["<user>", "<assistant>", "<system>"],
+)
+
+print("Tokenizer trained!")
+# Model and vocab will be saved as spm.model and spm.vocab in the specified path

Tokenizer/BPE/special_tokens_map.json

@@ -0,0 +1,53 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<user>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<assistant>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<system>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Tokenizer/BPE/spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:debb3ad91c0745bb304129ee9a0332a33c2bb1fffe7313d573608e5014ab69bb
+size 747364

Tokenizer/BPE/test_tokenizer.py

@@ -0,0 +1,51 @@
+from transformers import AutoTokenizer
+tok = AutoTokenizer.from_pretrained("./Tokenizer/BPE")
+
+
+text1 = "Hello world! <user> write code </s>"
+text2 = "myHTTPRequestHandler is calling process_payment_v2"
+text3 = "methylphenidate hydrochloride dopamine reuptake modulation"
+text4 = "hello 🔥🔥🔥💀💀"
+text5 = "https://github.com/Avinash-MiniLLM?tab=repos"
+
+
+print(text1)
+print(text2)
+print(text3)
+print(text4)
+print(text5)
+
+print(tok.tokenize(text1))
+print(tok.tokenize(text2))
+print(tok.tokenize(text3))
+print(tok.tokenize(text4))
+print(tok.tokenize(text5))
+
+
+ids1 = tok.encode(text1)
+ids2 = tok.encode(text2)
+ids3 = tok.encode(text3)
+ids4 = tok.encode(text4)
+ids5 = tok.encode(text5)
+
+print(ids1)
+print(tok.decode(ids1))
+print(tok.decode(ids1, skip_special_tokens=True))
+
+print(ids2)
+print(tok.decode(ids2))
+print(tok.decode(ids2, skip_special_tokens=True))
+
+print(ids3)
+print(tok.decode(ids3))
+print(tok.decode(ids3, skip_special_tokens=True))
+
+ids4 = tok.encode(text4)
+print(ids4)
+print(tok.decode(ids4))
+print(tok.decode(ids4, skip_special_tokens=True))
+
+ids5 = tok.encode(text5)
+print(ids5)
+print(tok.decode(ids5))
+print(tok.decode(ids5, skip_special_tokens=True))

Tokenizer/BPE/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:debb3ad91c0745bb304129ee9a0332a33c2bb1fffe7313d573608e5014ab69bb
+size 747364

Tokenizer/BPE/tokenizer_config.json

@@ -0,0 +1,80 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<user>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<assistant>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<system>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<user>",
+    "<assistant>",
+    "<system>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

Tokenizer/README.md

@@ -0,0 +1,160 @@
+# Tokenizer Module
+
+This module handles all tokenization tasks for the Mini-LLM project, converting raw text into numerical tokens that the model can process.
+
+## Overview
+
+The tokenizer uses **SentencePiece** with **Byte Pair Encoding (BPE)** to create a 32,000 token vocabulary. BPE is the same algorithm used by GPT-3, GPT-4, and LLaMA models.
+
+## Directory Structure
+
+```
+Tokenizer/
+├── BPE/                      # BPE tokenizer artifacts
+│   ├── spm.model            # Trained SentencePiece model
+│   ├── spm.vocab            # Vocabulary file
+│   ├── tokenizer.json       # HuggingFace format
+│   ├── tokenizer_config.json
+│   └── special_tokens_map.json
+├── Unigram/                 # Unigram tokenizer (baseline)
+│   └── ...
+├── train_spm_bpe.py         # Train BPE tokenizer
+├── train_spm_unigram.py     # Train Unigram tokenizer
+└── convert_to_hf.py         # Convert to HuggingFace format
+```
+
+## How It Works
+
+### 1. Training the Tokenizer
+
+**Script**: `train_spm_bpe.py`
+
+```python
+import sentencepiece as spm
+
+spm.SentencePieceTrainer.Train(
+    input="data/raw/merged_text/corpus.txt",
+    model_prefix="Tokenizer/BPE/spm",
+    vocab_size=32000,
+    model_type="bpe",
+    byte_fallback=True,  # Handles emojis, special chars
+    character_coverage=1.0,
+    user_defined_symbols=["<user>", "<assistant>", "<system>"]
+)
+```
+
+**What happens:**
+1. Reads raw text corpus
+2. Learns byte-pair merges (e.g., "th" + "e" → "the")
+3. Builds 32,000 most frequent tokens
+4. Saves model to `spm.model`
+
+### 2. Example: Tokenization Process
+
+**Input Text:**
+```
+"Hello world! <user> write code </s>"
+```
+
+**Tokenization Steps:**
+
+```
+┌─────────────────────────────────────────┐
+│ 1. Text Input                           │
+│    "Hello world! <user> write code"     │
+└─────────────────────────────────────────┘
+              ↓
+┌─────────────────────────────────────────┐
+│ 2. BPE Segmentation                     │
+│    ['H', 'ello', '▁world', '!',         │
+│     '▁', '<user>', '▁write', '▁code']   │
+└─────────────────────────────────────────┘
+              ↓
+┌─────────────────────────────────────────┐
+│ 3. Token IDs                            │
+│    [334, 3855, 288, 267, 2959,          │
+│     354, 267, 12397]                    │
+└─────────────────────────────────────────┘
+```
+
+**Key Features:**
+- `▁` represents space (SentencePiece convention)
+- Special tokens like `<user>` are preserved
+- Byte fallback handles emojis: 🔥 → `<0xF0><0x9F><0x94><0xA5>`
+
+### 3. Converting to HuggingFace Format
+
+**Script**: `convert_to_hf.py`
+
+```python
+from transformers import LlamaTokenizerFast
+
+tokenizer = LlamaTokenizerFast(vocab_file="Tokenizer/BPE/spm.model")
+tokenizer.add_special_tokens({
+    'bos_token': '<s>',
+    'eos_token': '</s>',
+    'unk_token': '<unk>',
+    'pad_token': '<pad>'
+})
+tokenizer.save_pretrained("Tokenizer/BPE")
+```
+
+This creates `tokenizer.json` and config files compatible with HuggingFace Transformers.
+
+## Usage
+
+### Load Tokenizer
+
+```python
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("Tokenizer/BPE")
+```
+
+### Encode Text
+
+```python
+text = "Hello world!"
+ids = tokenizer.encode(text)
+# Output: [1, 334, 3855, 288, 267, 2]
+#         [<s>, H, ello, ▁world, !, </s>]
+```
+
+### Decode IDs
+
+```python
+decoded = tokenizer.decode(ids)
+# Output: "<s> Hello world! </s>"
+
+decoded = tokenizer.decode(ids, skip_special_tokens=True)
+# Output: "Hello world!"
+```
+
+## BPE vs Unigram
+
+| Feature | BPE | Unigram |
+|---------|-----|---------|
+| **Algorithm** | Merge frequent pairs | Probabilistic segmentation |
+| **Emoji Handling** | ✅ Byte fallback | ❌ Creates `<unk>` |
+| **URL Handling** | ✅ Clean splits | ⚠️ Unstable |
+| **Used By** | GPT-3, GPT-4, LLaMA | BERT, T5 |
+| **Recommendation** | ✅ **Primary** | Baseline only |
+
+## Vocabulary Statistics
+
+- **Total Tokens**: 32,000
+- **Special Tokens**: 4 (`<s>`, `</s>`, `<unk>`, `<pad>`)
+- **User-Defined**: 3 (`<user>`, `<assistant>`, `<system>`)
+- **Coverage**: 100% (byte fallback ensures no `<unk>`)
+
+## Performance
+
+- **Compression Ratio**: ~3.5 bytes/token (English text)
+- **Tokenization Speed**: ~1M tokens/second
+- **Vocab Usage**: ~70% of tokens used in typical corpus
+
+## References
+
+- [SentencePiece Documentation](https://github.com/google/sentencepiece)
+- [BPE Paper (Sennrich et al., 2016)](https://arxiv.org/abs/1508.07909)
+- [Tokenizer Comparison Report](../tokenizer_report.md)

Tokenizer/Unigram/special_tokens_map.json

@@ -0,0 +1,53 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<user>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<assistant>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<system>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

Tokenizer/Unigram/spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:247ddb0f3561179d04614a59d7eb594da59ad881575a6d3860f859be9b709508
+size 768238

Tokenizer/Unigram/test_tokenizer.py

@@ -0,0 +1,51 @@
+from transformers import AutoTokenizer
+tok = AutoTokenizer.from_pretrained("./Tokenizer/Unigram")
+
+
+text1 = "Hello world! <user> write code </s>"
+text2 = "myHTTPRequestHandler is calling process_payment_v2"
+text3 = "methylphenidate hydrochloride dopamine reuptake modulation"
+text4 = "hello 🔥🔥🔥💀💀"
+text5 = "https://github.com/Avinash-MiniLLM?tab=repos"
+
+
+print(text1)
+print(text2)
+print(text3)
+print(text4)
+print(text5)
+
+print(tok.tokenize(text1))
+print(tok.tokenize(text2))
+print(tok.tokenize(text3))
+print(tok.tokenize(text4))
+print(tok.tokenize(text5))
+
+
+ids1 = tok.encode(text1)
+ids2 = tok.encode(text2)
+ids3 = tok.encode(text3)
+ids4 = tok.encode(text4)
+ids5 = tok.encode(text5)
+
+print(ids1)
+print(tok.decode(ids1))
+print(tok.decode(ids1, skip_special_tokens=True))
+
+print(ids2)
+print(tok.decode(ids2))
+print(tok.decode(ids2, skip_special_tokens=True))
+
+print(ids3)
+print(tok.decode(ids3))
+print(tok.decode(ids3, skip_special_tokens=True))
+
+ids4 = tok.encode(text4)
+print(ids4)
+print(tok.decode(ids4))
+print(tok.decode(ids4, skip_special_tokens=True))
+
+ids5 = tok.encode(text5)
+print(ids5)
+print(tok.decode(ids5))
+print(tok.decode(ids5, skip_special_tokens=True))

Tokenizer/Unigram/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:247ddb0f3561179d04614a59d7eb594da59ad881575a6d3860f859be9b709508
+size 768238

Tokenizer/Unigram/tokenizer_config.json

@@ -0,0 +1,80 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<user>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<assistant>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<system>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<user>",
+    "<assistant>",
+    "<system>"
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

Tokenizer/convert_to_hf.py

@@ -0,0 +1,18 @@
+from transformers import LlamaTokenizerFast
+
+# Load the raw spm model
+tokenizer = LlamaTokenizerFast(vocab_file="/home/aviinashh/projects/Mini-LLM/Tokenizer/BPE/spm.model")
+
+# Add your special tokens manually to the HF config part
+tokenizer.add_special_tokens({
+    "bos_token": "<s>",
+    "eos_token": "</s>",
+    "unk_token": "<unk>",
+    "pad_token": "<pad>",
+    "additional_special_tokens": ["<user>", "<assistant>", "<system>"]
+})
+
+# Save the json version
+tokenizer.save_pretrained("Tokenizer/")
+
+print("Converted to tokenizer.json successfully!")

Tokenizer/test_tokenizer.py

@@ -0,0 +1,9 @@
+from transformers import AutoTokenizer
+tok = AutoTokenizer.from_pretrained(".")
+print(tok.tokenize("Hello world! <user> write code </s>"))
+
+text = "Hello world! <user> write code </s>"
+ids = tok.encode(text)
+print(ids)
+print(tok.decode(ids))
+print(tok.decode(ids, skip_special_tokens=True))

Tokenizer/train_spm_bpe.py

@@ -0,0 +1,18 @@
+import sentencepiece as spm
+
+spm.SentencePieceTrainer.Train(
+    input="/home/aviinashh/projects/Mini-LLM/data/raw/merged_text/corpus.txt",
+    model_prefix="/home/aviinashh/projects/Mini-LLM/Tokenizer/BPE/spm",
+    vocab_size=32000,
+    model_type="bpe",
+    byte_fallback=True,
+    character_coverage=1.0,
+    unk_id=0,
+    bos_id=1,
+    eos_id=2,
+    pad_id=3,
+    user_defined_symbols=["<user>", "<assistant>", "<system>"],
+)
+
+print("Tokenizer trained!")
+# Model and vocab will be saved as spm.model and spm.vocab in the specified path

Tokenizer/train_spm_unigram.py

@@ -0,0 +1,17 @@
+import sentencepiece as spm
+
+spm.SentencePieceTrainer.Train(
+    input="/home/aviinashh/projects/Mini-LLM/data/raw/merged_text/corpus.txt",
+    model_prefix="/home/aviinashh/projects/Mini-LLM/Tokenizer/spm",
+    vocab_size=32000,
+    model_type="unigram",
+    character_coverage=1.0,
+    unk_id=0,
+    bos_id=1,
+    eos_id=2,
+    pad_id=3,
+    user_defined_symbols=["<user>", "<assistant>", "<system>"],
+)
+
+print("Tokenizer trained!")
+# Model and vocab will be saved as spm.model and spm.vocab in the specified path