NLP & Transformers — From Tokenization to Fine-Tuning

Raw text → Normalize → Tokenize → Vectorize → Model → Decode → Output

P(c|d) ∝ P(c) × Π P(w|c)   for each word w in d

log P(c|d) ∝ log P(c) + Σ log P(w|c)

# Two semantically identical sentences get very different BoW vectors:
doc1 = "The automobile is fast"
doc2 = "The car is rapid"

# BoW with these 6 unique words:
# [automobile, fast, car, rapid, the, is]
# doc1: [1, 1, 0, 0, 1, 1]
# doc2: [0, 0, 1, 1, 1, 1]
# Cosine similarity ≈ 0.5  (not even close to 1.0)

# Using gensim (requires pip install gensim)
from gensim.models import Word2Vec
import gensim.downloader as api

# Train from scratch on custom text
sentences = [
    ["machine", "learning", "models", "learn", "from", "data"],
    ["deep", "learning", "neural", "networks", "transform", "features"],
    ["python", "code", "runs", "machine", "learning", "algorithms"],
]

model = Word2Vec(
    sentences,
    vector_size=100,   # Embedding dimension
    window=5,          # Context window size
    min_count=1,       # Minimum word frequency
    workers=4,         # Parallel training threads
    sg=1,              # 1=Skip-gram, 0=CBOW
    epochs=100,
)

# Word similarity
print(model.wv.most_similar("machine", topn=5))
# [('learning', 0.98), ('deep', 0.91), ('algorithms', 0.88), ...]

# Word analogy: king - man + woman = queen
result = model.wv.most_similar(
    positive=["king", "woman"],
    negative=["man"],
    topn=1,
)
print(result)  # [('queen', 0.89)]

# Load pre-trained Google News embeddings (300-dim, 3M words)
wv = api.load("word2vec-google-news-300")
print(wv.most_similar("transformer", topn=5))

from gensim.models import FastText

model = FastText(sentences, vector_size=100, window=5, min_count=1)

# fastText can embed words it never saw in training:
# "transformerized" → sum of character n-grams including "transform"
vec = model.wv["transformerized"]   # Works even though unseen!

Attention(Q, K, V) = softmax(QKᵀ / √d_k) × V

┌─────────────────────────────────────────────┐
│              Transformer Block               │
│                                             │
│  Input x ─────────────────────────────┐    │
│      ↓                                │    │
│  MultiHead Attention (x, x, x)        │    │
│      ↓                                │    │
│  Add & LayerNorm ←─────────────────── x    │
│      ↓                                │    │
│  Feed-Forward Network                 │    │
│   (Linear → ReLU → Linear)            │    │
│      ↓                                │    │
│  Add & LayerNorm ←─────────────────── ┘    │
│      ↓                                     │
│  Output (same shape as Input)              │
└─────────────────────────────────────────────┘

MultiHead(Q,K,V) = Concat(head₁,...,headₕ) × Wᴼ
where headᵢ = Attention(Q×Wᵢᴼ, K×Wᵢᴷ, V×Wᵢᵛ)

from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

# Single sentence
text = "The model didn't converge during training."
encoding = tokenizer(
    text,
    return_tensors="pt",      # PyTorch tensors
    padding=True,
    truncation=True,
    max_length=128,
)
print(encoding.keys())
# dict_keys(['input_ids', 'token_type_ids', 'attention_mask'])

# input_ids: integer token IDs
# attention_mask: 1 for real tokens, 0 for padding
# token_type_ids: 0 for sentence A, 1 for sentence B (BERT sentence pairs)

ids = encoding["input_ids"][0].tolist()
tokens = tokenizer.convert_ids_to_tokens(ids)
print(tokens)
# ['[CLS]', 'the', 'model', 'didn', "'", 't', 'converge', 'during', 'training', '.', '[SEP]']

# Batch encoding (handles padding automatically)
batch = tokenizer(
    ["Short text.", "A longer piece of text that needs padding."],
    return_tensors="pt",
    padding=True,       # Pad shorter sequences
    truncation=True,
    max_length=32,
)
print(batch["input_ids"].shape)   # torch.Size([2, 12])

# BPE subword splitting (handles OOV words)
rare = "Supercalifragilisticexpialidocious and transformerization"
toks = tokenizer.tokenize(rare)
print(toks)
# ['super', '##cali', '##fra', '##gil', '##istic', '##ex', '##pia', ...]

from transformers import pipeline

# Sentiment analysis
classifier = pipeline("sentiment-analysis",
                      model="distilbert-base-uncased-finetuned-sst-2-english")
results = classifier([
    "This movie was absolutely fantastic!",
    "I hated every minute of this terrible film.",
])
# [{'label': 'POSITIVE', 'score': 0.9998}, {'label': 'NEGATIVE', 'score': 0.9994}]

# Named Entity Recognition
ner = pipeline("ner",
               model="dslim/bert-base-NER",
               aggregation_strategy="simple")
entities = ner("Elon Musk founded SpaceX in California.")
# [{'entity_group': 'PER', 'word': 'Elon Musk', 'score': 0.998},
#  {'entity_group': 'ORG', 'word': 'SpaceX', 'score': 0.993}, ...]

# Zero-shot classification (no fine-tuning needed)
zero_shot = pipeline("zero-shot-classification",
                     model="facebook/bart-large-mnli")
result = zero_shot(
    "The interest rate hike will slow economic growth.",
    candidate_labels=["finance", "sports", "technology", "politics"],
)
# {'labels': ['finance', 'politics', 'technology', 'sports'],
#  'scores': [0.89, 0.07, 0.02, 0.01]}

# Summarization
summarizer = pipeline("summarization", model="sshleifer/distilbart-cnn-12-6")
text = "..." * 500   # Long article text
summary = summarizer(text, max_length=150, min_length=40, do_sample=False)
print(summary[0]["summary_text"])

# Question answering
qa = pipeline("question-answering",
              model="distilbert-base-cased-distilled-squad")
result = qa(
    question="What year did BERT come out?",
    context="BERT was published in 2018 by Google. It transformed NLP tasks."
)
# {'answer': '2018', 'score': 0.94, 'start': 25, 'end': 29}

# Text generation (GPT-style)
generator = pipeline("text-generation", model="gpt2")
output = generator("Deep learning is", max_new_tokens=50, num_return_sequences=2)

# Fill-mask (BERT-style)
fill_mask = pipeline("fill-mask", model="bert-base-uncased")
result = fill_mask("Paris is the [MASK] of France.")
# [{'sequence': 'Paris is the capital of France.', 'score': 0.97, ...}]

# Translation
translator = pipeline("translation_en_to_fr",
                       model="Helsinki-NLP/opus-mt-en-fr")
result = translator("Machine learning transforms everything.")
print(result[0]["translation_text"])  # "L'apprentissage automatique transforme tout."

from datasets import load_dataset
from transformers import (
    AutoTokenizer,
    AutoModelForSequenceClassification,
    TrainingArguments,
    Trainer,
    DataCollatorWithPadding,
)
import numpy as np
from sklearn.metrics import accuracy_score, f1_score

# ── 1. Load dataset ────────────────────────────────────────────────────────────
dataset = load_dataset("imdb")  # 50K movie reviews, binary sentiment

# ── 2. Tokenize ────────────────────────────────────────────────────────────────
checkpoint = "distilbert-base-uncased"   # 66M params, 40% smaller than BERT
tokenizer = AutoTokenizer.from_pretrained(checkpoint)

def tokenize_fn(batch):
    return tokenizer(
        batch["text"],
        truncation=True,
        max_length=256,    # Truncate at 256 tokens to save memory
    )

tokenized = dataset.map(tokenize_fn, batched=True, batch_size=1000,
                        remove_columns=["text"])
tokenized = tokenized.rename_column("label", "labels")

# ── 3. Data collator (dynamic padding per batch, more efficient) ───────────────
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

# ── 4. Load model ──────────────────────────────────────────────────────────────
model = AutoModelForSequenceClassification.from_pretrained(
    checkpoint,
    num_labels=2,
    id2label={0: "negative", 1: "positive"},
    label2id={"negative": 0, "positive": 1},
)

# ── 5. Metrics ─────────────────────────────────────────────────────────────────
def compute_metrics(eval_pred):
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=-1)
    return {
        "accuracy": accuracy_score(labels, preds),
        "f1":       f1_score(labels, preds, average="binary"),
    }

# ── 6. Training arguments ──────────────────────────────────────────────────────
args = TrainingArguments(
    output_dir="imdb-distilbert",
    num_train_epochs=3,
    per_device_train_batch_size=32,
    per_device_eval_batch_size=64,
    learning_rate=2e-5,              # Standard for fine-tuning: 1e-5 to 5e-5
    warmup_ratio=0.1,               # Warm up for 10% of steps
    weight_decay=0.01,              # L2 regularization
    evaluation_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
    metric_for_best_model="f1",
    fp16=True,                      # Mixed precision: 2x faster on modern GPUs
    dataloader_num_workers=4,
    report_to="none",               # Disable wandb/tensorboard for demo
)

# ── 7. Trainer ─────────────────────────────────────────────────────────────────
trainer = Trainer(
    model=model,
    args=args,
    train_dataset=tokenized["train"].select(range(10000)),  # Subset for demo
    eval_dataset=tokenized["test"].select(range(2000)),
    tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_metrics,
)

trainer.train()
# Epoch 1/3: loss=0.28, accuracy=0.898, f1=0.897
# Epoch 2/3: loss=0.19, accuracy=0.921, f1=0.920
# Epoch 3/3: loss=0.14, accuracy=0.931, f1=0.930

# ── 8. Save and push to hub ────────────────────────────────────────────────────
trainer.save_model("imdb-distilbert-final")
tokenizer.save_pretrained("imdb-distilbert-final")

# Push to HuggingFace Hub
# model.push_to_hub("your-username/imdb-distilbert")
# tokenizer.push_to_hub("your-username/imdb-distilbert")

from peft import LoraConfig, get_peft_model, TaskType

config = LoraConfig(
    task_type=TaskType.SEQ_CLS,
    r=16,               # Rank of adapter matrices (lower = fewer params)
    lora_alpha=32,      # Scaling factor
    lora_dropout=0.05,
    target_modules=["q_lin", "k_lin", "v_lin"],   # Which layers to adapt
)

peft_model = get_peft_model(model, config)
peft_model.print_trainable_parameters()
# trainable params: 147,456 || all params: 67,091,458 || trainable%: 0.22

# Train with standard Trainer — same API as full fine-tuning
# LoRA adds ~0.22% parameters but recovers most of full fine-tuning performance

from groq import Groq
import json
import time

client = Groq()  # Reads GROQ_API_KEY from env

# ── Basic completion ───────────────────────────────────────────────────────────
response = client.chat.completions.create(
    model="llama-3.3-70b-versatile",
    messages=[
        {"role": "system", "content": "You are an expert NLP engineer."},
        {"role": "user",   "content": "Explain BPE tokenization in 3 bullet points."},
    ],
    temperature=0.3,    # Low temperature for factual/technical content
    max_tokens=300,
)
text = response.choices[0].message.content
cost_in = response.usage.prompt_tokens
cost_out = response.usage.completion_tokens
print(f"Response ({cost_in}+{cost_out} tokens):\n{text}")

# ── Prompt engineering patterns ────────────────────────────────────────────────
# 1. Zero-shot
zero_shot = [{"role": "user", "content": "Classify: 'Shipment delayed by storm.' → category:"}]

# 2. Few-shot (3 examples train the format)
few_shot = [
    {"role": "system",    "content": "Classify customer support tickets."},
    {"role": "user",      "content": "My login stopped working yesterday."},
    {"role": "assistant", "content": "AUTH_ISSUE"},
    {"role": "user",      "content": "Invoice shows wrong amount charged."},
    {"role": "assistant", "content": "BILLING_ISSUE"},
    {"role": "user",      "content": "The dashboard loads slowly."},
    {"role": "assistant", "content": "PERFORMANCE_ISSUE"},
    {"role": "user",      "content": "I can't export my report to PDF."},  # New query
]

# 3. Chain-of-thought (improves complex reasoning by 30-50%)
cot = """
<task>Classify the sentiment of this review.</task>
<review>The product works well overall but the documentation is confusing.</review>
<thinking>
Think step by step:
1. Positive signals: "works well", "overall"
2. Negative signals: "documentation is confusing"
3. The positive aspects describe core functionality, negative is secondary
4. Overall tone: mixed, slightly positive
</thinking>
<output>{"sentiment": "mixed", "positive_score": 0.6, "aspects": ["functionality+", "docs-"]}</output>
"""

# ── JSON mode (structured output) ─────────────────────────────────────────────
struct_response = client.chat.completions.create(
    model="llama-3.3-70b-versatile",
    messages=[
        {"role": "system", "content": "Extract entities. Return JSON only."},
        {"role": "user",   "content": "Elon Musk's SpaceX launched Falcon 9 from Cape Canaveral."},
    ],
    response_format={"type": "json_object"},
)
data = json.loads(struct_response.choices[0].message.content)
# {"persons": ["Elon Musk"], "organizations": ["SpaceX"], "rockets": ["Falcon 9"], ...}

# ── Retry with exponential backoff ────────────────────────────────────────────
def call_with_retry(client, messages, model="llama-3.3-70b-versatile",
                    max_retries=3):
    for attempt in range(max_retries):
        try:
            return client.chat.completions.create(model=model, messages=messages)
        except Exception as e:
            if attempt == max_retries - 1:
                raise
            wait = 2 ** attempt   # 1s, 2s, 4s
            print(f"Attempt {attempt+1} failed: {e}. Retrying in {wait}s...")
            time.sleep(wait)

# ── Embeddings API ─────────────────────────────────────────────────────────────
# (Using OpenAI for embeddings — Groq doesn't offer embeddings API)
from openai import OpenAI
openai_client = OpenAI()

def embed(texts, model="text-embedding-3-small"):
    response = openai_client.embeddings.create(input=texts, model=model)
    return [item.embedding for item in response.data]

doc_embeddings = embed(["Python is great", "I love coding", "The market crashed"])
# Each embedding: list of 1536 floats
# Semantic search: cosine_similarity(query_embedding, doc_embeddings)

# ── Batched inference: process many texts efficiently ─────────────────────────
from transformers import pipeline
import torch

# Use batch_size parameter for throughput
classifier = pipeline(
    "text-classification",
    model="distilbert-base-uncased-finetuned-sst-2-english",
    device=0 if torch.cuda.is_available() else -1,
    batch_size=32,    # Process 32 texts per forward pass
)

texts = ["text1", "text2", ...] * 1000   # 1000 texts
results = classifier(texts)   # ~8x faster than batch_size=1

# ── Quantization with bitsandbytes ────────────────────────────────────────────
from transformers import AutoModelForCausalLM, BitsAndBytesConfig

# INT8 quantization: ~2x memory reduction, minimal quality loss
quantization_config = BitsAndBytesConfig(load_in_8bit=True)

# INT4 quantization: ~4x memory reduction
quantization_config_4bit = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_use_double_quant=True,   # Nested quantization
    bnb_4bit_quant_type="nf4",        # NormalFloat4 — better than uniform INT4
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-3.2-3B",
    quantization_config=quantization_config_4bit,
    device_map="auto",
)
# 3B model: 12GB FP16 → 3GB INT4

# ── Embedding caching ─────────────────────────────────────────────────────────
import hashlib
import json
import os

class EmbeddingCache:
    def __init__(self, cache_dir=".embedding_cache"):
        self.cache_dir = cache_dir
        os.makedirs(cache_dir, exist_ok=True)

    def _key(self, text, model):
        return hashlib.md5(f"{model}:{text}".encode()).hexdigest()

    def get(self, text, model):
        path = os.path.join(self.cache_dir, self._key(text, model) + ".json")
        if os.path.exists(path):
            with open(path) as f:
                return json.load(f)
        return None

    def set(self, text, model, embedding):
        path = os.path.join(self.cache_dir, self._key(text, model) + ".json")
        with open(path, "w") as f:
            json.dump(embedding, f)

# ── Long document handling: chunking strategies ───────────────────────────────
def chunk_fixed(text, chunk_size=512, overlap=50):
    """Fixed-size chunking with overlap."""
    words = text.split()
    chunks = []
    for i in range(0, len(words), chunk_size - overlap):
        chunk = " ".join(words[i:i + chunk_size])
        chunks.append(chunk)
    return chunks

def chunk_sentences(text, max_tokens=200):
    """Sentence-boundary chunking."""
    import re
    sentences = re.split(r'(?<=[.!?])\s+', text)
    chunks = []
    current = []
    current_len = 0
    for sent in sentences:
        sent_len = len(sent.split())
        if current_len + sent_len > max_tokens and current:
            chunks.append(" ".join(current))
            current = [sent]
            current_len = sent_len
        else:
            current.append(sent)
            current_len += sent_len
    if current:
        chunks.append(" ".join(current))
    return chunks