NLP for Beginners: How Computers Learn to Understand Language

Raw Text
    ↓
Preprocessing (cleaning, normalization)
    ↓
Tokenization (split into units)
    ↓
Feature Extraction (convert to numbers)
    ↓
Model (process the numerical representation)
    ↓
Output (classification, generation, etc.)

import re
import string

def preprocess_text(text):
    # Lowercase
    text = text.lower()
    
    # Remove URLs
    text = re.sub(r'http\S+|www\S+', '', text)
    
    # Remove special characters (keep letters and spaces)
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    
    # Remove extra whitespace
    text = ' '.join(text.split())
    
    return text

examples = [
    "Check out https://example.com for AMAZING deals!!!",
    "The movie wasn't good... or was it?? 🤔"
]

for text in examples:
    print(f"Original: {text}")
    print(f"Cleaned:  {preprocess_text(text)}\n")

# Simple word tokenization
text = "Natural language processing is fascinating."
tokens = text.split()  # ['Natural', 'language', 'processing', 'is', 'fascinating.']

# Better: using NLTK
import nltk
from nltk.tokenize import word_tokenize
nltk.download('punkt')

tokens = word_tokenize(text)
# ['Natural', 'language', 'processing', 'is', 'fascinating', '.']

# Subword tokenization (modern approach — used in BERT, GPT)
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
tokens = tokenizer.tokenize("unhappiness")
# ['un', '##happy', '##ness']  (## means continuation of a word)

from sklearn.feature_extraction.text import CountVectorizer

documents = [
    "I love this movie",
    "This movie is terrible",
    "Great film, loved it"
]

vectorizer = CountVectorizer()
X = vectorizer.fit_transform(documents)

print("Vocabulary:", vectorizer.vocabulary_)
print("Document-term matrix:\n", X.toarray())
# Each row = a document
# Each column = a word
# Value = word count in that document

from sklearn.feature_extraction.text import TfidfVectorizer

# TF-IDF weights words by how distinctive they are
# Common words (the, is, and) get low weight
# Distinctive words get high weight

tfidf = TfidfVectorizer(max_features=1000, stop_words='english')
X_tfidf = tfidf.fit_transform(documents)

# Word2Vec: words with similar meaning have similar vectors
# "king" - "man" + "woman" ≈ "queen"

from gensim.models import Word2Vec

# Train on your corpus
sentences = [["I", "love", "machine", "learning"],
             ["deep", "learning", "is", "powerful"]]
model = Word2Vec(sentences, vector_size=100, window=5, min_count=1)

# Get vector for a word
vector = model.wv['learning']  # 100-dimensional vector

# Find similar words
similar = model.wv.most_similar('learning', topn=5)

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import classification_report

# Sample dataset (in practice, use real labeled data)
# You can download movie reviews dataset from sklearn or NLTK
from sklearn.datasets import fetch_20newsgroups

# Or create a simple dataset
data = {
    'text': [
        "This movie was absolutely fantastic! Loved every minute.",
        "Terrible film, waste of two hours of my life.",
        "Pretty decent, some good moments but also slow parts.",
        "Outstanding performance by all actors. Highly recommend!",
        "Boring and predictable. Skip this one.",
        "A masterpiece of storytelling and cinematography.",
        "Not worth watching. Very disappointing."
    ],
    'sentiment': [1, 0, 1, 1, 0, 1, 0]  # 1=positive, 0=negative
}
df = pd.DataFrame(data)

# Split data
X_train, X_test, y_train, y_test = train_test_split(
    df['text'], df['sentiment'], test_size=0.3, random_state=42
)

# TF-IDF vectorization
vectorizer = TfidfVectorizer(
    max_features=5000,
    ngram_range=(1, 2),  # Include bigrams like "not good", "very bad"
    stop_words='english'
)
X_train_tfidf = vectorizer.fit_transform(X_train)
X_test_tfidf = vectorizer.transform(X_test)

# Train classifier
clf = LogisticRegression(random_state=42)
clf.fit(X_train_tfidf, y_train)

# Evaluate
y_pred = clf.predict(X_test_tfidf)
print(classification_report(y_test, y_pred, target_names=['Negative', 'Positive']))

# Predict on new text
def predict_sentiment(text):
    tfidf = vectorizer.transform([text])
    pred = clf.predict(tfidf)[0]
    prob = clf.predict_proba(tfidf)[0]
    sentiment = "Positive" if pred == 1 else "Negative"
    confidence = max(prob)
    return sentiment, confidence

sentiment, conf = predict_sentiment("The special effects were amazing but the plot was weak")
print(f"Sentiment: {sentiment} ({conf:.2%} confidence)")

Traditional NLP: Words are independent
"The bank approved the loan" — "bank" is just a word

Transformer NLP: Context-aware representations
"The bank approved the loan" — "bank" is understood as financial
"The river bank was eroded" — "bank" is understood as geographic

from transformers import pipeline

# Sentiment analysis with pretrained BERT
sentiment_analyzer = pipeline(
    "sentiment-analysis",
    model="distilbert-base-uncased-finetuned-sst-2-english"
)

texts = [
    "I absolutely loved this product, works perfectly!",
    "Complete waste of money, broke after one week.",
    "It's okay, nothing special but does the job."
]

for text in texts:
    result = sentiment_analyzer(text)[0]
    print(f"Text: {text}")
    print(f"Sentiment: {result['label']} ({result['score']:.3f})\n")

Text: I absolutely loved this product, works perfectly!
Sentiment: POSITIVE (0.9998)

Text: Complete waste of money, broke after one week.
Sentiment: NEGATIVE (0.9998)

Text: It's okay, nothing special but does the job.
Sentiment: POSITIVE (0.7123)

from transformers import AutoModelForSequenceClassification, AutoTokenizer
from transformers import TrainingArguments, Trainer
import torch
from torch.utils.data import Dataset

class TextDataset(Dataset):
    def __init__(self, texts, labels, tokenizer, max_length=512):
        self.encodings = tokenizer(
            texts, 
            truncation=True, 
            padding=True, 
            max_length=max_length
        )
        self.labels = labels
    
    def __len__(self):
        return len(self.labels)
    
    def __getitem__(self, idx):
        item = {k: torch.tensor(v[idx]) for k, v in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

# Initialize model and tokenizer
model_name = "distilbert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)

# Create datasets
train_dataset = TextDataset(train_texts, train_labels, tokenizer)
test_dataset = TextDataset(test_texts, test_labels, tokenizer)

# Training arguments
training_args = TrainingArguments(
    output_dir='./results',
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=32,
    warmup_steps=100,
    weight_decay=0.01,
    evaluation_strategy='epoch'
)

# Train
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=test_dataset
)
trainer.train()

import spacy

# Load English language model
nlp = spacy.load("en_core_web_sm")

text = "Apple Inc. was founded by Steve Jobs in Cupertino, California in 1976."
doc = nlp(text)

# Named Entity Recognition
print("Entities:")
for ent in doc.ents:
    print(f"  {ent.text}: {ent.label_} ({spacy.explain(ent.label_)})")
# Output:
# Apple Inc.: ORG (Companies, agencies, institutions)
# Steve Jobs: PERSON (People, including fictional)
# Cupertino: GPE (Countries, cities, states)
# California: GPE
# 1976: DATE

# Part-of-speech tagging
print("\nParts of speech:")
for token in doc:
    print(f"  {token.text:15} {token.pos_:6} {token.dep_}")

Beginner:
1. Python and pandas basics (2-4 weeks)
2. Text preprocessing with NLTK (1 week)
3. Traditional ML for text: TF-IDF + Logistic Regression
4. First project: sentiment classifier on movie reviews

Intermediate:
5. Word embeddings: Word2Vec, GloVe
6. Introduction to Transformers (3Blue1Brown attention video)
7. Hugging Face Transformers library
8. Fine-tune BERT on your own dataset

Advanced:
9. Build custom NLP pipelines with spaCy
10. Implement Transformer architecture from scratch
11. Explore NLP research on Hugging Face/Papers With Code