Python Web Scraping Guide 2026 — BeautifulSoup, Requests & Playwright

pip install requests beautifulsoup4 lxml playwright
playwright install chromium

import requests
from bs4 import BeautifulSoup

def scrape_page(url: str) -> BeautifulSoup:
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
    }
    response = requests.get(url, headers=headers, timeout=10)
    response.raise_for_status()  # Raises exception for 4xx/5xx errors
    return BeautifulSoup(response.content, "lxml")

# Example: scrape a Wikipedia page
soup = scrape_page("https://en.wikipedia.org/wiki/Python_(programming_language)")
title = soup.find("h1", id="firstHeading").text
print(f"Title: {title}")

soup = scrape_page("https://books.toscrape.com")

# Find by tag
all_h1 = soup.find_all("h1")

# Find by CSS class
books = soup.find_all("article", class_="product_pod")

# Find by attribute
link = soup.find("a", href=True)

# CSS selector (most flexible)
prices = soup.select("p.price_color")
titles = soup.select("h3 > a")

for i, (title, price) in enumerate(zip(titles[:5], prices[:5])):
    print(f"{i+1}. {title['title']} — {price.text.strip()}")

import requests
from bs4 import BeautifulSoup
import csv
import time

BASE_URL = "https://books.toscrape.com/catalogue/"

def scrape_books(max_pages: int = 5) -> list[dict]:
    books = []
    
    for page in range(1, max_pages + 1):
        url = f"https://books.toscrape.com/catalogue/page-{page}.html"
        soup = scrape_page(url)
        
        for article in soup.select("article.product_pod"):
            title = article.select_one("h3 > a")["title"]
            price = article.select_one("p.price_color").text.strip()
            rating_word = article.select_one("p.star-rating")["class"][1]
            
            books.append({
                "title": title,
                "price": price,
                "rating": rating_word,
            })
        
        print(f"Scraped page {page} — {len(books)} books so far")
        time.sleep(1)  # Be polite — don't hammer the server
    
    return books

def save_to_csv(books: list[dict], filename: str) -> None:
    with open(filename, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=["title", "price", "rating"])
        writer.writeheader()
        writer.writerows(books)
    print(f"Saved {len(books)} books to {filename}")

books = scrape_books(max_pages=3)
save_to_csv(books, "books.csv")

from playwright.sync_api import sync_playwright

def scrape_dynamic(url: str) -> str:
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        
        # Set realistic viewport and user agent
        page.set_viewport_size({"width": 1280, "height": 720})
        
        page.goto(url, wait_until="networkidle")  # Wait for all requests to finish
        content = page.content()  # Get full rendered HTML
        
        browser.close()
        return content

from playwright.sync_api import sync_playwright
from bs4 import BeautifulSoup

def scrape_spa(url: str, wait_selector: str) -> BeautifulSoup:
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(url)
        
        # Wait until the specific element appears
        page.wait_for_selector(wait_selector, timeout=10000)
        
        html = page.content()
        browser.close()
    
    return BeautifulSoup(html, "lxml")

# Example: wait for a product grid to load
soup = scrape_spa("https://example-shop.com/products", ".product-grid")
products = soup.select(".product-card")

from playwright.sync_api import sync_playwright

def search_and_scrape(query: str) -> list[dict]:
    results = []
    
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto("https://books.toscrape.com")
        
        # Type in a search box
        page.fill("input[name='q']", query)
        page.press("input[name='q']", "Enter")
        page.wait_for_load_state("networkidle")
        
        # Extract results
        for item in page.query_selector_all(".product_pod"):
            title = item.query_selector("h3 > a").get_attribute("title")
            price = item.query_selector(".price_color").inner_text()
            results.append({"title": title, "price": price})
        
        browser.close()
    
    return results

import requests
from bs4 import BeautifulSoup
import time

def scrape_all_pages(base_url: str) -> list[dict]:
    items = []
    page = 1
    
    while True:
        url = f"{base_url}?page={page}"
        response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10)
        
        if response.status_code == 404:
            print(f"Reached end at page {page}")
            break
        
        soup = BeautifulSoup(response.content, "lxml")
        products = soup.select(".product-item")
        
        if not products:
            break  # No more items
        
        for product in products:
            items.append({
                "name": product.select_one(".name").text.strip(),
                "price": product.select_one(".price").text.strip(),
            })
        
        print(f"Page {page}: {len(products)} items")
        page += 1
        time.sleep(0.5)  # Rate limiting
    
    return items

import csv

def save_csv(data: list[dict], filename: str) -> None:
    if not data:
        return
    with open(filename, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=data[0].keys())
        writer.writeheader()
        writer.writerows(data)

import json

def save_json(data: list[dict], filename: str) -> None:
    with open(filename, "w", encoding="utf-8") as f:
        json.dump(data, f, indent=2, ensure_ascii=False)

import sqlite3

def save_to_db(data: list[dict], db_path: str, table: str) -> None:
    conn = sqlite3.connect(db_path)
    cursor = conn.cursor()
    
    if data:
        columns = ", ".join(data[0].keys())
        placeholders = ", ".join(["?" for _ in data[0]])
        cursor.execute(f"CREATE TABLE IF NOT EXISTS {table} ({columns})")
        
        rows = [tuple(row.values()) for row in data]
        cursor.executemany(f"INSERT INTO {table} VALUES ({placeholders})", rows)
    
    conn.commit()
    conn.close()
    print(f"Saved {len(data)} rows to {db_path}")

import urllib.robotparser

def is_allowed(url: str, user_agent: str = "*") -> bool:
    rp = urllib.robotparser.RobotFileParser()
    from urllib.parse import urljoin, urlparse
    base = f"{urlparse(url).scheme}://{urlparse(url).netloc}"
    rp.set_url(urljoin(base, "/robots.txt"))
    rp.read()
    return rp.can_fetch(user_agent, url)

import time
import random

def polite_get(url: str, min_delay: float = 1.0, max_delay: float = 3.0) -> requests.Response:
    time.sleep(random.uniform(min_delay, max_delay))
    response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=15)
    return response