Ollama Tutorial: Run LLMs Locally on Your Computer (Complete Setup Guide)

# Download from ollama.com or use Homebrew
brew install ollama

# Start Ollama service
ollama serve

# Or download the Mac app (installs as menubar app)
# https://ollama.com/download/mac

# One-line install
curl -fsSL https://ollama.com/install.sh | sh

# Start service
systemctl start ollama  # systemd

# Or run manually
ollama serve

# winget
winget install Ollama.Ollama

# Or download from ollama.com/download/windows

ollama --version
# ollama version 0.3.x

# Test with a quick run
ollama run phi3 "What is 2+2? Answer in one sentence."

# Pull a model (downloads, doesn't run yet)
ollama pull llama3.1

# Pull specific size
ollama pull llama3.1:70b         # 70B (needs 40GB VRAM)
ollama pull llama3.1:8b          # 8B (needs 8GB RAM)
ollama pull llama3.1:405b-fp16   # Full precision 405B (research only)

# Run models interactively
ollama run llama3.1              # Chat mode
ollama run mistral
ollama run phi3
ollama run codellama             # Coding specialist
ollama run gemma2:9b

# One-shot query (not interactive)
ollama run llama3.1 "Explain transformer attention in one paragraph"

# List downloaded models
ollama list

# Remove a model (free disk space)
ollama rm mistral

# Show model details
ollama show llama3.1

8GB RAM (CPU only):
  ollama pull phi3:mini           # 3.8B, fast, good quality
  ollama pull gemma2:2b           # Google Gemma 2, excellent for size

8-16GB RAM / 8GB VRAM:
  ollama pull llama3.1:8b         # Best 8B general model
  ollama pull mistral             # Great for commercial use (Apache 2.0)
  ollama pull codellama           # Coding tasks
  
24GB VRAM (RTX 3090/4090):
  ollama pull llama3.1:70b        # Near GPT-3.5 quality
  ollama pull mixtral:8x7b        # Mixtral MoE, very capable

Apple Silicon (M1/M2/M3, 16GB+):
  ollama pull llama3.1:8b         # Very fast on Metal GPU
  ollama pull llama3.1:70b        # M2 Ultra 192GB can handle this

# Generate endpoint (simple completion)
curl http://localhost:11434/api/generate -d '{
  "model": "llama3.1",
  "prompt": "Why is the sky blue?",
  "stream": false
}'

# Chat endpoint (conversation)
curl http://localhost:11434/api/chat -d '{
  "model": "llama3.1",
  "messages": [
    {"role": "user", "content": "What is machine learning?"}
  ],
  "stream": false
}'

# pip install ollama

import ollama

# Simple generation
response = ollama.generate(
    model="llama3.1",
    prompt="Explain neural networks in simple terms.",
)
print(response["response"])

# Chat completion
response = ollama.chat(
    model="llama3.1",
    messages=[
        {"role": "system", "content": "You are a helpful Python tutor."},
        {"role": "user", "content": "Explain list comprehensions with an example."}
    ]
)
print(response["message"]["content"])

# Streaming response
stream = ollama.chat(
    model="llama3.1",
    messages=[{"role": "user", "content": "Write a bubble sort function"}],
    stream=True
)

for chunk in stream:
    print(chunk["message"]["content"], end="", flush=True)
print()  # Newline at end

# List available models
models = ollama.list()
for model in models["models"]:
    print(f"{model['name']}: {model['size'] / 1e9:.1f} GB")

# Embeddings (for semantic search)
embedding = ollama.embeddings(
    model="nomic-embed-text",  # Pull first: ollama pull nomic-embed-text
    prompt="Hello world"
)
print(f"Embedding dimensions: {len(embedding['embedding'])}")

from openai import OpenAI

# Point to local Ollama instead of OpenAI
client = OpenAI(
    base_url="http://localhost:11434/v1",
    api_key="ollama"  # Required but value doesn't matter
)

# Exact same code as OpenAI — just different base_url
response = client.chat.completions.create(
    model="llama3.1",  # Use any Ollama model name
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "What are the main benefits of Python?"}
    ],
    temperature=0.7
)

print(response.choices[0].message.content)
print(f"Input tokens: {response.usage.prompt_tokens}")
print(f"Output tokens: {response.usage.completion_tokens}")

# Streaming
stream = client.chat.completions.create(
    model="mistral",
    messages=[{"role": "user", "content": "Write a FastAPI hello world endpoint"}],
    stream=True
)

for chunk in stream:
    content = chunk.choices[0].delta.content
    if content:
        print(content, end="", flush=True)

# Create a Modelfile
cat > Modelfile << 'EOF'
FROM llama3.1

# Set a custom system prompt
SYSTEM """
You are an expert Python developer who:
- Writes clean, Pythonic code following PEP 8
- Always includes type hints
- Adds brief docstrings for non-obvious functions
- Prefers standard library solutions when available
- Points out potential issues and edge cases
"""

# Lower temperature for more deterministic code
PARAMETER temperature 0.2

# Larger context for long files
PARAMETER num_ctx 8192

# Stop tokens
PARAMETER stop "<|start_header_id|>"
PARAMETER stop "<|end_header_id|>"
PARAMETER stop "<|eot_id|>"
EOF

# Build the custom model
ollama create python-expert -f Modelfile

# Run your custom model
ollama run python-expert "Write a function to parse CSV files with error handling"

# List to confirm it's there
ollama list
# NAMES           ID            SIZE    MODIFIED
# python-expert   abc123...     4.7 GB  2 minutes ago
# llama3.1        ...           4.7 GB  ...

# Customer service assistant
cat > customer-service.Modelfile << 'EOF'
FROM mistral

SYSTEM """
You are a helpful customer service representative for TechStore.
Store Hours: Monday-Friday 9AM-6PM EST, Saturday 10AM-4PM EST
Return Policy: 30 days with receipt, items must be unused
Warranty: 1 year manufacturer warranty on all electronics
Shipping: Free standard shipping on orders over $50

Always be polite, empathetic, and solution-focused.
If you don't know the answer, say so and offer to escalate.
"""

PARAMETER temperature 0.3
PARAMETER num_ctx 4096
EOF

ollama create customer-service -f customer-service.Modelfile

# Code reviewer
cat > code-reviewer.Modelfile << 'EOF'
FROM codellama

SYSTEM """
You are a senior code reviewer. For any code shared:
1. Identify bugs and logic errors
2. Point out security vulnerabilities
3. Suggest performance improvements
4. Note readability/maintainability issues
5. Give specific examples of improved code

Be constructive and specific. Format findings as numbered lists.
"""

PARAMETER temperature 0.1
EOF

ollama create code-reviewer -f code-reviewer.Modelfile

# pip install ollama chromadb langchain-community

from langchain_community.llms import Ollama
from langchain_community.embeddings import OllamaEmbeddings
from langchain_community.vectorstores import Chroma
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.chains import RetrievalQA
from langchain_community.document_loaders import DirectoryLoader, TextLoader

# 1. Load documents
loader = DirectoryLoader("./my_docs/", glob="**/*.txt", loader_cls=TextLoader)
documents = loader.load()

# 2. Split documents
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
chunks = splitter.split_documents(documents)

# 3. Create local embeddings (no API key needed)
# ollama pull nomic-embed-text
embeddings = OllamaEmbeddings(model="nomic-embed-text")

# 4. Create vector store
vectorstore = Chroma.from_documents(
    documents=chunks,
    embedding=embeddings,
    persist_directory="./local_chroma"
)

# 5. Create RAG chain with local LLM
llm = Ollama(model="llama3.1", temperature=0)

qa_chain = RetrievalQA.from_chain_type(
    llm=llm,
    retriever=vectorstore.as_retriever(search_kwargs={"k": 3})
)

# 6. Query — fully local, no API costs
result = qa_chain.invoke({"query": "What are the main topics in these documents?"})
print(result["result"])

# GPU acceleration is automatic when GPU is detected
# Check what's being used
ollama ps  # Shows running models and which GPU layer they're using

# Increase GPU layers for faster inference
OLLAMA_NUM_GPU=999 ollama serve  # Use all available GPU layers

# Keep model loaded in memory (avoid reload delay between requests)
# Set OLLAMA_KEEP_ALIVE duration (default: 5m)
OLLAMA_KEEP_ALIVE=1h ollama serve

# For Apple Silicon: ensure Metal GPU is used
# It's automatic, but check with:
ollama run llama3.1 --verbose "hi" 2>&1 | grep "metal"