On this page

HTTP and Requests

12 min read TextCh. 5 — Python for Web

HTTP and Requests

Modern applications rarely operate in isolation — they consume REST APIs, send webhooks, call third-party services, and process external data. Python's ecosystem offers excellent HTTP clients: the time-tested requests library and the modern httpx library which supports both synchronous and asynchronous usage. In this lesson you will learn how to make HTTP requests, handle responses and errors robustly, and interact with real-world JSON APIs.

HTTP Fundamentals

Before diving into code, a quick refresher on HTTP:

  • GET — retrieve data. Parameters go in the URL query string.
  • POST — send data to create a resource. Body contains the payload.
  • PUT — replace a resource entirely.
  • PATCH — partially update a resource.
  • DELETE — remove a resource.

A response has a status code (200 OK, 201 Created, 400 Bad Request, 401 Unauthorized, 404 Not Found, 500 Internal Server Error), headers, and a body.

Installing `requests`

pip install requests

Making GET Requests

import requests

# Simple GET request
response = requests.get("https://httpbin.org/get")

# Status code
print(response.status_code)    # 200
print(response.ok)             # True if status_code < 400

# Response body
print(response.text)           # raw string body
print(response.json())         # parse JSON body to dict (raises if not valid JSON)

# Response headers
print(response.headers["Content-Type"])   # application/json
print(response.headers.get("X-Request-Id"))

# URL that was actually requested (after redirects)
print(response.url)

Query Parameters

import requests

params = {
    "q": "python tutorial",
    "page": 1,
    "per_page": 10,
}

# requests builds the query string automatically:
# https://api.example.com/search?q=python+tutorial&page=1&per_page=10
response = requests.get("https://httpbin.org/get", params=params)
data = response.json()
print(data["args"])   # {'page': '1', 'per_page': '10', 'q': 'python tutorial'}

POST Requests — Sending JSON

import requests
import json

# Sending JSON — use json= parameter (sets Content-Type automatically)
payload = {
    "title": "Buy groceries",
    "completed": False,
    "userId": 1,
}

response = requests.post(
    "https://jsonplaceholder.typicode.com/todos",
    json=payload,
)

print(response.status_code)   # 201
created = response.json()
print(created)
# {'title': 'Buy groceries', 'completed': False, 'userId': 1, 'id': 201}

# Sending form data — use data= parameter
form_response = requests.post(
    "https://httpbin.org/post",
    data={"username": "alice", "password": "secret"},
)
print(form_response.json()["form"])

Headers and Authentication

import requests

# Custom headers
headers = {
    "User-Agent": "MyApp/1.0",
    "Accept": "application/json",
    "X-Custom-Header": "value",
}

response = requests.get("https://httpbin.org/headers", headers=headers)
print(response.json())

# Bearer token authentication
token = "your-jwt-token-here"
auth_headers = {"Authorization": f"Bearer {token}"}
response = requests.get("https://api.example.com/profile", headers=auth_headers)

# Basic authentication
from requests.auth import HTTPBasicAuth

response = requests.get(
    "https://httpbin.org/basic-auth/user/pass",
    auth=HTTPBasicAuth("user", "pass"),
)
print(response.status_code)   # 200

Error Handling

Always handle HTTP errors explicitly:

import requests
from requests.exceptions import ConnectionError, Timeout, HTTPError

def fetch_user(user_id: int) -> dict | None:
    try:
        response = requests.get(
            f"https://jsonplaceholder.typicode.com/users/{user_id}",
            timeout=10,   # seconds — always set a timeout!
        )
        response.raise_for_status()   # raises HTTPError for 4xx/5xx
        return response.json()

    except ConnectionError:
        print("Network error — could not connect to server")
        return None
    except Timeout:
        print("Request timed out after 10 seconds")
        return None
    except HTTPError as e:
        print(f"HTTP error {e.response.status_code}: {e.response.text}")
        return None

user = fetch_user(1)
if user:
    print(f"Name: {user['name']}, Email: {user['email']}")

missing = fetch_user(9999)   # 404 Not Found

tip type: warning title: "Always set a timeout"

Never make a requests call without a timeout parameter. Without it, your program will hang indefinitely if the server is slow or unresponsive. A good default for most APIs is timeout=10 (seconds). For long-running operations, use timeout=(3.05, 30) — a tuple of (connect timeout, read timeout).

Using Sessions

A Session object reuses connections (HTTP Keep-Alive) and persists settings across requests — important for performance when making multiple requests to the same host:

import requests

# Session reuses the underlying TCP connection
with requests.Session() as session:
    # Set default headers for all requests in this session
    session.headers.update({
        "User-Agent": "MyClient/1.0",
        "Authorization": "Bearer my-token",
    })

    # All requests use the session's headers automatically
    users = session.get("https://jsonplaceholder.typicode.com/users").json()
    todos = session.get("https://jsonplaceholder.typicode.com/todos?userId=1").json()
    posts = session.get("https://jsonplaceholder.typicode.com/posts?userId=1").json()

    print(f"Users: {len(users)}")
    print(f"Todos for user 1: {len(todos)}")
    print(f"Posts for user 1: {len(posts)}")

The Modern Alternative: `httpx`

httpx is a next-generation HTTP client that is API-compatible with requests but adds async support, HTTP/2, and better type annotations:

pip install httpx

Synchronous `httpx`

import httpx

# Largely compatible with requests
with httpx.Client(timeout=10.0, base_url="https://jsonplaceholder.typicode.com") as client:
    users_response = client.get("/users")
    users_response.raise_for_status()
    users = users_response.json()

    post_response = client.post("/posts", json={"title": "Hello", "body": "World", "userId": 1})
    post_response.raise_for_status()
    new_post = post_response.json()

print(f"Total users: {len(users)}")
print(f"Created post ID: {new_post['id']}")

Asynchronous `httpx`

import asyncio
import httpx

async def fetch_all_users() -> list[dict]:
    async with httpx.AsyncClient(timeout=10.0) as client:
        response = await client.get("https://jsonplaceholder.typicode.com/users")
        response.raise_for_status()
        return response.json()

async def fetch_user_posts(user_id: int, client: httpx.AsyncClient) -> list[dict]:
    response = await client.get(
        "https://jsonplaceholder.typicode.com/posts",
        params={"userId": user_id},
    )
    response.raise_for_status()
    return response.json()

async def main() -> None:
    async with httpx.AsyncClient(timeout=10.0) as client:
        # Fetch all users first
        users_response = await client.get("https://jsonplaceholder.typicode.com/users")
        users = users_response.json()

        # Fetch posts for all users concurrently
        tasks = [fetch_user_posts(user["id"], client) for user in users[:3]]
        all_posts = await asyncio.gather(*tasks)

        for user, posts in zip(users[:3], all_posts):
            print(f"{user['name']}: {len(posts)} posts")

asyncio.run(main())

tip type: info title: "httpx for new projects"

For new projects, prefer httpx over requests. It has the same familiar API, better type annotations, built-in async support, and HTTP/2 support. Use httpx.Client for synchronous code and httpx.AsyncClient for async code (with FastAPI, for example).

Practical Example: A GitHub API Client

import httpx
from typing import NamedTuple

class GitHubRepo(NamedTuple):
    name: str
    stars: int
    forks: int
    language: str | None
    url: str

class GitHubClient:
    BASE_URL = "https://api.github.com"

    def __init__(self, token: str | None = None) -> None:
        headers = {"Accept": "application/vnd.github.v3+json"}
        if token:
            headers["Authorization"] = f"Bearer {token}"
        self._client = httpx.Client(
            base_url=self.BASE_URL,
            headers=headers,
            timeout=15.0,
        )

    def get_user_repos(self, username: str, limit: int = 10) -> list[GitHubRepo]:
        response = self._client.get(
            f"/users/{username}/repos",
            params={"per_page": limit, "sort": "stars"},
        )
        response.raise_for_status()
        return [
            GitHubRepo(
                name=r["name"],
                stars=r["stargazers_count"],
                forks=r["forks_count"],
                language=r.get("language"),
                url=r["html_url"],
            )
            for r in response.json()
        ]

    def close(self) -> None:
        self._client.close()

    def __enter__(self) -> "GitHubClient":
        return self

    def __exit__(self, *args) -> None:
        self.close()

# Usage
with GitHubClient() as github:
    repos = github.get_user_repos("python", limit=5)
    for repo in sorted(repos, key=lambda r: r.stars, reverse=True):
        print(f"  {repo.stars:>6,} stars — {repo.name} ({repo.language})")

nextSteps

  • intro-to-fastapi