Skip to content
  • Sponsor TheAlgorithms/Python

  • Notifications You must be signed in to change notification settings
  • Fork 47k

Lukazlim: Replace dependency requests with httpx #12744

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 14, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions DIRECTORY.md
Original file line number Diff line number Diff line change
@@ -899,6 +899,7 @@
* [N Body Simulation](physics/n_body_simulation.py)
* [Newtons Law Of Gravitation](physics/newtons_law_of_gravitation.py)
* [Newtons Second Law Of Motion](physics/newtons_second_law_of_motion.py)
* [Orbital Transfer Work](physics/orbital_transfer_work.py)
* [Period Of Pendulum](physics/period_of_pendulum.py)
* [Photoelectric Effect](physics/photoelectric_effect.py)
* [Potential Energy](physics/potential_energy.py)
12 changes: 10 additions & 2 deletions machine_learning/linear_regression.py
Original file line number Diff line number Diff line change
@@ -8,16 +8,24 @@
Rating). We try to best fit a line through dataset and estimate the parameters.
"""

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# "numpy",
# ]
# ///

import httpx
import numpy as np
import requests


def collect_dataset():
"""Collect dataset of CSGO
The dataset contains ADR vs Rating of a Player
:return : dataset obtained from the link, as matrix
"""
response = requests.get(
response = httpx.get(
"https://raw.githubusercontent.com/yashLadha/The_Math_of_Intelligence/"
"master/Week1/ADRvsRating.csv",
timeout=10,
8 changes: 4 additions & 4 deletions physics/speeds_of_gas_molecules.py
Original file line number Diff line number Diff line change
@@ -59,9 +59,9 @@ def avg_speed_of_molecule(temperature: float, molar_mass: float) -> float:
Examples:
>>> avg_speed_of_molecule(273, 0.028) # nitrogen at 273 K
454.3488755020387
454.3488755062257
>>> avg_speed_of_molecule(300, 0.032) # oxygen at 300 K
445.52572733919885
445.5257273433045
>>> avg_speed_of_molecule(-273, 0.028) # invalid temperature
Traceback (most recent call last):
...
@@ -87,9 +87,9 @@ def mps_speed_of_molecule(temperature: float, molar_mass: float) -> float:
Examples:
>>> mps_speed_of_molecule(273, 0.028) # nitrogen at 273 K
402.65620701908966
402.65620702280023
>>> mps_speed_of_molecule(300, 0.032) # oxygen at 300 K
394.836895549922
394.8368955535605
>>> mps_speed_of_molecule(-273, 0.028) # invalid temperature
Traceback (most recent call last):
...
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@ classifiers = [
dependencies = [
"beautifulsoup4>=4.12.3",
"fake-useragent>=1.5.1",
"httpx>=0.28.1",
"imageio>=2.36.1",
"keras>=3.7",
"lxml>=5.3",
@@ -19,7 +20,6 @@ dependencies = [
"opencv-python>=4.10.0.84",
"pandas>=2.2.3",
"pillow>=11",
"requests>=2.32.3",
"rich>=13.9.4",
"scikit-learn>=1.5.2",
"sphinx-pyproject>=0.3",
@@ -42,8 +42,8 @@ docs = [
"sphinx-pyproject>=0.3",
]
euler-validate = [
"httpx>=0.28.1",
"numpy>=2.1.3",
"requests>=2.32.3",
]

[tool.ruff]
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
beautifulsoup4
fake-useragent
httpx
imageio
keras
lxml
@@ -8,7 +9,6 @@ numpy
opencv-python
pandas
pillow
requests
rich
scikit-learn
sphinx-pyproject
6 changes: 3 additions & 3 deletions scripts/validate_solutions.py
Original file line number Diff line number Diff line change
@@ -3,8 +3,8 @@
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# "pytest",
# "requests",
# ]
# ///

@@ -15,8 +15,8 @@
import pathlib
from types import ModuleType

import httpx
import pytest
import requests

PROJECT_EULER_DIR_PATH = pathlib.Path.cwd().joinpath("project_euler")
PROJECT_EULER_ANSWERS_PATH = pathlib.Path.cwd().joinpath(
@@ -66,7 +66,7 @@ def added_solution_file_path() -> list[pathlib.Path]:
"Accept": "application/vnd.github.v3+json",
"Authorization": "token " + os.environ["GITHUB_TOKEN"],
}
files = requests.get(get_files_url(), headers=headers, timeout=10).json()
files = httpx.get(get_files_url(), headers=headers, timeout=10).json()
for file in files:
filepath = pathlib.Path.cwd().joinpath(file["filename"])
if (
1,108 changes: 582 additions & 526 deletions uv.lock

Large diffs are not rendered by default.

13 changes: 10 additions & 3 deletions web_programming/co2_emission.py
Original file line number Diff line number Diff line change
@@ -2,22 +2,29 @@
Get CO2 emission data from the UK CarbonIntensity API
"""

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

from datetime import date

import requests
import httpx

BASE_URL = "https://api.carbonintensity.org.uk/intensity"


# Emission in the last half hour
def fetch_last_half_hour() -> str:
last_half_hour = requests.get(BASE_URL, timeout=10).json()["data"][0]
last_half_hour = httpx.get(BASE_URL, timeout=10).json()["data"][0]
return last_half_hour["intensity"]["actual"]


# Emissions in a specific date range
def fetch_from_to(start, end) -> list:
return requests.get(f"{BASE_URL}/{start}/{end}", timeout=10).json()["data"]
return httpx.get(f"{BASE_URL}/{start}/{end}", timeout=10).json()["data"]


if __name__ == "__main__":
12 changes: 10 additions & 2 deletions web_programming/covid_stats_via_xpath.py
Original file line number Diff line number Diff line change
@@ -4,9 +4,17 @@
more convenient to use in Python web projects (e.g. Django or Flask-based)
"""

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# "lxml",
# ]
# ///

from typing import NamedTuple

import requests
import httpx
from lxml import html


@@ -19,7 +27,7 @@ class CovidData(NamedTuple):
def covid_stats(url: str = "https://www.worldometers.info/coronavirus/") -> CovidData:
xpath_str = '//div[@class = "maincounter-number"]/span/text()'
return CovidData(
*html.fromstring(requests.get(url, timeout=10).content).xpath(xpath_str)
*html.fromstring(httpx.get(url, timeout=10).content).xpath(xpath_str)
)


18 changes: 16 additions & 2 deletions web_programming/crawl_google_results.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,28 @@
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "fake-useragent",
# "httpx",
# ]
# ///

import sys
import webbrowser

import requests
import httpx
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

if __name__ == "__main__":
print("Googling.....")
url = "https://www.google.com/search?q=" + " ".join(sys.argv[1:])
res = requests.get(url, headers={"UserAgent": UserAgent().random}, timeout=10)
res = httpx.get(
url,
headers={"UserAgent": UserAgent().random},
timeout=10,
follow_redirects=True,
)
# res.raise_for_status()
with open("project1a.html", "wb") as out_file: # only for knowing the class
for data in res.iter_content(10000):
12 changes: 10 additions & 2 deletions web_programming/crawl_google_scholar_citation.py
Original file line number Diff line number Diff line change
@@ -3,7 +3,15 @@
using title and year of publication, and volume and pages of journal.
"""

import requests
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "httpx",
# ]
# ///

import httpx
from bs4 import BeautifulSoup


@@ -12,7 +20,7 @@ def get_citation(base_url: str, params: dict) -> str:
Return the citation number.
"""
soup = BeautifulSoup(
requests.get(base_url, params=params, timeout=10).content, "html.parser"
httpx.get(base_url, params=params, timeout=10).content, "html.parser"
)
div = soup.find("div", attrs={"class": "gs_ri"})
anchors = div.find("div", attrs={"class": "gs_fl"}).find_all("a")
11 changes: 9 additions & 2 deletions web_programming/currency_converter.py
Original file line number Diff line number Diff line change
@@ -3,9 +3,16 @@
https://www.amdoren.com
"""

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

import os

import requests
import httpx

URL_BASE = "https://www.amdoren.com/api/currency.php"

@@ -176,7 +183,7 @@ def convert_currency(
params = locals()
# from is a reserved keyword
params["from"] = params.pop("from_")
res = requests.get(URL_BASE, params=params, timeout=10).json()
res = httpx.get(URL_BASE, params=params, timeout=10).json()
return str(res["amount"]) if res["error"] == 0 else res["error_message"]


14 changes: 11 additions & 3 deletions web_programming/current_stock_price.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,12 @@
import requests
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "httpx",
# ]
# ///

import httpx
from bs4 import BeautifulSoup

"""
@@ -20,8 +28,8 @@ def stock_price(symbol: str = "AAPL") -> str:
True
"""
url = f"https://finance.yahoo.com/quote/{symbol}?p={symbol}"
yahoo_finance_source = requests.get(
url, headers={"USER-AGENT": "Mozilla/5.0"}, timeout=10
yahoo_finance_source = httpx.get(
url, headers={"USER-AGENT": "Mozilla/5.0"}, timeout=10, follow_redirects=True
).text
soup = BeautifulSoup(yahoo_finance_source, "html.parser")

13 changes: 10 additions & 3 deletions web_programming/current_weather.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
import requests
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

import httpx

# Put your API key(s) here
OPENWEATHERMAP_API_KEY = ""
@@ -19,13 +26,13 @@ def current_weather(location: str) -> list[dict]:
weather_data = []
if OPENWEATHERMAP_API_KEY:
params_openweathermap = {"q": location, "appid": OPENWEATHERMAP_API_KEY}
response_openweathermap = requests.get(
response_openweathermap = httpx.get(
OPENWEATHERMAP_URL_BASE, params=params_openweathermap, timeout=10
)
weather_data.append({"OpenWeatherMap": response_openweathermap.json()})
if WEATHERSTACK_API_KEY:
params_weatherstack = {"query": location, "access_key": WEATHERSTACK_API_KEY}
response_weatherstack = requests.get(
response_weatherstack = httpx.get(
WEATHERSTACK_URL_BASE, params=params_weatherstack, timeout=10
)
weather_data.append({"Weatherstack": response_weatherstack.json()})
12 changes: 10 additions & 2 deletions web_programming/daily_horoscope.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,12 @@
import requests
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "httpx",
# ]
# ///

import httpx
from bs4 import BeautifulSoup


@@ -7,7 +15,7 @@ def horoscope(zodiac_sign: int, day: str) -> str:
"https://www.horoscope.com/us/horoscopes/general/"
f"horoscope-general-daily-{day}.aspx?sign={zodiac_sign}"
)
soup = BeautifulSoup(requests.get(url, timeout=10).content, "html.parser")
soup = BeautifulSoup(httpx.get(url, timeout=10).content, "html.parser")
return soup.find("div", class_="main-horoscope").p.text


12 changes: 10 additions & 2 deletions web_programming/download_images_from_google_query.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "httpx",
# ]
# ///

import json
import os
import re
import sys
import urllib.request

import requests
import httpx
from bs4 import BeautifulSoup

headers = {
@@ -39,7 +47,7 @@ def download_images_from_google_query(query: str = "dhaka", max_images: int = 5)
"ijn": "0",
}

html = requests.get(
html = httpx.get(
"https://www.google.com/search", params=params, headers=headers, timeout=10
)
soup = BeautifulSoup(html.text, "html.parser")
21 changes: 17 additions & 4 deletions web_programming/emails_from_url.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
"""Get the site emails from URL."""

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

from __future__ import annotations

__author__ = "Muhammad Umer Farooq"
@@ -13,7 +20,7 @@
from html.parser import HTMLParser
from urllib import parse

import requests
import httpx


class Parser(HTMLParser):
@@ -72,7 +79,7 @@ def emails_from_url(url: str = "https://github.com") -> list[str]:

try:
# Open URL
r = requests.get(url, timeout=10)
r = httpx.get(url, timeout=10, follow_redirects=True)

# pass the raw HTML to the parser to get links
parser.feed(r.text)
@@ -81,9 +88,15 @@ def emails_from_url(url: str = "https://github.com") -> list[str]:
valid_emails = set()
for link in parser.urls:
# open URL.
# read = requests.get(link)
# Check if the link is already absolute
if not link.startswith("http://") and not link.startswith("https://"):
# Prepend protocol only if link starts with domain, normalize otherwise
if link.startswith(domain):
link = f"https://{link}"
else:
link = parse.urljoin(f"https://{domain}", link)
try:
read = requests.get(link, timeout=10)
read = httpx.get(link, timeout=10, follow_redirects=True)
# Get the valid email.
emails = re.findall("[a-zA-Z0-9]+@" + domain, read.text)
# If not in list then append it.
21 changes: 15 additions & 6 deletions web_programming/fetch_anime_and_play.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
import requests
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "fake-useragent",
# "httpx",
# ]
# ///

import httpx
from bs4 import BeautifulSoup, NavigableString, Tag
from fake_useragent import UserAgent

BASE_URL = "https://ww1.gogoanime2.org"
BASE_URL = "https://ww7.gogoanime2.org"


def search_scraper(anime_name: str) -> list:
@@ -25,9 +34,9 @@ def search_scraper(anime_name: str) -> list:
"""

# concat the name to form the search url.
search_url = f"{BASE_URL}/search/{anime_name}"
search_url = f"{BASE_URL}/search?keyword={anime_name}"

response = requests.get(
response = httpx.get(
search_url, headers={"UserAgent": UserAgent().chrome}, timeout=10
) # request the url.

@@ -82,7 +91,7 @@ def search_anime_episode_list(episode_endpoint: str) -> list:

request_url = f"{BASE_URL}{episode_endpoint}"

response = requests.get(
response = httpx.get(
url=request_url, headers={"UserAgent": UserAgent().chrome}, timeout=10
)
response.raise_for_status()
@@ -133,7 +142,7 @@ def get_anime_episode(episode_endpoint: str) -> list:

episode_page_url = f"{BASE_URL}{episode_endpoint}"

response = requests.get(
response = httpx.get(
url=episode_page_url, headers={"User-Agent": UserAgent().chrome}, timeout=10
)
response.raise_for_status()
11 changes: 9 additions & 2 deletions web_programming/fetch_bbc_news.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,20 @@
# Created by sarathkaul on 12/11/19

import requests
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

import httpx

_NEWS_API = "https://newsapi.org/v1/articles?source=bbc-news&sortBy=top&apiKey="


def fetch_bbc_news(bbc_news_api_key: str) -> None:
# fetching a list of articles in json format
bbc_news_page = requests.get(_NEWS_API + bbc_news_api_key, timeout=10).json()
bbc_news_page = httpx.get(_NEWS_API + bbc_news_api_key, timeout=10).json()
# each article in the list is a dict
for i, article in enumerate(bbc_news_page["articles"], 1):
print(f"{i}.) {article['title']}")
13 changes: 10 additions & 3 deletions web_programming/fetch_github_info.py
Original file line number Diff line number Diff line change
@@ -18,12 +18,19 @@
export USER_TOKEN=""
"""

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

from __future__ import annotations

import os
from typing import Any

import requests
import httpx

BASE_URL = "https://api.github.com"

@@ -36,13 +43,13 @@

def fetch_github_info(auth_token: str) -> dict[Any, Any]:
"""
Fetch GitHub info of a user using the requests module
Fetch GitHub info of a user using the httpx module
"""
headers = {
"Authorization": f"token {auth_token}",
"Accept": "application/vnd.github.v3+json",
}
return requests.get(AUTHENTICATED_USER_ENDPOINT, headers=headers, timeout=10).json()
return httpx.get(AUTHENTICATED_USER_ENDPOINT, headers=headers, timeout=10).json()


if __name__ == "__main__": # pragma: no cover
14 changes: 10 additions & 4 deletions web_programming/fetch_jobs.py
Original file line number Diff line number Diff line change
@@ -2,20 +2,26 @@
Scraping jobs given job title and location from indeed website
"""

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "httpx",
# ]
# ///

from __future__ import annotations

from collections.abc import Generator

import requests
import httpx
from bs4 import BeautifulSoup

url = "https://www.indeed.co.in/jobs?q=mobile+app+development&l="


def fetch_jobs(location: str = "mumbai") -> Generator[tuple[str, str]]:
soup = BeautifulSoup(
requests.get(url + location, timeout=10).content, "html.parser"
)
soup = BeautifulSoup(httpx.get(url + location, timeout=10).content, "html.parser")
# This attribute finds out all the specifics listed in a job
for job in soup.find_all("div", attrs={"data-tn-component": "organicJob"}):
job_title = job.find("a", attrs={"data-tn-element": "jobTitle"}).text.strip()
13 changes: 10 additions & 3 deletions web_programming/fetch_quotes.py
Original file line number Diff line number Diff line change
@@ -6,19 +6,26 @@
https://zenquotes.io/
"""

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

import pprint

import requests
import httpx

API_ENDPOINT_URL = "https://zenquotes.io/api"


def quote_of_the_day() -> list:
return requests.get(API_ENDPOINT_URL + "/today", timeout=10).json()
return httpx.get(API_ENDPOINT_URL + "/today", timeout=10).json()


def random_quotes() -> list:
return requests.get(API_ENDPOINT_URL + "/random", timeout=10).json()
return httpx.get(API_ENDPOINT_URL + "/random", timeout=10).json()


if __name__ == "__main__":
13 changes: 11 additions & 2 deletions web_programming/get_amazon_product_data.py
Original file line number Diff line number Diff line change
@@ -4,9 +4,18 @@
information will include title, URL, price, ratings, and the discount available.
"""

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "httpx",
# "pandas",
# ]
# ///

from itertools import zip_longest

import requests
import httpx
from bs4 import BeautifulSoup
from pandas import DataFrame

@@ -25,7 +34,7 @@ def get_amazon_product_data(product: str = "laptop") -> DataFrame:
"Accept-Language": "en-US, en;q=0.5",
}
soup = BeautifulSoup(
requests.get(url, headers=header, timeout=10).text, features="lxml"
httpx.get(url, headers=header, timeout=10).text, features="lxml"
)
# Initialize a Pandas dataframe with the column titles
data_frame = DataFrame(
16 changes: 12 additions & 4 deletions web_programming/get_imdb_top_250_movies_csv.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,24 @@
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "httpx",
# ]
# ///

from __future__ import annotations

import csv

import requests
import httpx
from bs4 import BeautifulSoup


def get_imdb_top_250_movies(url: str = "") -> dict[str, float]:
url = url or "https://www.imdb.com/chart/top/?ref_=nv_mv_250"
soup = BeautifulSoup(requests.get(url, timeout=10).text, "html.parser")
titles = soup.find_all("td", attrs="titleColumn")
ratings = soup.find_all("td", class_="ratingColumn imdbRating")
soup = BeautifulSoup(httpx.get(url, timeout=10).text, "html.parser")
titles = soup.find_all("h3", class_="ipc-title__text")
ratings = soup.find_all("span", class_="ipc-rating-star--rating")
return {
title.a.text: float(rating.strong.text)
for title, rating in zip(titles, ratings)
13 changes: 10 additions & 3 deletions web_programming/get_ip_geolocation.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
import requests
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

import httpx


# Function to get geolocation data for an IP address
@@ -8,7 +15,7 @@ def get_ip_geolocation(ip_address: str) -> str:
url = f"https://ipinfo.io/{ip_address}/json"

# Send a GET request to the API
response = requests.get(url, timeout=10)
response = httpx.get(url, timeout=10)

# Check if the HTTP request was successful
response.raise_for_status()
@@ -23,7 +30,7 @@ def get_ip_geolocation(ip_address: str) -> str:
location = "Location data not found."

return location
except requests.exceptions.RequestException as e:
except httpx.RequestError as e:
# Handle network-related exceptions
return f"Request error: {e}"
except ValueError as e:
12 changes: 10 additions & 2 deletions web_programming/get_top_billionaires.py
Original file line number Diff line number Diff line change
@@ -3,9 +3,17 @@
This works for some of us but fails for others.
"""

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# "rich",
# ]
# ///

from datetime import UTC, date, datetime

import requests
import httpx
from rich import box
from rich import console as rich_console
from rich import table as rich_table
@@ -57,7 +65,7 @@ def get_forbes_real_time_billionaires() -> list[dict[str, int | str]]:
Returns:
List of top 10 realtime billionaires data.
"""
response_json = requests.get(API_URL, timeout=10).json()
response_json = httpx.get(API_URL, timeout=10).json()
return [
{
"Name": person["personName"],
13 changes: 10 additions & 3 deletions web_programming/get_top_hn_posts.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,26 @@
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

from __future__ import annotations

import requests
import httpx


def get_hackernews_story(story_id: str) -> dict:
url = f"https://hacker-news.firebaseio.com/v0/item/{story_id}.json?print=pretty"
return requests.get(url, timeout=10).json()
return httpx.get(url, timeout=10).json()


def hackernews_top_stories(max_stories: int = 10) -> list[dict]:
"""
Get the top max_stories posts from HackerNews - https://news.ycombinator.com/
"""
url = "https://hacker-news.firebaseio.com/v0/topstories.json?print=pretty"
story_ids = requests.get(url, timeout=10).json()[:max_stories]
story_ids = httpx.get(url, timeout=10).json()[:max_stories]
return [get_hackernews_story(story_id) for story_id in story_ids]


12 changes: 10 additions & 2 deletions web_programming/giphy.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@
#!/usr/bin/env python3
import requests

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

import httpx

giphy_api_key = "YOUR API KEY"
# Can be fetched from https://developers.giphy.com/dashboard/
@@ -11,7 +19,7 @@ def get_gifs(query: str, api_key: str = giphy_api_key) -> list:
"""
formatted_query = "+".join(query.split())
url = f"https://api.giphy.com/v1/gifs/search?q={formatted_query}&api_key={api_key}"
gifs = requests.get(url, timeout=10).json()["data"]
gifs = httpx.get(url, timeout=10).json()["data"]
return [gif["url"] for gif in gifs]


14 changes: 12 additions & 2 deletions web_programming/instagram_crawler.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,19 @@
#!/usr/bin/env python3

# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "fake-useragent",
# "httpx",
# ]
# ///

from __future__ import annotations

import json

import requests
import httpx
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

@@ -39,7 +49,7 @@ def get_json(self) -> dict:
"""
Return a dict of user information
"""
html = requests.get(self.url, headers=headers, timeout=10).text
html = httpx.get(self.url, headers=headers, timeout=10).text
scripts = BeautifulSoup(html, "html.parser").find_all("script")
try:
return extract_user_profile(scripts[4])
20 changes: 14 additions & 6 deletions web_programming/instagram_pic.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "httpx",
# ]
# ///

from datetime import UTC, datetime

import requests
import httpx
from bs4 import BeautifulSoup


@@ -15,9 +23,9 @@ def download_image(url: str) -> str:
A message indicating the result of the operation.
"""
try:
response = requests.get(url, timeout=10)
response = httpx.get(url, timeout=10)
response.raise_for_status()
except requests.exceptions.RequestException as e:
except httpx.RequestError as e:
return f"An error occurred during the HTTP request to {url}: {e!r}"

soup = BeautifulSoup(response.text, "html.parser")
@@ -30,13 +38,13 @@ def download_image(url: str) -> str:
return f"Image URL not found in meta tag {image_meta_tag}."

try:
image_data = requests.get(image_url, timeout=10).content
except requests.exceptions.RequestException as e:
image_data = httpx.get(image_url, timeout=10).content
except httpx.RequestError as e:
return f"An error occurred during the HTTP request to {image_url}: {e!r}"
if not image_data:
return f"Failed to download the image from {image_url}."

file_name = f"{datetime.now(tz=UTC).astimezone():%Y-%m-%d_%H:%M:%S}.jpg"
file_name = f"{datetime.now(tz=UTC).astimezone():%Y-%m-%d_%H-%M-%S}.jpg"
with open(file_name, "wb") as out_file:
out_file.write(image_data)
return f"Image downloaded and saved in the file {file_name}"
15 changes: 11 additions & 4 deletions web_programming/instagram_video.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,24 @@
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

from datetime import UTC, datetime

import requests
import httpx


def download_video(url: str) -> bytes:
base_url = "https://downloadgram.net/wp-json/wppress/video-downloader/video?url="
video_url = requests.get(base_url + url, timeout=10).json()[0]["urls"][0]["src"]
return requests.get(video_url, timeout=10).content
video_url = httpx.get(base_url + url, timeout=10)
return httpx.get(video_url, timeout=10).content


if __name__ == "__main__":
url = input("Enter Video/IGTV url: ").strip()
file_name = f"{datetime.now(tz=UTC).astimezone():%Y-%m-%d_%H:%M:%S}.mp4"
file_name = f"{datetime.now(tz=UTC).astimezone():%Y-%m-%d_%H-%M-%S}.mp4"
with open(file_name, "wb") as fp:
fp.write(download_video(url))
print(f"Done. Video saved to disk as {file_name}.")
17 changes: 11 additions & 6 deletions web_programming/nasa_data.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import shutil
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

import requests
import httpx


def get_apod_data(api_key: str) -> dict:
@@ -9,17 +14,17 @@ def get_apod_data(api_key: str) -> dict:
Get your API Key from: https://api.nasa.gov/
"""
url = "https://api.nasa.gov/planetary/apod"
return requests.get(url, params={"api_key": api_key}, timeout=10).json()
return httpx.get(url, params={"api_key": api_key}, timeout=10).json()


def save_apod(api_key: str, path: str = ".") -> dict:
apod_data = get_apod_data(api_key)
img_url = apod_data["url"]
img_name = img_url.split("/")[-1]
response = requests.get(img_url, stream=True, timeout=10)
response = httpx.get(img_url, timeout=10)

with open(f"{path}/{img_name}", "wb+") as img_file:
shutil.copyfileobj(response.raw, img_file)
img_file.write(response.content)
del response
return apod_data

@@ -29,7 +34,7 @@ def get_archive_data(query: str) -> dict:
Get the data of a particular query from NASA archives
"""
url = "https://images-api.nasa.gov/search"
return requests.get(url, params={"q": query}, timeout=10).json()
return httpx.get(url, params={"q": query}, timeout=10).json()


if __name__ == "__main__":
25 changes: 13 additions & 12 deletions web_programming/open_google_results.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "fake-useragent",
# "httpx",
# ]
# ///

import webbrowser
from sys import argv
from urllib.parse import parse_qs, quote

import requests
import httpx
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

@@ -13,26 +22,18 @@

url = f"https://www.google.com/search?q={query}&num=100"

res = requests.get(
res = httpx.get(
url,
headers={"User-Agent": str(UserAgent().random)},
timeout=10,
)

try:
link = (
BeautifulSoup(res.text, "html.parser")
.find("div", attrs={"class": "yuRUbf"})
.find("a")
.get("href")
)
link = BeautifulSoup(res.text, "html.parser").find("div").find("a").get("href")

except AttributeError:
link = parse_qs(
BeautifulSoup(res.text, "html.parser")
.find("div", attrs={"class": "kCrYT"})
.find("a")
.get("href")
BeautifulSoup(res.text, "html.parser").find("div").find("a").get("href")
)["url"][0]

webbrowser.open(link)
15 changes: 12 additions & 3 deletions web_programming/random_anime_character.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "fake-useragent",
# "httpx",
# ]
# ///

import os

import requests
import httpx
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

@@ -12,7 +21,7 @@ def save_image(image_url: str, image_title: str) -> None:
"""
Saves the image of anime character
"""
image = requests.get(image_url, headers=headers, timeout=10)
image = httpx.get(image_url, headers=headers, timeout=10)
with open(image_title, "wb") as file:
file.write(image.content)

@@ -22,7 +31,7 @@ def random_anime_character() -> tuple[str, str, str]:
Returns the Title, Description, and Image Title of a random anime character .
"""
soup = BeautifulSoup(
requests.get(URL, headers=headers, timeout=10).text, "html.parser"
httpx.get(URL, headers=headers, timeout=10).text, "html.parser"
)
title = soup.find("meta", attrs={"property": "og:title"}).attrs["content"]
image_url = soup.find("meta", attrs={"property": "og:image"}).attrs["content"]
11 changes: 9 additions & 2 deletions web_programming/recaptcha_verification.py
Original file line number Diff line number Diff line change
@@ -32,7 +32,14 @@
recaptcha verification.
"""

import requests
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

import httpx

try:
from django.contrib.auth import authenticate, login
@@ -56,7 +63,7 @@ def login_using_recaptcha(request):
client_key = request.POST.get("g-recaptcha-response")

# post recaptcha response to Google's recaptcha api
response = requests.post(
response = httpx.post(
url, data={"secret": secret_key, "response": client_key}, timeout=10
)
# if the recaptcha api verified our keys
16 changes: 12 additions & 4 deletions web_programming/reddit.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

from __future__ import annotations

import requests
import httpx

valid_terms = set(
"""approved_at_utc approved_by author_flair_background_color
@@ -28,13 +35,14 @@ def get_subreddit_data(
if invalid_search_terms := ", ".join(sorted(set(wanted_data) - valid_terms)):
msg = f"Invalid search term: {invalid_search_terms}"
raise ValueError(msg)
response = requests.get(
f"https://reddit.com/r/{subreddit}/{age}.json?limit={limit}",
response = httpx.get(
f"https://www.reddit.com/r/{subreddit}/{age}.json?limit={limit}",
headers={"User-agent": "A random string"},
timeout=10,
)
response.raise_for_status()
if response.status_code == 429:
raise requests.HTTPError(response=response)
raise httpx.HTTPError(response=response)

data = response.json()
if not wanted_data:
21 changes: 14 additions & 7 deletions web_programming/search_books_by_isbn.py
Original file line number Diff line number Diff line change
@@ -4,9 +4,16 @@
ISBN: https://en.wikipedia.org/wiki/International_Standard_Book_Number
"""

from json import JSONDecodeError # Workaround for requests.exceptions.JSONDecodeError
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

import requests
from json import JSONDecodeError

import httpx


def get_openlibrary_data(olid: str = "isbn/0140328726") -> dict:
@@ -25,7 +32,9 @@ def get_openlibrary_data(olid: str = "isbn/0140328726") -> dict:
if new_olid.count("/") != 1:
msg = f"{olid} is not a valid Open Library olid"
raise ValueError(msg)
return requests.get(f"https://openlibrary.org/{new_olid}.json", timeout=10).json()
return httpx.get(
f"https://openlibrary.org/{new_olid}.json", timeout=10, follow_redirects=True
).json()


def summarize_book(ol_book_data: dict) -> dict:
@@ -36,16 +45,14 @@ def summarize_book(ol_book_data: dict) -> dict:
"title": "Title",
"publish_date": "Publish date",
"authors": "Authors",
"number_of_pages": "Number of pages:",
"first_sentence": "First sentence",
"number_of_pages": "Number of pages",
"isbn_10": "ISBN (10)",
"isbn_13": "ISBN (13)",
}
data = {better_key: ol_book_data[key] for key, better_key in desired_keys.items()}
data["Authors"] = [
get_openlibrary_data(author["key"])["name"] for author in data["Authors"]
]
data["First sentence"] = data["First sentence"]["value"]
for key, value in data.items():
if isinstance(value, list):
data[key] = ", ".join(value)
@@ -71,5 +78,5 @@ def summarize_book(ol_book_data: dict) -> dict:
try:
book_summary = summarize_book(get_openlibrary_data(f"isbn/{isbn}"))
print("\n".join(f"{key}: {value}" for key, value in book_summary.items()))
except JSONDecodeError: # Workaround for requests.exceptions.RequestException:
except JSONDecodeError:
print(f"Sorry, there are no results for ISBN: {isbn}.")
11 changes: 9 additions & 2 deletions web_programming/slack_message.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
# Created by sarathkaul on 12/11/19

import requests
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "httpx",
# ]
# ///

import httpx


def send_slack_message(message_body: str, slack_url: str) -> None:
headers = {"Content-Type": "application/json"}
response = requests.post(
response = httpx.post(
slack_url, json={"text": message_body}, headers=headers, timeout=10
)
if response.status_code != 200:
4 changes: 2 additions & 2 deletions web_programming/test_fetch_github_info.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import json

import requests
import httpx

from .fetch_github_info import AUTHENTICATED_USER_ENDPOINT, fetch_github_info

@@ -21,7 +21,7 @@ def mock_response(*args, **kwargs):
assert "Accept" in kwargs["headers"]
return FakeResponse(b'{"login":"test","id":1}')

monkeypatch.setattr(requests, "get", mock_response)
monkeypatch.setattr(httpx, "get", mock_response)
result = fetch_github_info("token")
assert result["login"] == "test"
assert result["id"] == 1
26 changes: 19 additions & 7 deletions web_programming/world_covid19_stats.py
Original file line number Diff line number Diff line change
@@ -5,19 +5,31 @@
This data is being scrapped from 'https://www.worldometers.info/coronavirus/'.
"""

import requests
# /// script
# requires-python = ">=3.13"
# dependencies = [
# "beautifulsoup4",
# "httpx",
# ]
# ///

import httpx
from bs4 import BeautifulSoup


def world_covid19_stats(url: str = "https://www.worldometers.info/coronavirus") -> dict:
def world_covid19_stats(
url: str = "https://www.worldometers.info/coronavirus/",
) -> dict:
"""
Return a dict of current worldwide COVID-19 statistics
"""
soup = BeautifulSoup(requests.get(url, timeout=10).text, "html.parser")
keys = soup.findAll("h1")
values = soup.findAll("div", {"class": "maincounter-number"})
keys += soup.findAll("span", {"class": "panel-title"})
values += soup.findAll("div", {"class": "number-table-main"})
soup = BeautifulSoup(
httpx.get(url, timeout=10, follow_redirects=True).text, "html.parser"
)
keys = soup.find_all("h1")
values = soup.find_all("div", {"class": "maincounter-number"})
keys += soup.find_all("span", {"class": "panel-title"})
values += soup.find_all("div", {"class": "number-table-main"})
return {key.text.strip(): value.text.strip() for key, value in zip(keys, values)}