Web data collection

import os
import time
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from urllib.parse import quote, urlsplit
from urllib.robotparser import RobotFileParser

import requests

USER_AGENT = "example-collector/1.0 (+https://example.com/contact)"
PROXY = "http://{}-cc-US:{}@scale-us.netdash.io:2000".format(
    quote(os.environ["NETDASH_USERNAME"], safe=""), quote(os.environ["NETDASH_PASSWORD"], safe="")
)
PROXIES = {"http": PROXY, "https": PROXY}  # set both, or HTTPS requests bypass the proxy
DEFAULT_DELAY = 2  # seconds between requests to one site, unless robots.txt asks for more

session = requests.Session()
session.headers["User-Agent"] = USER_AGENT
robots = {}


def rules_for(url):
    """Fetch robots.txt once per site, through the same exit country."""
    site = "{0.scheme}://{0.netloc}".format(urlsplit(url))
    if site not in robots:
        parser = RobotFileParser()
        resp = session.get(site + "/robots.txt", proxies=PROXIES, timeout=30)
        if resp.ok:
            parser.parse(resp.text.splitlines())
        elif resp.status_code in (404, 410):
            parser.parse([])  # no robots.txt: nothing is disallowed
        else:
            parser.parse(["User-agent: *", "Disallow: /"])  # unclear answer: don't collect yet
        robots[site] = parser
    return robots[site]


def retry_after(value, default=60):
    """Retry-After is either a number of seconds or an HTTP date."""
    if value and value.strip().isdigit():
        return int(value)
    try:
        when = parsedate_to_datetime(value)
        return max(0, (when - datetime.now(timezone.utc)).total_seconds())
    except (TypeError, ValueError):
        return default


def fetch(url, attempts=3):
    rules = rules_for(url)
    if not rules.can_fetch(USER_AGENT, url):
        return None  # disallowed by robots.txt
    delay = max(DEFAULT_DELAY, rules.crawl_delay(USER_AGENT) or 0)
    for _ in range(attempts):
        time.sleep(delay)
        # Wrong username or password: the gateway answers 407, which requests raises as
        # ProxyError for https:// URLs (and raise_for_status() raises for http:// URLs).
        resp = session.get(url, proxies=PROXIES, timeout=30)
        if resp.status_code in (429, 503):
            delay = max(DEFAULT_DELAY, retry_after(resp.headers.get("Retry-After")))
            continue
        resp.raise_for_status()
        return resp
    return None  # gave up after three attempts; if a site keeps limiting you, lower your rate


if __name__ == "__main__":
    print(fetch("https://api.ipify.org?format=json").text)