SERP tracking by country

import json
import os
import time
from datetime import date
from html.parser import HTMLParser
from urllib.parse import quote, quote_plus, urlparse

import httpx

USER = os.environ["NETDASH_USERNAME"]
PASS = quote(os.environ["NETDASH_PASSWORD"], safe="")  # encoded: it sits inside a URL

# Market -> gateway and Accept-Language. US pairs with the Americas gateway, DE and GB with Europe.
MARKETS = {
    "US": ("scale-us.netdash.io:2000", "en-US,en;q=0.8"),
    "DE": ("scale-eu.netdash.io:2000", "de-DE,de;q=0.8"),
    "GB": ("scale-eu.netdash.io:2000", "en-GB,en;q=0.8"),
}
SEARCH_URL = "https://search.example/search?q={query}"  # example host: use a source that allows it
DOMAIN = "yourdomain.example"
KEYWORDS = ["project management software", "invoice template"]
PAUSE_SECONDS = 90  # a steady pace; spread the daily run over the day


class ResultLinks(HTMLParser):
    """Collects absolute links in page order. Replace with a parser for your results page."""

    def __init__(self):
        super().__init__()
        self.links = []

    def handle_starttag(self, tag, attrs):
        href = dict(attrs).get("href") or ""
        if tag == "a" and href.startswith("http"):
            self.links.append(href)


def client_for(cc):
    host, lang = MARKETS[cc]
    # Rotating: no session parameters, so every query stands alone.
    proxy = f"http://{USER}-cc-{cc}:{PASS}@{host}"
    return httpx.Client(proxy=proxy, timeout=30, headers={"Accept-Language": lang})


def position(html, domain):
    """1-based position and URL of the first result on `domain`, or (None, None)."""
    parser = ResultLinks()
    parser.feed(html)
    for rank, link in enumerate(parser.links, start=1):
        host = urlparse(link).hostname or ""
        if host == domain or host.endswith("." + domain):
            return rank, link
    return None, None


for keyword in KEYWORDS:
    for cc in MARKETS:
        with client_for(cc) as client:
            # A wrong username or password fails here with httpx.ProxyError (407).
            response = client.get(SEARCH_URL.format(query=quote_plus(keyword)))
        if response.status_code in (403, 429):
            raise SystemExit(f"{cc}: HTTP {response.status_code}. Stop, then review volume and terms.")
        response.raise_for_status()
        rank, url = position(response.text, DOMAIN)
        print(json.dumps({"date": date.today().isoformat(), "market": cc, "device": "desktop",
                          "keyword": keyword, "position": rank, "url": url}))
        time.sleep(PAUSE_SECONDS)