Use an HTTP proxy in Go with net/http

package main

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

// netdashClient returns a client that sends every request through a netdash gateway.
// Leave sessionID empty for a rotating session; set it for a sticky one.
func netdashClient(country, sessionID string, ttl int, gateway string) *http.Client {
	cc := strings.ToUpper(country)
	if cc == "UK" {
		cc = "GB"
	}
	username := os.Getenv("NETDASH_USERNAME") + "-cc-" + cc
	if sessionID != "" {
		username += fmt.Sprintf("-session-%s-ttl-%d", sessionID, ttl)
	}
	proxyURL := &url.URL{
		Scheme: "http",
		User:   url.UserPassword(username, os.Getenv("NETDASH_PASSWORD")),
		Host:   gateway, // e.g. "scale-us.netdash.io:2000"
	}
	return &http.Client{
		Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)},
		Timeout:   30 * time.Second,
	}
}

// newSessionID returns 8 random characters from 0-9 and a-f for a sticky session.
func newSessionID() (string, error) {
	b := make([]byte, 4)
	if _, err := rand.Read(b); err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}

package main

import (
	"fmt"
	"io"
	"log"
)

func main() {
	client := netdashClient("US", "", 0, "scale-us.netdash.io:2000")

	resp, err := client.Get("https://api.ipify.org?format=json")
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(resp.Status, string(body))
}

package main

import (
	"fmt"
	"io"
	"log"
)

func main() {
	sessionID, err := newSessionID()
	if err != nil {
		log.Fatal(err)
	}
	// One client per sticky session: German exit through the Europe gateway, 30-minute TTL
	client := netdashClient("DE", sessionID, 30, "scale-eu.netdash.io:2000")
	defer client.CloseIdleConnections()

	for i := 0; i < 2; i++ {
		resp, err := client.Get("https://api.ipify.org?format=json")
		if err != nil {
			log.Fatal(err)
		}
		body, err := io.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(string(body)) // the same exit IP both times
	}
}

package main

import (
	"context"
	"fmt"
	"io"
	"log"
	"net/http"
	"strconv"
	"time"
)

// getWithRetry retries network errors, 5xx and 429, three attempts at most, and honors
// Retry-After on 429. Every other status, including 403, goes back to the caller.
func getWithRetry(ctx context.Context, client *http.Client, target string) (*http.Response, error) {
	wait := time.Second
	for attempt := 1; ; attempt++ {
		req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
		if err != nil {
			return nil, err
		}
		resp, err := client.Do(req)
		retryable := err != nil || resp.StatusCode >= 500 || resp.StatusCode == http.StatusTooManyRequests
		if !retryable || attempt == 3 {
			return resp, err
		}
		delay := wait
		if resp != nil {
			if s, convErr := strconv.Atoi(resp.Header.Get("Retry-After")); convErr == nil && resp.StatusCode == http.StatusTooManyRequests {
				delay = time.Duration(s) * time.Second
			}
			resp.Body.Close()
		}
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case <-time.After(delay):
		}
		wait *= 2
	}
}

func main() {
	client := netdashClient("US", "", 0, "scale-us.netdash.io:2000")
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
	defer cancel()

	resp, err := getWithRetry(ctx, client, "https://api.ipify.org?format=json")
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusForbidden {
		log.Fatal("403: stop here, then check the site's terms or use its official API")
	}
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(resp.Status, string(body))
}

package main

import (
	"context"
	"fmt"
	"io"
	"net/http"
	"sync"
	"time"
)

func main() {
	client := netdashClient("US", "", 0, "scale-us.netdash.io:2000")
	urls := make([]string, 0, 10)
	for i := 1; i <= 10; i++ {
		urls = append(urls, fmt.Sprintf("https://example.com/catalog?page=%d", i))
	}

	ctx, stop := context.WithCancel(context.Background())
	defer stop()
	pace := time.NewTicker(500 * time.Millisecond) // at most two requests per second to this site
	defer pace.Stop()

	jobs := make(chan string, len(urls))
	for _, u := range urls {
		jobs <- u
	}
	close(jobs)

	results := make(chan string)
	var wg sync.WaitGroup
	for w := 0; w < 4; w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for u := range jobs {
				select {
				case <-ctx.Done():
					return
				case <-pace.C:
				}
				req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
				if err != nil {
					results <- fmt.Sprintf("%s: %v", u, err)
					continue
				}
				resp, err := client.Do(req)
				if err != nil {
					results <- fmt.Sprintf("%s: %v", u, err)
					continue
				}
				n, _ := io.Copy(io.Discard, resp.Body)
				resp.Body.Close()
				if resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests {
					stop() // the site is pushing back: stop, then check its terms and limits
				}
				results <- fmt.Sprintf("%s: %s, %d bytes", u, resp.Status, n)
			}
		}()
	}
	go func() {
		wg.Wait()
		close(results)
	}()
	for r := range results {
		fmt.Println(r)
	}
}