How to Bypass CAPTCHA with Residential Proxies

import requests
import random
import time

# netdash residential proxies - clean IP reputation
PROXY = "http://username:password@gate.netdash.io:8080"

proxies = {
    "http": PROXY,
    "https": PROXY
}

# Realistic browser headers (critical for bypassing detection)
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
    "Accept-Encoding": "gzip, deflate, br",
    "DNT": "1",
    "Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "none",
    "Cache-Control": "max-age=0"
}

def scrape_with_captcha_avoidance(url):
    """
    Scrape a URL while minimizing CAPTCHA triggers
    """
    try:
        # Add random delay to mimic human behavior
        time.sleep(random.uniform(2, 5))
        
        response = requests.get(
            url,
            proxies=proxies,
            headers=headers,
            timeout=15,
            allow_redirects=True
        )
        
        # Check for CAPTCHA indicators
        if "captcha" in response.text.lower() or response.status_code == 403:
            print(f"⚠️  CAPTCHA encountered (rare with residential proxies)")
            return None
        
        if response.status_code == 200:
            print(f"✓ Success: {url[:50]}... (No CAPTCHA)")
            return response.text
        
    except requests.exceptions.RequestException as e:
        print(f"✗ Error: {e}")
        return None

# Example: Scrape multiple pages with CAPTCHA avoidance
urls = [
    "https://example.com/product/1",
    "https://example.com/product/2",
    "https://example.com/product/3",
    # ... thousands more
]

success_count = 0
captcha_count = 0

for url in urls:
    result = scrape_with_captcha_avoidance(url)
    
    if result:
        success_count += 1
        # Process your data here
    else:
        captcha_count += 1

print(f"\nResults: {success_count} successful, {captcha_count} CAPTCHAs")
print(f"Success rate: {(success_count / len(urls)) * 100:.1f}%")

import requests
import random
import time

class CaptchaAvoidanceScraper:
    def __init__(self, proxy_url):
        self.proxy_url = proxy_url
        self.session = requests.Session()
        
        # Maintain realistic headers
        self.session.headers.update({
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
            "Accept-Language": "en-US,en;q=0.9",
            "Accept-Encoding": "gzip, deflate, br",
            "DNT": "1"
        })
        
    def get_with_rotation(self, url, session_id=None):
        """
        Fetch URL with automatic IP rotation (per-request)
        OR use sticky session to maintain same IP
        """
        
        # Option 1: Rotate IP every request (maximum stealth)
        # Each request uses a NEW residential IP
        proxy = self.proxy_url
        
        # Option 2: Sticky session (same IP for multiple requests)
        # Useful for login flows or shopping carts
        if session_id:
            proxy = self.proxy_url.replace(
                "username",
                f"username-session-{session_id}"
            )
        
        proxies = {"http": proxy, "https": proxy}
        
        # Random delay: 2-5 seconds (mimic human reading time)
        time.sleep(random.uniform(2, 5))
        
        try:
            response = self.session.get(
                url,
                proxies=proxies,
                timeout=15
            )
            
            # Check response
            if response.status_code == 200:
                if "captcha" not in response.text.lower():
                    return response.text
                else:
                    print(f"⚠️  CAPTCHA detected on {url}")
                    return None
            else:
                print(f"Status {response.status_code}: {url}")
                return None
                
        except Exception as e:
            print(f"Error: {e}")
            return None

# Usage
scraper = CaptchaAvoidanceScraper(
    proxy_url="http://username:password@gate.netdash.io:8080"
)

# Scrape 1000 pages with rotating residential IPs
for i in range(1, 1001):
    url = f"https://example.com/page/{i}"
    html = scraper.get_with_rotation(url)
    
    if html:
        # Extract data
        print(f"Page {i}: Scraped successfully (No CAPTCHA)")

# Expected result with netdash residential proxies:
# 950-980 pages succeed without CAPTCHA (95-98% success rate)
# 20-50 pages encounter CAPTCHA (2-5% encounter rate)

import axios from 'axios';
import { HttpsProxyAgent } from 'https-proxy-agent';

const proxyUrl = 'http://username:password@gate.netdash.io:8080';
const proxyAgent = new HttpsProxyAgent(proxyUrl);

async function scrapeWithCaptchaAvoidance(url: string) {
  // Random delay: 2-4 seconds
  await new Promise(resolve => 
    setTimeout(resolve, 2000 + Math.random() * 2000)
  );
  
  try {
    const response = await axios.get(url, {
      httpsAgent: proxyAgent,
      httpAgent: proxyAgent,
      headers: {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.9',
        'DNT': '1'
      },
      timeout: 15000
    });
    
    // Check for CAPTCHA
    if (response.data.toLowerCase().includes('captcha')) {
      console.log('⚠️  CAPTCHA encountered (rare)');
      return null;
    }
    
    console.log(`✓ Success: ${url.substring(0, 50)}...`);
    return response.data;
    
  } catch (error) {
    console.error(`Error: ${error.message}`);
    return null;
  }
}

// Scrape multiple pages
async function scrapeMultiple() {
  const urls = Array.from(
    { length: 100 }, 
    (_, i) => `https://example.com/product/${i + 1}`
  );
  
  let successCount = 0;
  
  for (const url of urls) {
    const result = await scrapeWithCaptchaAvoidance(url);
    if (result) successCount++;
  }
  
  console.log(`\nSuccess rate: ${successCount}/${urls.length} (${(successCount/urls.length*100).toFixed(1)}%)`);
}

scrapeMultiple();