Web Scraping with Python & Proxies

# Core scraping libraries
pip install requests beautifulsoup4 lxml

# For advanced scraping
pip install scrapy playwright

# For data processing
pip install pandas

# Verify installation
python -c "import requests; print('Requests:', requests.__version__)"

import requests
from bs4 import BeautifulSoup
import time
import random

# netdash residential proxy configuration
PROXY = "http://username:password@gate.netdash.io:8080"

proxies = {
    "http": PROXY,
    "https": PROXY
}

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
    "Accept-Encoding": "gzip, deflate, br",
    "DNT": "1",
    "Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1"
}

def scrape_page(url):
    """
    Scrape a single page with residential proxy
    """
    try:
        # Add human-like delay
        time.sleep(random.uniform(2, 5))
        
        response = requests.get(
            url,
            proxies=proxies,
            headers=headers,
            timeout=15
        )
        
        if response.status_code == 200:
            soup = BeautifulSoup(response.text, 'lxml')
            
            # Extract data (example: product details)
            title = soup.find('h1', class_='product-title')
            price = soup.find('span', class_='price')
            description = soup.find('div', class_='description')
            
            return {
                'title': title.text.strip() if title else None,
                'price': price.text.strip() if price else None,
                'description': description.text.strip() if description else None,
                'url': url
            }
        else:
            print(f"Failed: {response.status_code}")
            return None
            
    except Exception as e:
        print(f"Error scraping {url}: {e}")
        return None

# Example: Scrape multiple product pages
product_urls = [
    "https://example.com/product/1",
    "https://example.com/product/2",
    "https://example.com/product/3",
    # ... add more URLs
]

products = []
for url in product_urls:
    data = scrape_page(url)
    if data:
        products.append(data)
        print(f"✓ Scraped: {data['title']}")

print(f"\nTotal products scraped: {len(products)}")

import requests
from bs4 import BeautifulSoup
import time
import random
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

class ProductScraper:
    def __init__(self, proxy_url):
        self.proxies = {
            "http": proxy_url,
            "https": proxy_url
        }
        
        # Setup session with retry strategy
        self.session = requests.Session()
        retry_strategy = Retry(
            total=3,
            backoff_factor=2,
            status_forcelist=[429, 500, 502, 503, 504]
        )
        adapter = HTTPAdapter(max_retries=retry_strategy)
        self.session.mount("http://", adapter)
        self.session.mount("https://", adapter)
        
        self.session.headers.update({
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
        })
    
    def scrape(self, url, max_retries=3):
        """
        Scrape with automatic retries and error handling
        """
        for attempt in range(max_retries):
            try:
                # Random delay (2-5 seconds)
                time.sleep(random.uniform(2, 5))
                
                response = self.session.get(
                    url,
                    proxies=self.proxies,
                    timeout=15
                )
                
                if response.status_code == 200:
                    return self._parse_response(response)
                elif response.status_code == 403:
                    print(f"⚠️  Blocked (attempt {attempt + 1}/{max_retries})")
                    continue
                else:
                    print(f"Status {response.status_code}")
                    continue
                    
            except requests.exceptions.Timeout:
                print(f"Timeout (attempt {attempt + 1}/{max_retries})")
                continue
            except requests.exceptions.RequestException as e:
                print(f"Error: {e}")
                continue
        
        return None
    
    def _parse_response(self, response):
        """
        Parse HTML and extract data
        """
        soup = BeautifulSoup(response.text, 'lxml')
        
        # Your parsing logic here
        data = {
            'title': soup.find('h1', class_='title').text.strip(),
            'price': soup.find('span', class_='price').text.strip(),
            # ... extract more fields
        }
        
        return data

# Usage
scraper = ProductScraper(
    proxy_url="http://username:password@gate.netdash.io:8080"
)

urls = ["https://example.com/product/1", "..."]
results = [scraper.scrape(url) for url in urls]

print(f"Scraped {len([r for r in results if r])} / {len(urls)} pages")

# Scrapy settings with netdash residential proxies

# Enable proxy middleware
DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,
}

# Configure residential proxy
PROXY = 'http://username:password@gate.netdash.io:8080'

# Respectful scraping settings
CONCURRENT_REQUESTS = 16
DOWNLOAD_DELAY = 2  # 2 seconds between requests
RANDOMIZE_DOWNLOAD_DELAY = True

# User-Agent rotation
USER_AGENT_LIST = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36',
]

# Retry settings
RETRY_TIMES = 3
RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 429]

# Auto-throttle (adjust speed based on response)
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 2
AUTOTHROTTLE_MAX_DELAY = 10
AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0

import scrapy
from scrapy.http import Request

class ProductSpider(scrapy.Spider):
    name = 'products'
    start_urls = ['https://example.com/products?page=1']
    
    # Custom settings for this spider
    custom_settings = {
        'CONCURRENT_REQUESTS': 16,
        'DOWNLOAD_DELAY': 2,
    }
    
    def start_requests(self):
        """
        Add proxy to all requests
        """
        proxy = 'http://username:password@gate.netdash.io:8080'
        
        for url in self.start_urls:
            yield Request(
                url,
                callback=self.parse,
                meta={'proxy': proxy},
                dont_filter=True
            )
    
    def parse(self, response):
        """
        Parse product listing page
        """
        # Extract product links
        product_links = response.css('a.product-link::attr(href)').getall()
        
        for link in product_links:
            yield response.follow(
                link,
                callback=self.parse_product,
                meta={'proxy': response.meta['proxy']}
            )
        
        # Follow pagination
        next_page = response.css('a.next-page::attr(href)').get()
        if next_page:
            yield response.follow(
                next_page,
                callback=self.parse,
                meta={'proxy': response.meta['proxy']}
            )
    
    def parse_product(self, response):
        """
        Parse individual product page
        """
        yield {
            'title': response.css('h1.product-title::text').get(),
            'price': response.css('span.price::text').get(),
            'description': response.css('div.description::text').get(),
            'rating': response.css('span.rating::text').get(),
            'reviews': response.css('span.review-count::text').get(),
            'url': response.url,
        }

# Run spider:
# scrapy crawl products -o products.json