2026年如何用 Python 抓取 Google Trends 数据

python -m venv venv && venv\Scripts\activate

pip install --upgrade pip && pip install selenium undetected-chromedriver requests pandas numpy pillow typing-extensions

cd %USERPROFILE%\Desktop && type nul > google_trends_scraper.py

import time
import zipfile
import tempfile
import os
import math
import json
import random
import re
import logging
import requests
import pandas as pd
from PIL import Image
import numpy as np
import io
from typing import Dict, Any
from urllib.parse import unquote, quote
from datetime import datetime
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import undetected_chromedriver as uc

# === Setup Logging ===
logging.basicConfig(
    level=logging.INFO, 
    format='[%(asctime)s] %(message)s',
    datefmt='%H:%M:%S'
)

# === Globals ===
HEADLESS = False
MAX_RETRIES = 5
GOOGLE_BACKOFF_STEPS = [10, 20, 30, 60, 120]  # Increased backoff times
HUMAN_TYPING_SPEED = (0.08, 0.3)
RANDOM_MOUSE_MOVEMENTS = True

# === Trends Target ===
GEO = "GB-LON"  # London, UK
GEO_HUMAN_NAME = "London, United Kingdom"
KEYWORD = "windsor castle"

class GoogleTrendsParser:
    """Parse REAL Google Trends data from actual page content"""
    
    def parse(self, text_content: str) -> Dict[str, Any]:
        """Parse actual Google Trends page content (not hypothetical patterns)"""
        try:
            result = {
                'search_term': KEYWORD,
                'location': GEO_HUMAN_NAME,
                'url': '',
                'interest_over_time': {'y_axis': [], 'x_axis': []},
                'interest_by_subregion': {}
            }
            
            lines = self._clean_text(text_content)
            
            # 1. Extract what's ACTUALLY on the page
            result['search_term'] = self._extract_actual_search_term(lines)
            result['location'] = self._extract_actual_location(lines)
            
            # 2. Extract chart data from page
            chart_data = self._extract_chart_data(lines)
            if chart_data:
                result['interest_over_time'] = chart_data
            
            # 3. Extract regional data
            regional_data = self._extract_regional_data(text_content)
            if regional_data:
                result['interest_by_subregion'] = regional_data
            else:
                logging.info(" No regional data found on page - returning empty dict")
            
            logging.info(f" Parser found: '{result['search_term']}' in {result['location']}")
            if result['interest_by_subregion']:
                logging.info(f" Found {len(result['interest_by_subregion'])} regions")
            
            return result
            
        except Exception as e:
            logging.error(f"Parser error: {e}")
            # Return minimal structure
            return {
                'search_term': KEYWORD,
                'location': GEO_HUMAN_NAME,
                'interest_over_time': {'y_axis': [], 'x_axis': []},
                'interest_by_subregion': {}
            }
    
    def _clean_text(self, text: str) -> list:
        """Clean and split text."""
        return [line.strip() for line in text.split('\n') if line.strip()]
    
    def _extract_actual_search_term(self, lines: list) -> str:
        """
        Extract the search term that's ACTUALLY displayed on Google Trends.
        Not looking for "Search term:" which doesn't exist.
        """
        # Google Trends shows the term prominently at the top
        for i, line in enumerate(lines[:10]):  # Check first 10 lines
            if line and len(line) < 100:  # Reasonable length for a search term
                # Exclude common UI elements
                exclude_words = ['Explore', 'Compare', 'Trends', 'Google', 'Interest', 
                                'Location', 'Time', 'Download', 'Share', 'Embed', 'Home']
                
                if not any(word in line for word in exclude_words):
                    # Likely the search term
                    return line.strip()
        
        return KEYWORD  # Fallback
    
    def _extract_actual_location(self, lines: list) -> str:
        """Extract location from page - SPECIFICALLY FOR UK/LONDON"""
        for line in lines:
            line_lower = line.lower()
            # Check for UK indicators first
            if "london" in line_lower or "gb-lon" in line_lower:
                return "London, UK"
            elif "united kingdom" in line_lower or "uk" in line_lower:
                return "United Kingdom"
            elif "england" in line_lower:
                return "England"
            # Check for location dropdown text
            elif "location" in line_lower:
                idx = lines.index(line)
                # Check next few lines for location
                for i in range(idx + 1, min(idx + 4, len(lines))):
                    next_line = lines[i].lower()
                    if any(indicator in next_line for indicator in 
                           ['london', 'united kingdom', 'uk', 'england']):
                        return lines[i].strip()
        
        # If not found, check the URL
        return "United Kingdom"
    
    def _extract_chart_data(self, lines: list) -> Dict[str, Any]:
        """
        Extract chart data from page.
        Google Trends shows: months and values like "100", "75", "50", etc.
        """
        result = {'y_axis': [], 'x_axis': []}
        
        # Look for month names (chart x-axis)
        months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 
                 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
        
        month_years_found = []
        for line in lines:
            for month in months:
                if month in line:
                    # Extract potential date pattern
                    date_match = re.search(rf'({month})\s*(\d{{4}})?', line)
                    if date_match:
                        date_str = date_match.group(0)
                        if date_str not in month_years_found:
                            month_years_found.append(date_str)
        
        result['x_axis'] = month_years_found[:12]  # Up to 12 months
        
        # Look for chart values (0-100 range)
        chart_values = []
        for line in lines:
            # Find numbers in 0-100 range (common for Google Trends)
            numbers = re.findall(r'\b(\d{1,3})\b', line)
            for num in numbers:
                if num.isdigit():
                    value = int(num)
                    if 0 <= value <= 100:
                        if value not in chart_values:  # Avoid duplicates
                            chart_values.append(value)
        
        # Sort and limit
        chart_values.sort()
        result['y_axis'] = chart_values[:20]  # Up to 20 values
        
        return result
    
    def _extract_regional_data(self, text_content: str) -> Dict[str, int]:
        """
        Extract regional interest data from Google Trends page.
        Only extracts what's actually on the page - no fake data.
        """
        result = {}
        
        # UK cities/regions to look for (expanded list)
        uk_regions = [
            "London", "Birmingham", "Manchester", "Glasgow", "Liverpool", 
            "Leeds", "Sheffield", "Bristol", "Edinburgh", "Cardiff",
            "Newcastle", "Leicester", "Coventry", "Nottingham", "Southampton",
            "Belfast", "Aberdeen", "Cambridge", "Oxford", "Brighton",
            "Portsmouth", "Plymouth", "Swansea", "York", "Bath",
            "Wales", "Scotland", "Northern Ireland", "England"
        ]
        
        # Split text into lines for analysis
        lines = [line.strip() for line in text_content.split('\n') if line.strip()]
        
        # Method 1: Look for "Interest by subregion" or "Interest by region" section
        interest_section_start = -1
        for i, line in enumerate(lines):
            line_lower = line.lower()
            if ('interest by subregion' in line_lower or 
                'interest by region' in line_lower or
                'interest by city' in line_lower):
                interest_section_start = i
                logging.info(" Found regional interest section")
                break
        
        # If we found the section, extract data from it
        if interest_section_start != -1:
            # Look for regional data in the next 50 lines after the section header
            section_end = min(interest_section_start + 50, len(lines))
            for j in range(interest_section_start + 1, section_end):
                current_line = lines[j]
                current_line_lower = current_line.lower()
                
                # Skip empty lines or lines that are too long (probably not region data)
                if not current_line or len(current_line) > 50:
                    continue
                
                # Check if this line contains a UK region
                for region in uk_regions:
                    if region.lower() in current_line_lower:
                        # Look for a number in this line
                        numbers_in_line = re.findall(r'\b(\d{1,3})\b', current_line)
                        
                        # If no number in this line, check the next line
                        if not numbers_in_line and j + 1 < len(lines):
                            next_line = lines[j + 1]
                            numbers_in_line = re.findall(r'\b(\d{1,3})\b', next_line)
                        
                        # Process found numbers
                        for num in numbers_in_line:
                            if num.isdigit():
                                value = int(num)
                                # Validate it's a reasonable interest value (0-100)
                                if 0 <= value <= 100:
                                    # Don't overwrite if we already have this region
                                    if region not in result:
                                        result[region] = value
                                        logging.info(f"   Found {region}: {value}")
                                    break
        
        # Method 2: If no structured section found, scan for region+number patterns
        if not result:
            logging.info(" Scanning for regional data patterns...")
            for i, line in enumerate(lines):
                if len(line) > 50:  # Skip long lines (not likely region data)
                    continue
                    
                for region in uk_regions:
                    if region.lower() in line.lower():
                        # Extract numbers from this line
                        numbers = re.findall(r'\b(\d{1,3})\b', line)
                        valid_numbers = [int(n) for n in numbers if n.isdigit() and 0 <= int(n) <= 100]
                        
                        if valid_numbers:
                            # Use the largest valid number
                            best_value = max(valid_numbers)
                            if region not in result:  # Don't overwrite
                                result[region] = best_value
                                logging.info(f"   Found {region}: {best_value}")
        
        # Method 3: Check for specific regional data in the Related topics/queries section
        # Sometimes regional data appears near related topics
        if not result:
            related_section_found = False
            for i, line in enumerate(lines):
                if 'related topics' in line.lower() or 'related queries' in line.lower():
                    related_section_found = True
                    # Check lines before this section for regional data
                    for j in range(max(0, i - 10), i):
                        check_line = lines[j]
                        for region in uk_regions:
                            if region.lower() in check_line.lower():
                                numbers = re.findall(r'\b(\d{1,3})\b', check_line)
                                valid_numbers = [int(n) for n in numbers if n.isdigit() and 0 <= int(n) <= 100]
                                if valid_numbers and region not in result:
                                    result[region] = max(valid_numbers)
                    break
        
        # Sort by value (descending) and limit to top regions
        if result:
            sorted_result = dict(sorted(result.items(), key=lambda x: x[1], reverse=True))
            # Take top regions (max 15)
            top_count = min(15, len(sorted_result))
            return dict(list(sorted_result.items())[:top_count])
        
        return result

def handle_cookie_consent(driver, timeout=6):
    """
    Handle Google cookie/consent UI if present.
    Returns True if clicked something, False otherwise.
    """
    logging.info("Checking for cookie consent dialog...")

    def try_click(elem):
        try:
            driver.execute_script("arguments[0].scrollIntoView({block:'center'});", elem)
        except:
            pass
        try:
            elem.click()
            return True
        except:
            try:
                driver.execute_script("arguments[0].click();", elem)
                return True
            except:
                return False

    # Common Google button ids that sometimes exist
    id_candidates = ["L2AGLb", "W0wltc"]  # L2AGLb often = accept, W0wltc often = reject (varies)
    for bid in id_candidates:
        try:
            elem = WebDriverWait(driver, 1).until(EC.presence_of_element_located((By.ID, bid)))
            if elem.is_displayed() and elem.is_enabled():
                logging.info(f" Found consent button by id: {bid} text='{elem.text.strip()}'")
                if try_click(elem):
                    time.sleep(1.5)
                    return True
        except:
            pass

    # XPath candidates (robust text matching)
    xpath_candidates = [
        "//button[contains(., 'Reject all')]",
        "//button[contains(., 'Accept all')]",
        "//button[contains(., 'I agree')]",
        "//div[@role='dialog']//button[contains(., 'Reject')]",
        "//div[@role='dialog']//button[contains(., 'Accept')]",
        "//form//button[contains(., 'Reject')]",
        "//form//button[contains(., 'Accept')]",
    ]

    # CSS candidates
    css_candidates = [
        "div[role='dialog'] button",
        "form button",
        "button[aria-label*='Accept']",
        "button[aria-label*='Reject']",
    ]

    # 1) Try on main document
    end = time.time() + timeout
    while time.time() < end:
        for xp in xpath_candidates:
            try:
                elems = driver.find_elements(By.XPATH, xp)
                for e in elems:
                    if e.is_displayed() and e.is_enabled():
                        txt = (e.text or "").strip()
                        if txt:
                            logging.info(f" Found consent button (xpath): '{txt}'")
                        if try_click(e):
                            time.sleep(1.5)
                            return True
            except:
                pass

        for cs in css_candidates:
            try:
                elems = driver.find_elements(By.CSS_SELECTOR, cs)
                for e in elems:
                    if e.is_displayed() and e.is_enabled():
                        txt = (e.text or "").strip().lower()
                        if any(k in txt for k in ["accept", "reject", "agree"]):
                            logging.info(f" Found consent button (css): '{e.text.strip()}'")
                            if try_click(e):
                                time.sleep(1.5)
                                return True
            except:
                pass

        time.sleep(0.25)

    # 2) Try inside consent iframes
    try:
        frames = driver.find_elements(By.CSS_SELECTOR, "iframe[src*='consent'], iframe[name*='consent'], iframe[title*='consent']")
        for fr in frames:
            try:
                driver.switch_to.frame(fr)
                for xp in xpath_candidates:
                    elems = driver.find_elements(By.XPATH, xp)
                    for e in elems:
                        if e.is_displayed() and e.is_enabled():
                            logging.info(f" Found consent button in iframe: '{(e.text or '').strip()}'")
                            if try_click(e):
                                driver.switch_to.default_content()
                                time.sleep(1.5)
                                return True
                driver.switch_to.default_content()
            except:
                try:
                    driver.switch_to.default_content()
                except:
                    pass
    except:
        pass

    logging.info("No cookie consent dialog found")
    return False

def set_google_consent_cookies(driver):
    """
    Set Google consent cookies.
    Call this AFTER navigating to https://www.google.com (or /ncr).
    """
    try:
        logging.info("Setting Google consent cookies...")

        # Only do this if we are on a google domain
        if "google." not in (driver.current_url or ""):
            logging.warning(f" Not on a Google domain ({driver.current_url}). Skipping cookie injection.")
            return False

        consent_cookie = {
            "name": "CONSENT",
            "value": "YES+GB.en-GB+V9+BX",
            "domain": ".google.com",
            "path": "/",
            "secure": True,
            "httpOnly": False,
        }

        socs_cookie = {
            "name": "SOCS",
            "value": "CAISHAgCEhJnd3NfMjAyNTAxMjEtMF9SQzIaAmVuIAEaBgiA_LCgBg",
            "domain": ".google.com",
            "path": "/",
            "secure": True,
            "httpOnly": False,
        }

        driver.add_cookie(consent_cookie)
        driver.add_cookie(socs_cookie)

        logging.info("Injected CONSENT and SOCS cookies")
        return True

    except Exception as e:
        logging.error(f" Failed to set consent cookies: {e}")
        return False

def extract_trends_data_from_current_page(driver) -> Dict[str, Any]:
    """
    Extract Google Trends data from the CURRENT page.
    """
    logging.info("Extracting data from Google Trends page...")
    
    try:
        # Get page text
        body = driver.find_element(By.TAG_NAME, "body")
        page_text = body.text
        
        # Also get URL for context
        current_url = driver.current_url
        
        logging.info(f" Current URL: {current_url}")
        logging.info(f" Page text length: {len(page_text)} characters")
        
        # Debug: Save page text for analysis
        with open("debug_page_text.txt", "w", encoding="utf-8") as f:
            f.write(f"URL: {current_url}\n")
            f.write(f"Time: {datetime.now()}\n")
            f.write("-" * 80 + "\n")
            f.write(page_text[:2000])  # First 2000 chars
        
        # Parse with IMPROVED parser
        parser = GoogleTrendsParser()
        parsed_data = parser.parse(page_text)
        
        # Always add URL
        parsed_data['url'] = current_url
        
        # Log what we got
        logging.info(f" Extracted data for: '{parsed_data.get('search_term', 'Unknown')}'")
        logging.info(f" Location: {parsed_data.get('location', 'Unknown')}")
        
        regions = parsed_data.get('interest_by_subregion', {})
        if regions:
            logging.info(f"  Regional data ({len(regions)} regions):")
            for region, value in regions.items():
                logging.info(f"   • {region}: {value}")
        else:
            logging.info("No regional data found")
        
        # Take screenshot for visual verification
        timestamp = datetime.now().strftime('%H%M%S')
        screenshot_name = f"extraction_{timestamp}.png"
        driver.save_screenshot(screenshot_name)
        logging.info(f"📸 Saved screenshot: {screenshot_name}")
        
        return parsed_data
        
    except Exception as e:
        logging.error(f" Failed to extract page data: {e}")
        
        # Emergency debug
        try:
            timestamp = datetime.now().strftime('%H%M%S')
            driver.save_screenshot(f"error_extraction_{timestamp}.png")
            
            # Save error page
            with open(f"error_page_{timestamp}.txt", "w", encoding="utf-8") as f:
                f.write(f"Error: {e}\n")
                f.write(f"URL: {driver.current_url}\n")
                f.write(driver.page_source[:5000])
        except:
            pass
        
        return {
            'search_term': KEYWORD,
            'location': GEO_HUMAN_NAME,
            'url': driver.current_url if 'driver' in locals() else '',
            'interest_over_time': {'y_axis': [], 'x_axis': []},
            'interest_by_subregion': {}
        }

# === London Proxy Pool ===
PROXIES = [
    "ultra.marsproxies.com:44443",
]

# === London Identity Pool (3 Profiles) ===
IDENTITY_POOL = [
    # 1) Windows + Chrome (Desktop) - ENHANCED
    {
        "timezone": "Europe/London",
        "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
        "viewport": (1366, 768),
        "platform": "Win32",
        "hardware_concurrency": 8,
        "max_touch_points": 0,
        "language": "en-GB,en;q=0.9",
        # Add these for better stealth:
        "webgl_vendor": "Google Inc. (Intel)",  # Common for Chrome on Intel PCs
        "webgl_renderer": "ANGLE (Intel, Intel(R) UHD Graphics...)",  # Example
        "device_memory": 8,  # In GB
        "do_not_track": 0,  # 0 = DNT not enabled (common)
        "canvas_hash": "needs_calculated_value",  # You'll need to generate this
        "audio_hash": "needs_calculated_value",   # You'll need to generate this
    },

    # 2) macOS + Safari (Desktop) - ENHANCED
    {
        "timezone": "Europe/London",
        "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15",
        "viewport": (1440, 900),
        "platform": "MacIntel",
        "hardware_concurrency": 4,  # Consider 8 for Apple Silicon
        "max_touch_points": 0,
        "language": "en-GB,en;q=0.9",
        # Add these for better stealth:
        "webgl_vendor": "Apple Inc.",
        "webgl_renderer": "Apple M1 Pro",  # Match your hardware choice
        "device_memory": 8,  # In GB
        "do_not_track": 1,  # Safari more commonly has DNT enabled
        "canvas_hash": "needs_calculated_value",
        "audio_hash": "needs_calculated_value",
    },
]

# === Cookie Management ===
def save_cookies(driver, filename="cookies.json"):
    """Save cookies to file"""
    try:
        cookies = driver.get_cookies()
        with open(filename, 'w') as f:
            json.dump(cookies, f, indent=2)
        logging.info(f" Saved cookies to {filename}")
        return True
    except Exception as e:
        logging.error(f" Failed to save cookies: {e}")
        return False

def load_cookies(driver, filename="cookies.json"):
    """Load cookies from file"""
    try:
        with open(filename, 'r') as f:
            cookies = json.load(f)
        
        # Navigate to domain first
        driver.get("https://trends.google.com")
        time.sleep(2)
        
        for cookie in cookies:
            try:
                driver.add_cookie(cookie)
            except:
                pass
        
        logging.info(f" Loaded {len(cookies)} cookies from {filename}")
        return True
    except Exception as e:
        logging.debug(f"Could not load cookies: {e}")
        return False

# === Smart Backoff Logic ===
def smart_backoff(min_wait=1, max_wait=5, reason="", driver=None):
    """
    Intelligent waiting with exponential backoff and jitter
    """
    # Add more randomness - sometimes wait much longer
    if random.random() < 0.15:  # 15% chance of longer wait
        min_wait = min_wait * 1.5
        max_wait = max_wait * 2
    
    wait_time = random.uniform(min_wait, max_wait)
    
    # Add micro-variations during wait
    if wait_time > 1.5 and driver and random.random() < 0.4:
        # Break long waits into smaller chunks with micro-movements
        chunks = max(2, int(wait_time / 0.7))
        chunk_time = wait_time / chunks
        
        for i in range(chunks):
            time.sleep(chunk_time)
            # Occasionally trigger tiny mouse movements during long waits
            if random.random() < 0.3:
                try:
                    driver.execute_script("""
                        var evt = new MouseEvent('mousemove', {
                            clientX: Math.random() * window.innerWidth,
                            clientY: Math.random() * window.innerHeight,
                            bubbles: true
                        });
                        document.dispatchEvent(evt);
                    """)
                except:
                    pass
    else:
        time.sleep(wait_time)
    
    if reason:
        logging.info(f" {reason} (waiting {wait_time:.1f}s)")
    return wait_time

def human_type_with_smart_correction(element, text, context="search", driver=None):
    """
    Human-ish typing that ALWAYS ends with the exact desired text in the input.
    If driver is provided, we can use JS as a last-resort clear.
    """
    logging.info(f" Typing '{text}' ({context})")

    def hard_clear():
        element.click()
        smart_backoff(0.15, 0.35)

        # Try select-all + backspace (both control and command)
        for combo in (Keys.CONTROL + "a", Keys.COMMAND + "a"):
            try:
                element.send_keys(combo)
                smart_backoff(0.05, 0.15)
                element.send_keys(Keys.BACKSPACE)
                smart_backoff(0.1, 0.25)
                if not (element.get_attribute("value") or "").strip():
                    return True
            except:
                pass

        # Try element.clear()
        try:
            element.clear()
            smart_backoff(0.1, 0.25)
            if not (element.get_attribute("value") or "").strip():
                return True
        except:
            pass

        # JS last resort if driver is available
        if driver is not None:
            try:
                driver.execute_script("arguments[0].value = '';", element)
                smart_backoff(0.1, 0.2)
                if not (element.get_attribute("value") or "").strip():
                    return True
            except:
                pass

        return False

    # 1) Clear reliably
    if not hard_clear():
        logging.warning("Could not fully clear field, proceeding anyway")

    # 2) Type with small human errors, but keep it controlled
    error_chance = 0.03 if context == "search" else 0.02

    for ch in text:
        # occasional hesitation
        if random.random() < 0.05:
            smart_backoff(0.08, 0.25)

        if random.random() < error_chance and ch.isalpha():
            err_type = random.choice(["typo", "extra"])
            if err_type == "typo":
                wrong = random.choice("abcdefghijklmnopqrstuvwxyz")
                element.send_keys(wrong)
                smart_backoff(0.06, 0.18)
                element.send_keys(Keys.BACKSPACE)
                smart_backoff(0.05, 0.12)

            elif err_type == "extra":
                extra = random.choice("aeious")
                element.send_keys(extra)
                smart_backoff(0.05, 0.12)
                element.send_keys(Keys.BACKSPACE)
                smart_backoff(0.05, 0.12)

        element.send_keys(ch)
        smart_backoff(0.03, 0.12)

    # 3) Final verification + hard correction (guarantee)
    smart_backoff(0.25, 0.6, "Final verification")
    final_text = element.get_attribute("value") or ""

    logging.info(f" Typed: '{final_text}'")
    logging.info(f" Expected: '{text}'")

    if final_text.strip() != text.strip():
        logging.warning("Text mismatch. Hard-correcting...")
        hard_clear()
        for ch in text:
            element.send_keys(ch)
            smart_backoff(0.02, 0.06)

        final_text2 = element.get_attribute("value") or ""
        if final_text2.strip() != text.strip():
            raise RuntimeError(f"Typing correction failed. Got '{final_text2}' expected '{text}'")
        logging.info("Text corrected")

    else:
        logging.info("Text typed correctly")

    smart_backoff(0.2, 0.6)
    return True
# === Helper: Safe Element Scroll ===
def scroll_element_into_view(driver, element):
    """Safely scroll an element into view."""
    try:
        driver.execute_script("arguments[0].scrollIntoView({behavior: 'smooth', block: 'center'});", element)
        time.sleep(random.uniform(0.2, 0.5))
        return True
    except:
        return False

def set_timeframe(driver, timeframe="Past 12 months"):
    """
    More flexible version of setting the timeframe on Google Trends
    """
    logging.info(f" Setting timeframe: {timeframe}")

    try:
        # STEP 1: Open the dropdown
        dropdown = WebDriverWait(driver, 15).until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, "md-select[aria-label^='Select time period']"))
        )
        logging.info(" Found timeframe dropdown")
        random_mouse_movement(driver, dropdown)
        smart_backoff(0.5, 1.0, "Hovering over timeframe dropdown", driver)
        dropdown.click()
        smart_backoff(1.5, 2.5, "Dropdown menu opening", driver)

        # STEP 2: Find all potential options (md-option and _md-text)
        options = driver.find_elements(By.CSS_SELECTOR, "md-option, div._md-text")
        logging.info(f" Found {len(options)} dropdown items")

        for option in options:
            try:
                if option.is_displayed() and timeframe.lower() in option.text.lower():
                    logging.info(f" Matching timeframe found: {option.text.strip()}")
                    random_mouse_movement(driver, option)
                    smart_backoff(0.5, 1.0, "Hovering over timeframe option", driver)

                    try:
                        option.click()
                    except Exception:
                        driver.execute_script("arguments[0].click();", option)
                    
                    logging.info(f" Timeframe '{timeframe}' selected")
                    smart_backoff(2.5, 4.0, "Waiting for charts to reload", driver)
                    return True
            except Exception as e:
                logging.debug(f" Option check error: {e}")
                continue

        logging.warning(f" Timeframe option not found: {timeframe}")
        return False

    except Exception as e:
        logging.error(f" Failed to set timeframe: {e}")
        return False

def verify_and_set_location(driver, target_location="London"):
    """
    Verify we're in the right location, set if not
    """
    logging.info(f" Verifying location: {target_location}")
    
    try:
        # Check current URL for geo parameter
        current_url = driver.current_url
        if "geo=" in current_url:
            geo_match = re.search(r'geo=([A-Za-z]{2}-[A-Za-z]{2,})', current_url)
            if geo_match:
                logging.info(f" Location already set in URL: {geo_match.group(1)}")
                return True
        
        # Check page text for location indicators
        page_text = driver.find_element(By.TAG_NAME, "body").text.lower()
        location_indicators = ["london", "united kingdom", "gb", "england", "uk"]

        
        for indicator in location_indicators:
            if indicator in page_text:
                logging.info(f" Location detected in page: {indicator}")
                return True
        
        # If location not detected, try to set it
        logging.info(" Location not detected, attempting to set...")
        
        # Look for location selector
        location_selectors = [
            "div[aria-label*='Location']",
            "div[aria-label*='location']",
            "button[aria-label*='Location']",
            "md-select[aria-label*='Location']",
            "//div[contains(text(), 'Worldwide') or contains(text(), 'Location')]",
            ".hierarchy-select",
            "div.custom-location-picker-select",
            "//div[@role='button' and contains(@aria-label, 'Location')]",
        ]
        
        location_button = None
        
        for selector in location_selectors:
            try:
                if selector.startswith("//"):
                    elements = driver.find_elements(By.XPATH, selector)
                else:
                    elements = driver.find_elements(By.CSS_SELECTOR, selector)
                
                for element in elements:
                    try:
                        if element.is_displayed():
                            text = element.text.lower()
                            if any(word in text for word in ['location', 'worldwide', 'country', 'region']):
                                location_button = element
                                logging.info(f" Found location selector")
                                break
                    except:
                        continue
                if location_button:
                    break
            except:
                continue
        
        if location_button:
           # Click location selector
            random_mouse_movement(driver, location_button)
            smart_backoff(0.8, 1.5)
            
            try:
                location_button.click()
            except:
                driver.execute_script("arguments[0].click();", location_button)
            
            logging.info("Opened location dropdown")
            smart_backoff(2, 4, "Waiting for location dropdown")
            
            # Search for Dubai
            try:
                location_search = driver.find_element(By.CSS_SELECTOR, "input[placeholder*='location'], input[placeholder*='Location'], input[aria-label*='search'], input[type='search']")
                location_search.clear()
                human_type_with_smart_correction(location_search, "London", context="location")
                smart_backoff(2, 4, "Waiting for location results")
                
                # Select London from results
                london_options = [
                    "//div[contains(text(), 'London')]",
                    "//md-option[contains(., 'London')]",
                    "//div[contains(@class, '_md-text') and contains(text(), 'London')]",
                    "//div[contains(text(), 'London, United Kingdom')]",
                ]
                
                for option_selector in london_options:
                    try:
                        elements = driver.find_elements(By.XPATH, option_selector)
                        for element in elements:
                            if element.is_displayed() and "london" in element.text.lower():
                                element.click()
                                logging.info("Selected London location")
                                smart_backoff(3, 6, "Waiting for location to apply")
                                return True
                    except:
                        continue
            except Exception as e:
                logging.warning(f" Could not search for location: {e}")
        
        logging.warning(" Could not set location via UI, might already be correct")
        return False
        
    except Exception as e:
        logging.error(f" Location verification failed: {e}")
        return False

# === Pre-warm Browser ===
def pre_warm_browser(driver):
    """Pre-warm the browser with human-like activity before actual scraping"""
    logging.info("Pre-warming browser with human-like activity...")
    
    # Visit a few neutral sites first
    neutral_sites = [
        "https://www.theguardian.com/uk",
        "https://en.wikipedia.org/wiki/London",
        "https://www.reddit.com/r/london",
    ]
    
    sites_to_visit = random.sample(neutral_sites, min(2, len(neutral_sites)))
    
    for site in sites_to_visit:
        try:
            site_name = site.split('//')[1].split('/')[0]
            logging.info(f" Visiting {site_name}")
            driver.get(site)
            smart_backoff(4, 8, f"Browsing {site_name}", driver)
            
            # Do some human-like scrolling
            scrolls = random.randint(2, 4)
            for scroll_num in range(scrolls):
                scroll_amount = random.randint(300, 700)
                direction = random.choice([-1, 1])
                driver.execute_script(f"window.scrollBy(0, {scroll_amount * direction});")
                smart_backoff(1, 3, f"Reading content (scroll {scroll_num+1}/{scrolls})", driver)
            
            # Occasionally click on a random safe link
            if random.random() < 0.4:
                try:
                    links = driver.find_elements(By.TAG_NAME, "a")
                    safe_links = []
                    for link in links:
                        try:
                            if link.is_displayed() and link.is_enabled():
                                href = link.get_attribute("href") or ""
                                text = link.text.lower()
                                # Avoid problematic links
                                if href and not any(x in href for x in ["javascript:", "mailto:", "#"]) and len(text) > 2:
                                    safe_links.append(link)
                        except:
                            continue
                    
                    if safe_links:
                        link = random.choice(safe_links[:15])
                        logging.info(f"🔗 Clicking on: {link.text[:50]}...")
                        driver.execute_script("arguments[0].click();", link)
                        smart_backoff(3, 7, "Following link", driver)
                        driver.back()
                        smart_backoff(2, 4, "Going back", driver)
                except:
                    pass
                    
        except Exception as e:
            logging.debug(f"Pre-warm site {site} failed: {e}")
            continue
    
    logging.info("Browser pre-warmed")

# === Random Window Resize ===
def random_window_resize(driver):
    """Randomly resize browser window to appear more human"""
    if random.random() < 0.25:  # 25% chance
        try:
            sizes = [
                (1366, 768),
                (1440, 900),
                (1536, 864),
                (1600, 900),
                (1920, 1080),
                (1280, 720),
            ]
            width, height = random.choice(sizes)
            driver.set_window_size(width, height)
            logging.debug(f" Resized window to {width}x{height}")
            smart_backoff(0.8, 2, "Window resized", driver)
        except:
            pass

# === Random Human Interactions ===
def simulate_human_scanning(driver, duration=2, intensity="light"):
    """
    Simulate human scanning/reading behavior WITH SAFETY CHECKS
    """
    logging.info(" Simulating human scanning...")
    
    intensities = {
        "very_light": {"scrolls": 0, "moves": 1, "clicks": 0},
        "light": {"scrolls": 1, "moves": 1, "clicks": 0},
        "medium": {"scrolls": 1, "moves": 2, "clicks": 0},
        "heavy": {"scrolls": 2, "moves": 3, "clicks": 1}
    }
    
    config = intensities.get(intensity, intensities["very_light"])
    
    # Random mouse movements - MORE NATURAL
    for i in range(config["moves"]):
        try:
            window_size = driver.get_window_size()
            start_x = random.randint(50, window_size['width'] - 50)
            start_y = random.randint(50, window_size['height'] - 50)
            end_x = random.randint(50, window_size['width'] - 50)
            end_y = random.randint(50, window_size['height'] - 50)
            
            # Create more natural mouse movement with curve
            driver.execute_script(f"""
                var startX = {start_x};
                var startY = {start_y};
                var endX = {end_x};
                var endY = {end_y};
                var steps = 15;
                var currentStep = 0;
                
                function moveStep() {{
                    if (currentStep <= steps) {{
                        var t = currentStep / steps;
                        // Bezier curve for natural movement
                        var x = startX + (endX - startX) * t;
                        var y = startY + (endY - startY) * t;
                        
                        // Add slight curve
                        x += Math.sin(t * Math.PI) * 20;
                        y += Math.cos(t * Math.PI) * 15;
                        
                        var evt = new MouseEvent('mousemove', {{
                            clientX: Math.round(x),
                            clientY: Math.round(y),
                            view: window,
                            bubbles: true
                        }});
                        document.dispatchEvent(evt);
                        
                        currentStep++;
                        setTimeout(moveStep, 20 + Math.random() * 15);
                    }}
                }}
                
                moveStep();
            """)
            
            # Wait for movement to complete plus some random time
            movement_time = 20 * 15 / 1000 + random.uniform(0.1, 0.4)
            time.sleep(movement_time)
            smart_backoff(0.2, 0.8, f"Mouse movement {i+1}/{config['moves']}", driver)
        except:
            pass
    
    # Random scrolling - WITH BOUNDS CHECK
    for i in range(config["scrolls"]):
        try:
            current_scroll = driver.execute_script("return window.pageYOffset;")
            window_height = driver.execute_script("return window.innerHeight;")
            document_height = driver.execute_script("return document.body.scrollHeight;")
            
            max_scroll = document_height - window_height
            direction = random.choice([-1, 1])
            
            if direction == 1 and current_scroll < max_scroll - 100:
                amount = random.randint(100, min(400, max_scroll - current_scroll - 50))
            elif direction == -1 and current_scroll > 100:
                amount = random.randint(100, min(400, current_scroll))
            else:
                direction = direction * -1
                amount = random.randint(100, 300)
            
            # Smooth scrolling
            steps = random.randint(3, 6)
            step_amount = amount / steps
            for step in range(steps):
                driver.execute_script(f"window.scrollBy(0, {step_amount * direction});")
                time.sleep(random.uniform(0.05, 0.15))
            
            smart_backoff(0.3, 1.2, f"Scrolling {i+1}/{config['scrolls']}", driver)
        except:
            pass
    
    # Random clicks (on safe elements) - WITH VISIBILITY CHECK (REDUCED)
    for i in range(config["clicks"]):
        try:
            safe_elements = driver.find_elements(By.CSS_SELECTOR, "body, div, span, p, a, button")
            visible_elements = []
            
            for element in safe_elements:
                try:
                    if element.is_displayed() and element.is_enabled():
                        location = element.location
                        size = element.size
                        if location and size:
                            text = element.text.lower()
                            # Avoid clicking on important elements
                            if not any(word in text for word in ['submit', 'login', 'sign', 'buy', 'purchase', 'download']):
                                visible_elements.append(element)
                except:
                    continue
            
            if visible_elements:
                element = random.choice(visible_elements[:5])  # Reduced from 10 to 5
                
                scroll_element_into_view(driver, element)
                smart_backoff(0.3, 0.8, "Preparing to click", driver)
                
                try:
                    actions = ActionChains(driver)
                    actions.move_to_element(element)
                    actions.click()
                    actions.perform()
                except:
                    driver.execute_script("arguments[0].click();", element)
                
                smart_backoff(0.3, 0.8, f"Random click {i+1}/{config['clicks']}", driver)
        except Exception as e:
            pass

# === Random Mouse Movements (Updated) ===
def random_mouse_movement(driver, element=None):
    if not RANDOM_MOUSE_MOVEMENTS:
        return
    
    try:
        window_size = driver.get_window_size()
        
        if element:
            try:
                scroll_element_into_view(driver, element)
                smart_backoff(0.3, 0.7, "Scrolled to element", driver)
                
                element_location = element.location
                if element_location:
                    # Move to element with natural motion
                    actions = ActionChains(driver)
                    
                    # Get element center
                    size = element.size
                    target_x = element_location['x'] + size['width'] / 2
                    target_y = element_location['y'] + size['height'] / 2
                    
                    # Add some offset for more natural movement
                    offset_x = random.randint(-10, 10)
                    offset_y = random.randint(-10, 10)
                    
                    # Move in natural curve
                    actions.move_to_element_with_offset(element, int(size['width']/2) + offset_x, int(size['height']/2) + offset_y)
                    actions.perform()
                    
                    smart_backoff(0.2, 0.5, "Moved to element", driver)
                    return
            except:
                pass
        
        # Random movement within viewport bounds
        actions = ActionChains(driver)
        
        target_x = random.randint(50, window_size['width'] - 50)
        target_y = random.randint(50, window_size['height'] - 50)
        
        steps = random.randint(3, 6)
        current_x, current_y = 0, 0
        
        for step in range(steps):
            step_x = current_x + (target_x - current_x) * (step + 1) / steps
            step_y = current_y + (target_y - current_y) * (step + 1) / steps
            
            # Add natural curve to movement
            curve_x = math.sin(step * 0.5) * 30
            curve_y = math.cos(step * 0.5) * 25
            
            step_x += curve_x + random.randint(-15, 15)
            step_y += curve_y + random.randint(-15, 15)
            
            # Ensure we stay within bounds
            step_x = max(10, min(step_x, window_size['width'] - 10))
            step_y = max(10, min(step_y, window_size['height'] - 10))
            
            actions.move_by_offset(int(step_x - current_x), int(step_y - current_y))
            current_x, current_y = step_x, step_y
            time.sleep(random.uniform(0.03, 0.1))
        
        actions.perform()
        smart_backoff(0.1, 0.3, "Random mouse movement", driver)
        
    except Exception as e:
        pass

# === Clean Exit ===
def clean_exit(driver):
    """Clean exit to look more human with proxy extension cleanup"""
    try:
        # Scroll to top
        driver.execute_script("window.scrollTo({top: 0, behavior: 'smooth'});")
        smart_backoff(1, 2, "Scrolling to top", driver)
        
        # Random mouse movement to top-left
        driver.execute_script("""
            var evt = new MouseEvent('mousemove', {
                clientX: 50,
                clientY: 50,
                view: window,
                bubbles: true
            });
            document.dispatchEvent(evt);
        """)
        
        # Wait a bit
        smart_backoff(2, 4, "Final pause before exit", driver)
        
        # Save cookies before quitting
        try:
            identity_hash = hash(str(driver.execute_script("return navigator.userAgent;")))
            cookie_file = f"cookies_{identity_hash}.json"
            save_cookies(driver, cookie_file)
        except:
            pass
        
        # Clean up proxy extension files
        try:
            if hasattr(driver, 'proxy_extension_cleanup'):
                cleanup = driver.proxy_extension_cleanup
                if os.path.exists(cleanup.get('extension_path', '')):
                    os.remove(cleanup['extension_path'])
                if os.path.exists(cleanup.get('temp_dir', '')):
                    os.rmdir(cleanup['temp_dir'])
                logging.info(" Cleaned up proxy extension files")
        except:
            pass
        
        # Close
        driver.quit()
        logging.info("Browser closed cleanly")
    except:
        try:
            driver.quit()
        except:
            pass

# === CAPTCHA Handling Functions ===
def detect_and_handle_captcha(driver):
    """
    Detect if CAPTCHA is present and handle it appropriately.
    Returns True if CAPTCHA was detected, False otherwise.
    """
    try:
        # Check for CAPTCHA iframes or elements
        captcha_selectors = [
            "iframe[src*='captcha']",
            "iframe[src*='recaptcha']",
            "div.g-recaptcha",
            "div.captcha",
            "div#captcha",
            "//div[contains(text(), 'CAPTCHA')]",
            "//div[contains(text(), 'captcha')]",
            "//div[contains(text(), 'Robot') or contains(text(), 'robot')]",
            "//img[contains(@src, 'captcha')]",
        ]
        
        captcha_detected = False
        
        # Check for CAPTCHA iframes
        for selector in captcha_selectors:
            try:
                if selector.startswith("//"):
                    elements = driver.find_elements(By.XPATH, selector)
                else:
                    elements = driver.find_elements(By.CSS_SELECTOR, selector)
                
                if elements and len(elements) > 0:
                    for element in elements:
                        try:
                            if element.is_displayed():
                                logging.warning(f" CAPTCHA detected with selector: {selector}")
                                captcha_detected = True
                                break
                        except:
                            continue
                if captcha_detected:
                    break
            except:
                continue
        
        # Check page text for CAPTCHA indicators
        if not captcha_detected:
            try:
                page_text = driver.find_element(By.TAG_NAME, "body").text.lower()
                captcha_indicators = [
                    "captcha",
                    "verify you are human",
                    "i'm not a robot",
                    "prove you're not a robot",
                    "security check",
                    "type the text",
                    "enter the characters",
                    "select all images with",
                    "verify you're human"
                ]
                
                for indicator in captcha_indicators:
                    if indicator in page_text:
                        logging.warning(f" CAPTCHA text detected: {indicator}")
                        captcha_detected = True
                        break
            except:
                pass
        
        if captcha_detected:
            # Visual CAPTCHA Alert
            logging.warning("=" * 60)
            logging.warning(" CAPTCHA DETECTED! ")
            logging.warning("=" * 60)
            logging.warning("Human intervention required!")
            logging.warning("The script will wait 60 seconds for you to solve it.")
            logging.warning("=" * 60)
            
            # Try to make system notification
            try:
                import platform
                system = platform.system()
                
                if system == "Darwin":  # macOS
                    import os
                    os.system(f"""
                        osascript -e 'display notification "CAPTCHA detected! Please solve it in the browser." with title "Google Trends Scraper" sound name "Submarine"'
                    """)
                elif system == "Linux":
                    import os
                    os.system('notify-send "Google Trends Scraper" "CAPTCHA detected! Please solve it in the browser."')
                elif system == "Windows":
                    # Windows notification
                    import ctypes
                    ctypes.windll.user32.MessageBoxW(0, 
                        "CAPTCHA detected!\n\nPlease solve the CAPTCHA in the browser window.\n\nThe script will wait 60 seconds.", 
                        "Google Trends Scraper", 
                        0x40 | 0x0)  # MB_ICONINFORMATION | MB_OK
            except:
                pass
            
            logging.warning(" CAPTCHA detected - human intervention required!")
            logging.info(" Waiting 60 seconds for manual CAPTCHA solving...")
            
            # Play a sound to alert user (if possible)
            try:
                import sys
                if sys.platform == "win32":
                    import winsound
                    winsound.Beep(1000, 1000)
                elif sys.platform == "darwin":  # macOS
                    import os
                    os.system('afplay /System/Library/Sounds/Ping.aiff')
            except:
                pass
            
            # Wait for user to solve CAPTCHA
            time.sleep(60)
            
            # Check if CAPTCHA is still there
            smart_backoff(5, 10, "Checking if CAPTCHA was solved", driver)
            
            # Verify if we can proceed
            try:
                page_text = driver.find_element(By.TAG_NAME, "body").text.lower()
                if any(indicator in page_text for indicator in ["captcha", "verify", "robot"]):
                    logging.error("CAPTCHA still present after wait")
                    return True
                else:
                    logging.info("CAPTCHA appears to be solved")
                    return False
            except:
                logging.info("Could not verify CAPTCHA status")
                return False
        
        return False
        
    except Exception as e:
        logging.error(f" Error in CAPTCHA detection: {e}")
        return False

# === Enhanced Block Detection ===
def detect_block(driver):
    try:
        # Wait for page to load
        time.sleep(2)
        
        html = driver.page_source.lower()
        page_text = driver.find_element(By.TAG_NAME, "body").text.lower()
        
        # STRICT block indicators only
        strict_block_indicators = [
            "unusual traffic from your computer network",
            "automated queries",
            "our systems have detected unusual traffic",
            "this page appears when google automatically detects requests",
            "we're sorry...",
            "we've detected unusual activity",
            "distressed search page",
        ]
        
        for indicator in strict_block_indicators:
            if indicator in html or indicator in page_text:
                logging.warning(f" STRICT Block detected: {indicator}")
                return True
        
        # Check for CAPTCHA separately
        if detect_and_handle_captcha(driver):
            return "CAPTCHA"
        
        # Check for rate limiting
        rate_limit_indicators = [
            "rate limit",
            "try again in a bit",
            "please try again later",
            "quota exceeded",
        ]
        
        for indicator in rate_limit_indicators:
            if indicator in page_text:
                logging.warning(f" Rate limit detected: {indicator}")
                return "RATE_LIMIT"
        
        return False
    except Exception as e:
        logging.error(f" Error in detect_block: {e}")
        return False

# === Safe Click Helper Function ===
def safe_click_with_retry(driver, element_name="Explore button", selector_type="xpath", selector="//span[contains(text(), 'Explore')]"):
    """
    Safely click an element with retry logic for stale element references
    """
    max_retries = 3
    
    for retry in range(max_retries):
        try:
            # Find the element fresh each time
            if selector_type == "xpath":
                element = WebDriverWait(driver, 5).until(
                    EC.element_to_be_clickable((By.XPATH, selector))
                )
            elif selector_type == "css":
                element = WebDriverWait(driver, 5).until(
                    EC.element_to_be_clickable((By.CSS_SELECTOR, selector))
                )
            
            # Try to click with multiple methods
            try:
                # Method 1: Normal click
                element.click()
                logging.info(f" Clicked {element_name} (normal click)")
                return True
            except:
                # Method 2: JavaScript click
                driver.execute_script("arguments[0].click();", element)
                logging.info(f" Clicked {element_name} (JavaScript)")
                return True
                
        except Exception as e:
            if retry < max_retries - 1:
                logging.debug(f"Retry {retry + 1}/{max_retries} for {element_name}: {str(e)[:100]}")
                smart_backoff(1, 2, f"Retrying {element_name}", driver)
            else:
                logging.warning(f" Failed to click {element_name} after {max_retries} retries")
                return False
    
    return False

def complete_human_journey(driver, keyword):
    """
    Complete human-like journey from Google search to Google Trends data
    """
    logging.info(" Starting complete human journey...")
    
    # PRE-WARM PHASE
    pre_warm_browser(driver)
    
    # Random window resize
    random_window_resize(driver)
    
    # PHASE 1: Start at Google homepage
    logging.info(" Phase 1: Starting at Google homepage")
    driver.get("https://www.google.com")
    smart_backoff(4, 8, "Loading Google homepage", driver)
    
    # === CRITICAL: HANDLE COOKIE CONSENT DIALOG ===
    cookie_handled = handle_cookie_consent(driver)
    if cookie_handled:
        logging.info("Cookie consent dialog handled")
        smart_backoff(2, 4, "After cookie handling", driver)
    else:
        logging.info("No cookie consent dialog found or already handled")
    
    # Human scanning of homepage
    simulate_human_scanning(driver, duration=2, intensity="very_light")
    
    # Find Google search box
    logging.info(" Looking for Google search box...")
    google_search_box = None
    google_selectors = [
        "textarea[name='q']",
        "input[name='q']",
        "textarea[title='Search']",
        "input[title='Search']",
        "textarea[aria-label='Search']",
        "input[aria-label='Search']",
        "textarea.gLFyf",
        "input.gLFyf",
    ]
    
    for selector in google_selectors:
        try:
            element = WebDriverWait(driver, 15).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, selector))
            )
            if element.is_displayed():
                google_search_box = element
                logging.info(f" Found Google search: {selector}")
                break
        except:
            continue
    
    if not google_search_box:
        raise Exception(" Could not find Google search box")
    
    # Type "Google trends" in Google search
    logging.info("⌨️ Typing 'Google trends' in Google search...")
    random_mouse_movement(driver, google_search_box)
    smart_backoff(0.5, 1.2, "Moving to search box", driver)
    
    google_search_box.click()
    smart_backoff(0.3, 0.8, "Clicked search box", driver)
    
    human_type_with_smart_correction(google_search_box, "google trends", context="search")
    
    # Wait for Google suggestions
    smart_backoff(2, 4, "Waiting for Google suggestions", driver)
    
    # Press Enter to search
    google_search_box.send_keys(Keys.RETURN)
    logging.info(" Searching for 'Google trends' on Google")
    smart_backoff(3, 6, "Loading Google search results", driver)
    
    # === ADDED: CAPTCHA CHECK RIGHT AFTER SEARCHING ===
    # Check for CAPTCHA after Google search
    captcha_found = detect_and_handle_captcha(driver)
    if captcha_found:
        logging.warning(" CAPTCHA encountered, trying alternative approach...")
        
        # Try to go back and search again
        try:
            driver.back()
            smart_backoff(3, 5, "Went back due to CAPTCHA", driver)
            
            # Try a different search approach
            google_search_box = driver.find_element(By.CSS_SELECTOR, "textarea[name='q'], input[name='q']")
            google_search_box.clear()
            smart_backoff(0.5, 1)
            
            # Type differently
            google_search_box.send_keys("Google Trends official")
            smart_backoff(1, 2)
            google_search_box.send_keys(Keys.RETURN)
            smart_backoff(4, 7, "Retrying search with different terms", driver)
        except:
            pass
    
    # PHASE 2: Scan search results
    logging.info(" Phase 2: Scanning Google search results")
    simulate_human_scanning(driver, duration=3, intensity="light")
    
    # Find and click the Google Trends result
    logging.info(" Looking for Google Trends link...")
    
    # Wait for search results to load
    smart_backoff(3, 5, "Waiting for search results", driver)
    
    # Look for the official Google Trends link
    trends_link = None
    trends_selectors = [
        "a[href*='trends.google.com']",
        "a:has(h3:contains('Google Trends'))",
        "//a[contains(@href, 'trends.google.com') and contains(., 'Trends')]",
        "a[ping*='/url?sa=t&source=web&rct=j&url=https://trends.google.com']",
        "//a[contains(@href, 'trends.google.com')]//h3[contains(text(), 'Trends')]",
    ]
    
    for selector in trends_selectors:
        try:
            if selector.startswith("//"):
                elements = driver.find_elements(By.XPATH, selector)
            else:
                elements = driver.find_elements(By.CSS_SELECTOR, selector)
            
            for element in elements:
                try:
                    if element.is_displayed():
                        # Get link text for verification
                        link_text = element.text.lower() if element.text else ""
                        href = element.get_attribute("href") or ""
                        
                        if "trends" in link_text or "trends.google.com" in href:
                            trends_link = element
                            logging.info(f" Found Google Trends link: {link_text[:50]}...")
                            logging.info(f"   URL: {href[:80]}...")
                            break
                except:
                    continue
            if trends_link:
                break
        except:
            continue
    
    if not trends_link:
        # Fallback: Look for any link containing "trends"
        try:
            all_links = driver.find_elements(By.TAG_NAME, "a")
            for link in all_links:
                try:
                    if link.is_displayed():
                        text = link.text.lower()
                        href = link.get_attribute("href") or ""
                        if "trends" in text and "google" in text:
                            trends_link = link
                            logging.info(f" Found fallback Trends link: {text[:50]}...")
                            break
                except:
                    continue
        except:
            pass
    
    if not trends_link:
        # Ultimate fallback: Go directly but simulate human behavior
        logging.warning(" Could not find Trends link, going directly")
        driver.get("https://trends.google.com")
        smart_backoff(4, 7, "Loading Google Trends directly", driver)
    else:
        # Click the link (human-like) - WITH INTERCEPTION HANDLING
        random_mouse_movement(driver, trends_link)
        smart_backoff(0.8, 1.5, "Hovering over Trends link", driver)
    
        try:
            # Try normal click first
            trends_link.click()
            logging.info("🔗 Clicked Google Trends link (normal click)")
        except Exception as e:
            if "element click intercepted" in str(e):
                logging.info(" Element click intercepted, trying JavaScript click...")
                try:
                    # Try JavaScript click
                    driver.execute_script("arguments[0].click();", trends_link)
                    logging.info(" Clicked Google Trends link (JavaScript click)")
                except Exception as js_e:
                    logging.warning(f" JavaScript click failed: {js_e}, trying scroll and click...")
                    try:
                        # Scroll element into view and try again
                        driver.execute_script("arguments[0].scrollIntoView({block: 'center', behavior: 'smooth'});", trends_link)
                        smart_backoff(0.8, 1.5, "Scrolled to link", driver)
                        trends_link.click()
                        logging.info(" Clicked Google Trends link (after scroll)")
                    except Exception as scroll_e:
                        logging.warning(f" All clicks failed, going directly: {scroll_e}")
                        driver.get("https://trends.google.com")
            else:
                # Some other error
                logging.warning(f" Click failed: {e}, going directly")
                driver.get("https://trends.google.com")
    
        smart_backoff(4, 8, "Loading Google Trends website", driver)
    
    # PHASE 3: On Google Trends website
    logging.info(" Phase 3: On Google Trends website")
    # === CHECK FOR COOKIE DIALOG ON GOOGLE TRENDS TOO ===
    cookie_handled_trends = handle_cookie_consent(driver)
    if cookie_handled_trends:
        logging.info(" Cookie consent handled on Google Trends")
        smart_backoff(2, 4, "After cookie handling", driver)
    # Simulate human exploring the page
    simulate_human_scanning(driver, duration=4, intensity="light")
    
    # Verify and set London location
    verify_and_set_location(driver, "London")
    
    # PHASE 4: Search for keyword on Google Trends
    logging.info(f" Phase 4: Searching for '{keyword}' on Google Trends")
    
    # === DEBUG: Print all search-like inputs first ===
    logging.info(" DEBUG: Looking for search inputs...")
    try:
        all_inputs = driver.find_elements(By.TAG_NAME, "input")
        logging.info(f" Found {len(all_inputs)} total input elements")
        
        visible_inputs = []
        for i, inp in enumerate(all_inputs[:15]):  # Check first 15
            try:
                if inp.is_displayed():
                    inp_id = inp.get_attribute("id") or "no-id"
                    inp_type = inp.get_attribute("type") or "no-type"
                    inp_class = inp.get_attribute("class") or "no-class"
                    inp_aria = inp.get_attribute("aria-label") or "no-aria"
                    inp_placeholder = inp.get_attribute("placeholder") or "no-placeholder"
                    
                    logging.info(f"  Input #{i}: id='{inp_id}', type='{inp_type}', aria='{inp_aria}'")
                    logging.info(f"     class='{inp_class[:50]}...', placeholder='{inp_placeholder}'")
                    
                    if inp_type == "text" and ("search" in inp_aria.lower() or "search" in inp_placeholder.lower()):
                        visible_inputs.append(inp)
                        logging.info(f"    This looks like a search box!")
            except:
                pass
        
        logging.info(f"🔍 Found {len(visible_inputs)} visible text inputs that might be search boxes")
    except:
        logging.warning(" Could not debug inputs")
    
    # Find the search box on Google Trends
    trends_search_box = None
    trends_search_selectors = [
        "input#i4",  # Always the correct one for real keyword search
        "input[jsname='YPqjbf'][aria-label='Search']:not([disabled])",  # Skip top disabled box
        "input[aria-label='Search']:not([disabled])",
        "input[placeholder*='earch']:not([disabled])",
        "input[type='text']:not([disabled])",
    ]
    
    for selector in trends_search_selectors:
        try:
            logging.info(f" Trying selector: {selector}")
            elements = driver.find_elements(By.CSS_SELECTOR, selector)
            logging.info(f" Found {len(elements)} elements with selector: {selector}")
            
            for element in elements:
                try:
                    if element.is_displayed() and element.is_enabled():
                        # Additional verification
                        el_id = element.get_attribute("id") or ""

                        element_aria = element.get_attribute("aria-label") or ""
                        el_class = element.get_attribute("class") or ""
                        el_aria = element.get_attribute("aria-label") or ""
                        if "explore topics" not in (element.get_attribute("value") or "").lower():
                            trends_search_box = element
                            logging.info(f" Selected search box: id={el_id}, aria={el_aria}")
                            break
                except Exception as e:
                    logging.debug(f"   Element check failed: {e}")
                    continue
            if trends_search_box:
                break
        except Exception as e:
            logging.debug(f"Selector {selector} failed: {e}")
            continue
    
    if not trends_search_box:
        # Last resort: Look for any input that looks like a search box
        logging.info(" Trying last resort: scanning all inputs...")
        try:
            all_inputs = driver.find_elements(By.TAG_NAME, "input")
            for inp in all_inputs:
                try:
                    if inp.is_displayed() and inp.get_attribute("type") == "text":
                        val = inp.get_attribute("value") or ""
                        aria = inp.get_attribute("aria-label") or ""
                        
                        if "explore topics" not in val.lower() and "search" in aria.lower():

                            trends_search_box = inp
                            logging.info(" Found fallback search box (filtered)")
                            break
                except:
                    continue
        except:
            pass
    
    if not trends_search_box:
        raise Exception(" Could not find search box on Google Trends")
    
    # Human-like interaction with search box
    logging.info(" Moving to search box...")
    random_mouse_movement(driver, trends_search_box)
    smart_backoff(0.8, 1.5, "Moving to search box", driver)
    
    trends_search_box.click()
    smart_backoff(0.5, 1.2, "Clicked search box", driver)
    
    # === ENHANCED: CLEAR THE SEARCH BOX PROPERLY ===
    logging.info(" Checking and clearing search box...")
    
    # Check if there's existing text
    existing_text = trends_search_box.get_attribute("value") or trends_search_box.text
    if existing_text and existing_text.strip():
        logging.info(f" Found existing text in search box: '{existing_text[:50]}...'")
        
        # Clear using multiple methods for reliability
        clear_methods = [
            lambda: trends_search_box.send_keys(Keys.CONTROL + 'a'),
            lambda: trends_search_box.send_keys(Keys.COMMAND + 'a'),
            lambda: trends_search_box.clear(),
            lambda: driver.execute_script("arguments[0].value = '';", trends_search_box)
        ]
        
        for method_num, clear_method in enumerate(clear_methods):
            try:
                clear_method()
                smart_backoff(0.2, 0.4, f"Clearing method {method_num + 1}")
                
                # Verify clear
                current_text = trends_search_box.get_attribute("value") or trends_search_box.text
                if not current_text or not current_text.strip():
                    logging.info(" Search box cleared successfully")
                    break
            except:
                continue
        
        # Final verification
        trends_search_box.click()
        for _ in range(5):
            trends_search_box.send_keys(Keys.BACKSPACE)
            smart_backoff(0.05, 0.1)
        
        # Send a space then backspace to ensure clear
        trends_search_box.send_keys(' ')
        smart_backoff(0.1, 0.2)
        trends_search_box.send_keys(Keys.BACKSPACE)
        smart_backoff(0.2, 0.4)
    else:
        logging.info(" Search box is already empty")
    
    # Type the keyword with human-like errors and corrections
    logging.info(f"⌨️ Typing '{keyword}' on Google Trends...")
    human_type_with_smart_correction(trends_search_box, keyword, context="trends_search")
    smart_backoff(2, 4, "Waiting for keyword suggestions", driver)

    # FIX: Scroll back up to remove 'Explore topics' from view
    try:
        logging.info(" Scrolling back to top to avoid 'Explore topics' interference...")
        driver.execute_script("window.scrollTo({ top: 0, behavior: 'smooth' });")
        smart_backoff(1.5, 2.8, "Scrolling to top", driver)
    except Exception as e:
        logging.warning(f" Could not scroll to top: {e}")

    
    # DEBUG: Take a screenshot to see what the page looks like
    try:
        driver.save_screenshot("debug_after_typing.png")
        logging.info(" Saved debug screenshot: debug_after_typing.png")
    except:
        pass
    
    # DEBUG: Check page text
    try:
        page_text = driver.find_element(By.TAG_NAME, "body").text.lower()
        if "explore" in page_text:
            logging.info(" 'Explore' text found somewhere on page")
        else:
            logging.info(" 'Explore' text NOT found on page")
            
        # Show first 500 chars of page
        logging.info(f" Page text preview: {page_text[:500]}...")
    except:
        pass
    
    # === ADD THIS: Close dropdown before looking for Explore button ===
    try:
        # Try to close any open dropdowns by clicking on a neutral area
        body = driver.find_element(By.TAG_NAME, "body")
        # Click near the top-left corner to avoid clicking on important elements
        actions = ActionChains(driver)
        actions.move_to_element_with_offset(body, 10, 10)
        actions.click()
        actions.perform()
        logging.info(" Clicked neutral area to close dropdowns")
        smart_backoff(1, 2, "Waiting for dropdowns to close")
    except:
        pass

    # ====== FIND AND CLICK CORRECT EXPLORE BUTTON ======
    logging.info(" Looking for CORRECT Explore button (not 'Explore topics')...")
    
    explore_clicked = False
    explore_button = None
    
    # Wait a bit for the button to appear after typing
    smart_backoff(2, 4, "Waiting for Explore button to appear", driver)
    
    # First, let's debug: show all Explore-like elements
    try:
        explore_elements = driver.find_elements(By.XPATH, "//*[contains(text(), 'Explore')]")
        logging.info(f" DEBUG: Found {len(explore_elements)} elements with 'Explore' text")
        for i, elem in enumerate(explore_elements[:5]):
            try:
                if elem.is_displayed():
                    elem_text = elem.text.strip()
                    elem_tag = elem.tag_name
                    elem_loc = elem.location
                    logging.info(f"  Element {i}: '{elem_text}' ({elem_tag}) at {elem_loc}")
            except:
                continue
    except:
        pass
    
    # Try specific selectors for the CORRECT Explore button
    correct_selectors = [
        "button:has(span.UywwFc-vQzf8d)",
        "button:has(span[jsname='V67aGc'])",
        "//button[.//span[text()='Explore'] and not(contains(., 'topics'))]",
        "//button[contains(., 'Explore') and not(contains(., 'topics'))]",
    ]
    
    for selector in correct_selectors:
        try:
            if selector.startswith("//"):
                elements = driver.find_elements(By.XPATH, selector)
            else:
                elements = driver.find_elements(By.CSS_SELECTOR, selector)
            
            for element in elements:
                try:
                    if element.is_displayed() and element.is_enabled():
                        text = element.text.strip().lower()
                        
                        # CRITICAL: Filter out "Explore topics" - we want JUST "Explore"
                        if "explore" in text and "topics" not in text:
                            explore_button = element
                            logging.info(f" Found CORRECT Explore button: '{element.text}'")
                            logging.info(f"   Using selector: {selector}")
                            
                            # Check if it's a span, find parent button
                            if explore_button.tag_name == 'span':
                                logging.info(" Found Explore span, looking for parent button...")
                                try:
                                    parent_button = explore_button.find_element(By.XPATH, "./ancestor::button[1]")
                                    if parent_button and parent_button.is_displayed():
                                        logging.info(" Found parent button for Explore span")
                                        explore_button = parent_button  # Use the button instead of span
                                except:
                                    pass  # Keep using the span if can't find parent
                            
                            break
                except Exception as e:
                    logging.debug(f"   Element check failed: {e}")
                    continue
            if explore_button:
                break
        except Exception as e:
            logging.debug(f"Selector {selector} failed: {e}")
            continue
    
    # If still not found, try scanning all buttons with better filtering
    if not explore_button:
        logging.warning(" Explore button not found with selectors, checking all buttons...")
        try:
            all_buttons = driver.find_elements(By.TAG_NAME, "button")
            logging.info(f" Found {len(all_buttons)} total buttons")
            
            for i, btn in enumerate(all_buttons[:15]):  # Check first 15
                try:
                    if btn.is_displayed() and btn.is_enabled():
                        btn_text = btn.text.strip().lower()
                        
                        # CRITICAL FILTER: Must have "explore" but NOT "topics"
                        if "explore" in btn_text and "topics" not in btn_text:
                            explore_button = btn
                            logging.info(f" Found Explore button by scanning: '{btn.text}'")
                            break
                except:
                    continue
        except:
            pass
    
    # ====== CLICK THE BUTTON ======
    if explore_button:
        # Double-check it's the right button
        button_text = explore_button.text.strip().lower()
        if "topics" in button_text:
            logging.error(" OOPS! Found WRONG 'Explore topics' button! Skipping...")
            explore_button = None
        else:
            # Human-like click
            logging.info(f" Clicking CORRECT Explore button: '{explore_button.text}'")
            random_mouse_movement(driver, explore_button)
            smart_backoff(0.5, 1.2, "Hovering over Explore button", driver)
            
            try:
                # Try multiple click methods
                click_success = False
                
                # Method 1: Normal click
                try:
                    explore_button.click()
                    click_success = True
                    logging.info(" Normal click successful")
                except:
                    # Method 2: JavaScript click
                    try:
                        driver.execute_script("arguments[0].click();", explore_button)
                        click_success = True
                        logging.info(" JavaScript click successful")
                    except:
                        # Method 3: ActionChains click
                        try:
                            ActionChains(driver).click(explore_button).perform()
                            click_success = True
                            logging.info(" ActionChains click successful")
                        except:
                            logging.warning(" All click methods failed")
                
                if click_success:
                    explore_clicked = True
                    smart_backoff(4, 7, "Waiting after Explore click", driver)
                    
                    # Check if we're on the right page
                    try:
                        WebDriverWait(driver, 10).until(
                            EC.presence_of_element_located((By.CSS_SELECTOR, "svg, canvas, div[jsname='hiK3ld']"))
                        )
                        logging.info(" Charts detected - success!")
                    except:
                        logging.info(" Charts not immediately detected, but continuing...")
                        
            except Exception as click_error:
                logging.warning(f" Error clicking Explore button: {click_error}")
                explore_clicked = False
    else:
        logging.warning(" CORRECT Explore button not found")
    
    # ====== FALLBACK: PRESS ENTER ======
    if not explore_clicked:
        logging.info(" Fallback: Pressing Enter on search box...")
        try:
            # Re-find search box
            trends_search_box = driver.find_element(
                By.CSS_SELECTOR, 
                "input[aria-label*='earch'], input[placeholder*='earch'], input[type='text']"
            )
            trends_search_box.send_keys(Keys.RETURN)
            logging.info(" Pressed Enter on search box")
            smart_backoff(6, 10, "Waiting for results after Enter", driver)
            explore_clicked = True  # Consider this success
        except Exception as e:
            logging.error(f" Could not press Enter: {e}")

    
    # Set the timeframe to "Past 12 months"
    set_timeframe(driver, "Past 12 months")
    
    # Check if charts loaded
    if not check_charts_loaded(driver):
        logging.warning(" Charts didn't load properly, trying recovery...")
        
        # Multiple recovery attempts
        for recovery_attempt in range(2):
            logging.info(f" Recovery attempt {recovery_attempt + 1}/2")
            smart_backoff(10, 15, f"Extended wait attempt {recovery_attempt + 1}", driver)
            
            # Try refreshing
            driver.refresh()
            smart_backoff(6, 10, "Page refreshed", driver)
            
            # Set timeframe again
            set_timeframe(driver, "Past 12 months")
            
            if check_charts_loaded(driver):
                logging.info(" Charts loaded after recovery")
                break
            
            if recovery_attempt == 0:
                # Try going back and searching again
                driver.back()
                smart_backoff(4, 7, "Went back to search", driver)
                
                # Find search box again and retry
                try:
                    trends_search_box = driver.find_element(By.CSS_SELECTOR, "input[aria-label='Search'][type='text']")
                    trends_search_box.click()
                    smart_backoff(1, 2)
                    trends_search_box.clear()
                    human_type_with_smart_correction(trends_search_box, keyword, context="trends_search")
                    smart_backoff(2, 4)
                    trends_search_box.send_keys(Keys.RETURN)
                    smart_backoff(8, 12, "Retried search", driver)
                except:
                    pass
    
    # PHASE 5: Wait for results with intelligent backoff
    logging.info(" Phase 5: Loading results with smart backoff...")
    
    # Progressive loading with increasing patience
    load_stages = [
        (random.uniform(3, 6), "Initial page load"),
        (random.uniform(2.5, 5), "Content rendering"),
        (random.uniform(2, 4), "Chart elements"),
        (random.uniform(1, 3), "Final touches"),
    ]
    
    for wait_time, stage in load_stages:
        logging.info(f"   {stage}...")
        time.sleep(wait_time)
        
        # Simulate human watching the load
        if random.random() < 0.3:
            simulate_human_scanning(driver, duration=1, intensity="very_light")
    
    # Final scan of the results page
    logging.info(" Final scan of results page...")
    simulate_human_scanning(driver, duration=3, intensity="light")
    
    # Verify we're in London
    page_source = driver.page_source.lower()
    if any(indicator in page_source for indicator in ["london", "united kingdom", "uk", "gb"]):
        logging.info(" Confirmed: Viewing London trends")
    else:
        logging.warning(" London location not clearly detected")
    
    # Additional wait for any final loading
    smart_backoff(3, 7, "Final page stabilization", driver)
    
    logging.info(" Complete human journey finished successfully!")
    
    # Extract data from the page
    trends_data = extract_trends_data_from_current_page(driver)
    
    return driver, trends_data  # Return both driver AND data