run.py

from env_bootstrap import ensure_venv
ensure_venv()


# Imports
import os
import sys
import time
import json
import random
import pathlib
from urllib.parse import quote, urlencode
from datetime import datetime
from pprint import pprint
from dotenv import load_dotenv

# Load .env file (credentials, etc.)
load_dotenv()

# Selenium Imports
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import NoSuchElementException, ElementClickInterceptedException, WebDriverException, ElementNotInteractableException

# -------------------------------------------------------------------------
# CONFIGURATION & SETTINGS
# -------------------------------------------------------------------------

def load_json_config(config_file="search_config.json"):
    """Load configuration from search_config.json."""
    if os.path.exists(config_file):
        try:
            with open(config_file, 'r') as f:
                return json.load(f)
        except Exception as e:
            print(f"Warning: failed to load {config_file}: {e}")
    return {}


def get_config_value(config_data, keys, default=None):
    value = config_data
    for key in keys:
        if isinstance(value, dict) and key in value:
            value = value[key]
        else:
            return default
    return value

# Load configuration
config_file_path = "search_config.json"
config = load_json_config(config_file_path)

# Global Settings
logs_folder_path = get_config_value(config, ["logging", "log_file"], "logs/bot.log")
if '/' in logs_folder_path:
    logs_folder_path = os.path.dirname(logs_folder_path) + '/'
else:
    logs_folder_path = 'logs/'
    
run_in_background = get_config_value(config, ["browser", "headless_mode"], False)
disable_extensions = False
safe_mode = True
stealth_mode = get_config_value(config, ["anti_detection", "use_stealth_mode"], False)
click_gap = get_config_value(config, ["delays_seconds", "min_delay"], 1.0)

# Credentials: env vars take priority, then config fallback
username = os.environ.get('LINKEDIN_USERNAME') or get_config_value(config, ["linkedin", "credentials", "username"], '')
password = os.environ.get('LINKEDIN_PASSWORD') or get_config_value(config, ["linkedin", "credentials", "password"], '')

if username:
    print(f"Loaded credentials for: {username}")
else:
    print("WARNING: No credentials found. Set LINKEDIN_USERNAME/LINKEDIN_PASSWORD in .env or search_config.json")

# -------------------------------------------------------------------------
# HELPERS
# -------------------------------------------------------------------------

def make_directories(paths: list[str]) -> None:
    '''
    Function to create missing directories
    '''
    for path in paths:
        path = os.path.expanduser(path)
        path = path.replace("//","/")
        if '.' in os.path.basename(path):
            path = os.path.dirname(path)
        if not path: continue
        try:
            if not os.path.exists(path):
                os.makedirs(path, exist_ok=True)
        except Exception as e:
            print(f'Error while creating directory "{path}": ', e)

def find_default_profile_directory() -> str | None:
    home = pathlib.Path.home()
    if sys.platform.startswith('win'):
        paths = [
            os.path.expandvars(r"%LOCALAPPDATA%\Google\Chrome\User Data"),
            os.path.expandvars(r"%USERPROFILE%\AppData\Local\Google\Chrome\User Data"),
            os.path.expandvars(r"%USERPROFILE%\Local Settings\Application Data\Google\Chrome\User Data")
        ]
    elif sys.platform.startswith('linux'):
        paths = [
            str(home / ".config" / "google-chrome"),
            str(home / ".var" / "app" / "com.google.Chrome" / "data" / ".config" / "google-chrome"),
        ]
    elif sys.platform == 'darwin':
        paths = [
            str(home / "Library" / "Application Support" / "Google" / "Chrome")
        ]
    else:
        return None
    for path_str in paths:
        if os.path.exists(path_str):
            return path_str
    return None

def print_lg(*msgs, end: str = "\n", pretty: bool = False, flush: bool = True):
    try:
        log_file = os.path.join(logs_folder_path, "log.txt")
        for message in msgs:
            pprint(message) if pretty else print(message, end=end, flush=flush)
            with open(log_file, 'a+', encoding="utf-8") as file:
                file.write(str(message) + end)
    except Exception as e:
        print(f"Logging failed: {e}")

def buffer(speed: int=0) -> None:
    if speed<=0: return
    elif speed <= 1 and speed < 2:
        time.sleep(random.randint(6,10)*0.1)
    elif speed <= 2 and speed < 3:
        time.sleep(random.randint(10,18)*0.1)
    else:
        time.sleep(random.randint(18,round(speed)*10)*0.1)

def manual_login_retry(is_logged_in_func, limit: int = 2) -> None:
    count = 0
    while not is_logged_in_func():
        print_lg("Seems like you're not logged in!")
        message = 'Please Log In manually in the browser.\nOnce you have successfully logged in, press Enter in this terminal to continue...'
        if count > limit:
            message = 'If you\'re seeing this message even after you logged in, press Enter to force continue. Seems like auto login confirmation failed!'
        count += 1
        print_lg(f"\n!!! ACTION REQUIRED !!!\n{message}\n")
        input("Press Enter to continue...")
        if count > limit: return

# -------------------------------------------------------------------------
# FINDERS & CLICKERS
# -------------------------------------------------------------------------

def scroll_to_view(driver, element, top: bool = False):
    if top:
        return driver.execute_script('arguments[0].scrollIntoView();', element)
    return driver.execute_script('arguments[0].scrollIntoView({block: "center", behavior: "smooth" });', element)

def is_checked(element) -> bool:
    try:
        if element.get_attribute("aria-checked") == "true" or element.get_attribute("checked") == "true":
            return True
        if element.tag_name == "input" and element.is_selected():
            return True
        return False
    except: return False

def wait_span_click(driver, text: str, time_wait: float=5.0, click: bool=True, scroll: bool=True, scrollTop: bool=False):
    if text:
        try:
            xpath = f'.//span[normalize-space(.)="{text}"] | .//label[normalize-space(.)="{text}"] | .//div[normalize-space(.)="{text}"]'
            button = WebDriverWait(driver,time_wait).until(EC.presence_of_element_located((By.XPATH, xpath)))
            if scroll: scroll_to_view(driver, button, scrollTop)
            if click:
                if not is_checked(button):
                    button.click()
                    buffer(click_gap)
            return button
        except Exception:
            return False

def text_input_by_ID(driver, id_val: str, value: str, time_wait: float=5.0):
    field = WebDriverWait(driver, time_wait).until(EC.presence_of_element_located((By.ID, id_val)))
    field.send_keys(Keys.CONTROL + "a")
    field.send_keys(value)

def find_by_class(driver, class_name: str, time_wait: float=5.0):
    return WebDriverWait(driver, time_wait).until(EC.presence_of_element_located((By.CLASS_NAME, class_name)))

def try_xp(driver, xpath: str, click: bool=True):
    try:
        if click:
            driver.find_element(By.XPATH, xpath).click()
            return True
        else:
            return driver.find_element(By.XPATH, xpath)
    except: return False

def try_linkText(driver, linkText: str):
    try:    return driver.find_element(By.LINK_TEXT, linkText)
    except:  return False

# -------------------------------------------------------------------------
# BROWSER & LOGIN LOGIC
# -------------------------------------------------------------------------

if stealth_mode:
    try:
        import undetected_chromedriver as uc
    except ImportError:
        print("Stealth mode requires 'undetected-chromedriver'. Installing it now...")
        try:
            import subprocess
            subprocess.check_call([sys.executable, "-m", "pip", "install", "undetected-chromedriver"])
            import undetected_chromedriver as uc
        except Exception as e:
            print(f"Failed to install undetected-chromedriver: {e}")
            stealth_mode = False

# -------------------------------------------------------------------------
# BROWSER & LOGIN LOGIC
# -------------------------------------------------------------------------

def setup_browser():
    make_directories([logs_folder_path])
    
    if stealth_mode:
        options = uc.ChromeOptions()
    else:
        options = Options()
        
    if run_in_background: 
        options.add_argument("--headless=new")
    if disable_extensions: 
        options.add_argument("--disable-extensions")
        
    # Stability options
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")
    # # # # # # # # # # options.add_argument("--remote-debugging-pipe")  # Causes Chrome to crash in some environments  # Causes Chrome to crash in some environments  # Causes Chrome to crash in some environments  # Causes Chrome to crash in some environments  # Causes Chrome to crash in some environments  # Causes Chrome to crash in some environments  # Causes Chrome to crash in some environments  # Causes Chrome to crash in some environments  # Causes Chrome to crash in some environments  # Causes Chrome to crash in some environments

    if safe_mode:
        print_lg("Safe Mode: Logging in with a guest profile.")
    else:
        # NOTE: To use your own profile, you must close ALL other Chrome instances first.
        # However, to be stable and avoid 'SessionNotCreatedException', we will default to a 
        # clean session unless explicitly configured otherwise.
        # Uncomment the lines below ONLY if you are sure Chrome is closed.
        
        # profile_dir = find_default_profile_directory()
        # if profile_dir:
        #     options.add_argument(f"--user-data-dir={profile_dir}")
        #     options.add_argument("--profile-directory=Default")
        
        # For now, we behave like the reference 'Auto_job_applier_linkedIn' and do NOT attach profile by default
        pass

    driver = None
    print_lg("Initializing WebDriver...")
    
    try:
        if stealth_mode:
            driver = uc.Chrome(options=options)
        else:
            from selenium.webdriver.chrome.service import Service
            from webdriver_manager.chrome import ChromeDriverManager
            
            print_lg("Resolving ChromeDriver via webdriver-manager...")
            service = Service(ChromeDriverManager().install())
            driver = webdriver.Chrome(service=service, options=options)
            
    except Exception as e:
        print_lg(f"Error initializing Chrome: {e}")
        print_lg("Ensure Chrome is installed. If using stealth_mode, try turning it off in config.")
        raise e

    try:
        # Move to 2nd monitor or maximize, similar to reference code
        driver.set_window_rect(x=0, y=0, width=1280, height=1080)
    except:
        driver.maximize_window()
        
    return driver, WebDriverWait(driver, 5), ActionChains(driver)

def is_logged_in_LN(driver) -> bool:
    if "linkedin.com/feed" in driver.current_url: return True
    if "linkedin.com/login" in driver.current_url: return False
    
    # If we see "Sign in" or "Join now" buttons, we are NOT logged in
    if try_linkText(driver, "Sign in"): return False
    if try_xp(driver, '//button[@type="submit" and contains(text(), "Sign in")]', click=False): return False
    if try_linkText(driver, "Join now"): return False
    
    # Double check for Feed
    try:
        driver.find_element(By.CLASS_NAME, "scaffold-layout__main") # Common feed container
        return True
    except:
        pass

    # If we are on the login page specifically
    if driver.find_elements(By.ID, "username") and driver.find_elements(By.ID, "password"):
        return False
        
    print_lg("Assuming user is logged in (No login identifiers found).")
    return True

def login_LN(driver, wait) -> None:
    print_lg("Navigating to LinkedIn Login...")
    driver.get("https://www.linkedin.com/login")
    
    # Wait for the login form to be ready (username field is the most reliable indicator)
    login_wait = WebDriverWait(driver, 15)
    try:
        login_wait.until(EC.presence_of_element_located((By.ID, "username")))
    except:
        # Retry once - LinkedIn sometimes shows interstitials
        print_lg("Login page slow to load, retrying...")
        time.sleep(3)
        driver.get("https://www.linkedin.com/login")
        try:
            login_wait.until(EC.presence_of_element_located((By.ID, "username")))
        except:
            print_lg("Could not load login page after retry.")

    try:
        text_input_by_ID(driver, "username", username, 5)
    except: print_lg("Couldn't find username field.")
        
    try:
        text_input_by_ID(driver, "password", password, 5)
    except: print_lg("Couldn't find password field.")
        
    try:
        driver.find_element(By.XPATH, '//button[@type="submit" and contains(text(), "Sign in")]').click()
    except:
        try:
            driver.find_element(By.XPATH, '//button[@type="submit"]').click()
        except:
            print_lg("Could not find Sign in button.")

    try:
        wait.until(EC.url_contains("linkedin.com/feed"))
        print_lg("Login successful!")
    except:
        # Check if we hit a verification challenge
        current_url = driver.current_url
        if 'checkpoint' in current_url or 'challenge' in current_url:
            print_lg("Verification challenge detected. Attempting auto-OTP from email...")
            try:
                from otp_email import fetch_otp
                code = fetch_otp()
                if code:
                    try:
                        input_field = driver.find_element(By.CSS_SELECTOR,
                            'input[name="pin"], input#input__email_verification_pin, input[type="text"]')
                        input_field.clear()
                        input_field.send_keys(code)
                        time.sleep(1)
                        submit_btn = driver.find_element(By.CSS_SELECTOR,
                            'button[type="submit"], button#email-pin-submit-button')
                        submit_btn.click()
                        time.sleep(5)
                        if is_logged_in_LN(driver):
                            print_lg("Login successful after OTP verification!")
                            return
                    except Exception as e:
                        print_lg(f"Auto-OTP submission failed: {e}")
                else:
                    print_lg("Auto-OTP: no code found in email.")
            except ImportError:
                print_lg("otp_email module not available. Configure IMAP settings in .env for auto-OTP.")
            except Exception as e:
                print_lg(f"Auto-OTP failed: {e}")
        
        print_lg("Login verification failed. Please log in manually.")
        manual_login_retry(lambda: is_logged_in_LN(driver), 2)

# -------------------------------------------------------------------------
# METRICS & BOT LOGIC
# -------------------------------------------------------------------------

class MetricsManager:
    def __init__(self, metrics_file="data/metrics.json"):
        self.metrics_file = metrics_file
        self.metrics = self.load_metrics()
        self.today = datetime.now().strftime("%Y-%m-%d")

    def load_metrics(self):
        if not os.path.exists(self.metrics_file):
            return {
                "total_requests": 0,
                "daily_requests": {},
                "weekly_requests": {},
                "last_run": "",
                "group_runs": []
            }
        try:
            with open(self.metrics_file, 'r') as f:
                return json.load(f)
        except:
            return {
                "total_requests": 0,
                "daily_requests": {},
                "weekly_requests": {},
                "last_run": "",
                "group_runs": []
            }

    def save_metrics(self):
        with open(self.metrics_file, 'w') as f:
            json.dump(self.metrics, f, indent=4)

    def get_daily_count(self):
        return self.metrics.get("daily_requests", {}).get(self.today, 0)

    def get_weekly_count(self):
        total = 0
        today = datetime.now().date()
        daily = self.metrics.get("daily_requests", {})
        for date_str, count in daily.items():
            try:
                d = datetime.strptime(date_str, "%Y-%m-%d").date()
                if (today - d).days < 7:
                    total += count
            except ValueError:
                continue
        return total

    def can_connect(self):
        max_daily = get_config_value(config, ["limits", "max_daily_invites"], 15)
        current = self.get_daily_count()
        if current >= max_daily:
            print_lg(f"Daily limit reached: {current}/{max_daily}")
            return False
        max_weekly = get_config_value(config, ["limits", "max_weekly_invites"], 80)
        weekly = self.get_weekly_count()
        if weekly >= max_weekly:
            print_lg(f"Weekly limit reached: {weekly}/{max_weekly}")
            return False
        return True

    def increment_requests(self, group_set_id=None):
        self.metrics["total_requests"] += 1
        
        if "daily_requests" not in self.metrics: self.metrics["daily_requests"] = {}
        daily = self.metrics["daily_requests"].get(self.today, 0)
        self.metrics["daily_requests"][self.today] = daily + 1

        # Track per group-set daily counts
        if group_set_id:
            gs_key = f"daily_requests_{group_set_id}"
            if gs_key not in self.metrics: self.metrics[gs_key] = {}
            gs_daily = self.metrics[gs_key].get(self.today, 0)
            self.metrics[gs_key][self.today] = gs_daily + 1
        
        self.metrics["last_run"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        self.metrics["last_group_set"] = group_set_id or ""
        self.save_metrics()
        print_lg(f"Request logged. Daily Total: {self.metrics['daily_requests'][self.today]}")

    def get_group_runs(self, group_set_id=None):
        runs = self.metrics.get("group_runs", [])
        if group_set_id:
            return [r for r in runs if isinstance(r, dict) and r.get("group_set") == group_set_id]
        return runs

    def add_group_run(self, entry):
        self.metrics.setdefault("group_runs", []).append(entry)
        self.save_metrics()

class LinkedInBot:
    def __init__(self, driver, wait):
        self.driver = driver
        self.wait = wait
        self.metrics = MetricsManager(get_config_value(config, ["monitoring", "metrics_file"], 'data/metrics.json'))

        # Determine run mode
        self.mode = config.get('mode', 'keywords')

        # Resolve groups from group_sets_config (for group_sets mode)
        gsc = config.get('group_sets_config', {})
        self.active_group_set_id = gsc.get('active_set', '')
        group_sets = gsc.get('sets', {})
        if self.active_group_set_id and self.active_group_set_id in group_sets:
            active_set = group_sets[self.active_group_set_id]
            self.groups = active_set.get('groups', [])
            self.active_group_set_name = active_set.get('name', self.active_group_set_id)
            print_lg(f"Using group set: '{self.active_group_set_name}' (id: {self.active_group_set_id})")
        else:
            self.groups = []
            self.active_group_set_id = 'default'
            self.active_group_set_name = 'Default'
            if group_sets:
                available = ', '.join(group_sets.keys())
                print_lg(f"WARNING: active_set '{gsc.get('active_set', '')}' not found. Available: {available}")

        # Load flat keywords list (for keywords mode)
        self.keywords = get_config_value(config, ["keywords_config", "keywords"], [])

        self.skip_keywords = [k.strip().lower() for k in get_config_value(config, ["filters", "skip_profiles_keywords"], []) if k.strip()]
        self.message_templates = get_config_value(config, ["messaging", "connection_message_templates"], [])
        self.search_url_base = get_config_value(
            config, ["search", "base_search_url"], 'https://www.linkedin.com/search/results/people/?keywords='
        )
        self.search_url_params = get_config_value(config, ["search", "url_params"], {})

    def run(self):
        if not self.metrics.can_connect():
            print_lg("Review limits in search_config.json")
            return

        print_lg(f"Mode: {self.mode}")
        print_lg(f"Loaded {len(self.skip_keywords)} skip keywords: {self.skip_keywords}")

        if self.mode == 'group_sets':
            self.run_group_sets_mode()
        else:
            self.run_keywords_mode()

    def run_keywords_mode(self):
        keywords = [k.strip() for k in self.keywords if k.strip()]
        if not keywords:
            print_lg("No keywords configured in keywords_config.keywords. Nothing to do.")
            return

        print_lg(f"Keywords mode: {len(keywords)} keywords to process")
        max_pages = get_config_value(config, ["pagination", "max_pages"], 10)
        total_sent = 0

        for keyword in keywords:
            if not self.metrics.can_connect():
                print_lg("Daily/weekly limit reached. Stopping.")
                break
            print_lg(f"Starting search for keyword: {keyword}")
            sent, search_url = self.search_and_connect_keyword(keyword, max_connections=999)
            total_sent += sent

        print_lg(f"Keywords mode complete. Total invites sent: {total_sent}")

    def run_group_sets_mode(self):
        print_lg(f"Active group set: '{self.active_group_set_name}' (id: {self.active_group_set_id})")
        print_lg(f"Groups in set: {len(self.groups)}")
        today_name = datetime.now().strftime("%A").lower()
        eligible_groups = [g for g in self.groups if self.should_run_group(g, today_name)]

        if not eligible_groups:
            print_lg(f"No groups scheduled for today ({today_name.title()}).")
            return

        for group in eligible_groups:
            keywords_csv = (group.get("keywords_csv") or "").strip()
            if not keywords_csv:
                keywords_list = group.get("keywords") or []
                if isinstance(keywords_list, list):
                    keywords_csv = ", ".join([k.strip() for k in keywords_list if str(k).strip()])
            if not keywords_csv:
                continue

            print_lg(f"Starting search for group: {group.get('name', group.get('id', 'unknown'))}")
            # Log group start immediately (before processing)
            self.start_group_run(group)
            connections_sent, search_urls = self.search_and_connect_group(group, keywords_csv)
            # Update and save the group run results
            self.update_group_run(search_urls, keywords_csv, connections_sent)

            if not self.metrics.can_connect():
                print_lg("Global daily limit reached. Stopping bot.")
                break

    def should_run_group(self, group, today_name):
        run_day = (group.get("run_day") or "").strip().lower()
        if run_day and run_day != today_name:
            return False
        completed_runs = self.metrics.get_group_runs(self.active_group_set_id)
        if not completed_runs:
            return True

        days_block = get_config_value(config, ["run_policy", "do_not_repeat_group_if_logged_within_days"], 1)
        today = datetime.now().date()
        today_str = today.strftime("%Y-%m-%d")
        invites_per_keyword = get_config_value(config, ["group_sets_config", "invites_per_keyword"], 5)
        keywords_limit = get_config_value(config, ["group_sets_config", "keywords_per_group"], 4)
        max_daily = get_config_value(config, ["limits", "max_daily_invites"], 15)
        group_target = invites_per_keyword * keywords_limit
        group_target = min(group_target, max_daily)

        today_total = 0
        has_today = False
        for entry in completed_runs:
            if not isinstance(entry, dict):
                continue
            if entry.get("group_id") != group.get("id"):
                continue
            date_str = entry.get("date")
            if not date_str:
                continue
            try:
                entry_date = datetime.strptime(date_str, "%Y-%m-%d").date()
            except ValueError:
                continue
            if date_str == today_str:
                has_today = True
                today_total += int(entry.get("connections_sent", 0) or 0)
                continue
            if (today - entry_date).days < days_block:
                return False
        if has_today:
            return today_total < group_target
        return True

    def search_and_connect_group(self, group, keywords_csv):
        keywords = [k.strip() for k in keywords_csv.split(",") if k.strip()]
        max_per_keyword = get_config_value(config, ["group_sets_config", "invites_per_keyword"], 5)
        keywords_limit = get_config_value(config, ["group_sets_config", "keywords_per_group"], len(keywords))
        keywords = keywords[:keywords_limit]
        total_sent = 0
        search_urls = []

        for keyword in keywords:
            if not self.metrics.can_connect():
                break
            print_lg(f"Starting search for keyword: {keyword}")
            sent, search_url = self.search_and_connect_keyword(keyword, max_per_keyword)
            total_sent += sent
            search_urls.append(search_url)

        return total_sent, search_urls

    def search_and_connect_keyword(self, keyword, max_connections):
        encoded_keyword = quote(keyword, safe="")
        search_url = f"{self.search_url_base}{encoded_keyword}"
        if self.search_url_params:
            search_url = f"{search_url}&{urlencode(self.search_url_params)}"
        print_lg(f"[NAVIGATE] Loading search URL: {search_url}")
        self.driver.get(search_url)
        time.sleep(get_config_value(config, ["delays_seconds", "page_load_delay"], 5))
        print_lg(f"[PAGE] Page loaded. Current URL: {self.driver.current_url}")
        print_lg(f"[PAGE] Page title: {self.driver.title}")

        connections_sent = self.process_page(max_connections)
        return connections_sent, search_url

    def go_to_next_page(self):
        """
        Click the 'Next' pagination button to go to the next page of results.
        Returns True if successfully navigated, False otherwise.
        """
        try:
            # Look for the Next button using multiple selectors
            next_btn = None
            selectors = [
                "//button[@data-testid='pagination-controls-next-button-visible']",
                "//button[normalize-space(.)='Next']",
                "//button[contains(@class, 'pagination')]//span[text()='Next']/ancestor::button"
            ]
            
            for selector in selectors:
                try:
                    next_btn = self.driver.find_element(By.XPATH, selector)
                    if next_btn and next_btn.is_displayed() and next_btn.is_enabled():
                        break
                except:
                    continue
            
            if not next_btn:
                print_lg("[PAGINATION] Next button not found or not clickable.")
                return False
            
            # Scroll to the pagination area
            scroll_to_view(self.driver, next_btn)
            time.sleep(1)
            
            # Click the Next button
            print_lg("[PAGINATION] Clicking 'Next' button...")
            self.driver.execute_script("arguments[0].click();", next_btn)
            
            # Wait for page to load (10 seconds as specified)
            print_lg("[PAGINATION] Waiting 10s for next page to load...")
            time.sleep(10)
            
            # Verify we're on a new page by checking URL or content change
            print_lg(f"[PAGINATION] Now on: {self.driver.current_url}")
            return True
            
        except Exception as e:
            print_lg(f"[PAGINATION] Error navigating to next page: {e}")
            return False

    def count_connect_buttons(self):
        """Count the number of Connect buttons/links on the current page."""
        try:
            connect_elements = self.driver.find_elements(By.XPATH,
                "//a[contains(@href, 'search-custom-invite')] | "
                "//button[contains(@aria-label, 'Invite')][contains(@aria-label, 'to connect')]"
            )
            return len(connect_elements)
        except:
            return 0

    def process_page(self, max_connections):
        connections_sent = 0
        max_pages = get_config_value(config, ["pagination", "max_pages"], 10)
        current_page = 1
        
        while current_page <= max_pages and connections_sent < max_connections:
            print_lg(f"[PAGE {current_page}] Processing page...")
            remaining = max_connections - connections_sent
            page_connections = self._process_single_page(remaining)
            connections_sent += page_connections
            
            if not self.metrics.can_connect():
                print_lg("[LIMIT] Daily limit reached, stopping pagination.")
                break
            
            # Check if we should go to next page
            remaining_buttons = self.count_connect_buttons()
            print_lg(f"[PAGE {current_page}] Processed {page_connections} connections. {remaining_buttons} Connect buttons remaining.")
            
            # If no more connect buttons or very few, try next page
            if connections_sent >= max_connections:
                break
            if remaining_buttons == 0:
                print_lg(f"[PAGINATION] No Connect buttons left on page {current_page}. Attempting next page...")
                if self.go_to_next_page():
                    current_page += 1
                    continue
                else:
                    print_lg("[PAGINATION] Cannot navigate to next page. Ending.")
                    break
            else:
                # Still have buttons, but we've processed what we could
                # This handles the case where buttons exist but were skipped
                break
        
        return connections_sent

    def _process_single_page(self, max_connections):
        """Process a single page of search results. Returns connections sent on this page."""
        connections_sent = 0
        # Allow page to settle
        print_lg("[PROCESS] Page settling (3s)...")
        time.sleep(3)
        
        # Scroll down and up to trigger lazy loading
        print_lg("[SCROLL] Scrolling to trigger lazy loading...")
        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight/2);")
        time.sleep(1)
        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
        self.driver.execute_script("window.scrollTo(0, 0);")
        time.sleep(2)
        print_lg("[SCROLL] Done scrolling.")
        
        # Updated XPath to include 'a' (anchor) tags as LinkedIn frequently renders these as links
        # We look for aria-labels like "Invite {Name} to connect" or inner text "Connect"
        xpath_query = """
            //button[contains(@aria-label, 'Invite')][contains(@aria-label, 'to connect')] |
            //a[contains(@aria-label, 'Invite')][contains(@aria-label, 'to connect')] |
            //button[normalize-space(.)='Connect'] |
            //a[normalize-space(.)='Connect']
        """
        
        buttons = self.driver.find_elements(By.XPATH, xpath_query)
        print_lg(f"[BUTTONS] Found {len(buttons)} 'Connect' buttons/links on this page.")
        
        # Debug: Log first few button details
        for idx, btn in enumerate(buttons[:3]):
            try:
                aria = btn.get_attribute('aria-label') or 'N/A'
                tag = btn.tag_name
                txt = btn.text[:30] if btn.text else 'N/A'
                print_lg(f"  [BTN {idx}] <{tag}> aria='{aria}' text='{txt}'")
            except: pass
        
        processed = set()
        while True:
            if not self.metrics.can_connect() or connections_sent >= max_connections:
                break

            # Re-find items to ensure freshness as the DOM might change
            current_buttons = self.driver.find_elements(By.XPATH, xpath_query)
            if not current_buttons:
                break

            button = None
            button_key = None
            for candidate in current_buttons:
                if not candidate.is_displayed():
                    continue
                try:
                    aria = candidate.get_attribute("aria-label") or ""
                    href = candidate.get_attribute("href") or ""
                    button_key = aria or href or getattr(candidate, "id", None)
                except Exception:
                    button_key = getattr(candidate, "id", None)

                if button_key and button_key in processed:
                    continue
                button = candidate
                break

            if not button:
                break

            # =====================================================
            # STEP 1: Find and read the profile card text
            # =====================================================
            card_text = ""
            skip_this_profile = False
            try:
                card = None
                selectors = [
                    # --- Selectors verified working against live LinkedIn (April 2026) ---
                    "./ancestor::a[contains(@href, '/in/')]",  # Profile card <a> wrapper (MOST RELIABLE)
                    "./ancestor::div[@data-display-contents='true'][string-length(.) > 50]",  # React display-contents wrapper
                    "./ancestor::div[string-length(.) > 50][string-length(.) < 500]",  # Text-length based fallback
                    # --- Legacy selectors (may work on older LinkedIn versions) ---
                    "./ancestor::li",  # Old LinkedIn structure
                    "./ancestor::*[@data-view-name='people-search-result']",  # LinkedIn data attribute
                    "./ancestor::div[contains(@class, 'entity-result__item')]",  # LinkedIn class
                    "./ancestor::div[contains(@class, 'reusable-search__result-container')]",  # Another LinkedIn class
                    "./ancestor::div[contains(@class, 'search-result')]",  # Generic search result
                    "./ancestor::div[contains(@class, 'artdeco-card')]",  # LinkedIn card class
                    "./ancestor::div[role='listitem']",  # ARIA role
                ]
                
                for selector in selectors:
                    try:
                        card = button.find_element(By.XPATH, selector)
                        if card:
                            break
                    except:
                        continue
                
                if card:
                    # Method 1: Try specific LinkedIn subtitle selectors (most reliable)
                    try:
                        subtitle_selectors = [
                            ".//span[contains(@class, 'entity-result__primary-subtitle')]",
                            ".//div[contains(@class, 'entity-result__primary-subtitle')]",
                            ".//span[contains(@class, 'entity-result__secondary-subtitle')]",
                            ".//div[contains(@class, 'entity-result__summary')]",
                            ".//p[contains(@class, 'entity-result')]",
                        ]
                        subtitle_texts = []
                        for sel in subtitle_selectors:
                            try:
                                elems = card.find_elements(By.XPATH, sel)
                                for elem in elems:
                                    if elem.text and elem.text.strip():
                                        subtitle_texts.append(elem.text.strip())
                            except:
                                continue
                        if subtitle_texts:
                            card_text = " ".join(subtitle_texts).lower()
                            print_lg(f"[VERIFY] Got profile text via subtitle selectors: {card_text[:80]}...")
                    except:
                        pass
                    
                    # Method 2: Try <p> tags
                    if not card_text or len(card_text.strip()) < 5:
                        p_texts = [p.text for p in card.find_elements(By.TAG_NAME, "p") if p.text]
                        if p_texts:
                            card_text = " ".join(p_texts).lower()
                            print_lg(f"[VERIFY] Got profile text via <p> tags: {card_text[:80]}...")
                    
                    # Method 3: Get all text from the card
                    if not card_text or len(card_text.strip()) < 5:
                        card_text = card.text.lower()
                        if card_text and len(card_text.strip()) >= 5:
                            print_lg(f"[VERIFY] Got profile text via full card text: {card_text[:80]}...")
                    
                    # Method 4: Try sibling/parent text
                    if not card_text or len(card_text.strip()) < 5:
                        try:
                            parent = button.find_element(By.XPATH, "./ancestor::div[1]")
                            siblings_text = parent.text.lower()
                            if siblings_text and len(siblings_text) > len(card_text or ''):
                                card_text = siblings_text
                                print_lg(f"[VERIFY] Got profile text via parent: {card_text[:80]}...")
                        except:
                            pass
                else:
                    print_lg("[VERIFY] SKIP: Could not find profile card element. No selector matched.")
                    skip_this_profile = True
                    
            except Exception as e:
                print_lg(f"[VERIFY] SKIP: Profile card read error: {e}")
                skip_this_profile = True

            # If card element wasn't found, skip for safety
            if skip_this_profile:
                if button_key:
                    processed.add(button_key)
                continue

            # =====================================================
            # STEP 2: Verification checks (THIS CODE NOW EXECUTES)
            # =====================================================

            # Check 2a: Skip keywords from config
            if card_text:
                found_keyword = next((k for k in self.skip_keywords if k in card_text), None)
                if found_keyword:
                    try:
                        name_log = card_text.split('\n')[0][:30] 
                    except:
                        name_log = "Unknown"
                    print_lg(f"[VERIFY] SKIP '{name_log}' - matched skip keyword: '{found_keyword}'")
                    if button_key:
                        processed.add(button_key)
                    continue
            
            # Check 2b: Skip non-AI roles (recruiters, HR, sales, etc.)
            non_ai_roles = [
                'recruiter', 'talent', 'hr ', 'human resources', 'headhunter',
                'sales', 'account executive', 'account manager', 'business development',
                'marketing', 'content', 'writer', 'editor', 'designer',
                'operations', 'admin', 'assistant', 'coordinator',
                'finance', 'accounting', 'legal', 'lawyer',
                'customer service', 'support', 'success manager'
            ]
            
            if card_text:
                has_non_ai_role = any(role in card_text for role in non_ai_roles)
                if has_non_ai_role:
                    print_lg(f"[VERIFY] SKIP - non-AI role detected: {card_text[:100]}...")
                    if button_key:
                        processed.add(button_key)
                    continue
            
            # Check 2c: AI keyword verification (only for AI group set)
            if self.active_group_set_id == 'ai':
                ai_keywords = [
                    'ai', 'artificial intelligence', 'machine learning', 'ml',
                    'deep learning', 'neural network', 'llm', 'large language model',
                    'generative ai', 'genai', 'nlp', 'natural language processing',
                    'computer vision', 'data science', 'data scientist',
                    'mlops', 'aiops', 'prompt engineer', 'ai engineer',
                    'research scientist', 'ai researcher', 'ai developer',
                    'ai product', 'ai solutions', 'ai strategy',  # AI-specific titles only
                ]
                
                if card_text:
                    has_ai_keyword = any(keyword in card_text for keyword in ai_keywords)
                    if not has_ai_keyword:
                        print_lg(f"[VERIFY] SKIP - no AI keywords in: {card_text[:100]}...")
                        if button_key:
                            processed.add(button_key)
                        continue
                    else:
                        print_lg(f"[VERIFY] PASS - AI keyword found in: {card_text[:80]}...")
                else:
                    # No card text at all - skip for safety
                    print_lg("[VERIFY] SKIP - no profile text available for AI verification.")
                    if button_key:
                        processed.add(button_key)
                    continue
            
            # Check 2d: Profile quality - skip very sparse profiles
            if card_text:
                word_count = len(card_text.split())
                if word_count < 3:
                    print_lg(f"[VERIFY] SKIP - too sparse ({word_count} words): {card_text[:100]}...")
                    if button_key:
                        processed.add(button_key)
                    continue

            # Extract Name
            first_name = "there"
            try:
                aria_label = button.get_attribute("aria-label") 
                if aria_label and "Invite" in aria_label and "to connect" in aria_label:
                    full_name = aria_label.split("Invite ")[1].split(" to connect")[0]
                    first_name = full_name.split(' ')[0]
                else:
                    # Fallback: Use card text (first line is usually the name)
                    if card_text:
                        full_name = card_text.split('\n')[0]
                        first_name = full_name.split(' ')[0]
                        if len(first_name) < 2: first_name = "there"
            except Exception as e:
                print_lg(f"Name extraction warning: {e}")

            print_lg(f"[CONNECT] Clicking Connect for {first_name}...")
            
            try:
                # Click Connect - use JS to avoid element interception
                self.driver.execute_script("arguments[0].click();", button)
                print_lg(f"[CONNECT] Click executed. Checking for modal or direct send...")
                time.sleep(1.5)
                
                # Handle Modal - pass button reference to detect direct send
                success = self.handle_connection_modal(first_name, connect_element=button)
                
                if success:
                    print_lg(f"[SUCCESS] Invite sent to {first_name}!")
                    self.metrics.increment_requests(self.active_group_set_id)
                    connections_sent += 1
                    buffer(get_config_value(config, ["delays_seconds", "min_delay"], 3))
                if button_key:
                    processed.add(button_key)
                    
                if not success:
                    print_lg(f"[FAIL] Connection failed for {first_name}, skipping increment.")
                    time.sleep(1)
                
            except Exception as e:
                if button_key:
                    processed.add(button_key)
                print_lg(f"Failed to process connection: {e}")
                continue
        return connections_sent

    def log_group_run(self, group, search_urls, keywords_csv, connections_sent):
        today = datetime.now().strftime("%Y-%m-%d")
        entry = {
            "date": today,
            "group_set": self.active_group_set_id,
            "group_set_name": self.active_group_set_name,
            "group_id": group.get("id"),
            "group_name": group.get("name"),
            "search_links": search_urls,
            "keywords_csv": keywords_csv,
            "connections_sent": connections_sent
        }
        self.metrics.add_group_run(entry)
    
    def start_group_run(self, group):
        """Log that a group run has started (with 0 connections initially)."""
        today = datetime.now().strftime("%Y-%m-%d")
        entry = {
            "date": today,
            "group_set": self.active_group_set_id,
            "group_set_name": self.active_group_set_name,
            "group_id": group.get("id"),
            "group_name": group.get("name"),
            "search_links": [],  # Will be filled later
            "keywords_csv": "",  # Will be filled later
            "connections_sent": 0,
            "status": "started"
        }
        # Store this temporarily, not in metrics yet
        self.current_group_run = entry
        print_lg(f"[GROUP START] Started group: {group.get('name')} (id: {group.get('id')})")
    
    def update_group_run(self, search_urls, keywords_csv, connections_sent):
        """Update the current group run with results and save to metrics."""
        if hasattr(self, 'current_group_run'):
            self.current_group_run["search_links"] = search_urls
            self.current_group_run["keywords_csv"] = keywords_csv
            self.current_group_run["connections_sent"] = connections_sent
            # Remove temporary status field
            if "status" in self.current_group_run:
                del self.current_group_run["status"]
            # Save to metrics
            self.metrics.add_group_run(self.current_group_run)
            print_lg(f"[GROUP COMPLETE] Group run logged: {connections_sent} connections sent")
            # Clear current group run
            del self.current_group_run

    def get_shadow_root_modal(self):
        """
        LinkedIn uses Shadow DOM for modals. This method accesses the shadow root
        to find modal elements that normal XPath/CSS selectors can't reach.
        """
        try:
            # Find the shadow host element
            shadow_host = self.driver.find_element(By.CSS_SELECTOR, 
                'div#interop-outlet[data-testid="interop-shadowdom"]'
            )
            # Get the shadow root
            shadow_root = shadow_host.shadow_root
            return shadow_root
        except Exception as e:
            print_lg(f"[SHADOW] Could not access shadow root: {e}")
            return None
    
    def handle_connection_modal(self, name, connect_element=None):
        """
        Returns True if invite was sent, False otherwise.
        Handles two flows:
        1. Modal flow - modal appears with "Add a note" / "Send without a note" options
        2. Fallback - check for "Pending" status if no modal detected
        
        Note: LinkedIn modals are in Shadow DOM, requiring special access.
        """
        try:
            print_lg(f"[MODAL] Waiting for connection modal...")
            modal_wait_time = 5  # seconds to wait for modal
            add_note_btn = None
            send_without_note_btn = None
            
            # Try to find modal in Shadow DOM
            for attempt in range(modal_wait_time * 2):  # Check every 0.5s
                time.sleep(0.5)
                
                shadow_root = self.get_shadow_root_modal()
                if shadow_root:
                    try:
                        # Look for modal container in shadow DOM
                        modal = shadow_root.find_element(By.CSS_SELECTOR, 
                            'div[data-test-modal-id="send-invite-modal"], div.artdeco-modal.send-invite'
                        )
                        if modal:
                            print_lg(f"[MODAL] Modal found in Shadow DOM!")
                            time.sleep(0.5)  # Let content load
                            
                            # Find buttons in shadow DOM
                            try:
                                add_note_btn = shadow_root.find_element(By.CSS_SELECTOR,
                                    'button[aria-label="Add a note"]'
                                )
                            except:
                                pass
                            
                            try:
                                send_without_note_btn = shadow_root.find_element(By.CSS_SELECTOR,
                                    'button[aria-label="Send without a note"]'
                                )
                            except:
                                pass
                            
                            if add_note_btn or send_without_note_btn:
                                print_lg(f"[MODAL] Found modal buttons!")
                                break
                    except:
                        pass
                
                # Also try regular DOM (in case LinkedIn changes behavior)
                try:
                    modal = self.driver.find_element(By.XPATH, 
                        "//div[contains(@class, 'send-invite')]//button[@aria-label='Add a note']"
                    )
                    if modal:
                        add_note_btn = modal
                        print_lg(f"[MODAL] Modal found in regular DOM!")
                        break
                except:
                    pass
            
            # If no modal found, check for Pending status
            if not add_note_btn and not send_without_note_btn:
                print_lg(f"[MODAL] No modal detected after {modal_wait_time}s. Checking for Pending status...")
                time.sleep(1)
                
                try:
                    pending = self.driver.find_elements(By.XPATH, 
                        f"//button[contains(@aria-label, 'Pending')] | "
                        f"//span[contains(text(), 'Pending')]"
                    )
                    if pending:
                        print_lg(f"[DIRECT SEND] Connection to {name} shows Pending status.")
                        return True
                except:
                    pass
                
                print_lg(f"[WARNING] Could not confirm connection status for {name}.")
                return False
            
            # --- MODAL FLOW ---
            send_without_note_config = bool(get_config_value(config, ["messaging", "send_without_note"], False))
            
            if send_without_note_config and send_without_note_btn:
                # Skip note, send directly from modal
                print_lg(f"[MODAL] SEND_WITHOUT_NOTE enabled, clicking 'Send without a note'...")
                send_without_note_btn.click()
                print_lg(f"[SUCCESS] Sent invite to {name} (without note).")
                time.sleep(2)
                return True
            
            # Add note flow
            if add_note_btn:
                template = random.choice(self.message_templates)
                message = template.replace("{name}", name).replace("{{name}}", name)
                
                print_lg(f"[MODAL] Clicking 'Add a note'...")
                add_note_btn.click()
                time.sleep(1)
                
                # Find textarea in shadow DOM
                shadow_root = self.get_shadow_root_modal()
                msg_box = None
                
                if shadow_root:
                    try:
                        msg_box = shadow_root.find_element(By.CSS_SELECTOR, 
                            'textarea[name="message"], textarea#custom-message, textarea'
                        )
                    except:
                        pass
                
                # Fallback to regular DOM
                if not msg_box:
                    try:
                        msg_box = WebDriverWait(self.driver, 3).until(
                            EC.presence_of_element_located((By.XPATH, 
                                "//textarea[@name='message'] | //textarea[@id='custom-message'] | //textarea"
                            ))
                        )
                    except:
                        pass
                
                if msg_box:
                    msg_box.send_keys(message)
                    print_lg(f"[MODAL] Typed message: {message[:50]}...")
                    time.sleep(1)
                else:
                    print_lg(f"[WARNING] Could not find message textarea")
                
                # Find and click Send button in shadow DOM
                send_btn = None
                if shadow_root:
                    try:
                        send_btn = shadow_root.find_element(By.CSS_SELECTOR,
                            'button[aria-label="Send invitation"], button.artdeco-button--primary'
                        )
                    except:
                        pass
                
                if not send_btn:
                    try:
                        send_btn = WebDriverWait(self.driver, 3).until(
                            EC.element_to_be_clickable((By.XPATH, 
                                "//button[@aria-label='Send invitation'] | //button[normalize-space(.)='Send']"
                            ))
                        )
                    except:
                        pass
                
                if send_btn:
                    send_btn.click()
                    print_lg(f"[SUCCESS] Sent invite to {name} with note.")
                    time.sleep(2)
                    return True
                else:
                    print_lg(f"[WARNING] Could not find Send button")
            
            # Fallback: try Send without note if Add note failed
            if send_without_note_btn:
                print_lg(f"[MODAL] Fallback: clicking 'Send without a note'...")
                send_without_note_btn.click()
                print_lg(f"[SUCCESS] Sent invite to {name} (without note - fallback).")
                time.sleep(2)
                return True
            
            return False
            
        except Exception as e:
            print_lg(f"Modal flow failed: {e}")
            pass

        # --- FALLBACK: Try shadow DOM one more time ---
        try:
            print_lg("[FALLBACK] Trying shadow DOM one more time...")
            shadow_root = self.get_shadow_root_modal()
            if shadow_root:
                send_btn = shadow_root.find_element(By.CSS_SELECTOR,
                    'button[aria-label="Send without a note"]'
                )
                send_btn.click()
                print_lg(f"[SUCCESS] Sent invite to {name} (without note - fallback).")
                return True
        except Exception as e:
            print_lg(f"Fallback failed: {e}")
            pass
            
        # --- CLEANUP (CLOSE MODAL) ---
        print_lg(f"Could not find Send buttons for {name}. Closing modal.")
        try:
            close_btn = self.driver.find_element(By.XPATH, "//button[@aria-label='Dismiss'] | //button[contains(@class, 'dismiss')]")
            close_btn.click()
        except: 
            pass
            
        return False


# -------------------------------------------------------------------------
# MAIN EXECUTION
# -------------------------------------------------------------------------

if __name__ == "__main__":
    if not username or not password:
        print("ERROR: LinkedIn credentials not found!")
        print("  1. Copy .env.example to .env:  cp .env.example .env")
        print("  2. Edit .env with your LinkedIn email and password")
        sys.exit(1)
    
    driver, wait, actions = setup_browser()
    
    try:
        # 1. Force navigation to LinkedIn main page immediately
        print_lg("Navigating to LinkedIn...")
        driver.get("https://www.linkedin.com/login")
        time.sleep(2)        

        if not is_logged_in_LN(driver):
            login_LN(driver, wait)
        else:
            print_lg("Already logged in!")
            
        print_lg("Login complete. Initializing Bot...")
        
        bot = LinkedInBot(driver, wait)
        bot.run()

        if not bot.metrics.can_connect():
            print_lg("Daily limit reached. Closing browser.")
            driver.quit()
            sys.exit(0)

        print_lg("Bot finished execution. Press Ctrl+C to exit and close browser.")
        # Keep browser open
        while True:
            time.sleep(1)
            
    except KeyboardInterrupt:
        print_lg("Closing browser...")
        driver.quit()
    except Exception as e:
        print_lg(f"An error occurred: {e}")