import yaml
from playwright.sync_api import sync_playwright
import nltk
from bs4 import BeautifulSoup
from nltk.tokenize import word_tokenize
nltk.download("punkt_tab")
import os

# Load settings
with open("config/settings.yml", "r") as file:
    settings = yaml.safe_load(file)
HEADLESS_MODE_JS = settings["debug"]["headless_mode"].get("js", True)  # Default to True

def fetch_js(url, debug=False):
    """Fetches JavaScript-rendered pages and handles interactions."""
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=HEADLESS_MODE_JS)
        page = browser.new_page()
        page.goto(url, wait_until="networkidle")

        # Scroll & click 'Load More'
        if needs_scrolling(page):
            scroll_page(page)
        if has_load_more_button(page):
            click_load_more(page)

        content = page.content()
        browser.close()

    if debug:
        save_debug_file(url, content)  # Now only two arguments

    return content

def needs_scrolling(page):
    """Detects if a page requires scrolling by checking if it dynamically loads more content."""
    prev_height = page.evaluate("document.body.scrollHeight")
    page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
    page.wait_for_timeout(2000)  # Wait for new content to load
    new_height = page.evaluate("document.body.scrollHeight")

    return new_height > prev_height  # If height increased, scrolling is needed

def scroll_page(page):
    """Scrolls down until no more content is being added."""
    for _ in range(5):  # Limit scrolling attempts to prevent infinite loops
        prev_height = page.evaluate("document.body.scrollHeight")
        page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        page.wait_for_timeout(2000)
        new_height = page.evaluate("document.body.scrollHeight")
        if new_height == prev_height:
            break  # Stop if no new content is loading

def has_load_more_button(page):
    """Checks if a 'Load More' button exists on the page."""
    button = page.query_selector("button, .load-more, .btn-load-more")
    return button is not None

def click_load_more(page):
    """Clicks 'Load More' buttons if they exist, while checking for popups."""
    for _ in range(3):  # Limit to 3 attempts
        button = page.query_selector("button, .load-more, .btn-load-more")

        # Check for modals/popups and try to close them
        close_popup(page)

        if button:
            try:
                button.click(timeout=5000)  # 5-second timeout to prevent long waits
                page.wait_for_timeout(2000)  # Wait for new content to load
                print("✅ Clicked 'Load More' button.")
            except Exception as e:
                print(f"❌ Failed to click 'Load More': {e}")
                break  # Stop trying if clicking fails
        else:
            print("⚠️ No 'Load More' button found.")
            break

from bs4 import BeautifulSoup

def close_popup(page):
    """Uses NLP to detect and remove popups dynamically."""
    page_html = page.content()
    soup = BeautifulSoup(page_html, "html.parser")

    popup_selectors = [
        "div[role='dialog']",  # Generic login popup
        ".modalOpened",        # CoinMarketCap's modal
        ".overlay",            # Overlays that block clicks
        ".popup",              # Other generic popups
    ]

    close_button_selectors = [
        "button[aria-label='Close']",  # CMC Close button
        "button.close",                # Generic close button
        ".modal-close",                # Another common class
    ]

    # Extract visible text
    visible_text = " ".join([tag.get_text(strip=True) for tag in soup.find_all("div")])

    # Tokenize the text
    words = word_tokenize(visible_text.lower())

    # Define words that indicate a login popup
    popup_keywords = {"login", "sign up", "sign in", "register", "account", "access", "continue"}

    if any(word in words for word in popup_keywords):
        print(f"⚠️ AI detected a login popup based on text analysis. Attempting to close...")

    for selector in popup_selectors:
        popup = page.query_selector(selector)
        if popup:
            print(f"⚠️ NLP detected popup: {selector}, attempting to close it...")

            # Try clicking the close button first
            for close_selector in close_button_selectors:
                close_button = page.query_selector(close_selector)
                if close_button:
                    try:
                        close_button.click()
                        page.wait_for_timeout(2000)  # Wait for it to close
                        print(f"✅ AI clicked close button: {close_selector}")
                        return
                    except Exception as e:
                        print(f"❌ AI failed to click close button: {close_selector} -> {e}")

            # No close button found? Try removing the popup
            print(f"⚠️ AI failed to find a close button, forcing removal of {selector}...")
            try:
                page.evaluate(f'document.querySelector("{selector}").remove();')
                page.wait_for_timeout(1000)  # Allow time for it to disappear
                print(f"✅ AI forced removal of popup: {selector}")
            except Exception as e:
                print(f"❌ AI failed to remove popup: {selector} -> {e}")



def save_debug_file(site_name, content):
    """Saves fetched JavaScript-rendered pages as .html instead of .js."""
    safe_name = (
        site_name.replace("https://", "")
        .replace("http://", "")
        .replace("/", "_")
        .replace(".", "_")  # Normalize periods
        .rstrip("_")  # Remove trailing underscores
    )
    
    file_path = f"tests/{safe_name}.html"  # Always save as .html

    with open(file_path, "w", encoding="utf-8") as f:
        f.write(content)

    print(f"✅ Debug file saved: {file_path}")


