import os
from bs4 import BeautifulSoup
import nltk
from nltk.tokenize import word_tokenize

# Ensure NLTK resources are downloaded
nltk.download("punkt")

def parse_html(html_content):
    """Extract token data dynamically using NLP instead of hardcoded class names."""
    from bs4 import BeautifulSoup
    import nltk
    from nltk.tokenize import word_tokenize

    soup = BeautifulSoup(html_content, "html.parser")

    # Extract all text from the page
    full_text = soup.get_text(separator=" ", strip=True)

    # DEBUG: Print first 1000 characters of extracted text
    print("\n🔍 DEBUG: Extracted Text from HTML:")
    print(full_text[:1000])  # Print first 1000 characters to see what the parser is working with

    # Tokenize the text (split it into words)
    words = word_tokenize(full_text)

    # Heuristic: A token name is usually a **capitalized word** with no numbers
    possible_names = [word for word in words if word.istitle() and word.isalpha()]
    possible_symbols = [word for word in words if word.isupper() and len(word) <= 5]

    tokens = []
    if possible_names:
        for i, name in enumerate(possible_names):
            symbol = possible_symbols[i] if i < len(possible_symbols) else "N/A"
            tokens.append({"name": name, "symbol": symbol})

    print(f"✅ Extracted {len(tokens)} tokens.")  # Debugging output
    return {"tokens": tokens}


def parse_from_file(filename):
    """Loads an HTML file from the tests directory and parses it."""
    filepath = os.path.join("tests", filename)
    if not os.path.exists(filepath):
        raise FileNotFoundError(f"❌ HTML file not found: {filepath}")

    with open(filepath, "r", encoding="utf-8") as file:
        html_content = file.read()

    return parse_html(html_content)