44 lines
1.5 KiB
Python
44 lines
1.5 KiB
Python
|
import sys
|
||
|
|
import os
|
||
|
|
|
||
|
|
# Add src/ to sys.path so Python can find scraper/
|
||
|
|
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "src")))
|
||
|
|
|
||
|
|
# Now import fetcher
|
||
|
|
from scraper.fetcher import Fetcher
|
||
|
|
|
||
|
|
# Initialize Fetcher
|
||
|
|
fetcher = Fetcher()
|
||
|
|
|
||
|
|
# Get all available sites from sites.yml
|
||
|
|
all_sites = list(fetcher.sites.keys())
|
||
|
|
|
||
|
|
def test_fetch_all_sites():
|
||
|
|
"""Test fetching all sites in sites.yml dynamically."""
|
||
|
|
for site_name in all_sites:
|
||
|
|
print(f"\n🚀 Fetching site: {site_name}...")
|
||
|
|
content = fetcher.fetch(site_name, debug=True)
|
||
|
|
assert content, f"❌ Failed to fetch data from {site_name}!"
|
||
|
|
print(f"✅ Successfully fetched data from {site_name}.")
|
||
|
|
|
||
|
|
def test_debug_files():
|
||
|
|
"""Check if the number of fetched sites matches the number of saved debug files."""
|
||
|
|
html_files = [f for f in os.listdir("tests") if f.endswith(".html")]
|
||
|
|
num_sites = len(all_sites)
|
||
|
|
|
||
|
|
print(f"📊 Found {len(html_files)} debug files, expected {num_sites}.")
|
||
|
|
|
||
|
|
if len(html_files) != num_sites:
|
||
|
|
print("\n❌ Mismatch in debug files:")
|
||
|
|
print(f"- Expected: {num_sites} files")
|
||
|
|
print(f"- Found: {len(html_files)} files")
|
||
|
|
assert False, "Number of debug files does not match number of sites."
|
||
|
|
else:
|
||
|
|
print("✅ Debug file count matches the number of sites in sites.yml.")
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
test_fetch_all_sites()
|
||
|
|
test_debug_files()
|
||
|
|
|
||
|
|
print("\n🎉 ALL TESTS COMPLETED SUCCESSFULLY! 🚀")
|