#!/usr/bin/env python3
"""
Research Infrastructure - Fixed Version
Bypasses anti-bot blocks using Playwright browser automation
Replaces blocked web_fetch/requests methods
"""

from playwright.sync_api import sync_playwright
import json
import time
from datetime import datetime

def get_research_sources():
    """Get current design research sources with working access methods"""
    return {
        'brand_new': {
            'url': 'https://underconsideration.com',
            'name': 'Brand New',
            'feed_path': '/archives/',
            'content_selector': '.entry'
        },
        'dezeen': {
            'url': 'https://dezeen.com/design',
            'name': 'Dezeen Design', 
            'feed_path': '/design/',
            'content_selector': '.dezeen-article'
        },
        'fast_company': {
            'url': 'https://fastcompany.com',
            'name': 'Fast Company',
            'feed_path': '/section/design',
            'content_selector': '.article'
        },
        'eye_on_design': {
            'url': 'https://eyeondesign.aiga.org',
            'name': 'AIGA Eye on Design',
            'feed_path': '/',
            'content_selector': '.post'
        },
        'its_nice_that': {
            'url': 'https://itsnicethat.com',
            'name': 'Its Nice That',
            'feed_path': '/articles',
            'content_selector': '.article-card'
        }
    }

def scrape_source_playwright(source_config, max_articles=5):
    """Scrape a design source using Playwright browser automation"""
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        context = browser.new_context(
            user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
        )
        page = context.new_page()
        
        try:
            # Navigate to source
            url = source_config['url'] + source_config['feed_path']
            page.goto(url, timeout=30000, wait_until='domcontentloaded')
            
            # Wait for content to load
            time.sleep(2)
            
            # Extract articles using selector
            articles = page.query_selector_all(source_config['content_selector'])
            
            results = []
            for i, article in enumerate(articles[:max_articles]):
                try:
                    title_elem = article.query_selector('h1, h2, h3, .title, .headline')
                    link_elem = article.query_selector('a')
                    
                    if title_elem and link_elem:
                        results.append({
                            'title': title_elem.inner_text().strip(),
                            'url': link_elem.get_attribute('href'),
                            'source': source_config['name']
                        })
                except:
                    continue
            
            browser.close()
            return results
            
        except Exception as e:
            browser.close()
            return {'error': str(e)}

def run_research_scan():
    """Run full research scan using fixed infrastructure"""
    sources = get_research_sources()
    results = {
        'scan_time': datetime.now().isoformat(),
        'status': 'success',
        'sources_scanned': len(sources),
        'articles': []
    }
    
    print(f"🔍 Scanning {len(sources)} design sources via Playwright...")
    
    for source_id, config in sources.items():
        print(f"📰 Scraping {config['name']}...")
        articles = scrape_source_playwright(config)
        
        if isinstance(articles, list):
            results['articles'].extend(articles)
            print(f"✅ Found {len(articles)} articles")
        else:
            print(f"❌ Error: {articles.get('error', 'Unknown error')}")
    
    # Save results
    with open('data/latest-research-scan.json', 'w') as f:
        json.dump(results, f, indent=2)
    
    print(f"\n✅ Research scan complete: {len(results['articles'])} articles collected")
    return results

if __name__ == "__main__":
    run_research_scan()