#!/usr/bin/env python3

import asyncio
import json
import sys
import re
from datetime import datetime, timedelta
from playwright.async_api import async_playwright
import argparse

# Improved source definitions with better selectors and fallbacks
SOURCES = [
    {
        'name': 'Brand New',
        'url': 'https://underconsideration.com/brandnew/',
        'selectors': {
            'articles': ['article', '.post', '.entry'],
            'title': ['h2 a', 'h1 a', '.entry-title a', '.post-title a'],
            'link': ['h2 a', 'h1 a', '.entry-title a', '.post-title a'],
            'date': ['time', '.entry-date', '.post-date'],
            'excerpt': ['p', '.entry-excerpt', '.post-excerpt']
        },
        'timeout': 10000
    },
    {
        'name': 'Creative Boom',
        'url': 'https://creativeboom.com/inspiration/',
        'selectors': {
            'articles': ['.post-item', 'article.post', '.entry', '.post'],
            'title': ['h2 a', 'h3 a', '.entry-title a', '.post-title a'],
            'link': ['h2 a', 'h3 a', '.entry-title a', '.post-title a'],
            'date': ['time', '.entry-date', '.post-date', '.meta-date'],
            'excerpt': ['.entry-excerpt', '.excerpt', '.entry-content p', 'p']
        },
        'timeout': 15000
    },
    {
        'name': "It's Nice That",
        'url': 'https://www.itsnicethat.com/',
        'selectors': {
            'articles': ['article', '.featured-article', '.post', '.entry'],
            'title': ['h1 a', 'h2 a', 'h3 a', '.title a'],
            'link': ['h1 a', 'h2 a', 'h3 a', '.title a'],
            'date': ['time', '.date', '.meta-date', '.publish-date'],
            'excerpt': ['.excerpt', '.description', '.summary', 'p']
        },
        'timeout': 12000
    },
    {
        'name': 'The Brand Identity',
        'url': 'https://the-brandidentity.com/',
        'selectors': {
            'articles': ['article', '.post', '.entry', '.project'],
            'title': ['h2 a', 'h1 a', '.title a', '.project-title a'],
            'link': ['h2 a', 'h1 a', '.title a', '.project-title a'],
            'date': ['time', '.date', '.meta-date', '.publish-date'],
            'excerpt': ['.excerpt', '.summary', '.entry-content p', 'p']
        },
        'timeout': 15000
    },
    {
        'name': 'Design Milk',  # More reliable alternative to Behance
        'url': 'https://design-milk.com/',
        'selectors': {
            'articles': ['article', '.post', '.entry'],
            'title': ['h2 a', 'h1 a', '.entry-title a'],
            'link': ['h2 a', 'h1 a', '.entry-title a'],
            'date': ['time', '.entry-date', '.post-date'],
            'excerpt': ['.entry-excerpt', '.excerpt', 'p']
        },
        'timeout': 10000
    }
]

async def scrape_source_improved(browser, source, max_articles=5):
    """Improved scraping with better error handling and multiple selector fallbacks"""
    print(f"📡 Scanning {source['name']}...", file=sys.stderr)
    
    page = await browser.new_page()
    
    # Set user agent to appear more like a regular browser
    await page.set_extra_http_headers({
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    })
    
    articles = []
    
    try:
        # Navigate with shorter timeout
        await page.goto(source['url'], wait_until='domcontentloaded', timeout=source['timeout'])
        
        # Wait for content
        await page.wait_for_timeout(2000)
        
        # Try different article selectors
        article_elements = []
        for selector in source['selectors']['articles']:
            elements = await page.query_selector_all(selector)
            if elements:
                article_elements = elements
                print(f"   Found {len(elements)} articles with selector '{selector}'", file=sys.stderr)
                break
        
        if not article_elements:
            print(f"   No articles found with any selector", file=sys.stderr)
            return []
        
        # Extract from found articles
        for i, article in enumerate(article_elements[:max_articles]):
            try:
                title = None
                href = None
                
                # Try title selectors
                for selector in source['selectors']['title']:
                    try:
                        element = await article.query_selector(selector)
                        if element:
                            title = await element.inner_text()
                            href = await element.get_attribute('href')
                            if title and href:
                                break
                    except:
                        continue
                
                if not title or not href:
                    continue
                
                # Normalize URL
                if not href.startswith('http'):
                    base_url = f"{source['url'].split('/')[0]}//{source['url'].split('/')[2]}"
                    href = base_url + (href if href.startswith('/') else '/' + href)
                
                # Try to get excerpt
                excerpt = ""
                for selector in source['selectors']['excerpt']:
                    try:
                        element = await article.query_selector(selector)
                        if element:
                            text = await element.inner_text()
                            if text and len(text.strip()) > 10:
                                excerpt = text.strip()[:250]
                                break
                    except:
                        continue
                
                articles.append({
                    'source': source['name'],
                    'title': title.strip(),
                    'url': href,
                    'date': datetime.now().strftime('%Y-%m-%d'),
                    'excerpt': excerpt,
                    'scraped_at': datetime.now().isoformat()
                })
                
            except Exception as e:
                print(f"   Error extracting article {i+1}: {str(e)[:50]}", file=sys.stderr)
                continue
        
        print(f"✅ {source['name']}: Successfully extracted {len(articles)} articles", file=sys.stderr)
        
    except Exception as e:
        print(f"❌ {source['name']}: {str(e)[:100]}", file=sys.stderr)
    
    finally:
        await page.close()
    
    return articles

def analyze_design_trends(articles):
    """Enhanced trend analysis with design-specific patterns"""
    
    trend_patterns = {
        'minimalism-evolution': {
            'keywords': ['minimal', 'clean', 'simple', 'white space', 'typography', 'geometric', 'sans serif'],
            'weight': 2
        },
        'bold-branding': {
            'keywords': ['bold', 'vibrant', 'neon', 'electric', 'loud', 'statement', 'impact'],
            'weight': 2
        },
        'retro-revival': {
            'keywords': ['retro', 'vintage', 'nostalgic', '90s', '80s', 'throwback', 'analog'],
            'weight': 2
        },
        'sustainable-design': {
            'keywords': ['sustainable', 'eco', 'green', 'renewable', 'carbon', 'climate', 'earth'],
            'weight': 3
        },
        'experiential-brands': {
            'keywords': ['experience', 'interactive', 'immersive', 'digital', 'AR', 'VR', 'engagement'],
            'weight': 3
        },
        'human-centered': {
            'keywords': ['human', 'authentic', 'personal', 'story', 'community', 'inclusive', 'diverse'],
            'weight': 2
        }
    }
    
    signals = []
    for article in articles:
        content = f"{article['title']} {article.get('excerpt', '')}".lower()
        
        for pattern_name, pattern in trend_patterns.items():
            score = 0
            matched = []
            
            for keyword in pattern['keywords']:
                if keyword in content:
                    score += pattern['weight']
                    matched.append(keyword)
            
            if score >= 4:  # Threshold for significance
                confidence = 'strong' if score >= 8 else 'moderate'
                
                signal = {
                    'title': article['title'],
                    'source': article['source'],
                    'url': article['url'],
                    'trend': pattern_name,
                    'confidence': confidence,
                    'score': score,
                    'indicators': matched,
                    'insight': f"Shows {pattern_name.replace('-', ' ')} trend through: {', '.join(matched[:3])}",
                    'date': article['date']
                }
                signals.append(signal)
    
    # Sort by score
    signals.sort(key=lambda x: x['score'], reverse=True)
    return signals[:8]

async def run_comprehensive_test():
    """Run comprehensive test of all sources"""
    print("🔧 Testing Fixed Research Infrastructure", file=sys.stderr)
    print(f"📅 {datetime.now().strftime('%Y-%m-%d %H:%M:%S UTC')}", file=sys.stderr)
    print("=" * 60, file=sys.stderr)
    
    results = {
        'test_timestamp': datetime.now().isoformat(),
        'infrastructure_version': '2.0-fixed',
        'sources_tested': [],
        'successful_sources': [],
        'failed_sources': [],
        'total_articles': 0,
        'trend_signals': [],
        'sample_articles': []
    }
    
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        
        for i, source in enumerate(SOURCES, 1):
            print(f"\n🔍 Testing source {i}/5: {source['name']}", file=sys.stderr)
            results['sources_tested'].append(source['name'])
            
            try:
                articles = await scrape_source_improved(browser, source)
                
                if articles:
                    results['successful_sources'].append(source['name'])
                    results['total_articles'] += len(articles)
                    results['sample_articles'].extend(articles)
                else:
                    results['failed_sources'].append(source['name'])
                    
            except Exception as e:
                print(f"   ❌ Failed: {str(e)[:50]}", file=sys.stderr)
                results['failed_sources'].append(source['name'])
        
        await browser.close()
    
    # Analyze trends from collected articles
    if results['sample_articles']:
        results['trend_signals'] = analyze_design_trends(results['sample_articles'])
    
    # Set status
    success_rate = len(results['successful_sources']) / len(SOURCES)
    if success_rate >= 0.8:
        status = 'operational'
    elif success_rate >= 0.4:
        status = 'degraded'
    else:
        status = 'critical'
    
    results['infrastructure_status'] = status
    results['success_rate'] = f"{success_rate:.1%}"
    
    # Print summary
    print(f"\n📊 Test Results Summary:", file=sys.stderr)
    print(f"   Sources tested: {len(results['sources_tested'])}", file=sys.stderr)
    print(f"   Successful: {len(results['successful_sources'])} ({results['success_rate']})", file=sys.stderr)
    print(f"   Total articles: {results['total_articles']}", file=sys.stderr)
    print(f"   Trend signals: {len(results['trend_signals'])}", file=sys.stderr)
    print(f"   Status: {status.upper()}", file=sys.stderr)
    
    if results['successful_sources']:
        print(f"   ✅ Working: {', '.join(results['successful_sources'])}", file=sys.stderr)
    if results['failed_sources']:
        print(f"   ❌ Failed: {', '.join(results['failed_sources'])}", file=sys.stderr)
    
    return results

if __name__ == "__main__":
    try:
        results = asyncio.run(run_comprehensive_test())
        print(json.dumps(results, indent=2))
    except Exception as e:
        print(f"💥 Infrastructure test failed: {str(e)}", file=sys.stderr)
        sys.exit(1)