"""
AI content filtering and relevance scoring
"""

struct ScoredStory
    story::Story
    ai_score::Float64
    matched_keywords::Vector{String}
    matched_companies::Vector{String}
end

function filter_ai_content(stories::Vector{Story}, filter_config::Dict)
    """Filter stories for AI relevance and return scored results"""
    
    scored_stories = ScoredStory[]
    threshold = filter_config["threshold"]
    
    for story in stories
        score, keywords, companies = calculate_ai_score(story, filter_config)
        
        if score >= threshold
            scored_story = ScoredStory(story, score, keywords, companies)
            push!(scored_stories, scored_story)
        end
    end
    
    # Sort by score (highest first)
    sort!(scored_stories, by=s -> s.ai_score, rev=true)
    
    return scored_stories
end

function calculate_ai_score(story::Story, filter_config::Dict)
    """Calculate AI relevance score for a story"""
    
    keywords = filter_config["keywords"]
    companies = filter_config["companies"]
    title_weight = filter_config["title_weight"]
    content_weight = filter_config["content_weight"]
    company_bonus = filter_config["company_bonus"]
    
    # Combine title and content for analysis
    title_lower = lowercase(story.title)
    content_lower = lowercase(story.summary)
    full_text = title_lower * " " * content_lower
    
    # Count keyword matches
    keyword_score = 0.0
    matched_keywords = String[]
    
    for keyword in keywords
        keyword_lower = lowercase(keyword)
        
        # Title matches get higher weight
        title_matches = count_matches(title_lower, keyword_lower)
        content_matches = count_matches(content_lower, keyword_lower)
        
        if title_matches > 0 || content_matches > 0
            push!(matched_keywords, keyword)
            keyword_score += (title_matches * title_weight) + (content_matches * content_weight)
        end
    end
    
    # Check for company mentions
    company_score = 0.0
    matched_companies = String[]
    
    for company in companies
        company_lower = lowercase(company)
        
        if contains(full_text, company_lower)
            push!(matched_companies, company)
            company_score += company_bonus
        end
    end
    
    # Combine scores and normalize
    raw_score = keyword_score + company_score
    
    # Normalize to 0-100 scale (this is a simplified normalization)
    normalized_score = min(100.0, raw_score * 10.0)
    
    return normalized_score, matched_keywords, matched_companies
end

function count_matches(text::String, pattern::String)
    """Count occurrences of pattern in text"""
    count = 0
    pos = 1
    
    while true
        found = findnext(pattern, text, pos)
        if found === nothing
            break
        end
        count += 1
        pos = last(found) + 1
    end
    
    return count
end

function is_likely_ai_content(title::String, summary::String)
    """Quick heuristic check for AI content"""
    
    text = lowercase(title * " " * summary)
    
    # Strong AI indicators
    strong_indicators = [
        "artificial intelligence", "machine learning", "neural network",
        "deep learning", "chatgpt", "gpt-", "openai", "anthropic"
    ]
    
    for indicator in strong_indicators
        if contains(text, indicator)
            return true
        end
    end
    
    return false
end

function deduplicate_stories(scored_stories::Vector{ScoredStory}, config::Dict)
    """Remove duplicate stories based on similarity"""
    
    if length(scored_stories) <= 1
        return scored_stories
    end
    
    threshold = config["similarity_threshold"]
    unique_stories = ScoredStory[]
    
    for story in scored_stories
        is_duplicate = false
        
        for existing in unique_stories
            similarity = calculate_similarity(story.story, existing.story, config)
            if similarity >= threshold
                is_duplicate = true
                break
            end
        end
        
        if !is_duplicate
            push!(unique_stories, story)
        end
    end
    
    return unique_stories
end

function calculate_similarity(story1::Story, story2::Story, config::Dict)
    """Calculate similarity between two stories"""
    
    title_weight = config["title_similarity_weight"]
    content_weight = config["content_similarity_weight"]
    
    # Calculate title similarity
    title_sim = text_similarity(story1.title, story2.title)
    
    # Calculate content similarity  
    content_sim = text_similarity(story1.summary, story2.summary)
    
    # Weighted average
    overall_sim = (title_sim * title_weight) + (content_sim * content_weight)
    
    return overall_sim
end

function text_similarity(text1::String, text2::String)
    """Calculate text similarity using Jaccard coefficient"""
    
    # Tokenize and normalize
    words1 = Set(split(lowercase(text1)))
    words2 = Set(split(lowercase(text2)))
    
    # Calculate Jaccard similarity
    intersection = length(intersect(words1, words2))
    union = length(union(words1, words2))
    
    if union == 0
        return 0.0
    end
    
    return intersection / union
end