"""
Digest compilation and story ranking
"""

struct DigestStory
    title::String
    url::String
    summary::String
    published::DateTime
    source::String
    ai_score::Float64
    rank::Int
    matched_keywords::Vector{String}
    matched_companies::Vector{String}
end

function compile_digest(scored_stories::Vector{ScoredStory}, max_stories::Int)
    """Compile final digest from scored stories"""
    
    # Deduplicate stories
    config = Dict(
        "similarity_threshold" => 0.8,
        "title_similarity_weight" => 0.7,
        "content_similarity_weight" => 0.3
    )
    
    unique_stories = deduplicate_stories(scored_stories, config)
    
    # Take top N stories
    top_stories = unique_stories[1:min(length(unique_stories), max_stories)]
    
    # Convert to digest format with ranking
    digest_stories = DigestStory[]
    
    for (rank, scored_story) in enumerate(top_stories)
        story = scored_story.story
        digest_story = DigestStory(
            story.title,
            story.url,
            truncate_summary(story.summary, 200),
            story.published,
            story.source,
            scored_story.ai_score,
            rank,
            scored_story.matched_keywords,
            scored_story.matched_companies
        )
        push!(digest_stories, digest_story)
    end
    
    return digest_stories
end

function truncate_summary(summary::String, max_length::Int)
    """Truncate summary to specified length"""
    if length(summary) <= max_length
        return summary
    end
    
    # Find last space before max_length
    truncated = summary[1:max_length]
    last_space = findlast(' ', truncated)
    
    if last_space !== nothing && last_space > max_length * 0.8
        truncated = summary[1:last_space]
    end
    
    return truncated * "..."
end

function rank_stories_by_engagement(stories::Vector{ScoredStory})
    """Re-rank stories considering engagement factors"""
    
    # This is a placeholder for more sophisticated ranking
    # Could include factors like:
    # - Time since publication (fresher = better)
    # - Source authority/reputation
    # - Social media metrics (if available)
    # - Click-through rates from previous digests
    
    return stories  # For now, keep AI score ranking
end

function get_trending_topics(digest_stories::Vector{DigestStory})
    """Extract trending topics from digest"""
    
    all_keywords = String[]
    all_companies = String[]
    
    for story in digest_stories
        append!(all_keywords, story.matched_keywords)
        append!(all_companies, story.matched_companies)
    end
    
    # Count frequency
    keyword_counts = count_items(all_keywords)
    company_counts = count_items(all_companies)
    
    # Return top trending items
    trending = Dict(
        "keywords" => get_top_items(keyword_counts, 5),
        "companies" => get_top_items(company_counts, 5)
    )
    
    return trending
end

function count_items(items::Vector{String})
    """Count frequency of items"""
    counts = Dict{String, Int}()
    
    for item in items
        counts[item] = get(counts, item, 0) + 1
    end
    
    return counts
end

function get_top_items(counts::Dict{String, Int}, n::Int)
    """Get top N items by frequency"""
    sorted_items = sort(collect(pairs(counts)), by=pair -> pair[2], rev=true)
    return [item[1] for item in sorted_items[1:min(length(sorted_items), n)]]
end

function generate_digest_metadata(digest_stories::Vector{DigestStory})
    """Generate metadata for the digest"""
    
    total_stories = length(digest_stories)
    avg_score = total_stories > 0 ? mean([s.ai_score for s in digest_stories]) : 0.0
    
    sources = unique([s.source for s in digest_stories])
    trending = get_trending_topics(digest_stories)
    
    latest_story = total_stories > 0 ? maximum([s.published for s in digest_stories]) : now()
    oldest_story = total_stories > 0 ? minimum([s.published for s in digest_stories]) : now()
    
    metadata = Dict(
        "total_stories" => total_stories,
        "average_ai_score" => round(avg_score, digits=1),
        "sources_count" => length(sources),
        "sources" => sources,
        "trending" => trending,
        "time_range" => Dict(
            "latest" => latest_story,
            "oldest" => oldest_story
        ),
        "generated_at" => now()
    )
    
    return metadata
end