import requests
from bs4 import BeautifulSoup
import xml.etree.ElementTree as ET
import json
import time
import sys
import argparse
from googlenewsdecoder import gnewsdecoder

HEADERS = {
    "User-Agent": "Mozilla/5.0"
}

# ========================
# GENERIC PARSER
# ========================

def parse_content(html):
    soup = BeautifulSoup(html, "html.parser")

    decompose_tags = [
        "script", "style", "nav", "footer", "header", "aside", 
        "form", "iframe", "noscript", "svg", "button", "menu",
        "figure", "ins", "dialog", ".sidebar", "#sidebar"
    ]
    for tag in soup(decompose_tags):
        tag.decompose()

    content_tags = ["p", "div", "article", "section", "span", "blockquote", "li", "td", "h1", "h2", "h3", "h4", "h5", "h6"]
    block_tags = ["p", "div", "article", "section", "ul", "ol", "table", "blockquote"]
    
    paragraphs = []
    for tag in soup.find_all(content_tags):
        if not tag.find(block_tags):
            text = tag.get_text(separator=" ", strip=True)
            if len(text) > 50:
                paragraphs.append(text)

    if not paragraphs:
        text_blocks = soup.get_text(separator='\n').split('\n')
        paragraphs = [t.strip() for t in text_blocks if len(t.strip()) > 50]

    seen = set()
    unique_paragraphs = []
    for p_text in paragraphs:
        if p_text not in seen:
            seen.add(p_text)
            unique_paragraphs.append(p_text)

    return " ".join(unique_paragraphs)

# ========================
# RSS FETCH
# ========================

def get_rss(keyword):
    url = f"https://news.google.com/rss/search?q={keyword}&hl=id&gl=ID&ceid=ID:id"
    
    res = requests.get(url)
    root = ET.fromstring(res.content)

    return [
        {
            "title": item.find("title").text,
            "link": item.find("link").text,
            "pubDate": item.find("pubDate").text,
            "source": item.find("source").text if item.find("source") is not None else "-"
        }
        for item in root.findall(".//item")
    ]

# ========================
# RESOLVE URL
# ========================

def resolve_url(url):
    try:
        res = requests.get(url, headers=HEADERS, timeout=10)
        decoded = gnewsdecoder(res.url)

        if decoded and decoded.get("status"):
            final_url = decoded.get("decoded_url")
            if final_url and "news.google.com" not in final_url:
                return final_url

        return None
    except:
        return None

# ========================
# CRAWLER
# ========================

def crawl(keyword="purwakarta", limit=5, offset=0):
    results = []

    for item in get_rss(keyword)[offset:offset+limit]:
        real_url = resolve_url(item["link"]) or item["link"]

        try:
            res = requests.get(real_url, headers=HEADERS)
            content = parse_content(res.text)

            results.append({
                "title": item["title"],
                "url": real_url,
                "published_at": item["pubDate"],
                "content": content,
                "keyword": keyword,
                "source": item["source"]
            })

            print(f"✔ {item['title']}", file=sys.stderr, flush=True)
            time.sleep(1)

        except Exception as e:
            print(f"❌ error: {e}", file=sys.stderr, flush=True)

    return results

# ========================
# MAIN
# ========================

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--keyword", type=str, default="purwakarta")
    parser.add_argument("--limit", type=int, default=1)
    parser.add_argument("--offset", type=int, default=0)
    args = parser.parse_args()

    print(f"🔎 Crawling for keyword: {args.keyword} (Limit: {args.limit}, Offset: {args.offset})", file=sys.stderr, flush=True)
    data = crawl(args.keyword, args.limit, args.offset)

    print(json.dumps(data, ensure_ascii=False))