135 lines
3.8 KiB
Python
135 lines
3.8 KiB
Python
import requests
|
|
from bs4 import BeautifulSoup
|
|
import xml.etree.ElementTree as ET
|
|
import json
|
|
import time
|
|
import sys
|
|
import argparse
|
|
from googlenewsdecoder import gnewsdecoder
|
|
|
|
HEADERS = {
|
|
"User-Agent": "Mozilla/5.0"
|
|
}
|
|
|
|
# ========================
|
|
# GENERIC PARSER
|
|
# ========================
|
|
|
|
def parse_content(html):
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
decompose_tags = [
|
|
"script", "style", "nav", "footer", "header", "aside",
|
|
"form", "iframe", "noscript", "svg", "button", "menu",
|
|
"figure", "ins", "dialog", ".sidebar", "#sidebar"
|
|
]
|
|
for tag in soup(decompose_tags):
|
|
tag.decompose()
|
|
|
|
content_tags = ["p", "div", "article", "section", "span", "blockquote", "li", "td", "h1", "h2", "h3", "h4", "h5", "h6"]
|
|
block_tags = ["p", "div", "article", "section", "ul", "ol", "table", "blockquote"]
|
|
|
|
paragraphs = []
|
|
for tag in soup.find_all(content_tags):
|
|
if not tag.find(block_tags):
|
|
text = tag.get_text(separator=" ", strip=True)
|
|
if len(text) > 50:
|
|
paragraphs.append(text)
|
|
|
|
if not paragraphs:
|
|
text_blocks = soup.get_text(separator='\n').split('\n')
|
|
paragraphs = [t.strip() for t in text_blocks if len(t.strip()) > 50]
|
|
|
|
seen = set()
|
|
unique_paragraphs = []
|
|
for p_text in paragraphs:
|
|
if p_text not in seen:
|
|
seen.add(p_text)
|
|
unique_paragraphs.append(p_text)
|
|
|
|
return " ".join(unique_paragraphs)
|
|
|
|
# ========================
|
|
# RSS FETCH
|
|
# ========================
|
|
|
|
def get_rss(keyword):
|
|
url = f"https://news.google.com/rss/search?q={keyword}&hl=id&gl=ID&ceid=ID:id"
|
|
|
|
res = requests.get(url)
|
|
root = ET.fromstring(res.content)
|
|
|
|
return [
|
|
{
|
|
"title": item.find("title").text,
|
|
"link": item.find("link").text,
|
|
"pubDate": item.find("pubDate").text,
|
|
"source": item.find("source").text if item.find("source") is not None else "-"
|
|
}
|
|
for item in root.findall(".//item")
|
|
]
|
|
|
|
# ========================
|
|
# RESOLVE URL
|
|
# ========================
|
|
|
|
def resolve_url(url):
|
|
try:
|
|
res = requests.get(url, headers=HEADERS, timeout=10)
|
|
decoded = gnewsdecoder(res.url)
|
|
|
|
if decoded and decoded.get("status"):
|
|
final_url = decoded.get("decoded_url")
|
|
if final_url and "news.google.com" not in final_url:
|
|
return final_url
|
|
|
|
return None
|
|
except:
|
|
return None
|
|
|
|
# ========================
|
|
# CRAWLER
|
|
# ========================
|
|
|
|
def crawl(keyword="purwakarta", limit=5, offset=0):
|
|
results = []
|
|
|
|
for item in get_rss(keyword)[offset:offset+limit]:
|
|
real_url = resolve_url(item["link"]) or item["link"]
|
|
|
|
try:
|
|
res = requests.get(real_url, headers=HEADERS)
|
|
content = parse_content(res.text)
|
|
|
|
results.append({
|
|
"title": item["title"],
|
|
"url": real_url,
|
|
"published_at": item["pubDate"],
|
|
"content": content,
|
|
"keyword": keyword,
|
|
"source": item["source"]
|
|
})
|
|
|
|
print(f"✔ {item['title']}", file=sys.stderr, flush=True)
|
|
time.sleep(1)
|
|
|
|
except Exception as e:
|
|
print(f"❌ error: {e}", file=sys.stderr, flush=True)
|
|
|
|
return results
|
|
|
|
# ========================
|
|
# MAIN
|
|
# ========================
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--keyword", type=str, default="purwakarta")
|
|
parser.add_argument("--limit", type=int, default=1)
|
|
parser.add_argument("--offset", type=int, default=0)
|
|
args = parser.parse_args()
|
|
|
|
print(f"🔎 Crawling for keyword: {args.keyword} (Limit: {args.limit}, Offset: {args.offset})", file=sys.stderr, flush=True)
|
|
data = crawl(args.keyword, args.limit, args.offset)
|
|
|
|
print(json.dumps(data, ensure_ascii=False)) |