refactor: remove news crawler service, command, and associated UI actions
This commit is contained in:
Yoga Pangestu
parent
711aedcf2b
commit
1977d08339
@ -1,61 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Theme;
|
||||
use App\Services\NewsCrawlerService;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class CrawlNewsCommand extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'app:crawl-news {keyword?} {--limit=10} {--all-themes}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Crawl news from Google News based on keywords or themes';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(NewsCrawlerService $service)
|
||||
{
|
||||
$limit = (int) $this->option('limit');
|
||||
$allThemes = $this->option('all-themes');
|
||||
$keyword = $this->argument('keyword');
|
||||
|
||||
if ($allThemes) {
|
||||
$themes = Theme::active()->get();
|
||||
if ($themes->isEmpty()) {
|
||||
$this->error('No active themes found.');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($themes as $theme) {
|
||||
$this->info("Crawling for theme: {$theme->name}");
|
||||
$count = $service->crawlFromGoogleNews($theme->name, $limit, [$theme->id]);
|
||||
$this->info("Saved {$count} news items for theme: {$theme->name}");
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (! $keyword) {
|
||||
$this->error('Please provide a keyword or use --all-themes');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
$this->info("Crawling for keyword: {$keyword}");
|
||||
$count = $service->crawlFromGoogleNews($keyword, $limit);
|
||||
$this->info("Saved {$count} news items for keyword: {$keyword}");
|
||||
}
|
||||
}
|
||||
@ -1,67 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Filament\Resources\Monitoring\MediaMonitorings\Actions;
|
||||
|
||||
use App\Filament\Support\CheerfulNotification;
|
||||
use App\Models\Theme;
|
||||
use App\Services\NewsCrawlerService;
|
||||
use Filament\Actions\Action;
|
||||
use Filament\Forms\Components\Select;
|
||||
use Filament\Forms\Components\TextInput;
|
||||
use Filament\Support\Enums\Width;
|
||||
|
||||
class CrawlNewsAction extends Action
|
||||
{
|
||||
public static function getDefaultName(): ?string
|
||||
{
|
||||
return 'crawlNews';
|
||||
}
|
||||
|
||||
protected function setUp(): void
|
||||
{
|
||||
parent::setUp();
|
||||
|
||||
$this->label('Crawl Berita')
|
||||
->color('slate')
|
||||
->schema([
|
||||
TextInput::make('keyword')
|
||||
->label('Kata Kunci')
|
||||
->placeholder('Pendidikan, UMKM, Pariwisata')
|
||||
->autocomplete(false)
|
||||
->autofocus()
|
||||
->required()
|
||||
->helperText('Jika kata kunci nya lebih dari 1, pisahkan dengan koma.'),
|
||||
|
||||
Select::make('theme_id')
|
||||
->label('Tema')
|
||||
->required()
|
||||
->options(Theme::active()->pluck('name', 'id'))
|
||||
->searchable()
|
||||
->multiple(),
|
||||
|
||||
TextInput::make('limit')
|
||||
->label('Jumlah Berita')
|
||||
->placeholder(10)
|
||||
->numeric()
|
||||
->required()
|
||||
->default(10)
|
||||
->autocomplete(false),
|
||||
])
|
||||
->action(function (array $data, NewsCrawlerService $service): void {
|
||||
$count = $service->crawlFromGoogleNews($data['keyword'], $data['limit'], (array) ($data['theme_id'] ?? []));
|
||||
|
||||
if ($count > 0) {
|
||||
CheerfulNotification::success(
|
||||
CheerfulNotification::getByKey('crawl.success'),
|
||||
CheerfulNotification::getByKey('news.content.fetch_success', ['count' => $count, 'data__keyword' => $data['keyword']])
|
||||
)->send();
|
||||
} else {
|
||||
CheerfulNotification::info(
|
||||
CheerfulNotification::getByKey('news.no_new_data'),
|
||||
CheerfulNotification::getByKey('news.content.fetch_empty', ['data__keyword' => $data['keyword']])
|
||||
)->send();
|
||||
}
|
||||
})
|
||||
->modalWidth(Width::Large);
|
||||
}
|
||||
}
|
||||
@ -3,7 +3,6 @@
|
||||
namespace App\Filament\Resources\Monitoring\MediaMonitorings\Pages;
|
||||
|
||||
use App\Filament\Exports\MediaMonitoringExcelExport;
|
||||
use App\Filament\Resources\Monitoring\MediaMonitorings\Actions\CrawlNewsAction;
|
||||
use App\Filament\Resources\Monitoring\MediaMonitorings\MediaMonitoringResource;
|
||||
use Filament\Actions\CreateAction;
|
||||
use Filament\Resources\Pages\ListRecords;
|
||||
@ -21,8 +20,6 @@ protected function getHeaderActions(): array
|
||||
CreateAction::make()
|
||||
->label('Tambah'),
|
||||
|
||||
CrawlNewsAction::make(),
|
||||
|
||||
ExportAction::make()
|
||||
->exports([
|
||||
MediaMonitoringExcelExport::make(),
|
||||
|
||||
@ -1,246 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
use App\Enums\Channel;
|
||||
use App\Models\MediaMonitoring;
|
||||
use Carbon\Carbon;
|
||||
use Illuminate\Http\Client\Pool;
|
||||
use Illuminate\Http\Client\Response;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Str;
|
||||
use Symfony\Component\DomCrawler\Crawler;
|
||||
|
||||
class NewsCrawlerService
|
||||
{
|
||||
/**
|
||||
* Crawl news with high performance using HTTP Pooling.
|
||||
*/
|
||||
public function crawlFromGoogleNews(string $keyword, int $limit = 10, array $themeIds = []): int
|
||||
{
|
||||
$searchQuery = $keyword.' Purwakarta';
|
||||
$url = 'https://news.google.com/rss/search?q='.urlencode($searchQuery).'&hl=id&gl=ID&ceid=ID:id';
|
||||
|
||||
try {
|
||||
$response = Http::get($url);
|
||||
if ($response->failed()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$xml = simplexml_load_string($response->body());
|
||||
if (! $xml) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
$items = [];
|
||||
foreach ($xml->channel->item as $item) {
|
||||
if (count($items) >= $limit) {
|
||||
break;
|
||||
}
|
||||
$items[] = [
|
||||
'google_link' => (string) $item->link,
|
||||
'title' => (string) $item->title,
|
||||
'pubDate' => (string) $item->pubDate,
|
||||
'source' => (string) $item->source,
|
||||
'description' => (string) $item->description,
|
||||
];
|
||||
}
|
||||
|
||||
// Phase 1: Parallel Resolving Links
|
||||
$resolvedLinks = Http::pool(
|
||||
fn (Pool $pool) => array_map(fn ($item) => $pool->withHeaders([
|
||||
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
])->get($item['google_link']), $items)
|
||||
);
|
||||
|
||||
$parsedData = [];
|
||||
$parsedData = [];
|
||||
foreach ($resolvedLinks as $key => $res) {
|
||||
// Check if $res is a response or an exception (ConnectionException)
|
||||
$originalLink = ($res instanceof Response && $res->successful())
|
||||
? (string) $res->effectiveUri()
|
||||
: $items[$key]['google_link'];
|
||||
|
||||
// Anti-Duplicate Check
|
||||
if (MediaMonitoring::where('link', $originalLink)->exists()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$parsedData[] = array_merge($items[$key], ['original_link' => $originalLink]);
|
||||
}
|
||||
|
||||
// Phase 2: Parallel Fetching Extra Content (Metadata)
|
||||
$contentResponses = Http::pool(
|
||||
fn (Pool $pool) => array_map(fn ($data) => $pool->timeout(5)->get($data['original_link']), $parsedData)
|
||||
);
|
||||
|
||||
$count = 0;
|
||||
foreach ($contentResponses as $key => $res) {
|
||||
try {
|
||||
if (! isset($parsedData[$key])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$itemData = $parsedData[$key];
|
||||
$extra = ['writer' => 'Redaksi', 'content' => null];
|
||||
|
||||
if ($res instanceof Response && $res->successful()) {
|
||||
$crawler = new Crawler($res->body());
|
||||
$extra['writer'] = $this->extractWriter($crawler);
|
||||
$extra['content'] = $this->extractContent($crawler);
|
||||
}
|
||||
|
||||
// Clean generic Google News text
|
||||
$finalContent = $this->cleanContent($extra['content'] ?: $itemData['description']);
|
||||
$cleanTitle = $this->cleanTitle($itemData['title']);
|
||||
|
||||
$monitoring = MediaMonitoring::create([
|
||||
'code' => $this->generateCode(),
|
||||
'media_name' => $itemData['source'] ?: 'Online Media',
|
||||
'title' => Str::limit($cleanTitle, 255),
|
||||
'channel' => Channel::WEBSITE->value,
|
||||
'writter' => Str::limit($extra['writer'] ?: 'Redaksi', 50),
|
||||
'link' => $itemData['original_link'],
|
||||
'quote' => Str::limit($finalContent, 500),
|
||||
'content' => $finalContent,
|
||||
'keyword' => Str::limit($searchQuery, 100),
|
||||
'release_date' => Carbon::parse($itemData['pubDate'])->format('Y-m-d'),
|
||||
]);
|
||||
|
||||
if (! empty($themeIds)) {
|
||||
$monitoring->themes()->syncWithoutDetaching($themeIds);
|
||||
}
|
||||
|
||||
$count++;
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('Failed to insert item', [
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString(),
|
||||
'link' => $itemData['original_link'] ?? 'unknown',
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
return $count;
|
||||
} catch (\Throwable $e) {
|
||||
Log::error('Crawler error', [
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString(),
|
||||
]);
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanTitle(string $title): string
|
||||
{
|
||||
if (Str::contains($title, ' - ')) {
|
||||
$parts = explode(' - ', $title);
|
||||
array_pop($parts);
|
||||
|
||||
return implode(' - ', $parts);
|
||||
}
|
||||
|
||||
return $title;
|
||||
}
|
||||
|
||||
private function extractWriter(Crawler $crawler): ?string
|
||||
{
|
||||
$selectors = ['meta[name="author"]', 'meta[property="article:author"]', '.author', '.writer'];
|
||||
foreach ($selectors as $selector) {
|
||||
$el = $crawler->filter($selector);
|
||||
if ($el->count()) {
|
||||
return $el->attr('content') ?: $el->text();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private function extractContent(Crawler $crawler): ?string
|
||||
{
|
||||
// Priority 1: Open Graph Description
|
||||
$ogDesc = $crawler->filter('meta[property="og:description"]');
|
||||
if ($ogDesc->count()) {
|
||||
$content = $ogDesc->attr('content');
|
||||
if (! $this->isGarbage($content)) {
|
||||
return $content;
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 2: Meta Description
|
||||
$metaDesc = $crawler->filter('meta[name="description"]');
|
||||
if ($metaDesc->count()) {
|
||||
$content = $metaDesc->attr('content');
|
||||
if (! $this->isGarbage($content)) {
|
||||
return $content;
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 3: First substantial paragraph
|
||||
try {
|
||||
$paragraphs = $crawler->filter('p');
|
||||
foreach ($paragraphs as $p) {
|
||||
$text = trim($p->textContent);
|
||||
if (strlen($text) > 100) {
|
||||
return $text;
|
||||
}
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
Log::warning('Failed to check paragraphs content', [
|
||||
'error' => $e->getMessage(),
|
||||
]);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private function isGarbage(string $content): bool
|
||||
{
|
||||
$garbage = [
|
||||
'Comprehensive, up-to-date news coverage',
|
||||
'Google News',
|
||||
'Baca berita tanpa iklan',
|
||||
];
|
||||
foreach ($garbage as $g) {
|
||||
if (Str::contains($content, $g)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function cleanContent(?string $content): string
|
||||
{
|
||||
if (! $content) {
|
||||
return 'Tidak ada kutipan berita.';
|
||||
}
|
||||
|
||||
$cleaned = strip_tags($content);
|
||||
$cleaned = preg_replace('/\s+/', ' ', $cleaned);
|
||||
|
||||
return trim($cleaned);
|
||||
}
|
||||
|
||||
private function generateCode(): string
|
||||
{
|
||||
// Use a lock-safe approach for sequential code
|
||||
return DB::transaction(function () {
|
||||
$latestCode = MediaMonitoring::where('code', 'LIKE', 'ADN%')
|
||||
->lockForUpdate()
|
||||
->orderBy('code', 'desc')
|
||||
->first();
|
||||
|
||||
$lastNumber = $latestCode
|
||||
? intval(substr($latestCode->code, 3))
|
||||
: 0;
|
||||
|
||||
$newNumber = str_pad($lastNumber + 1, 6, '0', STR_PAD_LEFT);
|
||||
|
||||
return 'ADN'.$newNumber;
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -778,12 +778,6 @@
|
||||
'formal' => 'Tidak Ada Data Rekap Konten',
|
||||
],
|
||||
],
|
||||
'crawl' => [
|
||||
'success' => [
|
||||
'cheerful' => 'Crawl Berhasil! 🕸️🚀',
|
||||
'formal' => 'Crawl Selesai',
|
||||
],
|
||||
],
|
||||
'news' => [
|
||||
'no_new_data' => [
|
||||
'cheerful' => 'Belum Ada Berita Baru 🔍',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user