refactor: remove news crawler service, command, and associated UI actions
This commit is contained in:
Yoga Pangestu
parent
711aedcf2b
commit
1977d08339
@ -1,61 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Console\Commands;
|
|
||||||
|
|
||||||
use App\Models\Theme;
|
|
||||||
use App\Services\NewsCrawlerService;
|
|
||||||
use Illuminate\Console\Command;
|
|
||||||
|
|
||||||
class CrawlNewsCommand extends Command
|
|
||||||
{
|
|
||||||
/**
|
|
||||||
* The name and signature of the console command.
|
|
||||||
*
|
|
||||||
* @var string
|
|
||||||
*/
|
|
||||||
protected $signature = 'app:crawl-news {keyword?} {--limit=10} {--all-themes}';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The console command description.
|
|
||||||
*
|
|
||||||
* @var string
|
|
||||||
*/
|
|
||||||
protected $description = 'Crawl news from Google News based on keywords or themes';
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Execute the console command.
|
|
||||||
*/
|
|
||||||
public function handle(NewsCrawlerService $service)
|
|
||||||
{
|
|
||||||
$limit = (int) $this->option('limit');
|
|
||||||
$allThemes = $this->option('all-themes');
|
|
||||||
$keyword = $this->argument('keyword');
|
|
||||||
|
|
||||||
if ($allThemes) {
|
|
||||||
$themes = Theme::active()->get();
|
|
||||||
if ($themes->isEmpty()) {
|
|
||||||
$this->error('No active themes found.');
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($themes as $theme) {
|
|
||||||
$this->info("Crawling for theme: {$theme->name}");
|
|
||||||
$count = $service->crawlFromGoogleNews($theme->name, $limit, [$theme->id]);
|
|
||||||
$this->info("Saved {$count} news items for theme: {$theme->name}");
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (! $keyword) {
|
|
||||||
$this->error('Please provide a keyword or use --all-themes');
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->info("Crawling for keyword: {$keyword}");
|
|
||||||
$count = $service->crawlFromGoogleNews($keyword, $limit);
|
|
||||||
$this->info("Saved {$count} news items for keyword: {$keyword}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,67 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Filament\Resources\Monitoring\MediaMonitorings\Actions;
|
|
||||||
|
|
||||||
use App\Filament\Support\CheerfulNotification;
|
|
||||||
use App\Models\Theme;
|
|
||||||
use App\Services\NewsCrawlerService;
|
|
||||||
use Filament\Actions\Action;
|
|
||||||
use Filament\Forms\Components\Select;
|
|
||||||
use Filament\Forms\Components\TextInput;
|
|
||||||
use Filament\Support\Enums\Width;
|
|
||||||
|
|
||||||
class CrawlNewsAction extends Action
|
|
||||||
{
|
|
||||||
public static function getDefaultName(): ?string
|
|
||||||
{
|
|
||||||
return 'crawlNews';
|
|
||||||
}
|
|
||||||
|
|
||||||
protected function setUp(): void
|
|
||||||
{
|
|
||||||
parent::setUp();
|
|
||||||
|
|
||||||
$this->label('Crawl Berita')
|
|
||||||
->color('slate')
|
|
||||||
->schema([
|
|
||||||
TextInput::make('keyword')
|
|
||||||
->label('Kata Kunci')
|
|
||||||
->placeholder('Pendidikan, UMKM, Pariwisata')
|
|
||||||
->autocomplete(false)
|
|
||||||
->autofocus()
|
|
||||||
->required()
|
|
||||||
->helperText('Jika kata kunci nya lebih dari 1, pisahkan dengan koma.'),
|
|
||||||
|
|
||||||
Select::make('theme_id')
|
|
||||||
->label('Tema')
|
|
||||||
->required()
|
|
||||||
->options(Theme::active()->pluck('name', 'id'))
|
|
||||||
->searchable()
|
|
||||||
->multiple(),
|
|
||||||
|
|
||||||
TextInput::make('limit')
|
|
||||||
->label('Jumlah Berita')
|
|
||||||
->placeholder(10)
|
|
||||||
->numeric()
|
|
||||||
->required()
|
|
||||||
->default(10)
|
|
||||||
->autocomplete(false),
|
|
||||||
])
|
|
||||||
->action(function (array $data, NewsCrawlerService $service): void {
|
|
||||||
$count = $service->crawlFromGoogleNews($data['keyword'], $data['limit'], (array) ($data['theme_id'] ?? []));
|
|
||||||
|
|
||||||
if ($count > 0) {
|
|
||||||
CheerfulNotification::success(
|
|
||||||
CheerfulNotification::getByKey('crawl.success'),
|
|
||||||
CheerfulNotification::getByKey('news.content.fetch_success', ['count' => $count, 'data__keyword' => $data['keyword']])
|
|
||||||
)->send();
|
|
||||||
} else {
|
|
||||||
CheerfulNotification::info(
|
|
||||||
CheerfulNotification::getByKey('news.no_new_data'),
|
|
||||||
CheerfulNotification::getByKey('news.content.fetch_empty', ['data__keyword' => $data['keyword']])
|
|
||||||
)->send();
|
|
||||||
}
|
|
||||||
})
|
|
||||||
->modalWidth(Width::Large);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -3,7 +3,6 @@
|
|||||||
namespace App\Filament\Resources\Monitoring\MediaMonitorings\Pages;
|
namespace App\Filament\Resources\Monitoring\MediaMonitorings\Pages;
|
||||||
|
|
||||||
use App\Filament\Exports\MediaMonitoringExcelExport;
|
use App\Filament\Exports\MediaMonitoringExcelExport;
|
||||||
use App\Filament\Resources\Monitoring\MediaMonitorings\Actions\CrawlNewsAction;
|
|
||||||
use App\Filament\Resources\Monitoring\MediaMonitorings\MediaMonitoringResource;
|
use App\Filament\Resources\Monitoring\MediaMonitorings\MediaMonitoringResource;
|
||||||
use Filament\Actions\CreateAction;
|
use Filament\Actions\CreateAction;
|
||||||
use Filament\Resources\Pages\ListRecords;
|
use Filament\Resources\Pages\ListRecords;
|
||||||
@ -21,8 +20,6 @@ protected function getHeaderActions(): array
|
|||||||
CreateAction::make()
|
CreateAction::make()
|
||||||
->label('Tambah'),
|
->label('Tambah'),
|
||||||
|
|
||||||
CrawlNewsAction::make(),
|
|
||||||
|
|
||||||
ExportAction::make()
|
ExportAction::make()
|
||||||
->exports([
|
->exports([
|
||||||
MediaMonitoringExcelExport::make(),
|
MediaMonitoringExcelExport::make(),
|
||||||
|
|||||||
@ -1,246 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
namespace App\Services;
|
|
||||||
|
|
||||||
use App\Enums\Channel;
|
|
||||||
use App\Models\MediaMonitoring;
|
|
||||||
use Carbon\Carbon;
|
|
||||||
use Illuminate\Http\Client\Pool;
|
|
||||||
use Illuminate\Http\Client\Response;
|
|
||||||
use Illuminate\Support\Facades\DB;
|
|
||||||
use Illuminate\Support\Facades\Http;
|
|
||||||
use Illuminate\Support\Facades\Log;
|
|
||||||
use Illuminate\Support\Str;
|
|
||||||
use Symfony\Component\DomCrawler\Crawler;
|
|
||||||
|
|
||||||
class NewsCrawlerService
|
|
||||||
{
|
|
||||||
/**
|
|
||||||
* Crawl news with high performance using HTTP Pooling.
|
|
||||||
*/
|
|
||||||
public function crawlFromGoogleNews(string $keyword, int $limit = 10, array $themeIds = []): int
|
|
||||||
{
|
|
||||||
$searchQuery = $keyword.' Purwakarta';
|
|
||||||
$url = 'https://news.google.com/rss/search?q='.urlencode($searchQuery).'&hl=id&gl=ID&ceid=ID:id';
|
|
||||||
|
|
||||||
try {
|
|
||||||
$response = Http::get($url);
|
|
||||||
if ($response->failed()) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
$xml = simplexml_load_string($response->body());
|
|
||||||
if (! $xml) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
$items = [];
|
|
||||||
foreach ($xml->channel->item as $item) {
|
|
||||||
if (count($items) >= $limit) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
$items[] = [
|
|
||||||
'google_link' => (string) $item->link,
|
|
||||||
'title' => (string) $item->title,
|
|
||||||
'pubDate' => (string) $item->pubDate,
|
|
||||||
'source' => (string) $item->source,
|
|
||||||
'description' => (string) $item->description,
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
// Phase 1: Parallel Resolving Links
|
|
||||||
$resolvedLinks = Http::pool(
|
|
||||||
fn (Pool $pool) => array_map(fn ($item) => $pool->withHeaders([
|
|
||||||
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
|
||||||
])->get($item['google_link']), $items)
|
|
||||||
);
|
|
||||||
|
|
||||||
$parsedData = [];
|
|
||||||
$parsedData = [];
|
|
||||||
foreach ($resolvedLinks as $key => $res) {
|
|
||||||
// Check if $res is a response or an exception (ConnectionException)
|
|
||||||
$originalLink = ($res instanceof Response && $res->successful())
|
|
||||||
? (string) $res->effectiveUri()
|
|
||||||
: $items[$key]['google_link'];
|
|
||||||
|
|
||||||
// Anti-Duplicate Check
|
|
||||||
if (MediaMonitoring::where('link', $originalLink)->exists()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
$parsedData[] = array_merge($items[$key], ['original_link' => $originalLink]);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Phase 2: Parallel Fetching Extra Content (Metadata)
|
|
||||||
$contentResponses = Http::pool(
|
|
||||||
fn (Pool $pool) => array_map(fn ($data) => $pool->timeout(5)->get($data['original_link']), $parsedData)
|
|
||||||
);
|
|
||||||
|
|
||||||
$count = 0;
|
|
||||||
foreach ($contentResponses as $key => $res) {
|
|
||||||
try {
|
|
||||||
if (! isset($parsedData[$key])) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
$itemData = $parsedData[$key];
|
|
||||||
$extra = ['writer' => 'Redaksi', 'content' => null];
|
|
||||||
|
|
||||||
if ($res instanceof Response && $res->successful()) {
|
|
||||||
$crawler = new Crawler($res->body());
|
|
||||||
$extra['writer'] = $this->extractWriter($crawler);
|
|
||||||
$extra['content'] = $this->extractContent($crawler);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clean generic Google News text
|
|
||||||
$finalContent = $this->cleanContent($extra['content'] ?: $itemData['description']);
|
|
||||||
$cleanTitle = $this->cleanTitle($itemData['title']);
|
|
||||||
|
|
||||||
$monitoring = MediaMonitoring::create([
|
|
||||||
'code' => $this->generateCode(),
|
|
||||||
'media_name' => $itemData['source'] ?: 'Online Media',
|
|
||||||
'title' => Str::limit($cleanTitle, 255),
|
|
||||||
'channel' => Channel::WEBSITE->value,
|
|
||||||
'writter' => Str::limit($extra['writer'] ?: 'Redaksi', 50),
|
|
||||||
'link' => $itemData['original_link'],
|
|
||||||
'quote' => Str::limit($finalContent, 500),
|
|
||||||
'content' => $finalContent,
|
|
||||||
'keyword' => Str::limit($searchQuery, 100),
|
|
||||||
'release_date' => Carbon::parse($itemData['pubDate'])->format('Y-m-d'),
|
|
||||||
]);
|
|
||||||
|
|
||||||
if (! empty($themeIds)) {
|
|
||||||
$monitoring->themes()->syncWithoutDetaching($themeIds);
|
|
||||||
}
|
|
||||||
|
|
||||||
$count++;
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
Log::error('Failed to insert item', [
|
|
||||||
'error' => $e->getMessage(),
|
|
||||||
'trace' => $e->getTraceAsString(),
|
|
||||||
'link' => $itemData['original_link'] ?? 'unknown',
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return $count;
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
Log::error('Crawler error', [
|
|
||||||
'error' => $e->getMessage(),
|
|
||||||
'trace' => $e->getTraceAsString(),
|
|
||||||
]);
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function cleanTitle(string $title): string
|
|
||||||
{
|
|
||||||
if (Str::contains($title, ' - ')) {
|
|
||||||
$parts = explode(' - ', $title);
|
|
||||||
array_pop($parts);
|
|
||||||
|
|
||||||
return implode(' - ', $parts);
|
|
||||||
}
|
|
||||||
|
|
||||||
return $title;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function extractWriter(Crawler $crawler): ?string
|
|
||||||
{
|
|
||||||
$selectors = ['meta[name="author"]', 'meta[property="article:author"]', '.author', '.writer'];
|
|
||||||
foreach ($selectors as $selector) {
|
|
||||||
$el = $crawler->filter($selector);
|
|
||||||
if ($el->count()) {
|
|
||||||
return $el->attr('content') ?: $el->text();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function extractContent(Crawler $crawler): ?string
|
|
||||||
{
|
|
||||||
// Priority 1: Open Graph Description
|
|
||||||
$ogDesc = $crawler->filter('meta[property="og:description"]');
|
|
||||||
if ($ogDesc->count()) {
|
|
||||||
$content = $ogDesc->attr('content');
|
|
||||||
if (! $this->isGarbage($content)) {
|
|
||||||
return $content;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Priority 2: Meta Description
|
|
||||||
$metaDesc = $crawler->filter('meta[name="description"]');
|
|
||||||
if ($metaDesc->count()) {
|
|
||||||
$content = $metaDesc->attr('content');
|
|
||||||
if (! $this->isGarbage($content)) {
|
|
||||||
return $content;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Priority 3: First substantial paragraph
|
|
||||||
try {
|
|
||||||
$paragraphs = $crawler->filter('p');
|
|
||||||
foreach ($paragraphs as $p) {
|
|
||||||
$text = trim($p->textContent);
|
|
||||||
if (strlen($text) > 100) {
|
|
||||||
return $text;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
Log::warning('Failed to check paragraphs content', [
|
|
||||||
'error' => $e->getMessage(),
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function isGarbage(string $content): bool
|
|
||||||
{
|
|
||||||
$garbage = [
|
|
||||||
'Comprehensive, up-to-date news coverage',
|
|
||||||
'Google News',
|
|
||||||
'Baca berita tanpa iklan',
|
|
||||||
];
|
|
||||||
foreach ($garbage as $g) {
|
|
||||||
if (Str::contains($content, $g)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
private function cleanContent(?string $content): string
|
|
||||||
{
|
|
||||||
if (! $content) {
|
|
||||||
return 'Tidak ada kutipan berita.';
|
|
||||||
}
|
|
||||||
|
|
||||||
$cleaned = strip_tags($content);
|
|
||||||
$cleaned = preg_replace('/\s+/', ' ', $cleaned);
|
|
||||||
|
|
||||||
return trim($cleaned);
|
|
||||||
}
|
|
||||||
|
|
||||||
private function generateCode(): string
|
|
||||||
{
|
|
||||||
// Use a lock-safe approach for sequential code
|
|
||||||
return DB::transaction(function () {
|
|
||||||
$latestCode = MediaMonitoring::where('code', 'LIKE', 'ADN%')
|
|
||||||
->lockForUpdate()
|
|
||||||
->orderBy('code', 'desc')
|
|
||||||
->first();
|
|
||||||
|
|
||||||
$lastNumber = $latestCode
|
|
||||||
? intval(substr($latestCode->code, 3))
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
$newNumber = str_pad($lastNumber + 1, 6, '0', STR_PAD_LEFT);
|
|
||||||
|
|
||||||
return 'ADN'.$newNumber;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -778,12 +778,6 @@
|
|||||||
'formal' => 'Tidak Ada Data Rekap Konten',
|
'formal' => 'Tidak Ada Data Rekap Konten',
|
||||||
],
|
],
|
||||||
],
|
],
|
||||||
'crawl' => [
|
|
||||||
'success' => [
|
|
||||||
'cheerful' => 'Crawl Berhasil! 🕸️🚀',
|
|
||||||
'formal' => 'Crawl Selesai',
|
|
||||||
],
|
|
||||||
],
|
|
||||||
'news' => [
|
'news' => [
|
||||||
'no_new_data' => [
|
'no_new_data' => [
|
||||||
'cheerful' => 'Belum Ada Berita Baru 🔍',
|
'cheerful' => 'Belum Ada Berita Baru 🔍',
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user