diff --git a/app/Console/Commands/CrawlNewsCommand.php b/app/Console/Commands/CrawlNewsCommand.php deleted file mode 100644 index 8d96af6..0000000 --- a/app/Console/Commands/CrawlNewsCommand.php +++ /dev/null @@ -1,61 +0,0 @@ -option('limit'); - $allThemes = $this->option('all-themes'); - $keyword = $this->argument('keyword'); - - if ($allThemes) { - $themes = Theme::active()->get(); - if ($themes->isEmpty()) { - $this->error('No active themes found.'); - - return; - } - - foreach ($themes as $theme) { - $this->info("Crawling for theme: {$theme->name}"); - $count = $service->crawlFromGoogleNews($theme->name, $limit, [$theme->id]); - $this->info("Saved {$count} news items for theme: {$theme->name}"); - } - - return; - } - - if (! $keyword) { - $this->error('Please provide a keyword or use --all-themes'); - - return; - } - - $this->info("Crawling for keyword: {$keyword}"); - $count = $service->crawlFromGoogleNews($keyword, $limit); - $this->info("Saved {$count} news items for keyword: {$keyword}"); - } -} diff --git a/app/Filament/Resources/Monitoring/MediaMonitorings/Actions/CrawlNewsAction.php b/app/Filament/Resources/Monitoring/MediaMonitorings/Actions/CrawlNewsAction.php deleted file mode 100644 index d4e8ae6..0000000 --- a/app/Filament/Resources/Monitoring/MediaMonitorings/Actions/CrawlNewsAction.php +++ /dev/null @@ -1,67 +0,0 @@ -label('Crawl Berita') - ->color('slate') - ->schema([ - TextInput::make('keyword') - ->label('Kata Kunci') - ->placeholder('Pendidikan, UMKM, Pariwisata') - ->autocomplete(false) - ->autofocus() - ->required() - ->helperText('Jika kata kunci nya lebih dari 1, pisahkan dengan koma.'), - - Select::make('theme_id') - ->label('Tema') - ->required() - ->options(Theme::active()->pluck('name', 'id')) - ->searchable() - ->multiple(), - - TextInput::make('limit') - ->label('Jumlah Berita') - ->placeholder(10) - ->numeric() - ->required() - ->default(10) - ->autocomplete(false), - ]) - ->action(function (array $data, NewsCrawlerService $service): void { - $count = $service->crawlFromGoogleNews($data['keyword'], $data['limit'], (array) ($data['theme_id'] ?? [])); - - if ($count > 0) { - CheerfulNotification::success( - CheerfulNotification::getByKey('crawl.success'), - CheerfulNotification::getByKey('news.content.fetch_success', ['count' => $count, 'data__keyword' => $data['keyword']]) - )->send(); - } else { - CheerfulNotification::info( - CheerfulNotification::getByKey('news.no_new_data'), - CheerfulNotification::getByKey('news.content.fetch_empty', ['data__keyword' => $data['keyword']]) - )->send(); - } - }) - ->modalWidth(Width::Large); - } -} diff --git a/app/Filament/Resources/Monitoring/MediaMonitorings/Pages/ListMediaMonitorings.php b/app/Filament/Resources/Monitoring/MediaMonitorings/Pages/ListMediaMonitorings.php index 84a28c4..ef3cf8b 100644 --- a/app/Filament/Resources/Monitoring/MediaMonitorings/Pages/ListMediaMonitorings.php +++ b/app/Filament/Resources/Monitoring/MediaMonitorings/Pages/ListMediaMonitorings.php @@ -3,7 +3,6 @@ namespace App\Filament\Resources\Monitoring\MediaMonitorings\Pages; use App\Filament\Exports\MediaMonitoringExcelExport; -use App\Filament\Resources\Monitoring\MediaMonitorings\Actions\CrawlNewsAction; use App\Filament\Resources\Monitoring\MediaMonitorings\MediaMonitoringResource; use Filament\Actions\CreateAction; use Filament\Resources\Pages\ListRecords; @@ -21,8 +20,6 @@ protected function getHeaderActions(): array CreateAction::make() ->label('Tambah'), - CrawlNewsAction::make(), - ExportAction::make() ->exports([ MediaMonitoringExcelExport::make(), diff --git a/app/Services/NewsCrawlerService.php b/app/Services/NewsCrawlerService.php deleted file mode 100644 index e2ace59..0000000 --- a/app/Services/NewsCrawlerService.php +++ /dev/null @@ -1,246 +0,0 @@ -failed()) { - return 0; - } - - $xml = simplexml_load_string($response->body()); - if (! $xml) { - return 0; - } - - $items = []; - foreach ($xml->channel->item as $item) { - if (count($items) >= $limit) { - break; - } - $items[] = [ - 'google_link' => (string) $item->link, - 'title' => (string) $item->title, - 'pubDate' => (string) $item->pubDate, - 'source' => (string) $item->source, - 'description' => (string) $item->description, - ]; - } - - // Phase 1: Parallel Resolving Links - $resolvedLinks = Http::pool( - fn (Pool $pool) => array_map(fn ($item) => $pool->withHeaders([ - 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', - ])->get($item['google_link']), $items) - ); - - $parsedData = []; - $parsedData = []; - foreach ($resolvedLinks as $key => $res) { - // Check if $res is a response or an exception (ConnectionException) - $originalLink = ($res instanceof Response && $res->successful()) - ? (string) $res->effectiveUri() - : $items[$key]['google_link']; - - // Anti-Duplicate Check - if (MediaMonitoring::where('link', $originalLink)->exists()) { - continue; - } - - $parsedData[] = array_merge($items[$key], ['original_link' => $originalLink]); - } - - // Phase 2: Parallel Fetching Extra Content (Metadata) - $contentResponses = Http::pool( - fn (Pool $pool) => array_map(fn ($data) => $pool->timeout(5)->get($data['original_link']), $parsedData) - ); - - $count = 0; - foreach ($contentResponses as $key => $res) { - try { - if (! isset($parsedData[$key])) { - continue; - } - - $itemData = $parsedData[$key]; - $extra = ['writer' => 'Redaksi', 'content' => null]; - - if ($res instanceof Response && $res->successful()) { - $crawler = new Crawler($res->body()); - $extra['writer'] = $this->extractWriter($crawler); - $extra['content'] = $this->extractContent($crawler); - } - - // Clean generic Google News text - $finalContent = $this->cleanContent($extra['content'] ?: $itemData['description']); - $cleanTitle = $this->cleanTitle($itemData['title']); - - $monitoring = MediaMonitoring::create([ - 'code' => $this->generateCode(), - 'media_name' => $itemData['source'] ?: 'Online Media', - 'title' => Str::limit($cleanTitle, 255), - 'channel' => Channel::WEBSITE->value, - 'writter' => Str::limit($extra['writer'] ?: 'Redaksi', 50), - 'link' => $itemData['original_link'], - 'quote' => Str::limit($finalContent, 500), - 'content' => $finalContent, - 'keyword' => Str::limit($searchQuery, 100), - 'release_date' => Carbon::parse($itemData['pubDate'])->format('Y-m-d'), - ]); - - if (! empty($themeIds)) { - $monitoring->themes()->syncWithoutDetaching($themeIds); - } - - $count++; - } catch (\Throwable $e) { - Log::error('Failed to insert item', [ - 'error' => $e->getMessage(), - 'trace' => $e->getTraceAsString(), - 'link' => $itemData['original_link'] ?? 'unknown', - ]); - } - } - - return $count; - } catch (\Throwable $e) { - Log::error('Crawler error', [ - 'error' => $e->getMessage(), - 'trace' => $e->getTraceAsString(), - ]); - - return 0; - } - } - - private function cleanTitle(string $title): string - { - if (Str::contains($title, ' - ')) { - $parts = explode(' - ', $title); - array_pop($parts); - - return implode(' - ', $parts); - } - - return $title; - } - - private function extractWriter(Crawler $crawler): ?string - { - $selectors = ['meta[name="author"]', 'meta[property="article:author"]', '.author', '.writer']; - foreach ($selectors as $selector) { - $el = $crawler->filter($selector); - if ($el->count()) { - return $el->attr('content') ?: $el->text(); - } - } - - return null; - } - - private function extractContent(Crawler $crawler): ?string - { - // Priority 1: Open Graph Description - $ogDesc = $crawler->filter('meta[property="og:description"]'); - if ($ogDesc->count()) { - $content = $ogDesc->attr('content'); - if (! $this->isGarbage($content)) { - return $content; - } - } - - // Priority 2: Meta Description - $metaDesc = $crawler->filter('meta[name="description"]'); - if ($metaDesc->count()) { - $content = $metaDesc->attr('content'); - if (! $this->isGarbage($content)) { - return $content; - } - } - - // Priority 3: First substantial paragraph - try { - $paragraphs = $crawler->filter('p'); - foreach ($paragraphs as $p) { - $text = trim($p->textContent); - if (strlen($text) > 100) { - return $text; - } - } - } catch (\Throwable $e) { - Log::warning('Failed to check paragraphs content', [ - 'error' => $e->getMessage(), - ]); - } - - return null; - } - - private function isGarbage(string $content): bool - { - $garbage = [ - 'Comprehensive, up-to-date news coverage', - 'Google News', - 'Baca berita tanpa iklan', - ]; - foreach ($garbage as $g) { - if (Str::contains($content, $g)) { - return true; - } - } - - return false; - } - - private function cleanContent(?string $content): string - { - if (! $content) { - return 'Tidak ada kutipan berita.'; - } - - $cleaned = strip_tags($content); - $cleaned = preg_replace('/\s+/', ' ', $cleaned); - - return trim($cleaned); - } - - private function generateCode(): string - { - // Use a lock-safe approach for sequential code - return DB::transaction(function () { - $latestCode = MediaMonitoring::where('code', 'LIKE', 'ADN%') - ->lockForUpdate() - ->orderBy('code', 'desc') - ->first(); - - $lastNumber = $latestCode - ? intval(substr($latestCode->code, 3)) - : 0; - - $newNumber = str_pad($lastNumber + 1, 6, '0', STR_PAD_LEFT); - - return 'ADN'.$newNumber; - }); - } -} diff --git a/lang/id/notif.php b/lang/id/notif.php index 9c0007d..10d3f07 100644 --- a/lang/id/notif.php +++ b/lang/id/notif.php @@ -778,12 +778,6 @@ 'formal' => 'Tidak Ada Data Rekap Konten', ], ], - 'crawl' => [ - 'success' => [ - 'cheerful' => 'Crawl Berhasil! πŸ•ΈοΈπŸš€', - 'formal' => 'Crawl Selesai', - ], - ], 'news' => [ 'no_new_data' => [ 'cheerful' => 'Belum Ada Berita Baru πŸ”',