diff --git a/app/Console/Commands/CrawlNewsCommand.php b/app/Console/Commands/CrawlNewsCommand.php new file mode 100644 index 0000000..8d96af6 --- /dev/null +++ b/app/Console/Commands/CrawlNewsCommand.php @@ -0,0 +1,61 @@ +option('limit'); + $allThemes = $this->option('all-themes'); + $keyword = $this->argument('keyword'); + + if ($allThemes) { + $themes = Theme::active()->get(); + if ($themes->isEmpty()) { + $this->error('No active themes found.'); + + return; + } + + foreach ($themes as $theme) { + $this->info("Crawling for theme: {$theme->name}"); + $count = $service->crawlFromGoogleNews($theme->name, $limit, [$theme->id]); + $this->info("Saved {$count} news items for theme: {$theme->name}"); + } + + return; + } + + if (! $keyword) { + $this->error('Please provide a keyword or use --all-themes'); + + return; + } + + $this->info("Crawling for keyword: {$keyword}"); + $count = $service->crawlFromGoogleNews($keyword, $limit); + $this->info("Saved {$count} news items for keyword: {$keyword}"); + } +} diff --git a/app/Filament/Resources/Monitoring/MediaMonitorings/MediaMonitoringResource.php b/app/Filament/Resources/Monitoring/MediaMonitorings/MediaMonitoringResource.php index db22de2..d91d1cb 100644 --- a/app/Filament/Resources/Monitoring/MediaMonitorings/MediaMonitoringResource.php +++ b/app/Filament/Resources/Monitoring/MediaMonitorings/MediaMonitoringResource.php @@ -29,7 +29,7 @@ class MediaMonitoringResource extends Resource protected static ?int $navigationSort = 12; - protected static ?string $recordTitleAttribute = 'media_name'; + protected static ?string $recordTitleAttribute = 'code'; protected static ?string $slug = 'monitoring/media-monitorings'; diff --git a/app/Filament/Resources/Monitoring/MediaMonitorings/Tables/MediaMonitoringsTable.php b/app/Filament/Resources/Monitoring/MediaMonitorings/Tables/MediaMonitoringsTable.php index a7b35fe..5d75c34 100644 --- a/app/Filament/Resources/Monitoring/MediaMonitorings/Tables/MediaMonitoringsTable.php +++ b/app/Filament/Resources/Monitoring/MediaMonitorings/Tables/MediaMonitoringsTable.php @@ -4,11 +4,18 @@ use App\Filament\Actions\DefaultBulkActions; use App\Filament\Columns\TimestampColumns; +use App\Models\Theme; +use App\Services\NewsCrawlerService; +use Filament\Actions\Action; use Filament\Actions\BulkActionGroup; use Filament\Actions\DeleteAction; use Filament\Actions\EditAction; use Filament\Actions\ForceDeleteAction; use Filament\Actions\RestoreAction; +use Filament\Forms\Components\Select; +use Filament\Forms\Components\TextInput; +use Filament\Notifications\Notification; +use Filament\Support\Enums\Width; use Filament\Support\Icons\Heroicon; use Filament\Tables\Columns\TextColumn; use Filament\Tables\Filters\TrashedFilter; @@ -95,6 +102,53 @@ public static function configure(Table $table): Table RestoreAction::make(), ]) + ->headerActions([ + Action::make('crawlNews') + ->label('Crawl Berita') + ->icon(Heroicon::MagnifyingGlass) + ->color('info') + ->schema([ + TextInput::make('keyword') + ->label('Kata Kunci') + ->placeholder('Pendidikan, UMKM, Pariwisata') + ->autofocus() + ->autocomplete('off') + ->required() + ->helperText('Jika kata kunci nya lebih dari 1, pisahkan dengan koma.'), + + Select::make('theme_id') + ->label('Tema') + ->required() + ->options(Theme::active()->pluck('name', 'id')) + ->searchable() + ->multiple(), + + TextInput::make('limit') + ->label('Jumlah Berita') + ->placeholder(10) + ->numeric() + ->required() + ->default(10), + ]) + ->action(function (array $data, NewsCrawlerService $service) { + $count = $service->crawlFromGoogleNews($data['keyword'], $data['limit'], $data['theme_id'] ?? []); + + if ($count > 0) { + Notification::make() + ->title('Crawl Selesai') + ->body("Berhasil mengambil {$count} berita baru tentang '{$data['keyword']}' di Purwakarta.") + ->success() + ->send(); + } else { + Notification::make() + ->title('Tidak Ada Berita Baru') + ->body("Tidak ditemukan berita baru untuk '{$data['keyword']}' yang belum ada di database.") + ->info() + ->send(); + } + }) + ->modalWidth(Width::Large), + ]) ->toolbarActions([ BulkActionGroup::make([ ...DefaultBulkActions::make('Monitoring Media'), diff --git a/app/Services/NewsCrawlerService.php b/app/Services/NewsCrawlerService.php new file mode 100644 index 0000000..8a636c3 --- /dev/null +++ b/app/Services/NewsCrawlerService.php @@ -0,0 +1,235 @@ +failed()) { + return 0; + } + + $xml = simplexml_load_string($response->body()); + if (! $xml) { + return 0; + } + + $items = []; + foreach ($xml->channel->item as $item) { + if (count($items) >= $limit) { + break; + } + $items[] = [ + 'google_link' => (string) $item->link, + 'title' => (string) $item->title, + 'pubDate' => (string) $item->pubDate, + 'source' => (string) $item->source, + 'description' => (string) $item->description, + ]; + } + + // Phase 1: Parallel Resolving Links + $resolvedLinks = Http::pool( + fn (Pool $pool) => array_map(fn ($item) => $pool->withHeaders([ + 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + ])->get($item['google_link']), $items) + ); + + $parsedData = []; + $parsedData = []; + foreach ($resolvedLinks as $key => $res) { + // Check if $res is a response or an exception (ConnectionException) + $originalLink = ($res instanceof \Illuminate\Http\Client\Response && $res->successful()) + ? (string) $res->effectiveUri() + : $items[$key]['google_link']; + + // Anti-Duplicate Check + if (MediaMonitoring::where('link', $originalLink)->exists()) { + continue; + } + + $parsedData[] = array_merge($items[$key], ['original_link' => $originalLink]); + } + + // Phase 2: Parallel Fetching Extra Content (Metadata) + $contentResponses = Http::pool( + fn (Pool $pool) => array_map(fn ($data) => $pool->timeout(5)->get($data['original_link']), $parsedData) + ); + + $count = 0; + foreach ($contentResponses as $key => $res) { + try { + if (! isset($parsedData[$key])) { + continue; + } + + $itemData = $parsedData[$key]; + $extra = ['writer' => 'Redaksi', 'content' => null]; + + if ($res instanceof \Illuminate\Http\Client\Response && $res->successful()) { + $crawler = new Crawler($res->body()); + $extra['writer'] = $this->extractWriter($crawler); + $extra['content'] = $this->extractContent($crawler); + } + + // Clean generic Google News text + $finalContent = $this->cleanContent($extra['content'] ?: $itemData['description']); + $cleanTitle = $this->cleanTitle($itemData['title']); + + $monitoring = MediaMonitoring::create([ + 'code' => $this->generateCode(), + 'media_name' => $itemData['source'] ?: 'Online Media', + 'title' => Str::limit($cleanTitle, 255), + 'channel' => Channel::WEBSITE->value, + 'writter' => Str::limit($extra['writer'] ?: 'Redaksi', 50), + 'link' => $itemData['original_link'], + 'news_page' => null, // Asked to be null if not specific pagination + 'quote' => Str::limit($finalContent, 500), + 'content' => $finalContent, + 'keyword' => Str::limit($searchQuery, 100), + 'release_date' => Carbon::parse($itemData['pubDate'])->format('Y-m-d'), + ]); + + if (! empty($themeIds)) { + $monitoring->themes()->syncWithoutDetaching($themeIds); + } + + $count++; + } catch (\Exception $e) { + Log::error('Failed to insert item: '.$e->getMessage()); + } + } + + return $count; + } catch (\Exception $e) { + Log::error('Crawler error: '.$e->getMessage()); + + return 0; + } + } + + private function cleanTitle(string $title): string + { + if (Str::contains($title, ' - ')) { + $parts = explode(' - ', $title); + array_pop($parts); + + return implode(' - ', $parts); + } + + return $title; + } + + private function extractWriter(Crawler $crawler): ?string + { + $selectors = ['meta[name="author"]', 'meta[property="article:author"]', '.author', '.writer']; + foreach ($selectors as $selector) { + $el = $crawler->filter($selector); + if ($el->count()) { + return $el->attr('content') ?: $el->text(); + } + } + + return null; + } + + private function extractContent(Crawler $crawler): ?string + { + // Priority 1: Open Graph Description + $ogDesc = $crawler->filter('meta[property="og:description"]'); + if ($ogDesc->count()) { + $content = $ogDesc->attr('content'); + if (! $this->isGarbage($content)) { + return $content; + } + } + + // Priority 2: Meta Description + $metaDesc = $crawler->filter('meta[name="description"]'); + if ($metaDesc->count()) { + $content = $metaDesc->attr('content'); + if (! $this->isGarbage($content)) { + return $content; + } + } + + // Priority 3: First substantial paragraph + try { + $paragraphs = $crawler->filter('p'); + foreach ($paragraphs as $p) { + $text = trim($p->textContent); + if (strlen($text) > 100) { + return $text; + } + } + } catch (\Exception $e) { + } + + return null; + } + + private function isGarbage(string $content): bool + { + $garbage = [ + 'Comprehensive, up-to-date news coverage', + 'Google News', + 'Baca berita tanpa iklan', + ]; + foreach ($garbage as $g) { + if (Str::contains($content, $g)) { + return true; + } + } + + return false; + } + + private function cleanContent(?string $content): string + { + if (! $content) { + return 'Tidak ada kutipan berita.'; + } + + $cleaned = strip_tags($content); + $cleaned = preg_replace('/\s+/', ' ', $cleaned); + + return trim($cleaned); + } + + private function generateCode(): string + { + // Use a lock-safe approach for sequential code + return \Illuminate\Support\Facades\DB::transaction(function () { + $latestCode = MediaMonitoring::where('code', 'LIKE', 'ADN%') + ->lockForUpdate() + ->orderBy('code', 'desc') + ->first(); + + $lastNumber = $latestCode + ? intval(substr($latestCode->code, 3)) + : 0; + + $newNumber = str_pad($lastNumber + 1, 6, '0', STR_PAD_LEFT); + + return 'ADN'.$newNumber; + }); + } +} diff --git a/composer.json b/composer.json index f217c1c..dde6f78 100644 --- a/composer.json +++ b/composer.json @@ -23,7 +23,9 @@ "spatie/laravel-settings": "^3.6", "spatie/laravel-sluggable": "^3.7", "spatie/laravel-tags": "^4.10", - "swisnl/filament-backgrounds": "^2.0" + "swisnl/filament-backgrounds": "^2.0", + "symfony/css-selector": "^7.4", + "symfony/dom-crawler": "^7.4" }, "require-dev": { "fakerphp/faker": "^1.23", diff --git a/composer.lock b/composer.lock index 1779b0c..9c6f306 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a8225cac91593cfe6465c7be71fbbdc6", + "content-hash": "9ef9bb9f083fbdfe2d22c8b092633d54", "packages": [ { "name": "anourvalar/eloquent-serialize", @@ -7414,6 +7414,78 @@ ], "time": "2024-09-25T14:21:43+00:00" }, + { + "name": "symfony/dom-crawler", + "version": "v7.4.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/dom-crawler.git", + "reference": "0c5e8f20c74c78172a8ee72b125909b505033597" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/dom-crawler/zipball/0c5e8f20c74c78172a8ee72b125909b505033597", + "reference": "0c5e8f20c74c78172a8ee72b125909b505033597", + "shasum": "" + }, + "require": { + "masterminds/html5": "^2.6", + "php": ">=8.2", + "symfony/deprecation-contracts": "^2.5|^3", + "symfony/polyfill-ctype": "~1.8", + "symfony/polyfill-mbstring": "~1.0" + }, + "require-dev": { + "symfony/css-selector": "^6.4|^7.0|^8.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\DomCrawler\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Eases DOM navigation for HTML and XML documents", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/dom-crawler/tree/v7.4.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2025-12-06T15:47:47+00:00" + }, { "name": "symfony/error-handler", "version": "v7.4.0",