feat: Add news crawling service and command with a UI action to the media monitoring table.

app/Console/Commands/CrawlNewsCommand.php

@@ -0,0 +1,61 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Theme;
+use App\Services\NewsCrawlerService;
+use Illuminate\Console\Command;
+
+class CrawlNewsCommand extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:crawl-news {keyword?} {--limit=10} {--all-themes}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Crawl news from Google News based on keywords or themes';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(NewsCrawlerService $service)
+    {
+        $limit = (int) $this->option('limit');
+        $allThemes = $this->option('all-themes');
+        $keyword = $this->argument('keyword');
+
+        if ($allThemes) {
+            $themes = Theme::active()->get();
+            if ($themes->isEmpty()) {
+                $this->error('No active themes found.');
+
+                return;
+            }
+
+            foreach ($themes as $theme) {
+                $this->info("Crawling for theme: {$theme->name}");
+                $count = $service->crawlFromGoogleNews($theme->name, $limit, [$theme->id]);
+                $this->info("Saved {$count} news items for theme: {$theme->name}");
+            }
+
+            return;
+        }
+
+        if (! $keyword) {
+            $this->error('Please provide a keyword or use --all-themes');
+
+            return;
+        }
+
+        $this->info("Crawling for keyword: {$keyword}");
+        $count = $service->crawlFromGoogleNews($keyword, $limit);
+        $this->info("Saved {$count} news items for keyword: {$keyword}");
+    }
+}

app/Filament/Resources/Monitoring/MediaMonitorings/MediaMonitoringResource.php

@@ -29,7 +29,7 @@ class MediaMonitoringResource extends Resource
 
     protected static ?int $navigationSort = 12;
 
-    protected static ?string $recordTitleAttribute = 'media_name';
+    protected static ?string $recordTitleAttribute = 'code';
 
     protected static ?string $slug = 'monitoring/media-monitorings';
 

app/Filament/Resources/Monitoring/MediaMonitorings/Tables/MediaMonitoringsTable.php

@@ -4,11 +4,18 @@
 
 use App\Filament\Actions\DefaultBulkActions;
 use App\Filament\Columns\TimestampColumns;
+use App\Models\Theme;
+use App\Services\NewsCrawlerService;
+use Filament\Actions\Action;
 use Filament\Actions\BulkActionGroup;
 use Filament\Actions\DeleteAction;
 use Filament\Actions\EditAction;
 use Filament\Actions\ForceDeleteAction;
 use Filament\Actions\RestoreAction;
+use Filament\Forms\Components\Select;
+use Filament\Forms\Components\TextInput;
+use Filament\Notifications\Notification;
+use Filament\Support\Enums\Width;
 use Filament\Support\Icons\Heroicon;
 use Filament\Tables\Columns\TextColumn;
 use Filament\Tables\Filters\TrashedFilter;
@@ -95,6 +102,53 @@ public static function configure(Table $table): Table
 
                 RestoreAction::make(),
             ])
+            ->headerActions([
+                Action::make('crawlNews')
+                    ->label('Crawl Berita')
+                    ->icon(Heroicon::MagnifyingGlass)
+                    ->color('info')
+                    ->schema([
+                        TextInput::make('keyword')
+                            ->label('Kata Kunci')
+                            ->placeholder('Pendidikan, UMKM, Pariwisata')
+                            ->autofocus()
+                            ->autocomplete('off')
+                            ->required()
+                            ->helperText('Jika kata kunci nya lebih dari 1, pisahkan dengan koma.'),
+
+                        Select::make('theme_id')
+                            ->label('Tema')
+                            ->required()
+                            ->options(Theme::active()->pluck('name', 'id'))
+                            ->searchable()
+                            ->multiple(),
+
+                        TextInput::make('limit')
+                            ->label('Jumlah Berita')
+                            ->placeholder(10)
+                            ->numeric()
+                            ->required()
+                            ->default(10),
+                    ])
+                    ->action(function (array $data, NewsCrawlerService $service) {
+                        $count = $service->crawlFromGoogleNews($data['keyword'], $data['limit'], $data['theme_id'] ?? []);
+
+                        if ($count > 0) {
+                            Notification::make()
+                                ->title('Crawl Selesai')
+                                ->body("Berhasil mengambil {$count} berita baru tentang '{$data['keyword']}' di Purwakarta.")
+                                ->success()
+                                ->send();
+                        } else {
+                            Notification::make()
+                                ->title('Tidak Ada Berita Baru')
+                                ->body("Tidak ditemukan berita baru untuk '{$data['keyword']}' yang belum ada di database.")
+                                ->info()
+                                ->send();
+                        }
+                    })
+                    ->modalWidth(Width::Large),
+            ])
             ->toolbarActions([
                 BulkActionGroup::make([
                     ...DefaultBulkActions::make('Monitoring Media'),

app/Services/NewsCrawlerService.php

@@ -0,0 +1,235 @@
+<?php
+
+namespace App\Services;
+
+use App\Enums\Channel;
+use App\Models\MediaMonitoring;
+use Carbon\Carbon;
+use Illuminate\Http\Client\Pool;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Str;
+use Symfony\Component\DomCrawler\Crawler;
+
+class NewsCrawlerService
+{
+    /**
+     * Crawl news with high performance using HTTP Pooling.
+     */
+    public function crawlFromGoogleNews(string $keyword, int $limit = 10, array $themeIds = []): int
+    {
+        $searchQuery = $keyword.' Purwakarta';
+        $url = 'https://news.google.com/rss/search?q='.urlencode($searchQuery).'&hl=id&gl=ID&ceid=ID:id';
+
+        try {
+            $response = Http::get($url);
+            if ($response->failed()) {
+                return 0;
+            }
+
+            $xml = simplexml_load_string($response->body());
+            if (! $xml) {
+                return 0;
+            }
+
+            $items = [];
+            foreach ($xml->channel->item as $item) {
+                if (count($items) >= $limit) {
+                    break;
+                }
+                $items[] = [
+                    'google_link' => (string) $item->link,
+                    'title' => (string) $item->title,
+                    'pubDate' => (string) $item->pubDate,
+                    'source' => (string) $item->source,
+                    'description' => (string) $item->description,
+                ];
+            }
+
+            // Phase 1: Parallel Resolving Links
+            $resolvedLinks = Http::pool(
+                fn (Pool $pool) => array_map(fn ($item) => $pool->withHeaders([
+                    'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+                ])->get($item['google_link']), $items)
+            );
+
+            $parsedData = [];
+            $parsedData = [];
+            foreach ($resolvedLinks as $key => $res) {
+                // Check if $res is a response or an exception (ConnectionException)
+                $originalLink = ($res instanceof \Illuminate\Http\Client\Response && $res->successful())
+                    ? (string) $res->effectiveUri()
+                    : $items[$key]['google_link'];
+
+                // Anti-Duplicate Check
+                if (MediaMonitoring::where('link', $originalLink)->exists()) {
+                    continue;
+                }
+
+                $parsedData[] = array_merge($items[$key], ['original_link' => $originalLink]);
+            }
+
+            // Phase 2: Parallel Fetching Extra Content (Metadata)
+            $contentResponses = Http::pool(
+                fn (Pool $pool) => array_map(fn ($data) => $pool->timeout(5)->get($data['original_link']), $parsedData)
+            );
+
+            $count = 0;
+            foreach ($contentResponses as $key => $res) {
+                try {
+                    if (! isset($parsedData[$key])) {
+                        continue;
+                    }
+
+                    $itemData = $parsedData[$key];
+                    $extra = ['writer' => 'Redaksi', 'content' => null];
+
+                    if ($res instanceof \Illuminate\Http\Client\Response && $res->successful()) {
+                        $crawler = new Crawler($res->body());
+                        $extra['writer'] = $this->extractWriter($crawler);
+                        $extra['content'] = $this->extractContent($crawler);
+                    }
+
+                    // Clean generic Google News text
+                    $finalContent = $this->cleanContent($extra['content'] ?: $itemData['description']);
+                    $cleanTitle = $this->cleanTitle($itemData['title']);
+
+                    $monitoring = MediaMonitoring::create([
+                        'code' => $this->generateCode(),
+                        'media_name' => $itemData['source'] ?: 'Online Media',
+                        'title' => Str::limit($cleanTitle, 255),
+                        'channel' => Channel::WEBSITE->value,
+                        'writter' => Str::limit($extra['writer'] ?: 'Redaksi', 50),
+                        'link' => $itemData['original_link'],
+                        'news_page' => null, // Asked to be null if not specific pagination
+                        'quote' => Str::limit($finalContent, 500),
+                        'content' => $finalContent,
+                        'keyword' => Str::limit($searchQuery, 100),
+                        'release_date' => Carbon::parse($itemData['pubDate'])->format('Y-m-d'),
+                    ]);
+
+                    if (! empty($themeIds)) {
+                        $monitoring->themes()->syncWithoutDetaching($themeIds);
+                    }
+
+                    $count++;
+                } catch (\Exception $e) {
+                    Log::error('Failed to insert item: '.$e->getMessage());
+                }
+            }
+
+            return $count;
+        } catch (\Exception $e) {
+            Log::error('Crawler error: '.$e->getMessage());
+
+            return 0;
+        }
+    }
+
+    private function cleanTitle(string $title): string
+    {
+        if (Str::contains($title, ' - ')) {
+            $parts = explode(' - ', $title);
+            array_pop($parts);
+
+            return implode(' - ', $parts);
+        }
+
+        return $title;
+    }
+
+    private function extractWriter(Crawler $crawler): ?string
+    {
+        $selectors = ['meta[name="author"]', 'meta[property="article:author"]', '.author', '.writer'];
+        foreach ($selectors as $selector) {
+            $el = $crawler->filter($selector);
+            if ($el->count()) {
+                return $el->attr('content') ?: $el->text();
+            }
+        }
+
+        return null;
+    }
+
+    private function extractContent(Crawler $crawler): ?string
+    {
+        // Priority 1: Open Graph Description
+        $ogDesc = $crawler->filter('meta[property="og:description"]');
+        if ($ogDesc->count()) {
+            $content = $ogDesc->attr('content');
+            if (! $this->isGarbage($content)) {
+                return $content;
+            }
+        }
+
+        // Priority 2: Meta Description
+        $metaDesc = $crawler->filter('meta[name="description"]');
+        if ($metaDesc->count()) {
+            $content = $metaDesc->attr('content');
+            if (! $this->isGarbage($content)) {
+                return $content;
+            }
+        }
+
+        // Priority 3: First substantial paragraph
+        try {
+            $paragraphs = $crawler->filter('p');
+            foreach ($paragraphs as $p) {
+                $text = trim($p->textContent);
+                if (strlen($text) > 100) {
+                    return $text;
+                }
+            }
+        } catch (\Exception $e) {
+        }
+
+        return null;
+    }
+
+    private function isGarbage(string $content): bool
+    {
+        $garbage = [
+            'Comprehensive, up-to-date news coverage',
+            'Google News',
+            'Baca berita tanpa iklan',
+        ];
+        foreach ($garbage as $g) {
+            if (Str::contains($content, $g)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private function cleanContent(?string $content): string
+    {
+        if (! $content) {
+            return 'Tidak ada kutipan berita.';
+        }
+
+        $cleaned = strip_tags($content);
+        $cleaned = preg_replace('/\s+/', ' ', $cleaned);
+
+        return trim($cleaned);
+    }
+
+    private function generateCode(): string
+    {
+        // Use a lock-safe approach for sequential code
+        return \Illuminate\Support\Facades\DB::transaction(function () {
+            $latestCode = MediaMonitoring::where('code', 'LIKE', 'ADN%')
+                ->lockForUpdate()
+                ->orderBy('code', 'desc')
+                ->first();
+
+            $lastNumber = $latestCode
+                ? intval(substr($latestCode->code, 3))
+                : 0;
+
+            $newNumber = str_pad($lastNumber + 1, 6, '0', STR_PAD_LEFT);
+
+            return 'ADN'.$newNumber;
+        });
+    }
+}

composer.json

@@ -23,7 +23,9 @@
         "spatie/laravel-settings": "^3.6",
         "spatie/laravel-sluggable": "^3.7",
         "spatie/laravel-tags": "^4.10",
-        "swisnl/filament-backgrounds": "^2.0"
+        "swisnl/filament-backgrounds": "^2.0",
+        "symfony/css-selector": "^7.4",
+        "symfony/dom-crawler": "^7.4"
     },
     "require-dev": {
         "fakerphp/faker": "^1.23",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "a8225cac91593cfe6465c7be71fbbdc6",
+    "content-hash": "9ef9bb9f083fbdfe2d22c8b092633d54",
     "packages": [
         {
             "name": "anourvalar/eloquent-serialize",
@@ -7414,6 +7414,78 @@
             ],
             "time": "2024-09-25T14:21:43+00:00"
         },
+        {
+            "name": "symfony/dom-crawler",
+            "version": "v7.4.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/symfony/dom-crawler.git",
+                "reference": "0c5e8f20c74c78172a8ee72b125909b505033597"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/symfony/dom-crawler/zipball/0c5e8f20c74c78172a8ee72b125909b505033597",
+                "reference": "0c5e8f20c74c78172a8ee72b125909b505033597",
+                "shasum": ""
+            },
+            "require": {
+                "masterminds/html5": "^2.6",
+                "php": ">=8.2",
+                "symfony/deprecation-contracts": "^2.5|^3",
+                "symfony/polyfill-ctype": "~1.8",
+                "symfony/polyfill-mbstring": "~1.0"
+            },
+            "require-dev": {
+                "symfony/css-selector": "^6.4|^7.0|^8.0"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Symfony\\Component\\DomCrawler\\": ""
+                },
+                "exclude-from-classmap": [
+                    "/Tests/"
+                ]
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Fabien Potencier",
+                    "email": "fabien@symfony.com"
+                },
+                {
+                    "name": "Symfony Community",
+                    "homepage": "https://symfony.com/contributors"
+                }
+            ],
+            "description": "Eases DOM navigation for HTML and XML documents",
+            "homepage": "https://symfony.com",
+            "support": {
+                "source": "https://github.com/symfony/dom-crawler/tree/v7.4.1"
+            },
+            "funding": [
+                {
+                    "url": "https://symfony.com/sponsor",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://github.com/fabpot",
+                    "type": "github"
+                },
+                {
+                    "url": "https://github.com/nicolas-grekas",
+                    "type": "github"
+                },
+                {
+                    "url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
+                    "type": "tidelift"
+                }
+            ],
+            "time": "2025-12-06T15:47:47+00:00"
+        },
         {
             "name": "symfony/error-handler",
             "version": "v7.4.0",