diskominfo/simedkom
diskominfo/simedkom
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add news crawling service and command with a UI action to the media monitoring table.

Browse Source
This commit is contained in:
Yoga Pangestu 2026-01-06 15:19:53 +07:00
parent c48bb24c68
commit dd100d1704
6 changed files with 427 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
app/Console/Commands/CrawlNewsCommand.php Normal file
View File

@ -0,0 +1,61 @@
<?php
namespace App\Console\Commands;
use App\Models\Theme;
use App\Services\NewsCrawlerService;
use Illuminate\Console\Command;
class CrawlNewsCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:crawl-news {keyword?} {--limit=10} {--all-themes}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Crawl news from Google News based on keywords or themes';
/**
* Execute the console command.
*/
public function handle(NewsCrawlerService $service)
{
$limit = (int) $this->option('limit');
$allThemes = $this->option('all-themes');
$keyword = $this->argument('keyword');
if ($allThemes) {
$themes = Theme::active()->get();
if ($themes->isEmpty()) {
$this->error('No active themes found.');
return;
}
foreach ($themes as $theme) {
$this->info("Crawling for theme: {$theme->name}");
$count = $service->crawlFromGoogleNews($theme->name, $limit, [$theme->id]);
$this->info("Saved {$count} news items for theme: {$theme->name}");
}
return;
}
if (! $keyword) {
$this->error('Please provide a keyword or use --all-themes');
return;
}
$this->info("Crawling for keyword: {$keyword}");
$count = $service->crawlFromGoogleNews($keyword, $limit);
$this->info("Saved {$count} news items for keyword: {$keyword}");
}
}

2
app/Filament/Resources/Monitoring/MediaMonitorings/MediaMonitoringResource.php
View File

@ -29,7 +29,7 @@ class MediaMonitoringResource extends Resource
protected static ?int $navigationSort = 12;
protected static ?string $recordTitleAttribute = 'media_name';
protected static ?string $recordTitleAttribute = 'code';
protected static ?string $slug = 'monitoring/media-monitorings';

54
app/Filament/Resources/Monitoring/MediaMonitorings/Tables/MediaMonitoringsTable.php
View File

@ -4,11 +4,18 @@
use App\Filament\Actions\DefaultBulkActions;
use App\Filament\Columns\TimestampColumns;
use App\Models\Theme;
use App\Services\NewsCrawlerService;
use Filament\Actions\Action;
use Filament\Actions\BulkActionGroup;
use Filament\Actions\DeleteAction;
use Filament\Actions\EditAction;
use Filament\Actions\ForceDeleteAction;
use Filament\Actions\RestoreAction;
use Filament\Forms\Components\Select;
use Filament\Forms\Components\TextInput;
use Filament\Notifications\Notification;
use Filament\Support\Enums\Width;
use Filament\Support\Icons\Heroicon;
use Filament\Tables\Columns\TextColumn;
use Filament\Tables\Filters\TrashedFilter;
@ -95,6 +102,53 @@ public static function configure(Table $table): Table
RestoreAction::make(),
])
->headerActions([
Action::make('crawlNews')
->label('Crawl Berita')
->icon(Heroicon::MagnifyingGlass)
->color('info')
->schema([
TextInput::make('keyword')
->label('Kata Kunci')
->placeholder('Pendidikan, UMKM, Pariwisata')
->autofocus()
->autocomplete('off')
->required()
->helperText('Jika kata kunci nya lebih dari 1, pisahkan dengan koma.'),
Select::make('theme_id')
->label('Tema')
->required()
->options(Theme::active()->pluck('name', 'id'))
->searchable()
->multiple(),
TextInput::make('limit')
->label('Jumlah Berita')
->placeholder(10)
->numeric()
->required()
->default(10),
])
->action(function (array $data, NewsCrawlerService $service) {
$count = $service->crawlFromGoogleNews($data['keyword'], $data['limit'], $data['theme_id'] ?? []);
if ($count > 0) {
Notification::make()
->title('Crawl Selesai')
->body("Berhasil mengambil {$count} berita baru tentang '{$data['keyword']}' di Purwakarta.")
->success()
->send();
} else {
Notification::make()
->title('Tidak Ada Berita Baru')
->body("Tidak ditemukan berita baru untuk '{$data['keyword']}' yang belum ada di database.")
->info()
->send();
}
})
->modalWidth(Width::Large),
])
->toolbarActions([
BulkActionGroup::make([
...DefaultBulkActions::make('Monitoring Media'),

235
app/Services/NewsCrawlerService.php Normal file
View File

@ -0,0 +1,235 @@
<?php
namespace App\Services;
use App\Enums\Channel;
use App\Models\MediaMonitoring;
use Carbon\Carbon;
use Illuminate\Http\Client\Pool;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;
use Symfony\Component\DomCrawler\Crawler;
class NewsCrawlerService
{
/**
* Crawl news with high performance using HTTP Pooling.
*/
public function crawlFromGoogleNews(string $keyword, int $limit = 10, array $themeIds = []): int
{
$searchQuery = $keyword.' Purwakarta';
$url = 'https://news.google.com/rss/search?q='.urlencode($searchQuery).'&hl=id&gl=ID&ceid=ID:id';
try {
$response = Http::get($url);
if ($response->failed()) {
return 0;
}
$xml = simplexml_load_string($response->body());
if (! $xml) {
return 0;
}
$items = [];
foreach ($xml->channel->item as $item) {
if (count($items) >= $limit) {
break;
}
$items[] = [
'google_link' => (string) $item->link,
'title' => (string) $item->title,
'pubDate' => (string) $item->pubDate,
'source' => (string) $item->source,
'description' => (string) $item->description,
];
}
// Phase 1: Parallel Resolving Links
$resolvedLinks = Http::pool(
fn (Pool $pool) => array_map(fn ($item) => $pool->withHeaders([
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
])->get($item['google_link']), $items)
);
$parsedData = [];
$parsedData = [];
foreach ($resolvedLinks as $key => $res) {
// Check if $res is a response or an exception (ConnectionException)
$originalLink = ($res instanceof \Illuminate\Http\Client\Response && $res->successful())
? (string) $res->effectiveUri()
: $items[$key]['google_link'];
// Anti-Duplicate Check
if (MediaMonitoring::where('link', $originalLink)->exists()) {
continue;
}
$parsedData[] = array_merge($items[$key], ['original_link' => $originalLink]);
}
// Phase 2: Parallel Fetching Extra Content (Metadata)
$contentResponses = Http::pool(
fn (Pool $pool) => array_map(fn ($data) => $pool->timeout(5)->get($data['original_link']), $parsedData)
);
$count = 0;
foreach ($contentResponses as $key => $res) {
try {
if (! isset($parsedData[$key])) {
continue;
}
$itemData = $parsedData[$key];
$extra = ['writer' => 'Redaksi', 'content' => null];
if ($res instanceof \Illuminate\Http\Client\Response && $res->successful()) {
$crawler = new Crawler($res->body());
$extra['writer'] = $this->extractWriter($crawler);
$extra['content'] = $this->extractContent($crawler);
}
// Clean generic Google News text
$finalContent = $this->cleanContent($extra['content'] ?: $itemData['description']);
$cleanTitle = $this->cleanTitle($itemData['title']);
$monitoring = MediaMonitoring::create([
'code' => $this->generateCode(),
'media_name' => $itemData['source'] ?: 'Online Media',
'title' => Str::limit($cleanTitle, 255),
'channel' => Channel::WEBSITE->value,
'writter' => Str::limit($extra['writer'] ?: 'Redaksi', 50),
'link' => $itemData['original_link'],
'news_page' => null, // Asked to be null if not specific pagination
'quote' => Str::limit($finalContent, 500),
'content' => $finalContent,
'keyword' => Str::limit($searchQuery, 100),
'release_date' => Carbon::parse($itemData['pubDate'])->format('Y-m-d'),
]);
if (! empty($themeIds)) {
$monitoring->themes()->syncWithoutDetaching($themeIds);
}
$count++;
} catch (\Exception $e) {
Log::error('Failed to insert item: '.$e->getMessage());
}
}
return $count;
} catch (\Exception $e) {
Log::error('Crawler error: '.$e->getMessage());
return 0;
}
}
private function cleanTitle(string $title): string
{
if (Str::contains($title, ' - ')) {
$parts = explode(' - ', $title);
array_pop($parts);
return implode(' - ', $parts);
}
return $title;
}
private function extractWriter(Crawler $crawler): ?string
{
$selectors = ['meta[name="author"]', 'meta[property="article:author"]', '.author', '.writer'];
foreach ($selectors as $selector) {
$el = $crawler->filter($selector);
if ($el->count()) {
return $el->attr('content') ?: $el->text();
}
}
return null;
}
private function extractContent(Crawler $crawler): ?string
{
// Priority 1: Open Graph Description
$ogDesc = $crawler->filter('meta[property="og:description"]');
if ($ogDesc->count()) {
$content = $ogDesc->attr('content');
if (! $this->isGarbage($content)) {
return $content;
}
}
// Priority 2: Meta Description
$metaDesc = $crawler->filter('meta[name="description"]');
if ($metaDesc->count()) {
$content = $metaDesc->attr('content');
if (! $this->isGarbage($content)) {
return $content;
}
}
// Priority 3: First substantial paragraph
try {
$paragraphs = $crawler->filter('p');
foreach ($paragraphs as $p) {
$text = trim($p->textContent);
if (strlen($text) > 100) {
return $text;
}
}
} catch (\Exception $e) {
}
return null;
}
private function isGarbage(string $content): bool
{
$garbage = [
'Comprehensive, up-to-date news coverage',
'Google News',
'Baca berita tanpa iklan',
];
foreach ($garbage as $g) {
if (Str::contains($content, $g)) {
return true;
}
}
return false;
}
private function cleanContent(?string $content): string
{
if (! $content) {
return 'Tidak ada kutipan berita.';
}
$cleaned = strip_tags($content);
$cleaned = preg_replace('/\s+/', ' ', $cleaned);
return trim($cleaned);
}
private function generateCode(): string
{
// Use a lock-safe approach for sequential code
return \Illuminate\Support\Facades\DB::transaction(function () {
$latestCode = MediaMonitoring::where('code', 'LIKE', 'ADN%')
->lockForUpdate()
->orderBy('code', 'desc')
->first();
$lastNumber = $latestCode
? intval(substr($latestCode->code, 3))
: 0;
$newNumber = str_pad($lastNumber + 1, 6, '0', STR_PAD_LEFT);
return 'ADN'.$newNumber;
});
}
}

4
composer.json
View File

@ -23,7 +23,9 @@
"spatie/laravel-settings": "^3.6",
"spatie/laravel-sluggable": "^3.7",
"spatie/laravel-tags": "^4.10",
"swisnl/filament-backgrounds": "^2.0"
"swisnl/filament-backgrounds": "^2.0",
"symfony/css-selector": "^7.4",
"symfony/dom-crawler": "^7.4"
},
"require-dev": {
"fakerphp/faker": "^1.23",

74
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "a8225cac91593cfe6465c7be71fbbdc6",
"content-hash": "9ef9bb9f083fbdfe2d22c8b092633d54",
"packages": [
{
"name": "anourvalar/eloquent-serialize",
@ -7414,6 +7414,78 @@
],
"time": "2024-09-25T14:21:43+00:00"
},
{
"name": "symfony/dom-crawler",
"version": "v7.4.1",
"source": {
"type": "git",
"url": "https://github.com/symfony/dom-crawler.git",
"reference": "0c5e8f20c74c78172a8ee72b125909b505033597"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/dom-crawler/zipball/0c5e8f20c74c78172a8ee72b125909b505033597",
"reference": "0c5e8f20c74c78172a8ee72b125909b505033597",
"shasum": ""
},
"require": {
"masterminds/html5": "^2.6",
"php": ">=8.2",
"symfony/deprecation-contracts": "^2.5|^3",
"symfony/polyfill-ctype": "~1.8",
"symfony/polyfill-mbstring": "~1.0"
},
"require-dev": {
"symfony/css-selector": "^6.4|^7.0|^8.0"
},
"type": "library",
"autoload": {
"psr-4": {
"Symfony\\Component\\DomCrawler\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Eases DOM navigation for HTML and XML documents",
"homepage": "https://symfony.com",
"support": {
"source": "https://github.com/symfony/dom-crawler/tree/v7.4.1"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://github.com/nicolas-grekas",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2025-12-06T15:47:47+00:00"
},
{
"name": "symfony/error-handler",
"version": "v7.4.0",