131 lines
3.9 KiB
PHP
131 lines
3.9 KiB
PHP
<?php
|
|
|
|
namespace App\Console\Commands;
|
|
|
|
use App\Enums\Channel;
|
|
use App\Models\MediaMonitoring;
|
|
use Carbon\Carbon;
|
|
use Illuminate\Console\Command;
|
|
use Symfony\Component\Process\Process;
|
|
|
|
class CrawlMediaMonitoringCommand extends Command
|
|
{
|
|
protected $signature = 'monitoring:crawl {--limit=20 : Maximum number of news to save globally}';
|
|
|
|
protected $description = 'Crawl data from Google News RSS based on active Website Themes or specific keywords';
|
|
|
|
protected $keywords = [
|
|
'Purwakarta',
|
|
'Pemkab Purwakarta',
|
|
'Bupati Purwakarta',
|
|
'Pendidikan',
|
|
'Pariwisata',
|
|
'Kriminalitas',
|
|
'Institusi Pemerintah/ Dinas',
|
|
'Diskominfo Purwakarta',
|
|
'Om Zein',
|
|
'Abang Ijo Hapidin',
|
|
'Wakil Bupati Purwakarta',
|
|
];
|
|
|
|
public function handle()
|
|
{
|
|
$globalLimit = (int) $this->option('limit');
|
|
$totalImported = 0;
|
|
|
|
$this->info('Starting crawler engine...');
|
|
|
|
$keywordsToCrawl = $this->keywords;
|
|
shuffle($keywordsToCrawl);
|
|
|
|
foreach ($keywordsToCrawl as $kw) {
|
|
$offset = 0;
|
|
|
|
while ($totalImported < $globalLimit) {
|
|
$remainingLimit = $globalLimit - $totalImported;
|
|
|
|
$this->info("\n--- Crawling: $kw (limit: $remainingLimit, offset: $offset) ---");
|
|
|
|
$command = ['python3', base_path('scripts/crawler.py'), '--keyword', $kw, '--limit', $remainingLimit, '--offset', $offset];
|
|
|
|
$process = new Process($command);
|
|
$process->setTimeout(600);
|
|
$process->run(function ($type, $buffer) {
|
|
if ($type === Process::ERR) {
|
|
$this->output->write($buffer);
|
|
}
|
|
});
|
|
|
|
if (! $process->isSuccessful()) {
|
|
$this->error($process->getErrorOutput());
|
|
break;
|
|
}
|
|
|
|
$output = $process->getOutput();
|
|
$newsData = json_decode($output, true);
|
|
|
|
if (json_last_error() !== JSON_ERROR_NONE || empty($newsData)) {
|
|
$this->warn("No more news found for '$kw'.");
|
|
break;
|
|
}
|
|
|
|
$importedCount = $this->saveToDatabase($newsData, $remainingLimit);
|
|
$totalImported += $importedCount;
|
|
|
|
$offset += $remainingLimit;
|
|
}
|
|
|
|
if ($totalImported >= $globalLimit) {
|
|
$this->info("\n✅ Reached global limit of $globalLimit new articles. Stopping crawl.");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
protected function saveToDatabase(array $newsData, int $maxToSave)
|
|
{
|
|
$count = 0;
|
|
foreach ($newsData as $item) {
|
|
if ($count >= $maxToSave) {
|
|
break;
|
|
}
|
|
|
|
if (MediaMonitoring::where('link', $item['url'])->where('title', $item['title'])->exists()) {
|
|
continue;
|
|
}
|
|
|
|
$host = parse_url($item['url'], PHP_URL_HOST);
|
|
$mediaName = str_replace('www.', '', $host);
|
|
|
|
MediaMonitoring::create([
|
|
'code' => $this->code(),
|
|
'media_name' => $mediaName,
|
|
'title' => $item['title'],
|
|
'channel' => Channel::WEBSITE,
|
|
'writter' => $item['source'] ?? '-',
|
|
'link' => $item['url'],
|
|
'content' => $item['content'],
|
|
'keyword' => $item['keyword'],
|
|
'release_date' => Carbon::parse($item['published_at'])->toDateString(),
|
|
]);
|
|
|
|
$count++;
|
|
}
|
|
|
|
$this->info("Imported $count new articles.");
|
|
|
|
return $count;
|
|
}
|
|
|
|
protected function code()
|
|
{
|
|
$latestCode = MediaMonitoring::max('code');
|
|
|
|
$lastNumber = $latestCode
|
|
? intval(substr($latestCode, 3))
|
|
: 0;
|
|
|
|
return 'ADN'.str_pad($lastNumber + 1, 6, '0', STR_PAD_LEFT);
|
|
}
|
|
}
|