option('limit'); $totalImported = 0; $this->info('Starting crawler engine...'); $keywordsToCrawl = $this->keywords; shuffle($keywordsToCrawl); foreach ($keywordsToCrawl as $kw) { $offset = 0; while ($totalImported < $globalLimit) { $remainingLimit = $globalLimit - $totalImported; $this->info("\n--- Crawling: $kw (limit: $remainingLimit, offset: $offset) ---"); $command = ['python3', base_path('scripts/crawler.py'), '--keyword', $kw, '--limit', $remainingLimit, '--offset', $offset]; $process = new Process($command); $process->setTimeout(600); $process->run(function ($type, $buffer) { if ($type === Process::ERR) { $this->output->write($buffer); } }); if (! $process->isSuccessful()) { $this->error($process->getErrorOutput()); break; } $output = $process->getOutput(); $newsData = json_decode($output, true); if (json_last_error() !== JSON_ERROR_NONE || empty($newsData)) { $this->warn("No more news found for '$kw'."); break; } $importedCount = $this->saveToDatabase($newsData, $remainingLimit); $totalImported += $importedCount; $offset += $remainingLimit; } if ($totalImported >= $globalLimit) { $this->info("\n✅ Reached global limit of $globalLimit new articles. Stopping crawl."); break; } } } protected function saveToDatabase(array $newsData, int $maxToSave) { $count = 0; foreach ($newsData as $item) { if ($count >= $maxToSave) { break; } if (MediaMonitoring::where('link', $item['url'])->where('title', $item['title'])->exists()) { continue; } $host = parse_url($item['url'], PHP_URL_HOST); $mediaName = str_replace('www.', '', $host); MediaMonitoring::create([ 'code' => $this->code(), 'media_name' => $mediaName, 'title' => $item['title'], 'channel' => Channel::WEBSITE, 'writter' => $item['source'] ?? '-', 'link' => $item['url'], 'content' => $item['content'], 'keyword' => $item['keyword'], 'release_date' => Carbon::parse($item['published_at'])->toDateString(), ]); $count++; } $this->info("Imported $count new articles."); return $count; } protected function code() { $latestCode = MediaMonitoring::max('code'); $lastNumber = $latestCode ? intval(substr($latestCode, 3)) : 0; return 'ADN'.str_pad($lastNumber + 1, 6, '0', STR_PAD_LEFT); } }