feat: implement automated media monitoring crawler with Python script and Laravel console command
This commit is contained in:
Yoga Pangestu
parent
ca5aa283d0
commit
4a1993a778
130
app/Console/Commands/CrawlMediaMonitoringCommand.php
Normal file
130
app/Console/Commands/CrawlMediaMonitoringCommand.php
Normal file
@ -0,0 +1,130 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Console\Commands;
|
||||||
|
|
||||||
|
use App\Enums\Channel;
|
||||||
|
use App\Models\MediaMonitoring;
|
||||||
|
use Carbon\Carbon;
|
||||||
|
use Illuminate\Console\Command;
|
||||||
|
use Symfony\Component\Process\Process;
|
||||||
|
|
||||||
|
class CrawlMediaMonitoringCommand extends Command
|
||||||
|
{
|
||||||
|
protected $signature = 'monitoring:crawl {--limit=20 : Maximum number of news to save globally}';
|
||||||
|
|
||||||
|
protected $description = 'Crawl data from Google News RSS based on active Website Themes or specific keywords';
|
||||||
|
|
||||||
|
protected $keywords = [
|
||||||
|
'Purwakarta',
|
||||||
|
'Pemkab Purwakarta',
|
||||||
|
'Bupati Purwakarta',
|
||||||
|
'Pendidikan',
|
||||||
|
'Pariwisata',
|
||||||
|
'Kriminalitas',
|
||||||
|
'Institusi Pemerintah/ Dinas',
|
||||||
|
'Diskominfo Purwakarta',
|
||||||
|
'Om Zein',
|
||||||
|
'Abang Ijo Hapidin',
|
||||||
|
'Wakil Bupati Purwakarta',
|
||||||
|
];
|
||||||
|
|
||||||
|
public function handle()
|
||||||
|
{
|
||||||
|
$globalLimit = (int) $this->option('limit');
|
||||||
|
$totalImported = 0;
|
||||||
|
|
||||||
|
$this->info('Starting crawler engine...');
|
||||||
|
|
||||||
|
$keywordsToCrawl = $this->keywords;
|
||||||
|
shuffle($keywordsToCrawl);
|
||||||
|
|
||||||
|
foreach ($keywordsToCrawl as $kw) {
|
||||||
|
$offset = 0;
|
||||||
|
|
||||||
|
while ($totalImported < $globalLimit) {
|
||||||
|
$remainingLimit = $globalLimit - $totalImported;
|
||||||
|
|
||||||
|
$this->info("\n--- Crawling: $kw (limit: $remainingLimit, offset: $offset) ---");
|
||||||
|
|
||||||
|
$command = ['python3', base_path('scripts/crawler.py'), '--keyword', $kw, '--limit', $remainingLimit, '--offset', $offset];
|
||||||
|
|
||||||
|
$process = new Process($command);
|
||||||
|
$process->setTimeout(600);
|
||||||
|
$process->run(function ($type, $buffer) {
|
||||||
|
if ($type === Process::ERR) {
|
||||||
|
$this->output->write($buffer);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (! $process->isSuccessful()) {
|
||||||
|
$this->error($process->getErrorOutput());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
$output = $process->getOutput();
|
||||||
|
$newsData = json_decode($output, true);
|
||||||
|
|
||||||
|
if (json_last_error() !== JSON_ERROR_NONE || empty($newsData)) {
|
||||||
|
$this->warn("No more news found for '$kw'.");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
$importedCount = $this->saveToDatabase($newsData, $remainingLimit);
|
||||||
|
$totalImported += $importedCount;
|
||||||
|
|
||||||
|
$offset += $remainingLimit;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($totalImported >= $globalLimit) {
|
||||||
|
$this->info("\n✅ Reached global limit of $globalLimit new articles. Stopping crawl.");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function saveToDatabase(array $newsData, int $maxToSave)
|
||||||
|
{
|
||||||
|
$count = 0;
|
||||||
|
foreach ($newsData as $item) {
|
||||||
|
if ($count >= $maxToSave) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (MediaMonitoring::where('link', $item['url'])->where('title', $item['title'])->exists()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$host = parse_url($item['url'], PHP_URL_HOST);
|
||||||
|
$mediaName = str_replace('www.', '', $host);
|
||||||
|
|
||||||
|
MediaMonitoring::create([
|
||||||
|
'code' => $this->code(),
|
||||||
|
'media_name' => $mediaName,
|
||||||
|
'title' => $item['title'],
|
||||||
|
'channel' => Channel::WEBSITE,
|
||||||
|
'writter' => $item['source'] ?? '-',
|
||||||
|
'link' => $item['url'],
|
||||||
|
'content' => $item['content'],
|
||||||
|
'keyword' => $item['keyword'],
|
||||||
|
'release_date' => Carbon::parse($item['published_at'])->toDateString(),
|
||||||
|
]);
|
||||||
|
|
||||||
|
$count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->info("Imported $count new articles.");
|
||||||
|
|
||||||
|
return $count;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function code()
|
||||||
|
{
|
||||||
|
$latestCode = MediaMonitoring::max('code');
|
||||||
|
|
||||||
|
$lastNumber = $latestCode
|
||||||
|
? intval(substr($latestCode, 3))
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
return 'ADN'.str_pad($lastNumber + 1, 6, '0', STR_PAD_LEFT);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -15,6 +15,7 @@
|
|||||||
"barryvdh/laravel-dompdf": "^3.1",
|
"barryvdh/laravel-dompdf": "^3.1",
|
||||||
"bezhansalleh/filament-shield": "^4.0",
|
"bezhansalleh/filament-shield": "^4.0",
|
||||||
"bilfeldt/laravel-route-statistics": "^4.2",
|
"bilfeldt/laravel-route-statistics": "^4.2",
|
||||||
|
"crwlr/crawler": "^3.5",
|
||||||
"filament/filament": "^4.0",
|
"filament/filament": "^4.0",
|
||||||
"filament/spatie-laravel-media-library-plugin": "^4.0",
|
"filament/spatie-laravel-media-library-plugin": "^4.0",
|
||||||
"filament/spatie-laravel-settings-plugin": "^4.4",
|
"filament/spatie-laravel-settings-plugin": "^4.4",
|
||||||
|
|||||||
778
composer.lock
generated
778
composer.lock
generated
@ -4,7 +4,7 @@
|
|||||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||||
"This file is @generated automatically"
|
"This file is @generated automatically"
|
||||||
],
|
],
|
||||||
"content-hash": "116c8e60c2cac8f2a9bac2ede4d16dea",
|
"content-hash": "b29f415e8e8d01a4f83dfbf06c166e32",
|
||||||
"packages": [
|
"packages": [
|
||||||
{
|
{
|
||||||
"name": "achyutn/filament-log-viewer",
|
"name": "achyutn/filament-log-viewer",
|
||||||
@ -91,6 +91,60 @@
|
|||||||
],
|
],
|
||||||
"time": "2026-01-30T17:48:57+00:00"
|
"time": "2026-01-30T17:48:57+00:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "adbario/php-dot-notation",
|
||||||
|
"version": "3.5.0",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/adbario/php-dot-notation.git",
|
||||||
|
"reference": "aa09db5a7a365a8d2d8dd7edfa3cd501d1a8acb0"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/adbario/php-dot-notation/zipball/aa09db5a7a365a8d2d8dd7edfa3cd501d1a8acb0",
|
||||||
|
"reference": "aa09db5a7a365a8d2d8dd7edfa3cd501d1a8acb0",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"ext-json": "*",
|
||||||
|
"php": "^7.4 || ^8.0"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"phpstan/phpstan": "^2.1",
|
||||||
|
"phpunit/phpunit": "^9.5",
|
||||||
|
"squizlabs/php_codesniffer": "^3.7"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"files": [
|
||||||
|
"src/helpers.php"
|
||||||
|
],
|
||||||
|
"psr-4": {
|
||||||
|
"Adbar\\": "src"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Riku Särkinen",
|
||||||
|
"email": "riku@adbar.io"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "PHP dot notation access to arrays",
|
||||||
|
"homepage": "https://github.com/adbario/php-dot-notation",
|
||||||
|
"keywords": [
|
||||||
|
"ArrayAccess",
|
||||||
|
"dotnotation"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"issues": "https://github.com/adbario/php-dot-notation/issues",
|
||||||
|
"source": "https://github.com/adbario/php-dot-notation/tree/3.5.0"
|
||||||
|
},
|
||||||
|
"time": "2026-04-04T04:29:49+00:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "anourvalar/eloquent-serialize",
|
"name": "anourvalar/eloquent-serialize",
|
||||||
"version": "1.3.5",
|
"version": "1.3.5",
|
||||||
@ -1251,6 +1305,142 @@
|
|||||||
],
|
],
|
||||||
"time": "2024-07-16T11:13:48+00:00"
|
"time": "2024-07-16T11:13:48+00:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "chrome-php/chrome",
|
||||||
|
"version": "v1.15.0",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/chrome-php/chrome.git",
|
||||||
|
"reference": "5b2ee677f34ed4cd9463e49dc59c4989d0fe40f3"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/chrome-php/chrome/zipball/5b2ee677f34ed4cd9463e49dc59c4989d0fe40f3",
|
||||||
|
"reference": "5b2ee677f34ed4cd9463e49dc59c4989d0fe40f3",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"chrome-php/wrench": "^1.8",
|
||||||
|
"evenement/evenement": "^3.0.1",
|
||||||
|
"monolog/monolog": "^1.27.1 || ^2.8 || ^3.2",
|
||||||
|
"php": "^7.4.15 || ^8.0.2",
|
||||||
|
"psr/log": "^1.1 || ^2.0 || ^3.0",
|
||||||
|
"symfony/filesystem": "^5.4 || ^6.0 || ^7.0 || ^8.0",
|
||||||
|
"symfony/polyfill-mbstring": "^1.26",
|
||||||
|
"symfony/process": "^5.4 || ^6.0 || ^7.0 || ^8.0"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"bamarni/composer-bin-plugin": "^1.8.2",
|
||||||
|
"phpunit/phpunit": "^9.6.3 || ^10.0.12",
|
||||||
|
"symfony/var-dumper": "^5.4 || ^6.0 || ^7.0 || ^8.0"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"extra": {
|
||||||
|
"bamarni-bin": {
|
||||||
|
"bin-links": true,
|
||||||
|
"forward-command": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"HeadlessChromium\\": "src/"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Graham Campbell",
|
||||||
|
"email": "hello@gjcampbell.co.uk",
|
||||||
|
"homepage": "https://github.com/GrahamCampbell"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Enrico Dias",
|
||||||
|
"email": "enrico@enricodias.com",
|
||||||
|
"homepage": "https://github.com/enricodias"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Instrument headless chrome/chromium instances from PHP",
|
||||||
|
"keywords": [
|
||||||
|
"browser",
|
||||||
|
"chrome",
|
||||||
|
"chromium",
|
||||||
|
"crawl",
|
||||||
|
"headless",
|
||||||
|
"pdf",
|
||||||
|
"puppeteer",
|
||||||
|
"screenshot"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"issues": "https://github.com/chrome-php/chrome/issues",
|
||||||
|
"source": "https://github.com/chrome-php/chrome/tree/v1.15.0"
|
||||||
|
},
|
||||||
|
"time": "2025-12-27T12:23:40+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "chrome-php/wrench",
|
||||||
|
"version": "v1.8.0",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/chrome-php/wrench.git",
|
||||||
|
"reference": "bd67a315cf2143ba30598339ad1b6c346db96c99"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/chrome-php/wrench/zipball/bd67a315cf2143ba30598339ad1b6c346db96c99",
|
||||||
|
"reference": "bd67a315cf2143ba30598339ad1b6c346db96c99",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"ext-sockets": "*",
|
||||||
|
"php": "^7.4.15 || ^8.0.2",
|
||||||
|
"psr/log": "^1.1 || ^2.0 || ^3.0",
|
||||||
|
"symfony/polyfill-php80": "^1.26"
|
||||||
|
},
|
||||||
|
"conflict": {
|
||||||
|
"wrench/wrench": "*"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"bamarni/composer-bin-plugin": "^1.8.2",
|
||||||
|
"phpunit/phpunit": "^9.6.3 || ^10.0.12"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"extra": {
|
||||||
|
"bamarni-bin": {
|
||||||
|
"bin-links": true,
|
||||||
|
"forward-command": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Wrench\\": "src/"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Graham Campbell",
|
||||||
|
"email": "hello@gjcampbell.co.uk",
|
||||||
|
"homepage": "https://github.com/GrahamCampbell"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "A simple PHP WebSocket implementation",
|
||||||
|
"keywords": [
|
||||||
|
"WebSockets",
|
||||||
|
"hybi",
|
||||||
|
"websocket"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"issues": "https://github.com/chrome-php/wrench/issues",
|
||||||
|
"source": "https://github.com/chrome-php/wrench/tree/v1.8.0"
|
||||||
|
},
|
||||||
|
"time": "2025-12-27T11:29:35+00:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "composer/pcre",
|
"name": "composer/pcre",
|
||||||
"version": "3.3.2",
|
"version": "3.3.2",
|
||||||
@ -1407,6 +1597,472 @@
|
|||||||
],
|
],
|
||||||
"time": "2025-08-20T19:15:30+00:00"
|
"time": "2025-08-20T19:15:30+00:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "crwlr/crawler",
|
||||||
|
"version": "v3.5.6",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/crwlrsoft/crawler.git",
|
||||||
|
"reference": "d6680f9e698a050f3a5eded2fc8c6776d6043a3b"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/crwlrsoft/crawler/zipball/d6680f9e698a050f3a5eded2fc8c6776d6043a3b",
|
||||||
|
"reference": "d6680f9e698a050f3a5eded2fc8c6776d6043a3b",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"adbario/php-dot-notation": "^3.1",
|
||||||
|
"chrome-php/chrome": "^1.7",
|
||||||
|
"crwlr/html-2-text": "^0.1.0",
|
||||||
|
"crwlr/robots-txt": "^1.1",
|
||||||
|
"crwlr/schema-org": "^0.2|^0.3",
|
||||||
|
"crwlr/url": "^2.1",
|
||||||
|
"crwlr/utils": "^1.2",
|
||||||
|
"ext-dom": "*",
|
||||||
|
"guzzlehttp/guzzle": "^7.4",
|
||||||
|
"php": "^8.1",
|
||||||
|
"psr/log": "^2.0|^3.0",
|
||||||
|
"psr/simple-cache": "^1.0|^2.0|^3.0",
|
||||||
|
"symfony/css-selector": "^6.0|^7.0",
|
||||||
|
"symfony/dom-crawler": "^6.0|^7.0"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"friendsofphp/php-cs-fixer": "^3.57",
|
||||||
|
"mockery/mockery": "^1.5",
|
||||||
|
"pestphp/pest": "^2.3|^3.0|^4.0",
|
||||||
|
"phpstan/extension-installer": "^1.1",
|
||||||
|
"phpstan/phpstan": "^1.4|^2.0",
|
||||||
|
"phpstan/phpstan-mockery": "^1.0|^2.0",
|
||||||
|
"phpstan/phpstan-phpunit": "^1.0|^2.0",
|
||||||
|
"spatie/invade": "^2.0",
|
||||||
|
"symfony/process": "^6.0|^7.0"
|
||||||
|
},
|
||||||
|
"suggest": {
|
||||||
|
"ext-zlib": "Needed to uncompress compressed responses",
|
||||||
|
"voku/portable-ascii": "^2.0"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Crwlr\\Crawler\\": "src/"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Christian Olear",
|
||||||
|
"homepage": "https://www.otsch.codes",
|
||||||
|
"role": "Developer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Web crawling and scraping library.",
|
||||||
|
"homepage": "https://www.crwlr.software/packages/crawler",
|
||||||
|
"keywords": [
|
||||||
|
"bot",
|
||||||
|
"crawl",
|
||||||
|
"crawler",
|
||||||
|
"crawling",
|
||||||
|
"crwlr",
|
||||||
|
"scrape",
|
||||||
|
"scraper",
|
||||||
|
"scraping",
|
||||||
|
"web"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"docs": "https://www.crwlr.software/packages/crawler",
|
||||||
|
"issues": "https://github.com/crwlrsoft/crawler/issues",
|
||||||
|
"source": "https://github.com/crwlrsoft/crawler"
|
||||||
|
},
|
||||||
|
"funding": [
|
||||||
|
{
|
||||||
|
"url": "https://github.com/sponsors/otsch",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"time": "2026-01-05T11:13:18+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crwlr/html-2-text",
|
||||||
|
"version": "v0.1.2",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/crwlrsoft/html-2-text.git",
|
||||||
|
"reference": "5109fb909cee28d187ea14523bcdfca03b6e9e72"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/crwlrsoft/html-2-text/zipball/5109fb909cee28d187ea14523bcdfca03b6e9e72",
|
||||||
|
"reference": "5109fb909cee28d187ea14523bcdfca03b6e9e72",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"ext-dom": "*",
|
||||||
|
"ext-iconv": "*",
|
||||||
|
"ext-libxml": "*",
|
||||||
|
"masterminds/html5": "^2.8",
|
||||||
|
"php": "^8.1"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"friendsofphp/php-cs-fixer": "^3.57",
|
||||||
|
"pestphp/pest": "^2.19|^3.0",
|
||||||
|
"phpstan/phpstan": "^1.10"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Crwlr\\Html2Text\\": "src/"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Christian Olear",
|
||||||
|
"homepage": "https://www.otsch.codes",
|
||||||
|
"role": "Developer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Convert HTML to formatted plain text.",
|
||||||
|
"homepage": "https://www.crwlr.software/packages/html-2-text",
|
||||||
|
"keywords": [
|
||||||
|
"converter",
|
||||||
|
"crwlr",
|
||||||
|
"html",
|
||||||
|
"html to text",
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"docs": "https://www.crwlr.software/packages/html-2-text",
|
||||||
|
"issues": "https://github.com/crwlrsoft/html-2-text/issues",
|
||||||
|
"source": "https://github.com/crwlrsoft/html-2-text"
|
||||||
|
},
|
||||||
|
"funding": [
|
||||||
|
{
|
||||||
|
"url": "https://github.com/sponsors/otsch",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"time": "2024-11-06T17:55:46+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crwlr/query-string",
|
||||||
|
"version": "v1.0.3",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/crwlrsoft/query-string.git",
|
||||||
|
"reference": "4e71cae0c942fe1ce2ae22506bd46ecdfa5b4f0e"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/crwlrsoft/query-string/zipball/4e71cae0c942fe1ce2ae22506bd46ecdfa5b4f0e",
|
||||||
|
"reference": "4e71cae0c942fe1ce2ae22506bd46ecdfa5b4f0e",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"php": "^8.0"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"friendsofphp/php-cs-fixer": "^3.8",
|
||||||
|
"pestphp/pest": "^1.21",
|
||||||
|
"phpstan/phpstan": "^1.7"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Crwlr\\QueryString\\": "src/"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Christian Olear",
|
||||||
|
"homepage": "https://www.otsch.codes",
|
||||||
|
"role": "Developer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "A library for convenient handling of query strings used in HTTP requests.",
|
||||||
|
"homepage": "https://www.crwlr.software/packages/query-string",
|
||||||
|
"keywords": [
|
||||||
|
"crwlr",
|
||||||
|
"http",
|
||||||
|
"query",
|
||||||
|
"query string",
|
||||||
|
"query-string",
|
||||||
|
"request",
|
||||||
|
"string"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"docs": "https://www.crwlr.software/packages/query-string",
|
||||||
|
"issues": "https://github.com/crwlrsoft/query-string/issues",
|
||||||
|
"source": "https://github.com/crwlrsoft/query-string"
|
||||||
|
},
|
||||||
|
"time": "2023-04-12T21:31:58+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crwlr/robots-txt",
|
||||||
|
"version": "v1.1.2",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/crwlrsoft/robots-txt.git",
|
||||||
|
"reference": "420aea3518b9a4109defb72940a504d15ea0d26a"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/crwlrsoft/robots-txt/zipball/420aea3518b9a4109defb72940a504d15ea0d26a",
|
||||||
|
"reference": "420aea3518b9a4109defb72940a504d15ea0d26a",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"crwlr/url": "^1.0|^2.0",
|
||||||
|
"php": "^8.0"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"friendsofphp/php-cs-fixer": "^3.57",
|
||||||
|
"mockery/mockery": "^1.4",
|
||||||
|
"phpstan/phpstan": "^1.1",
|
||||||
|
"phpunit/phpunit": "^9.0",
|
||||||
|
"sempro/phpunit-pretty-print": "^1.4"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Crwlr\\RobotsTxt\\": "src"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Christian Olear",
|
||||||
|
"homepage": "https://www.otsch.codes",
|
||||||
|
"role": "Developer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Robots Exclusion Standard/Protocol Parser for Web Crawling/Scraping",
|
||||||
|
"homepage": "https://www.crwlr.software/packages/robots-txt",
|
||||||
|
"keywords": [
|
||||||
|
"bot",
|
||||||
|
"bots",
|
||||||
|
"crawler",
|
||||||
|
"crwlr",
|
||||||
|
"exclusion",
|
||||||
|
"parser",
|
||||||
|
"protocol",
|
||||||
|
"robots",
|
||||||
|
"robots.txt",
|
||||||
|
"scraper",
|
||||||
|
"scraping",
|
||||||
|
"spider",
|
||||||
|
"standard"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"docs": "https://www.crwlr.software/packages/robots-txt",
|
||||||
|
"issues": "https://github.com/crwlrsoft/robots-txt/issues",
|
||||||
|
"source": "https://github.com/crwlrsoft/robots-txt"
|
||||||
|
},
|
||||||
|
"funding": [
|
||||||
|
{
|
||||||
|
"url": "https://github.com/sponsors/otsch",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"time": "2025-01-27T17:32:13+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crwlr/schema-org",
|
||||||
|
"version": "v0.3.4",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/crwlrsoft/schema-org.git",
|
||||||
|
"reference": "d55229470e641d1e3ecda3fc2e8b893d1ba2fad0"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/crwlrsoft/schema-org/zipball/d55229470e641d1e3ecda3fc2e8b893d1ba2fad0",
|
||||||
|
"reference": "d55229470e641d1e3ecda3fc2e8b893d1ba2fad0",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"crwlr/utils": "^1.0",
|
||||||
|
"php": "^8.0",
|
||||||
|
"psr/log": "^2.0|^3.0",
|
||||||
|
"spatie/schema-org": "~3.23.0",
|
||||||
|
"symfony/dom-crawler": "^6.0|^7.0"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"friendsofphp/php-cs-fixer": "^3.57",
|
||||||
|
"pestphp/pest": "^1.22|^2.0|^3.0",
|
||||||
|
"phpstan/phpstan": "^1.8"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Crwlr\\SchemaOrg\\": "src/"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Christian Olear",
|
||||||
|
"homepage": "https://www.otsch.codes",
|
||||||
|
"role": "Developer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Extract schema.org structured data from HTML documents.",
|
||||||
|
"homepage": "https://www.crwlr.software/packages/schema-org",
|
||||||
|
"keywords": [
|
||||||
|
"JSON-LD",
|
||||||
|
"crwlr",
|
||||||
|
"data",
|
||||||
|
"org",
|
||||||
|
"schema",
|
||||||
|
"schema.org",
|
||||||
|
"structured"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"docs": "https://www.crwlr.software/packages/schema-org",
|
||||||
|
"issues": "https://github.com/crwlrsoft/schema-org/issues",
|
||||||
|
"source": "https://github.com/crwlrsoft/schema-org"
|
||||||
|
},
|
||||||
|
"funding": [
|
||||||
|
{
|
||||||
|
"url": "https://github.com/sponsors/otsch",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"time": "2025-04-08T11:22:10+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crwlr/url",
|
||||||
|
"version": "v2.2.0",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/crwlrsoft/url.git",
|
||||||
|
"reference": "227b673b7120197be8961050798426e9628dead3"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/crwlrsoft/url/zipball/227b673b7120197be8961050798426e9628dead3",
|
||||||
|
"reference": "227b673b7120197be8961050798426e9628dead3",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"crwlr/query-string": "^1.0",
|
||||||
|
"php": "^8.0",
|
||||||
|
"psr/http-message": "^1.0 || ^2.0",
|
||||||
|
"symfony/polyfill-intl-idn": "^1.11"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"friendsofphp/php-cs-fixer": "^3.4",
|
||||||
|
"pestphp/pest": "^1.22|^2.0|^3.0|^4.0",
|
||||||
|
"phpstan/phpstan": "^1.8|^2.0"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Crwlr\\Url\\": "src"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Christian Olear",
|
||||||
|
"homepage": "https://www.otsch.codes",
|
||||||
|
"role": "Developer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Swiss Army knife for URLs.",
|
||||||
|
"homepage": "https://www.crwlr.software/packages/url",
|
||||||
|
"keywords": [
|
||||||
|
"UriInterface",
|
||||||
|
"components",
|
||||||
|
"crwlr",
|
||||||
|
"idn",
|
||||||
|
"parse",
|
||||||
|
"parser",
|
||||||
|
"parsing",
|
||||||
|
"psr-7",
|
||||||
|
"rfc3986",
|
||||||
|
"rfc3987",
|
||||||
|
"uri",
|
||||||
|
"url",
|
||||||
|
"urls"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"docs": "https://www.crwlr.software/packages/url",
|
||||||
|
"issues": "https://github.com/crwlrsoft/url/issues",
|
||||||
|
"source": "https://github.com/crwlrsoft/url"
|
||||||
|
},
|
||||||
|
"time": "2026-01-06T19:06:56+00:00"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crwlr/utils",
|
||||||
|
"version": "v1.2.0",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/crwlrsoft/utils.git",
|
||||||
|
"reference": "c8b36051d44aa32f3108bd7408ed294c7eea9aab"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/crwlrsoft/utils/zipball/c8b36051d44aa32f3108bd7408ed294c7eea9aab",
|
||||||
|
"reference": "c8b36051d44aa32f3108bd7408ed294c7eea9aab",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"php": "^8.0"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"friendsofphp/php-cs-fixer": "^3.57",
|
||||||
|
"pestphp/pest": "^1.22|^2.0|^3.0",
|
||||||
|
"phpstan/phpstan": "^1.8"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Crwlr\\Utils\\": "src/"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Christian Olear",
|
||||||
|
"homepage": "https://www.otsch.codes",
|
||||||
|
"role": "Developer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Utilities that are needed in multiple crawler packages.",
|
||||||
|
"keywords": [
|
||||||
|
"crwlr",
|
||||||
|
"json",
|
||||||
|
"utils"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"issues": "https://github.com/crwlrsoft/utils/issues",
|
||||||
|
"source": "https://github.com/crwlrsoft/utils"
|
||||||
|
},
|
||||||
|
"time": "2024-12-08T14:20:52+00:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "danharrin/date-format-converter",
|
"name": "danharrin/date-format-converter",
|
||||||
"version": "v0.3.1",
|
"version": "v0.3.1",
|
||||||
@ -2088,6 +2744,53 @@
|
|||||||
],
|
],
|
||||||
"time": "2025-03-06T22:45:56+00:00"
|
"time": "2025-03-06T22:45:56+00:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "evenement/evenement",
|
||||||
|
"version": "v3.0.2",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/igorw/evenement.git",
|
||||||
|
"reference": "0a16b0d71ab13284339abb99d9d2bd813640efbc"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/igorw/evenement/zipball/0a16b0d71ab13284339abb99d9d2bd813640efbc",
|
||||||
|
"reference": "0a16b0d71ab13284339abb99d9d2bd813640efbc",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"php": ">=7.0"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"phpunit/phpunit": "^9 || ^6"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Evenement\\": "src/"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Igor Wiedler",
|
||||||
|
"email": "igor@wiedler.ch"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Événement is a very simple event dispatching library for PHP",
|
||||||
|
"keywords": [
|
||||||
|
"event-dispatcher",
|
||||||
|
"event-emitter"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"issues": "https://github.com/igorw/evenement/issues",
|
||||||
|
"source": "https://github.com/igorw/evenement/tree/v3.0.2"
|
||||||
|
},
|
||||||
|
"time": "2023-08-08T05:53:35+00:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "ezyang/htmlpurifier",
|
"name": "ezyang/htmlpurifier",
|
||||||
"version": "v4.19.0",
|
"version": "v4.19.0",
|
||||||
@ -8288,6 +8991,79 @@
|
|||||||
],
|
],
|
||||||
"time": "2025-02-20T15:51:22+00:00"
|
"time": "2025-02-20T15:51:22+00:00"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "spatie/schema-org",
|
||||||
|
"version": "3.23.1",
|
||||||
|
"source": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/spatie/schema-org.git",
|
||||||
|
"reference": "a8dc1b6fcdd06afc1ab084c3ead9b7a4c3d7a35d"
|
||||||
|
},
|
||||||
|
"dist": {
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://api.github.com/repos/spatie/schema-org/zipball/a8dc1b6fcdd06afc1ab084c3ead9b7a4c3d7a35d",
|
||||||
|
"reference": "a8dc1b6fcdd06afc1ab084c3ead9b7a4c3d7a35d",
|
||||||
|
"shasum": ""
|
||||||
|
},
|
||||||
|
"require": {
|
||||||
|
"ext-json": "*",
|
||||||
|
"php": "^8.0"
|
||||||
|
},
|
||||||
|
"require-dev": {
|
||||||
|
"friendsofphp/php-cs-fixer": "^3.6",
|
||||||
|
"graham-campbell/analyzer": "^4.2",
|
||||||
|
"illuminate/collections": "^8.62.0",
|
||||||
|
"league/flysystem": "^2.3.0 || ^3.0",
|
||||||
|
"pestphp/pest": "^1.21",
|
||||||
|
"symfony/console": "^5.3.7 || 6.0",
|
||||||
|
"twig/twig": "^3.3.3"
|
||||||
|
},
|
||||||
|
"type": "library",
|
||||||
|
"autoload": {
|
||||||
|
"psr-4": {
|
||||||
|
"Spatie\\SchemaOrg\\": "src"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"notification-url": "https://packagist.org/downloads/",
|
||||||
|
"license": [
|
||||||
|
"MIT"
|
||||||
|
],
|
||||||
|
"authors": [
|
||||||
|
{
|
||||||
|
"name": "Sebastian De Deyne",
|
||||||
|
"email": "sebastian@spatie.be",
|
||||||
|
"homepage": "https://spatie.be",
|
||||||
|
"role": "Developer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Tom Witkowski",
|
||||||
|
"email": "dev.gummibeer@gmail.com",
|
||||||
|
"homepage": "https://gummibeer.de",
|
||||||
|
"role": "Developer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "A fluent builder Schema.org types and ld+json generator",
|
||||||
|
"homepage": "https://github.com/spatie/schema-org",
|
||||||
|
"keywords": [
|
||||||
|
"schema-org",
|
||||||
|
"spatie"
|
||||||
|
],
|
||||||
|
"support": {
|
||||||
|
"issues": "https://github.com/spatie/schema-org/issues",
|
||||||
|
"source": "https://github.com/spatie/schema-org/tree/3.23.1"
|
||||||
|
},
|
||||||
|
"funding": [
|
||||||
|
{
|
||||||
|
"url": "https://spatie.be/open-source/support-us",
|
||||||
|
"type": "custom"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://github.com/spatie",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"time": "2025-01-31T14:54:12+00:00"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "spatie/shiki-php",
|
"name": "spatie/shiki-php",
|
||||||
"version": "2.3.3",
|
"version": "2.3.3",
|
||||||
|
|||||||
@ -3,3 +3,4 @@
|
|||||||
use Illuminate\Support\Facades\Schedule;
|
use Illuminate\Support\Facades\Schedule;
|
||||||
|
|
||||||
Schedule::command('cooperation:update-status')->daily();
|
Schedule::command('cooperation:update-status')->daily();
|
||||||
|
Schedule::command('monitoring:crawl')->dailyAt('12:00');
|
||||||
|
|||||||
135
scripts/crawler.py
Normal file
135
scripts/crawler.py
Normal file
@ -0,0 +1,135 @@
|
|||||||
|
import requests
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import sys
|
||||||
|
import argparse
|
||||||
|
from googlenewsdecoder import gnewsdecoder
|
||||||
|
|
||||||
|
HEADERS = {
|
||||||
|
"User-Agent": "Mozilla/5.0"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ========================
|
||||||
|
# GENERIC PARSER
|
||||||
|
# ========================
|
||||||
|
|
||||||
|
def parse_content(html):
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
|
||||||
|
decompose_tags = [
|
||||||
|
"script", "style", "nav", "footer", "header", "aside",
|
||||||
|
"form", "iframe", "noscript", "svg", "button", "menu",
|
||||||
|
"figure", "ins", "dialog", ".sidebar", "#sidebar"
|
||||||
|
]
|
||||||
|
for tag in soup(decompose_tags):
|
||||||
|
tag.decompose()
|
||||||
|
|
||||||
|
content_tags = ["p", "div", "article", "section", "span", "blockquote", "li", "td", "h1", "h2", "h3", "h4", "h5", "h6"]
|
||||||
|
block_tags = ["p", "div", "article", "section", "ul", "ol", "table", "blockquote"]
|
||||||
|
|
||||||
|
paragraphs = []
|
||||||
|
for tag in soup.find_all(content_tags):
|
||||||
|
if not tag.find(block_tags):
|
||||||
|
text = tag.get_text(separator=" ", strip=True)
|
||||||
|
if len(text) > 50:
|
||||||
|
paragraphs.append(text)
|
||||||
|
|
||||||
|
if not paragraphs:
|
||||||
|
text_blocks = soup.get_text(separator='\n').split('\n')
|
||||||
|
paragraphs = [t.strip() for t in text_blocks if len(t.strip()) > 50]
|
||||||
|
|
||||||
|
seen = set()
|
||||||
|
unique_paragraphs = []
|
||||||
|
for p_text in paragraphs:
|
||||||
|
if p_text not in seen:
|
||||||
|
seen.add(p_text)
|
||||||
|
unique_paragraphs.append(p_text)
|
||||||
|
|
||||||
|
return " ".join(unique_paragraphs)
|
||||||
|
|
||||||
|
# ========================
|
||||||
|
# RSS FETCH
|
||||||
|
# ========================
|
||||||
|
|
||||||
|
def get_rss(keyword):
|
||||||
|
url = f"https://news.google.com/rss/search?q={keyword}&hl=id&gl=ID&ceid=ID:id"
|
||||||
|
|
||||||
|
res = requests.get(url)
|
||||||
|
root = ET.fromstring(res.content)
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"title": item.find("title").text,
|
||||||
|
"link": item.find("link").text,
|
||||||
|
"pubDate": item.find("pubDate").text,
|
||||||
|
"source": item.find("source").text if item.find("source") is not None else "-"
|
||||||
|
}
|
||||||
|
for item in root.findall(".//item")
|
||||||
|
]
|
||||||
|
|
||||||
|
# ========================
|
||||||
|
# RESOLVE URL
|
||||||
|
# ========================
|
||||||
|
|
||||||
|
def resolve_url(url):
|
||||||
|
try:
|
||||||
|
res = requests.get(url, headers=HEADERS, timeout=10)
|
||||||
|
decoded = gnewsdecoder(res.url)
|
||||||
|
|
||||||
|
if decoded and decoded.get("status"):
|
||||||
|
final_url = decoded.get("decoded_url")
|
||||||
|
if final_url and "news.google.com" not in final_url:
|
||||||
|
return final_url
|
||||||
|
|
||||||
|
return None
|
||||||
|
except:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# ========================
|
||||||
|
# CRAWLER
|
||||||
|
# ========================
|
||||||
|
|
||||||
|
def crawl(keyword="purwakarta", limit=5, offset=0):
|
||||||
|
results = []
|
||||||
|
|
||||||
|
for item in get_rss(keyword)[offset:offset+limit]:
|
||||||
|
real_url = resolve_url(item["link"]) or item["link"]
|
||||||
|
|
||||||
|
try:
|
||||||
|
res = requests.get(real_url, headers=HEADERS)
|
||||||
|
content = parse_content(res.text)
|
||||||
|
|
||||||
|
results.append({
|
||||||
|
"title": item["title"],
|
||||||
|
"url": real_url,
|
||||||
|
"published_at": item["pubDate"],
|
||||||
|
"content": content,
|
||||||
|
"keyword": keyword,
|
||||||
|
"source": item["source"]
|
||||||
|
})
|
||||||
|
|
||||||
|
print(f"✔ {item['title']}", file=sys.stderr, flush=True)
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"❌ error: {e}", file=sys.stderr, flush=True)
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
# ========================
|
||||||
|
# MAIN
|
||||||
|
# ========================
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--keyword", type=str, default="purwakarta")
|
||||||
|
parser.add_argument("--limit", type=int, default=1)
|
||||||
|
parser.add_argument("--offset", type=int, default=0)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
print(f"🔎 Crawling for keyword: {args.keyword} (Limit: {args.limit}, Offset: {args.offset})", file=sys.stderr, flush=True)
|
||||||
|
data = crawl(args.keyword, args.limit, args.offset)
|
||||||
|
|
||||||
|
print(json.dumps(data, ensure_ascii=False))
|
||||||
Loading…
Reference in New Issue
Block a user