feat: implement automated media monitoring crawler with Python script and Laravel console command
This commit is contained in:
Yoga Pangestu
parent
ca5aa283d0
commit
4a1993a778
130
app/Console/Commands/CrawlMediaMonitoringCommand.php
Normal file
130
app/Console/Commands/CrawlMediaMonitoringCommand.php
Normal file
@ -0,0 +1,130 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Enums\Channel;
|
||||
use App\Models\MediaMonitoring;
|
||||
use Carbon\Carbon;
|
||||
use Illuminate\Console\Command;
|
||||
use Symfony\Component\Process\Process;
|
||||
|
||||
class CrawlMediaMonitoringCommand extends Command
|
||||
{
|
||||
protected $signature = 'monitoring:crawl {--limit=20 : Maximum number of news to save globally}';
|
||||
|
||||
protected $description = 'Crawl data from Google News RSS based on active Website Themes or specific keywords';
|
||||
|
||||
protected $keywords = [
|
||||
'Purwakarta',
|
||||
'Pemkab Purwakarta',
|
||||
'Bupati Purwakarta',
|
||||
'Pendidikan',
|
||||
'Pariwisata',
|
||||
'Kriminalitas',
|
||||
'Institusi Pemerintah/ Dinas',
|
||||
'Diskominfo Purwakarta',
|
||||
'Om Zein',
|
||||
'Abang Ijo Hapidin',
|
||||
'Wakil Bupati Purwakarta',
|
||||
];
|
||||
|
||||
public function handle()
|
||||
{
|
||||
$globalLimit = (int) $this->option('limit');
|
||||
$totalImported = 0;
|
||||
|
||||
$this->info('Starting crawler engine...');
|
||||
|
||||
$keywordsToCrawl = $this->keywords;
|
||||
shuffle($keywordsToCrawl);
|
||||
|
||||
foreach ($keywordsToCrawl as $kw) {
|
||||
$offset = 0;
|
||||
|
||||
while ($totalImported < $globalLimit) {
|
||||
$remainingLimit = $globalLimit - $totalImported;
|
||||
|
||||
$this->info("\n--- Crawling: $kw (limit: $remainingLimit, offset: $offset) ---");
|
||||
|
||||
$command = ['python3', base_path('scripts/crawler.py'), '--keyword', $kw, '--limit', $remainingLimit, '--offset', $offset];
|
||||
|
||||
$process = new Process($command);
|
||||
$process->setTimeout(600);
|
||||
$process->run(function ($type, $buffer) {
|
||||
if ($type === Process::ERR) {
|
||||
$this->output->write($buffer);
|
||||
}
|
||||
});
|
||||
|
||||
if (! $process->isSuccessful()) {
|
||||
$this->error($process->getErrorOutput());
|
||||
break;
|
||||
}
|
||||
|
||||
$output = $process->getOutput();
|
||||
$newsData = json_decode($output, true);
|
||||
|
||||
if (json_last_error() !== JSON_ERROR_NONE || empty($newsData)) {
|
||||
$this->warn("No more news found for '$kw'.");
|
||||
break;
|
||||
}
|
||||
|
||||
$importedCount = $this->saveToDatabase($newsData, $remainingLimit);
|
||||
$totalImported += $importedCount;
|
||||
|
||||
$offset += $remainingLimit;
|
||||
}
|
||||
|
||||
if ($totalImported >= $globalLimit) {
|
||||
$this->info("\n✅ Reached global limit of $globalLimit new articles. Stopping crawl.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected function saveToDatabase(array $newsData, int $maxToSave)
|
||||
{
|
||||
$count = 0;
|
||||
foreach ($newsData as $item) {
|
||||
if ($count >= $maxToSave) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (MediaMonitoring::where('link', $item['url'])->where('title', $item['title'])->exists()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$host = parse_url($item['url'], PHP_URL_HOST);
|
||||
$mediaName = str_replace('www.', '', $host);
|
||||
|
||||
MediaMonitoring::create([
|
||||
'code' => $this->code(),
|
||||
'media_name' => $mediaName,
|
||||
'title' => $item['title'],
|
||||
'channel' => Channel::WEBSITE,
|
||||
'writter' => $item['source'] ?? '-',
|
||||
'link' => $item['url'],
|
||||
'content' => $item['content'],
|
||||
'keyword' => $item['keyword'],
|
||||
'release_date' => Carbon::parse($item['published_at'])->toDateString(),
|
||||
]);
|
||||
|
||||
$count++;
|
||||
}
|
||||
|
||||
$this->info("Imported $count new articles.");
|
||||
|
||||
return $count;
|
||||
}
|
||||
|
||||
protected function code()
|
||||
{
|
||||
$latestCode = MediaMonitoring::max('code');
|
||||
|
||||
$lastNumber = $latestCode
|
||||
? intval(substr($latestCode, 3))
|
||||
: 0;
|
||||
|
||||
return 'ADN'.str_pad($lastNumber + 1, 6, '0', STR_PAD_LEFT);
|
||||
}
|
||||
}
|
||||
@ -15,6 +15,7 @@
|
||||
"barryvdh/laravel-dompdf": "^3.1",
|
||||
"bezhansalleh/filament-shield": "^4.0",
|
||||
"bilfeldt/laravel-route-statistics": "^4.2",
|
||||
"crwlr/crawler": "^3.5",
|
||||
"filament/filament": "^4.0",
|
||||
"filament/spatie-laravel-media-library-plugin": "^4.0",
|
||||
"filament/spatie-laravel-settings-plugin": "^4.4",
|
||||
|
||||
778
composer.lock
generated
778
composer.lock
generated
@ -4,7 +4,7 @@
|
||||
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
|
||||
"This file is @generated automatically"
|
||||
],
|
||||
"content-hash": "116c8e60c2cac8f2a9bac2ede4d16dea",
|
||||
"content-hash": "b29f415e8e8d01a4f83dfbf06c166e32",
|
||||
"packages": [
|
||||
{
|
||||
"name": "achyutn/filament-log-viewer",
|
||||
@ -91,6 +91,60 @@
|
||||
],
|
||||
"time": "2026-01-30T17:48:57+00:00"
|
||||
},
|
||||
{
|
||||
"name": "adbario/php-dot-notation",
|
||||
"version": "3.5.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/adbario/php-dot-notation.git",
|
||||
"reference": "aa09db5a7a365a8d2d8dd7edfa3cd501d1a8acb0"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/adbario/php-dot-notation/zipball/aa09db5a7a365a8d2d8dd7edfa3cd501d1a8acb0",
|
||||
"reference": "aa09db5a7a365a8d2d8dd7edfa3cd501d1a8acb0",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-json": "*",
|
||||
"php": "^7.4 || ^8.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpstan/phpstan": "^2.1",
|
||||
"phpunit/phpunit": "^9.5",
|
||||
"squizlabs/php_codesniffer": "^3.7"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"files": [
|
||||
"src/helpers.php"
|
||||
],
|
||||
"psr-4": {
|
||||
"Adbar\\": "src"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Riku Särkinen",
|
||||
"email": "riku@adbar.io"
|
||||
}
|
||||
],
|
||||
"description": "PHP dot notation access to arrays",
|
||||
"homepage": "https://github.com/adbario/php-dot-notation",
|
||||
"keywords": [
|
||||
"ArrayAccess",
|
||||
"dotnotation"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/adbario/php-dot-notation/issues",
|
||||
"source": "https://github.com/adbario/php-dot-notation/tree/3.5.0"
|
||||
},
|
||||
"time": "2026-04-04T04:29:49+00:00"
|
||||
},
|
||||
{
|
||||
"name": "anourvalar/eloquent-serialize",
|
||||
"version": "1.3.5",
|
||||
@ -1251,6 +1305,142 @@
|
||||
],
|
||||
"time": "2024-07-16T11:13:48+00:00"
|
||||
},
|
||||
{
|
||||
"name": "chrome-php/chrome",
|
||||
"version": "v1.15.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/chrome-php/chrome.git",
|
||||
"reference": "5b2ee677f34ed4cd9463e49dc59c4989d0fe40f3"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/chrome-php/chrome/zipball/5b2ee677f34ed4cd9463e49dc59c4989d0fe40f3",
|
||||
"reference": "5b2ee677f34ed4cd9463e49dc59c4989d0fe40f3",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"chrome-php/wrench": "^1.8",
|
||||
"evenement/evenement": "^3.0.1",
|
||||
"monolog/monolog": "^1.27.1 || ^2.8 || ^3.2",
|
||||
"php": "^7.4.15 || ^8.0.2",
|
||||
"psr/log": "^1.1 || ^2.0 || ^3.0",
|
||||
"symfony/filesystem": "^5.4 || ^6.0 || ^7.0 || ^8.0",
|
||||
"symfony/polyfill-mbstring": "^1.26",
|
||||
"symfony/process": "^5.4 || ^6.0 || ^7.0 || ^8.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"bamarni/composer-bin-plugin": "^1.8.2",
|
||||
"phpunit/phpunit": "^9.6.3 || ^10.0.12",
|
||||
"symfony/var-dumper": "^5.4 || ^6.0 || ^7.0 || ^8.0"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"bamarni-bin": {
|
||||
"bin-links": true,
|
||||
"forward-command": false
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"HeadlessChromium\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Graham Campbell",
|
||||
"email": "hello@gjcampbell.co.uk",
|
||||
"homepage": "https://github.com/GrahamCampbell"
|
||||
},
|
||||
{
|
||||
"name": "Enrico Dias",
|
||||
"email": "enrico@enricodias.com",
|
||||
"homepage": "https://github.com/enricodias"
|
||||
}
|
||||
],
|
||||
"description": "Instrument headless chrome/chromium instances from PHP",
|
||||
"keywords": [
|
||||
"browser",
|
||||
"chrome",
|
||||
"chromium",
|
||||
"crawl",
|
||||
"headless",
|
||||
"pdf",
|
||||
"puppeteer",
|
||||
"screenshot"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/chrome-php/chrome/issues",
|
||||
"source": "https://github.com/chrome-php/chrome/tree/v1.15.0"
|
||||
},
|
||||
"time": "2025-12-27T12:23:40+00:00"
|
||||
},
|
||||
{
|
||||
"name": "chrome-php/wrench",
|
||||
"version": "v1.8.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/chrome-php/wrench.git",
|
||||
"reference": "bd67a315cf2143ba30598339ad1b6c346db96c99"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/chrome-php/wrench/zipball/bd67a315cf2143ba30598339ad1b6c346db96c99",
|
||||
"reference": "bd67a315cf2143ba30598339ad1b6c346db96c99",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-sockets": "*",
|
||||
"php": "^7.4.15 || ^8.0.2",
|
||||
"psr/log": "^1.1 || ^2.0 || ^3.0",
|
||||
"symfony/polyfill-php80": "^1.26"
|
||||
},
|
||||
"conflict": {
|
||||
"wrench/wrench": "*"
|
||||
},
|
||||
"require-dev": {
|
||||
"bamarni/composer-bin-plugin": "^1.8.2",
|
||||
"phpunit/phpunit": "^9.6.3 || ^10.0.12"
|
||||
},
|
||||
"type": "library",
|
||||
"extra": {
|
||||
"bamarni-bin": {
|
||||
"bin-links": true,
|
||||
"forward-command": false
|
||||
}
|
||||
},
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Wrench\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Graham Campbell",
|
||||
"email": "hello@gjcampbell.co.uk",
|
||||
"homepage": "https://github.com/GrahamCampbell"
|
||||
}
|
||||
],
|
||||
"description": "A simple PHP WebSocket implementation",
|
||||
"keywords": [
|
||||
"WebSockets",
|
||||
"hybi",
|
||||
"websocket"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/chrome-php/wrench/issues",
|
||||
"source": "https://github.com/chrome-php/wrench/tree/v1.8.0"
|
||||
},
|
||||
"time": "2025-12-27T11:29:35+00:00"
|
||||
},
|
||||
{
|
||||
"name": "composer/pcre",
|
||||
"version": "3.3.2",
|
||||
@ -1407,6 +1597,472 @@
|
||||
],
|
||||
"time": "2025-08-20T19:15:30+00:00"
|
||||
},
|
||||
{
|
||||
"name": "crwlr/crawler",
|
||||
"version": "v3.5.6",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/crwlrsoft/crawler.git",
|
||||
"reference": "d6680f9e698a050f3a5eded2fc8c6776d6043a3b"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/crwlrsoft/crawler/zipball/d6680f9e698a050f3a5eded2fc8c6776d6043a3b",
|
||||
"reference": "d6680f9e698a050f3a5eded2fc8c6776d6043a3b",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"adbario/php-dot-notation": "^3.1",
|
||||
"chrome-php/chrome": "^1.7",
|
||||
"crwlr/html-2-text": "^0.1.0",
|
||||
"crwlr/robots-txt": "^1.1",
|
||||
"crwlr/schema-org": "^0.2|^0.3",
|
||||
"crwlr/url": "^2.1",
|
||||
"crwlr/utils": "^1.2",
|
||||
"ext-dom": "*",
|
||||
"guzzlehttp/guzzle": "^7.4",
|
||||
"php": "^8.1",
|
||||
"psr/log": "^2.0|^3.0",
|
||||
"psr/simple-cache": "^1.0|^2.0|^3.0",
|
||||
"symfony/css-selector": "^6.0|^7.0",
|
||||
"symfony/dom-crawler": "^6.0|^7.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"friendsofphp/php-cs-fixer": "^3.57",
|
||||
"mockery/mockery": "^1.5",
|
||||
"pestphp/pest": "^2.3|^3.0|^4.0",
|
||||
"phpstan/extension-installer": "^1.1",
|
||||
"phpstan/phpstan": "^1.4|^2.0",
|
||||
"phpstan/phpstan-mockery": "^1.0|^2.0",
|
||||
"phpstan/phpstan-phpunit": "^1.0|^2.0",
|
||||
"spatie/invade": "^2.0",
|
||||
"symfony/process": "^6.0|^7.0"
|
||||
},
|
||||
"suggest": {
|
||||
"ext-zlib": "Needed to uncompress compressed responses",
|
||||
"voku/portable-ascii": "^2.0"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Crwlr\\Crawler\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Christian Olear",
|
||||
"homepage": "https://www.otsch.codes",
|
||||
"role": "Developer"
|
||||
}
|
||||
],
|
||||
"description": "Web crawling and scraping library.",
|
||||
"homepage": "https://www.crwlr.software/packages/crawler",
|
||||
"keywords": [
|
||||
"bot",
|
||||
"crawl",
|
||||
"crawler",
|
||||
"crawling",
|
||||
"crwlr",
|
||||
"scrape",
|
||||
"scraper",
|
||||
"scraping",
|
||||
"web"
|
||||
],
|
||||
"support": {
|
||||
"docs": "https://www.crwlr.software/packages/crawler",
|
||||
"issues": "https://github.com/crwlrsoft/crawler/issues",
|
||||
"source": "https://github.com/crwlrsoft/crawler"
|
||||
},
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://github.com/sponsors/otsch",
|
||||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2026-01-05T11:13:18+00:00"
|
||||
},
|
||||
{
|
||||
"name": "crwlr/html-2-text",
|
||||
"version": "v0.1.2",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/crwlrsoft/html-2-text.git",
|
||||
"reference": "5109fb909cee28d187ea14523bcdfca03b6e9e72"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/crwlrsoft/html-2-text/zipball/5109fb909cee28d187ea14523bcdfca03b6e9e72",
|
||||
"reference": "5109fb909cee28d187ea14523bcdfca03b6e9e72",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-dom": "*",
|
||||
"ext-iconv": "*",
|
||||
"ext-libxml": "*",
|
||||
"masterminds/html5": "^2.8",
|
||||
"php": "^8.1"
|
||||
},
|
||||
"require-dev": {
|
||||
"friendsofphp/php-cs-fixer": "^3.57",
|
||||
"pestphp/pest": "^2.19|^3.0",
|
||||
"phpstan/phpstan": "^1.10"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Crwlr\\Html2Text\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Christian Olear",
|
||||
"homepage": "https://www.otsch.codes",
|
||||
"role": "Developer"
|
||||
}
|
||||
],
|
||||
"description": "Convert HTML to formatted plain text.",
|
||||
"homepage": "https://www.crwlr.software/packages/html-2-text",
|
||||
"keywords": [
|
||||
"converter",
|
||||
"crwlr",
|
||||
"html",
|
||||
"html to text",
|
||||
"text"
|
||||
],
|
||||
"support": {
|
||||
"docs": "https://www.crwlr.software/packages/html-2-text",
|
||||
"issues": "https://github.com/crwlrsoft/html-2-text/issues",
|
||||
"source": "https://github.com/crwlrsoft/html-2-text"
|
||||
},
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://github.com/sponsors/otsch",
|
||||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2024-11-06T17:55:46+00:00"
|
||||
},
|
||||
{
|
||||
"name": "crwlr/query-string",
|
||||
"version": "v1.0.3",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/crwlrsoft/query-string.git",
|
||||
"reference": "4e71cae0c942fe1ce2ae22506bd46ecdfa5b4f0e"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/crwlrsoft/query-string/zipball/4e71cae0c942fe1ce2ae22506bd46ecdfa5b4f0e",
|
||||
"reference": "4e71cae0c942fe1ce2ae22506bd46ecdfa5b4f0e",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": "^8.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"friendsofphp/php-cs-fixer": "^3.8",
|
||||
"pestphp/pest": "^1.21",
|
||||
"phpstan/phpstan": "^1.7"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Crwlr\\QueryString\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Christian Olear",
|
||||
"homepage": "https://www.otsch.codes",
|
||||
"role": "Developer"
|
||||
}
|
||||
],
|
||||
"description": "A library for convenient handling of query strings used in HTTP requests.",
|
||||
"homepage": "https://www.crwlr.software/packages/query-string",
|
||||
"keywords": [
|
||||
"crwlr",
|
||||
"http",
|
||||
"query",
|
||||
"query string",
|
||||
"query-string",
|
||||
"request",
|
||||
"string"
|
||||
],
|
||||
"support": {
|
||||
"docs": "https://www.crwlr.software/packages/query-string",
|
||||
"issues": "https://github.com/crwlrsoft/query-string/issues",
|
||||
"source": "https://github.com/crwlrsoft/query-string"
|
||||
},
|
||||
"time": "2023-04-12T21:31:58+00:00"
|
||||
},
|
||||
{
|
||||
"name": "crwlr/robots-txt",
|
||||
"version": "v1.1.2",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/crwlrsoft/robots-txt.git",
|
||||
"reference": "420aea3518b9a4109defb72940a504d15ea0d26a"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/crwlrsoft/robots-txt/zipball/420aea3518b9a4109defb72940a504d15ea0d26a",
|
||||
"reference": "420aea3518b9a4109defb72940a504d15ea0d26a",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"crwlr/url": "^1.0|^2.0",
|
||||
"php": "^8.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"friendsofphp/php-cs-fixer": "^3.57",
|
||||
"mockery/mockery": "^1.4",
|
||||
"phpstan/phpstan": "^1.1",
|
||||
"phpunit/phpunit": "^9.0",
|
||||
"sempro/phpunit-pretty-print": "^1.4"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Crwlr\\RobotsTxt\\": "src"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Christian Olear",
|
||||
"homepage": "https://www.otsch.codes",
|
||||
"role": "Developer"
|
||||
}
|
||||
],
|
||||
"description": "Robots Exclusion Standard/Protocol Parser for Web Crawling/Scraping",
|
||||
"homepage": "https://www.crwlr.software/packages/robots-txt",
|
||||
"keywords": [
|
||||
"bot",
|
||||
"bots",
|
||||
"crawler",
|
||||
"crwlr",
|
||||
"exclusion",
|
||||
"parser",
|
||||
"protocol",
|
||||
"robots",
|
||||
"robots.txt",
|
||||
"scraper",
|
||||
"scraping",
|
||||
"spider",
|
||||
"standard"
|
||||
],
|
||||
"support": {
|
||||
"docs": "https://www.crwlr.software/packages/robots-txt",
|
||||
"issues": "https://github.com/crwlrsoft/robots-txt/issues",
|
||||
"source": "https://github.com/crwlrsoft/robots-txt"
|
||||
},
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://github.com/sponsors/otsch",
|
||||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2025-01-27T17:32:13+00:00"
|
||||
},
|
||||
{
|
||||
"name": "crwlr/schema-org",
|
||||
"version": "v0.3.4",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/crwlrsoft/schema-org.git",
|
||||
"reference": "d55229470e641d1e3ecda3fc2e8b893d1ba2fad0"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/crwlrsoft/schema-org/zipball/d55229470e641d1e3ecda3fc2e8b893d1ba2fad0",
|
||||
"reference": "d55229470e641d1e3ecda3fc2e8b893d1ba2fad0",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"crwlr/utils": "^1.0",
|
||||
"php": "^8.0",
|
||||
"psr/log": "^2.0|^3.0",
|
||||
"spatie/schema-org": "~3.23.0",
|
||||
"symfony/dom-crawler": "^6.0|^7.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"friendsofphp/php-cs-fixer": "^3.57",
|
||||
"pestphp/pest": "^1.22|^2.0|^3.0",
|
||||
"phpstan/phpstan": "^1.8"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Crwlr\\SchemaOrg\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Christian Olear",
|
||||
"homepage": "https://www.otsch.codes",
|
||||
"role": "Developer"
|
||||
}
|
||||
],
|
||||
"description": "Extract schema.org structured data from HTML documents.",
|
||||
"homepage": "https://www.crwlr.software/packages/schema-org",
|
||||
"keywords": [
|
||||
"JSON-LD",
|
||||
"crwlr",
|
||||
"data",
|
||||
"org",
|
||||
"schema",
|
||||
"schema.org",
|
||||
"structured"
|
||||
],
|
||||
"support": {
|
||||
"docs": "https://www.crwlr.software/packages/schema-org",
|
||||
"issues": "https://github.com/crwlrsoft/schema-org/issues",
|
||||
"source": "https://github.com/crwlrsoft/schema-org"
|
||||
},
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://github.com/sponsors/otsch",
|
||||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2025-04-08T11:22:10+00:00"
|
||||
},
|
||||
{
|
||||
"name": "crwlr/url",
|
||||
"version": "v2.2.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/crwlrsoft/url.git",
|
||||
"reference": "227b673b7120197be8961050798426e9628dead3"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/crwlrsoft/url/zipball/227b673b7120197be8961050798426e9628dead3",
|
||||
"reference": "227b673b7120197be8961050798426e9628dead3",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"crwlr/query-string": "^1.0",
|
||||
"php": "^8.0",
|
||||
"psr/http-message": "^1.0 || ^2.0",
|
||||
"symfony/polyfill-intl-idn": "^1.11"
|
||||
},
|
||||
"require-dev": {
|
||||
"friendsofphp/php-cs-fixer": "^3.4",
|
||||
"pestphp/pest": "^1.22|^2.0|^3.0|^4.0",
|
||||
"phpstan/phpstan": "^1.8|^2.0"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Crwlr\\Url\\": "src"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Christian Olear",
|
||||
"homepage": "https://www.otsch.codes",
|
||||
"role": "Developer"
|
||||
}
|
||||
],
|
||||
"description": "Swiss Army knife for URLs.",
|
||||
"homepage": "https://www.crwlr.software/packages/url",
|
||||
"keywords": [
|
||||
"UriInterface",
|
||||
"components",
|
||||
"crwlr",
|
||||
"idn",
|
||||
"parse",
|
||||
"parser",
|
||||
"parsing",
|
||||
"psr-7",
|
||||
"rfc3986",
|
||||
"rfc3987",
|
||||
"uri",
|
||||
"url",
|
||||
"urls"
|
||||
],
|
||||
"support": {
|
||||
"docs": "https://www.crwlr.software/packages/url",
|
||||
"issues": "https://github.com/crwlrsoft/url/issues",
|
||||
"source": "https://github.com/crwlrsoft/url"
|
||||
},
|
||||
"time": "2026-01-06T19:06:56+00:00"
|
||||
},
|
||||
{
|
||||
"name": "crwlr/utils",
|
||||
"version": "v1.2.0",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/crwlrsoft/utils.git",
|
||||
"reference": "c8b36051d44aa32f3108bd7408ed294c7eea9aab"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/crwlrsoft/utils/zipball/c8b36051d44aa32f3108bd7408ed294c7eea9aab",
|
||||
"reference": "c8b36051d44aa32f3108bd7408ed294c7eea9aab",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": "^8.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"friendsofphp/php-cs-fixer": "^3.57",
|
||||
"pestphp/pest": "^1.22|^2.0|^3.0",
|
||||
"phpstan/phpstan": "^1.8"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Crwlr\\Utils\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Christian Olear",
|
||||
"homepage": "https://www.otsch.codes",
|
||||
"role": "Developer"
|
||||
}
|
||||
],
|
||||
"description": "Utilities that are needed in multiple crawler packages.",
|
||||
"keywords": [
|
||||
"crwlr",
|
||||
"json",
|
||||
"utils"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/crwlrsoft/utils/issues",
|
||||
"source": "https://github.com/crwlrsoft/utils"
|
||||
},
|
||||
"time": "2024-12-08T14:20:52+00:00"
|
||||
},
|
||||
{
|
||||
"name": "danharrin/date-format-converter",
|
||||
"version": "v0.3.1",
|
||||
@ -2088,6 +2744,53 @@
|
||||
],
|
||||
"time": "2025-03-06T22:45:56+00:00"
|
||||
},
|
||||
{
|
||||
"name": "evenement/evenement",
|
||||
"version": "v3.0.2",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/igorw/evenement.git",
|
||||
"reference": "0a16b0d71ab13284339abb99d9d2bd813640efbc"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/igorw/evenement/zipball/0a16b0d71ab13284339abb99d9d2bd813640efbc",
|
||||
"reference": "0a16b0d71ab13284339abb99d9d2bd813640efbc",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"php": ">=7.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"phpunit/phpunit": "^9 || ^6"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Evenement\\": "src/"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Igor Wiedler",
|
||||
"email": "igor@wiedler.ch"
|
||||
}
|
||||
],
|
||||
"description": "Événement is a very simple event dispatching library for PHP",
|
||||
"keywords": [
|
||||
"event-dispatcher",
|
||||
"event-emitter"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/igorw/evenement/issues",
|
||||
"source": "https://github.com/igorw/evenement/tree/v3.0.2"
|
||||
},
|
||||
"time": "2023-08-08T05:53:35+00:00"
|
||||
},
|
||||
{
|
||||
"name": "ezyang/htmlpurifier",
|
||||
"version": "v4.19.0",
|
||||
@ -8288,6 +8991,79 @@
|
||||
],
|
||||
"time": "2025-02-20T15:51:22+00:00"
|
||||
},
|
||||
{
|
||||
"name": "spatie/schema-org",
|
||||
"version": "3.23.1",
|
||||
"source": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/spatie/schema-org.git",
|
||||
"reference": "a8dc1b6fcdd06afc1ab084c3ead9b7a4c3d7a35d"
|
||||
},
|
||||
"dist": {
|
||||
"type": "zip",
|
||||
"url": "https://api.github.com/repos/spatie/schema-org/zipball/a8dc1b6fcdd06afc1ab084c3ead9b7a4c3d7a35d",
|
||||
"reference": "a8dc1b6fcdd06afc1ab084c3ead9b7a4c3d7a35d",
|
||||
"shasum": ""
|
||||
},
|
||||
"require": {
|
||||
"ext-json": "*",
|
||||
"php": "^8.0"
|
||||
},
|
||||
"require-dev": {
|
||||
"friendsofphp/php-cs-fixer": "^3.6",
|
||||
"graham-campbell/analyzer": "^4.2",
|
||||
"illuminate/collections": "^8.62.0",
|
||||
"league/flysystem": "^2.3.0 || ^3.0",
|
||||
"pestphp/pest": "^1.21",
|
||||
"symfony/console": "^5.3.7 || 6.0",
|
||||
"twig/twig": "^3.3.3"
|
||||
},
|
||||
"type": "library",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Spatie\\SchemaOrg\\": "src"
|
||||
}
|
||||
},
|
||||
"notification-url": "https://packagist.org/downloads/",
|
||||
"license": [
|
||||
"MIT"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Sebastian De Deyne",
|
||||
"email": "sebastian@spatie.be",
|
||||
"homepage": "https://spatie.be",
|
||||
"role": "Developer"
|
||||
},
|
||||
{
|
||||
"name": "Tom Witkowski",
|
||||
"email": "dev.gummibeer@gmail.com",
|
||||
"homepage": "https://gummibeer.de",
|
||||
"role": "Developer"
|
||||
}
|
||||
],
|
||||
"description": "A fluent builder Schema.org types and ld+json generator",
|
||||
"homepage": "https://github.com/spatie/schema-org",
|
||||
"keywords": [
|
||||
"schema-org",
|
||||
"spatie"
|
||||
],
|
||||
"support": {
|
||||
"issues": "https://github.com/spatie/schema-org/issues",
|
||||
"source": "https://github.com/spatie/schema-org/tree/3.23.1"
|
||||
},
|
||||
"funding": [
|
||||
{
|
||||
"url": "https://spatie.be/open-source/support-us",
|
||||
"type": "custom"
|
||||
},
|
||||
{
|
||||
"url": "https://github.com/spatie",
|
||||
"type": "github"
|
||||
}
|
||||
],
|
||||
"time": "2025-01-31T14:54:12+00:00"
|
||||
},
|
||||
{
|
||||
"name": "spatie/shiki-php",
|
||||
"version": "2.3.3",
|
||||
|
||||
@ -3,3 +3,4 @@
|
||||
use Illuminate\Support\Facades\Schedule;
|
||||
|
||||
Schedule::command('cooperation:update-status')->daily();
|
||||
Schedule::command('monitoring:crawl')->dailyAt('12:00');
|
||||
|
||||
135
scripts/crawler.py
Normal file
135
scripts/crawler.py
Normal file
@ -0,0 +1,135 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import xml.etree.ElementTree as ET
|
||||
import json
|
||||
import time
|
||||
import sys
|
||||
import argparse
|
||||
from googlenewsdecoder import gnewsdecoder
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0"
|
||||
}
|
||||
|
||||
# ========================
|
||||
# GENERIC PARSER
|
||||
# ========================
|
||||
|
||||
def parse_content(html):
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
|
||||
decompose_tags = [
|
||||
"script", "style", "nav", "footer", "header", "aside",
|
||||
"form", "iframe", "noscript", "svg", "button", "menu",
|
||||
"figure", "ins", "dialog", ".sidebar", "#sidebar"
|
||||
]
|
||||
for tag in soup(decompose_tags):
|
||||
tag.decompose()
|
||||
|
||||
content_tags = ["p", "div", "article", "section", "span", "blockquote", "li", "td", "h1", "h2", "h3", "h4", "h5", "h6"]
|
||||
block_tags = ["p", "div", "article", "section", "ul", "ol", "table", "blockquote"]
|
||||
|
||||
paragraphs = []
|
||||
for tag in soup.find_all(content_tags):
|
||||
if not tag.find(block_tags):
|
||||
text = tag.get_text(separator=" ", strip=True)
|
||||
if len(text) > 50:
|
||||
paragraphs.append(text)
|
||||
|
||||
if not paragraphs:
|
||||
text_blocks = soup.get_text(separator='\n').split('\n')
|
||||
paragraphs = [t.strip() for t in text_blocks if len(t.strip()) > 50]
|
||||
|
||||
seen = set()
|
||||
unique_paragraphs = []
|
||||
for p_text in paragraphs:
|
||||
if p_text not in seen:
|
||||
seen.add(p_text)
|
||||
unique_paragraphs.append(p_text)
|
||||
|
||||
return " ".join(unique_paragraphs)
|
||||
|
||||
# ========================
|
||||
# RSS FETCH
|
||||
# ========================
|
||||
|
||||
def get_rss(keyword):
|
||||
url = f"https://news.google.com/rss/search?q={keyword}&hl=id&gl=ID&ceid=ID:id"
|
||||
|
||||
res = requests.get(url)
|
||||
root = ET.fromstring(res.content)
|
||||
|
||||
return [
|
||||
{
|
||||
"title": item.find("title").text,
|
||||
"link": item.find("link").text,
|
||||
"pubDate": item.find("pubDate").text,
|
||||
"source": item.find("source").text if item.find("source") is not None else "-"
|
||||
}
|
||||
for item in root.findall(".//item")
|
||||
]
|
||||
|
||||
# ========================
|
||||
# RESOLVE URL
|
||||
# ========================
|
||||
|
||||
def resolve_url(url):
|
||||
try:
|
||||
res = requests.get(url, headers=HEADERS, timeout=10)
|
||||
decoded = gnewsdecoder(res.url)
|
||||
|
||||
if decoded and decoded.get("status"):
|
||||
final_url = decoded.get("decoded_url")
|
||||
if final_url and "news.google.com" not in final_url:
|
||||
return final_url
|
||||
|
||||
return None
|
||||
except:
|
||||
return None
|
||||
|
||||
# ========================
|
||||
# CRAWLER
|
||||
# ========================
|
||||
|
||||
def crawl(keyword="purwakarta", limit=5, offset=0):
|
||||
results = []
|
||||
|
||||
for item in get_rss(keyword)[offset:offset+limit]:
|
||||
real_url = resolve_url(item["link"]) or item["link"]
|
||||
|
||||
try:
|
||||
res = requests.get(real_url, headers=HEADERS)
|
||||
content = parse_content(res.text)
|
||||
|
||||
results.append({
|
||||
"title": item["title"],
|
||||
"url": real_url,
|
||||
"published_at": item["pubDate"],
|
||||
"content": content,
|
||||
"keyword": keyword,
|
||||
"source": item["source"]
|
||||
})
|
||||
|
||||
print(f"✔ {item['title']}", file=sys.stderr, flush=True)
|
||||
time.sleep(1)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ error: {e}", file=sys.stderr, flush=True)
|
||||
|
||||
return results
|
||||
|
||||
# ========================
|
||||
# MAIN
|
||||
# ========================
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--keyword", type=str, default="purwakarta")
|
||||
parser.add_argument("--limit", type=int, default=1)
|
||||
parser.add_argument("--offset", type=int, default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"🔎 Crawling for keyword: {args.keyword} (Limit: {args.limit}, Offset: {args.offset})", file=sys.stderr, flush=True)
|
||||
data = crawl(args.keyword, args.limit, args.offset)
|
||||
|
||||
print(json.dumps(data, ensure_ascii=False))
|
||||
Loading…
Reference in New Issue
Block a user