From 4a1993a7784f065b4bd4ef428b59cd4212b30329 Mon Sep 17 00:00:00 2001
From: Yoga Pangestu <ypangestu976@gmail.com>
Date: Sat, 11 Apr 2026 13:47:18 +0700
Subject: [PATCH] feat: implement automated media monitoring crawler with
 Python script and Laravel console command

---
 .../Commands/CrawlMediaMonitoringCommand.php  | 130 +++
 composer.json                                 |   1 +
 composer.lock                                 | 778 +++++++++++++++++-
 routes/console.php                            |   1 +
 scripts/crawler.py                            | 135 +++
 5 files changed, 1044 insertions(+), 1 deletion(-)
 create mode 100644 app/Console/Commands/CrawlMediaMonitoringCommand.php
 create mode 100644 scripts/crawler.py

diff --git a/app/Console/Commands/CrawlMediaMonitoringCommand.php b/app/Console/Commands/CrawlMediaMonitoringCommand.php
new file mode 100644
index 0000000..731dd70
--- /dev/null
+++ b/app/Console/Commands/CrawlMediaMonitoringCommand.php
@@ -0,0 +1,130 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Enums\Channel;
+use App\Models\MediaMonitoring;
+use Carbon\Carbon;
+use Illuminate\Console\Command;
+use Symfony\Component\Process\Process;
+
+class CrawlMediaMonitoringCommand extends Command
+{
+    protected $signature = 'monitoring:crawl {--limit=20 : Maximum number of news to save globally}';
+
+    protected $description = 'Crawl data from Google News RSS based on active Website Themes or specific keywords';
+
+    protected $keywords = [
+        'Purwakarta',
+        'Pemkab Purwakarta',
+        'Bupati Purwakarta',
+        'Pendidikan',
+        'Pariwisata',
+        'Kriminalitas',
+        'Institusi Pemerintah/ Dinas',
+        'Diskominfo Purwakarta',
+        'Om Zein',
+        'Abang Ijo Hapidin',
+        'Wakil Bupati Purwakarta',
+    ];
+
+    public function handle()
+    {
+        $globalLimit = (int) $this->option('limit');
+        $totalImported = 0;
+
+        $this->info('Starting crawler engine...');
+
+        $keywordsToCrawl = $this->keywords;
+        shuffle($keywordsToCrawl);
+
+        foreach ($keywordsToCrawl as $kw) {
+            $offset = 0;
+
+            while ($totalImported < $globalLimit) {
+                $remainingLimit = $globalLimit - $totalImported;
+
+                $this->info("\n--- Crawling: $kw (limit: $remainingLimit, offset: $offset) ---");
+
+                $command = ['python3', base_path('scripts/crawler.py'), '--keyword', $kw, '--limit', $remainingLimit, '--offset', $offset];
+
+                $process = new Process($command);
+                $process->setTimeout(600);
+                $process->run(function ($type, $buffer) {
+                    if ($type === Process::ERR) {
+                        $this->output->write($buffer);
+                    }
+                });
+
+                if (! $process->isSuccessful()) {
+                    $this->error($process->getErrorOutput());
+                    break;
+                }
+
+                $output = $process->getOutput();
+                $newsData = json_decode($output, true);
+
+                if (json_last_error() !== JSON_ERROR_NONE || empty($newsData)) {
+                    $this->warn("No more news found for '$kw'.");
+                    break;
+                }
+
+                $importedCount = $this->saveToDatabase($newsData, $remainingLimit);
+                $totalImported += $importedCount;
+
+                $offset += $remainingLimit;
+            }
+
+            if ($totalImported >= $globalLimit) {
+                $this->info("\n✅ Reached global limit of $globalLimit new articles. Stopping crawl.");
+                break;
+            }
+        }
+    }
+
+    protected function saveToDatabase(array $newsData, int $maxToSave)
+    {
+        $count = 0;
+        foreach ($newsData as $item) {
+            if ($count >= $maxToSave) {
+                break;
+            }
+
+            if (MediaMonitoring::where('link', $item['url'])->where('title', $item['title'])->exists()) {
+                continue;
+            }
+
+            $host = parse_url($item['url'], PHP_URL_HOST);
+            $mediaName = str_replace('www.', '', $host);
+
+            MediaMonitoring::create([
+                'code' => $this->code(),
+                'media_name' => $mediaName,
+                'title' => $item['title'],
+                'channel' => Channel::WEBSITE,
+                'writter' => $item['source'] ?? '-',
+                'link' => $item['url'],
+                'content' => $item['content'],
+                'keyword' => $item['keyword'],
+                'release_date' => Carbon::parse($item['published_at'])->toDateString(),
+            ]);
+
+            $count++;
+        }
+
+        $this->info("Imported $count new articles.");
+
+        return $count;
+    }
+
+    protected function code()
+    {
+        $latestCode = MediaMonitoring::max('code');
+
+        $lastNumber = $latestCode
+            ? intval(substr($latestCode, 3))
+            : 0;
+
+        return 'ADN'.str_pad($lastNumber + 1, 6, '0', STR_PAD_LEFT);
+    }
+}
diff --git a/composer.json b/composer.json
index 024bc6a..f8713b5 100644
--- a/composer.json
+++ b/composer.json
@@ -15,6 +15,7 @@
         "barryvdh/laravel-dompdf": "^3.1",
         "bezhansalleh/filament-shield": "^4.0",
         "bilfeldt/laravel-route-statistics": "^4.2",
+        "crwlr/crawler": "^3.5",
         "filament/filament": "^4.0",
         "filament/spatie-laravel-media-library-plugin": "^4.0",
         "filament/spatie-laravel-settings-plugin": "^4.4",
diff --git a/composer.lock b/composer.lock
index aa2be4d..01ede7a 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "116c8e60c2cac8f2a9bac2ede4d16dea",
+    "content-hash": "b29f415e8e8d01a4f83dfbf06c166e32",
     "packages": [
         {
             "name": "achyutn/filament-log-viewer",
@@ -91,6 +91,60 @@
             ],
             "time": "2026-01-30T17:48:57+00:00"
         },
+        {
+            "name": "adbario/php-dot-notation",
+            "version": "3.5.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/adbario/php-dot-notation.git",
+                "reference": "aa09db5a7a365a8d2d8dd7edfa3cd501d1a8acb0"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/adbario/php-dot-notation/zipball/aa09db5a7a365a8d2d8dd7edfa3cd501d1a8acb0",
+                "reference": "aa09db5a7a365a8d2d8dd7edfa3cd501d1a8acb0",
+                "shasum": ""
+            },
+            "require": {
+                "ext-json": "*",
+                "php": "^7.4 || ^8.0"
+            },
+            "require-dev": {
+                "phpstan/phpstan": "^2.1",
+                "phpunit/phpunit": "^9.5",
+                "squizlabs/php_codesniffer": "^3.7"
+            },
+            "type": "library",
+            "autoload": {
+                "files": [
+                    "src/helpers.php"
+                ],
+                "psr-4": {
+                    "Adbar\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Riku Särkinen",
+                    "email": "riku@adbar.io"
+                }
+            ],
+            "description": "PHP dot notation access to arrays",
+            "homepage": "https://github.com/adbario/php-dot-notation",
+            "keywords": [
+                "ArrayAccess",
+                "dotnotation"
+            ],
+            "support": {
+                "issues": "https://github.com/adbario/php-dot-notation/issues",
+                "source": "https://github.com/adbario/php-dot-notation/tree/3.5.0"
+            },
+            "time": "2026-04-04T04:29:49+00:00"
+        },
         {
             "name": "anourvalar/eloquent-serialize",
             "version": "1.3.5",
@@ -1251,6 +1305,142 @@
             ],
             "time": "2024-07-16T11:13:48+00:00"
         },
+        {
+            "name": "chrome-php/chrome",
+            "version": "v1.15.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/chrome-php/chrome.git",
+                "reference": "5b2ee677f34ed4cd9463e49dc59c4989d0fe40f3"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/chrome-php/chrome/zipball/5b2ee677f34ed4cd9463e49dc59c4989d0fe40f3",
+                "reference": "5b2ee677f34ed4cd9463e49dc59c4989d0fe40f3",
+                "shasum": ""
+            },
+            "require": {
+                "chrome-php/wrench": "^1.8",
+                "evenement/evenement": "^3.0.1",
+                "monolog/monolog": "^1.27.1 || ^2.8 || ^3.2",
+                "php": "^7.4.15 || ^8.0.2",
+                "psr/log": "^1.1 || ^2.0 || ^3.0",
+                "symfony/filesystem": "^5.4 || ^6.0 || ^7.0 || ^8.0",
+                "symfony/polyfill-mbstring": "^1.26",
+                "symfony/process": "^5.4 || ^6.0 || ^7.0 || ^8.0"
+            },
+            "require-dev": {
+                "bamarni/composer-bin-plugin": "^1.8.2",
+                "phpunit/phpunit": "^9.6.3 || ^10.0.12",
+                "symfony/var-dumper": "^5.4 || ^6.0 || ^7.0 || ^8.0"
+            },
+            "type": "library",
+            "extra": {
+                "bamarni-bin": {
+                    "bin-links": true,
+                    "forward-command": false
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "HeadlessChromium\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Graham Campbell",
+                    "email": "hello@gjcampbell.co.uk",
+                    "homepage": "https://github.com/GrahamCampbell"
+                },
+                {
+                    "name": "Enrico Dias",
+                    "email": "enrico@enricodias.com",
+                    "homepage": "https://github.com/enricodias"
+                }
+            ],
+            "description": "Instrument headless chrome/chromium instances from PHP",
+            "keywords": [
+                "browser",
+                "chrome",
+                "chromium",
+                "crawl",
+                "headless",
+                "pdf",
+                "puppeteer",
+                "screenshot"
+            ],
+            "support": {
+                "issues": "https://github.com/chrome-php/chrome/issues",
+                "source": "https://github.com/chrome-php/chrome/tree/v1.15.0"
+            },
+            "time": "2025-12-27T12:23:40+00:00"
+        },
+        {
+            "name": "chrome-php/wrench",
+            "version": "v1.8.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/chrome-php/wrench.git",
+                "reference": "bd67a315cf2143ba30598339ad1b6c346db96c99"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/chrome-php/wrench/zipball/bd67a315cf2143ba30598339ad1b6c346db96c99",
+                "reference": "bd67a315cf2143ba30598339ad1b6c346db96c99",
+                "shasum": ""
+            },
+            "require": {
+                "ext-sockets": "*",
+                "php": "^7.4.15 || ^8.0.2",
+                "psr/log": "^1.1 || ^2.0 || ^3.0",
+                "symfony/polyfill-php80": "^1.26"
+            },
+            "conflict": {
+                "wrench/wrench": "*"
+            },
+            "require-dev": {
+                "bamarni/composer-bin-plugin": "^1.8.2",
+                "phpunit/phpunit": "^9.6.3 || ^10.0.12"
+            },
+            "type": "library",
+            "extra": {
+                "bamarni-bin": {
+                    "bin-links": true,
+                    "forward-command": false
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "Wrench\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Graham Campbell",
+                    "email": "hello@gjcampbell.co.uk",
+                    "homepage": "https://github.com/GrahamCampbell"
+                }
+            ],
+            "description": "A simple PHP WebSocket implementation",
+            "keywords": [
+                "WebSockets",
+                "hybi",
+                "websocket"
+            ],
+            "support": {
+                "issues": "https://github.com/chrome-php/wrench/issues",
+                "source": "https://github.com/chrome-php/wrench/tree/v1.8.0"
+            },
+            "time": "2025-12-27T11:29:35+00:00"
+        },
         {
             "name": "composer/pcre",
             "version": "3.3.2",
@@ -1407,6 +1597,472 @@
             ],
             "time": "2025-08-20T19:15:30+00:00"
         },
+        {
+            "name": "crwlr/crawler",
+            "version": "v3.5.6",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/crwlrsoft/crawler.git",
+                "reference": "d6680f9e698a050f3a5eded2fc8c6776d6043a3b"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/crwlrsoft/crawler/zipball/d6680f9e698a050f3a5eded2fc8c6776d6043a3b",
+                "reference": "d6680f9e698a050f3a5eded2fc8c6776d6043a3b",
+                "shasum": ""
+            },
+            "require": {
+                "adbario/php-dot-notation": "^3.1",
+                "chrome-php/chrome": "^1.7",
+                "crwlr/html-2-text": "^0.1.0",
+                "crwlr/robots-txt": "^1.1",
+                "crwlr/schema-org": "^0.2|^0.3",
+                "crwlr/url": "^2.1",
+                "crwlr/utils": "^1.2",
+                "ext-dom": "*",
+                "guzzlehttp/guzzle": "^7.4",
+                "php": "^8.1",
+                "psr/log": "^2.0|^3.0",
+                "psr/simple-cache": "^1.0|^2.0|^3.0",
+                "symfony/css-selector": "^6.0|^7.0",
+                "symfony/dom-crawler": "^6.0|^7.0"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^3.57",
+                "mockery/mockery": "^1.5",
+                "pestphp/pest": "^2.3|^3.0|^4.0",
+                "phpstan/extension-installer": "^1.1",
+                "phpstan/phpstan": "^1.4|^2.0",
+                "phpstan/phpstan-mockery": "^1.0|^2.0",
+                "phpstan/phpstan-phpunit": "^1.0|^2.0",
+                "spatie/invade": "^2.0",
+                "symfony/process": "^6.0|^7.0"
+            },
+            "suggest": {
+                "ext-zlib": "Needed to uncompress compressed responses",
+                "voku/portable-ascii": "^2.0"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Crwlr\\Crawler\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Christian Olear",
+                    "homepage": "https://www.otsch.codes",
+                    "role": "Developer"
+                }
+            ],
+            "description": "Web crawling and scraping library.",
+            "homepage": "https://www.crwlr.software/packages/crawler",
+            "keywords": [
+                "bot",
+                "crawl",
+                "crawler",
+                "crawling",
+                "crwlr",
+                "scrape",
+                "scraper",
+                "scraping",
+                "web"
+            ],
+            "support": {
+                "docs": "https://www.crwlr.software/packages/crawler",
+                "issues": "https://github.com/crwlrsoft/crawler/issues",
+                "source": "https://github.com/crwlrsoft/crawler"
+            },
+            "funding": [
+                {
+                    "url": "https://github.com/sponsors/otsch",
+                    "type": "github"
+                }
+            ],
+            "time": "2026-01-05T11:13:18+00:00"
+        },
+        {
+            "name": "crwlr/html-2-text",
+            "version": "v0.1.2",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/crwlrsoft/html-2-text.git",
+                "reference": "5109fb909cee28d187ea14523bcdfca03b6e9e72"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/crwlrsoft/html-2-text/zipball/5109fb909cee28d187ea14523bcdfca03b6e9e72",
+                "reference": "5109fb909cee28d187ea14523bcdfca03b6e9e72",
+                "shasum": ""
+            },
+            "require": {
+                "ext-dom": "*",
+                "ext-iconv": "*",
+                "ext-libxml": "*",
+                "masterminds/html5": "^2.8",
+                "php": "^8.1"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^3.57",
+                "pestphp/pest": "^2.19|^3.0",
+                "phpstan/phpstan": "^1.10"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Crwlr\\Html2Text\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Christian Olear",
+                    "homepage": "https://www.otsch.codes",
+                    "role": "Developer"
+                }
+            ],
+            "description": "Convert HTML to formatted plain text.",
+            "homepage": "https://www.crwlr.software/packages/html-2-text",
+            "keywords": [
+                "converter",
+                "crwlr",
+                "html",
+                "html to text",
+                "text"
+            ],
+            "support": {
+                "docs": "https://www.crwlr.software/packages/html-2-text",
+                "issues": "https://github.com/crwlrsoft/html-2-text/issues",
+                "source": "https://github.com/crwlrsoft/html-2-text"
+            },
+            "funding": [
+                {
+                    "url": "https://github.com/sponsors/otsch",
+                    "type": "github"
+                }
+            ],
+            "time": "2024-11-06T17:55:46+00:00"
+        },
+        {
+            "name": "crwlr/query-string",
+            "version": "v1.0.3",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/crwlrsoft/query-string.git",
+                "reference": "4e71cae0c942fe1ce2ae22506bd46ecdfa5b4f0e"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/crwlrsoft/query-string/zipball/4e71cae0c942fe1ce2ae22506bd46ecdfa5b4f0e",
+                "reference": "4e71cae0c942fe1ce2ae22506bd46ecdfa5b4f0e",
+                "shasum": ""
+            },
+            "require": {
+                "php": "^8.0"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^3.8",
+                "pestphp/pest": "^1.21",
+                "phpstan/phpstan": "^1.7"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Crwlr\\QueryString\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Christian Olear",
+                    "homepage": "https://www.otsch.codes",
+                    "role": "Developer"
+                }
+            ],
+            "description": "A library for convenient handling of query strings used in HTTP requests.",
+            "homepage": "https://www.crwlr.software/packages/query-string",
+            "keywords": [
+                "crwlr",
+                "http",
+                "query",
+                "query string",
+                "query-string",
+                "request",
+                "string"
+            ],
+            "support": {
+                "docs": "https://www.crwlr.software/packages/query-string",
+                "issues": "https://github.com/crwlrsoft/query-string/issues",
+                "source": "https://github.com/crwlrsoft/query-string"
+            },
+            "time": "2023-04-12T21:31:58+00:00"
+        },
+        {
+            "name": "crwlr/robots-txt",
+            "version": "v1.1.2",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/crwlrsoft/robots-txt.git",
+                "reference": "420aea3518b9a4109defb72940a504d15ea0d26a"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/crwlrsoft/robots-txt/zipball/420aea3518b9a4109defb72940a504d15ea0d26a",
+                "reference": "420aea3518b9a4109defb72940a504d15ea0d26a",
+                "shasum": ""
+            },
+            "require": {
+                "crwlr/url": "^1.0|^2.0",
+                "php": "^8.0"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^3.57",
+                "mockery/mockery": "^1.4",
+                "phpstan/phpstan": "^1.1",
+                "phpunit/phpunit": "^9.0",
+                "sempro/phpunit-pretty-print": "^1.4"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Crwlr\\RobotsTxt\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Christian Olear",
+                    "homepage": "https://www.otsch.codes",
+                    "role": "Developer"
+                }
+            ],
+            "description": "Robots Exclusion Standard/Protocol Parser for Web Crawling/Scraping",
+            "homepage": "https://www.crwlr.software/packages/robots-txt",
+            "keywords": [
+                "bot",
+                "bots",
+                "crawler",
+                "crwlr",
+                "exclusion",
+                "parser",
+                "protocol",
+                "robots",
+                "robots.txt",
+                "scraper",
+                "scraping",
+                "spider",
+                "standard"
+            ],
+            "support": {
+                "docs": "https://www.crwlr.software/packages/robots-txt",
+                "issues": "https://github.com/crwlrsoft/robots-txt/issues",
+                "source": "https://github.com/crwlrsoft/robots-txt"
+            },
+            "funding": [
+                {
+                    "url": "https://github.com/sponsors/otsch",
+                    "type": "github"
+                }
+            ],
+            "time": "2025-01-27T17:32:13+00:00"
+        },
+        {
+            "name": "crwlr/schema-org",
+            "version": "v0.3.4",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/crwlrsoft/schema-org.git",
+                "reference": "d55229470e641d1e3ecda3fc2e8b893d1ba2fad0"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/crwlrsoft/schema-org/zipball/d55229470e641d1e3ecda3fc2e8b893d1ba2fad0",
+                "reference": "d55229470e641d1e3ecda3fc2e8b893d1ba2fad0",
+                "shasum": ""
+            },
+            "require": {
+                "crwlr/utils": "^1.0",
+                "php": "^8.0",
+                "psr/log": "^2.0|^3.0",
+                "spatie/schema-org": "~3.23.0",
+                "symfony/dom-crawler": "^6.0|^7.0"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^3.57",
+                "pestphp/pest": "^1.22|^2.0|^3.0",
+                "phpstan/phpstan": "^1.8"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Crwlr\\SchemaOrg\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Christian Olear",
+                    "homepage": "https://www.otsch.codes",
+                    "role": "Developer"
+                }
+            ],
+            "description": "Extract schema.org structured data from HTML documents.",
+            "homepage": "https://www.crwlr.software/packages/schema-org",
+            "keywords": [
+                "JSON-LD",
+                "crwlr",
+                "data",
+                "org",
+                "schema",
+                "schema.org",
+                "structured"
+            ],
+            "support": {
+                "docs": "https://www.crwlr.software/packages/schema-org",
+                "issues": "https://github.com/crwlrsoft/schema-org/issues",
+                "source": "https://github.com/crwlrsoft/schema-org"
+            },
+            "funding": [
+                {
+                    "url": "https://github.com/sponsors/otsch",
+                    "type": "github"
+                }
+            ],
+            "time": "2025-04-08T11:22:10+00:00"
+        },
+        {
+            "name": "crwlr/url",
+            "version": "v2.2.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/crwlrsoft/url.git",
+                "reference": "227b673b7120197be8961050798426e9628dead3"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/crwlrsoft/url/zipball/227b673b7120197be8961050798426e9628dead3",
+                "reference": "227b673b7120197be8961050798426e9628dead3",
+                "shasum": ""
+            },
+            "require": {
+                "crwlr/query-string": "^1.0",
+                "php": "^8.0",
+                "psr/http-message": "^1.0 || ^2.0",
+                "symfony/polyfill-intl-idn": "^1.11"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^3.4",
+                "pestphp/pest": "^1.22|^2.0|^3.0|^4.0",
+                "phpstan/phpstan": "^1.8|^2.0"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Crwlr\\Url\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Christian Olear",
+                    "homepage": "https://www.otsch.codes",
+                    "role": "Developer"
+                }
+            ],
+            "description": "Swiss Army knife for URLs.",
+            "homepage": "https://www.crwlr.software/packages/url",
+            "keywords": [
+                "UriInterface",
+                "components",
+                "crwlr",
+                "idn",
+                "parse",
+                "parser",
+                "parsing",
+                "psr-7",
+                "rfc3986",
+                "rfc3987",
+                "uri",
+                "url",
+                "urls"
+            ],
+            "support": {
+                "docs": "https://www.crwlr.software/packages/url",
+                "issues": "https://github.com/crwlrsoft/url/issues",
+                "source": "https://github.com/crwlrsoft/url"
+            },
+            "time": "2026-01-06T19:06:56+00:00"
+        },
+        {
+            "name": "crwlr/utils",
+            "version": "v1.2.0",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/crwlrsoft/utils.git",
+                "reference": "c8b36051d44aa32f3108bd7408ed294c7eea9aab"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/crwlrsoft/utils/zipball/c8b36051d44aa32f3108bd7408ed294c7eea9aab",
+                "reference": "c8b36051d44aa32f3108bd7408ed294c7eea9aab",
+                "shasum": ""
+            },
+            "require": {
+                "php": "^8.0"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^3.57",
+                "pestphp/pest": "^1.22|^2.0|^3.0",
+                "phpstan/phpstan": "^1.8"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Crwlr\\Utils\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Christian Olear",
+                    "homepage": "https://www.otsch.codes",
+                    "role": "Developer"
+                }
+            ],
+            "description": "Utilities that are needed in multiple crawler packages.",
+            "keywords": [
+                "crwlr",
+                "json",
+                "utils"
+            ],
+            "support": {
+                "issues": "https://github.com/crwlrsoft/utils/issues",
+                "source": "https://github.com/crwlrsoft/utils"
+            },
+            "time": "2024-12-08T14:20:52+00:00"
+        },
         {
             "name": "danharrin/date-format-converter",
             "version": "v0.3.1",
@@ -2088,6 +2744,53 @@
             ],
             "time": "2025-03-06T22:45:56+00:00"
         },
+        {
+            "name": "evenement/evenement",
+            "version": "v3.0.2",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/igorw/evenement.git",
+                "reference": "0a16b0d71ab13284339abb99d9d2bd813640efbc"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/igorw/evenement/zipball/0a16b0d71ab13284339abb99d9d2bd813640efbc",
+                "reference": "0a16b0d71ab13284339abb99d9d2bd813640efbc",
+                "shasum": ""
+            },
+            "require": {
+                "php": ">=7.0"
+            },
+            "require-dev": {
+                "phpunit/phpunit": "^9 || ^6"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Evenement\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Igor Wiedler",
+                    "email": "igor@wiedler.ch"
+                }
+            ],
+            "description": "Événement is a very simple event dispatching library for PHP",
+            "keywords": [
+                "event-dispatcher",
+                "event-emitter"
+            ],
+            "support": {
+                "issues": "https://github.com/igorw/evenement/issues",
+                "source": "https://github.com/igorw/evenement/tree/v3.0.2"
+            },
+            "time": "2023-08-08T05:53:35+00:00"
+        },
         {
             "name": "ezyang/htmlpurifier",
             "version": "v4.19.0",
@@ -8288,6 +8991,79 @@
             ],
             "time": "2025-02-20T15:51:22+00:00"
         },
+        {
+            "name": "spatie/schema-org",
+            "version": "3.23.1",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/spatie/schema-org.git",
+                "reference": "a8dc1b6fcdd06afc1ab084c3ead9b7a4c3d7a35d"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/spatie/schema-org/zipball/a8dc1b6fcdd06afc1ab084c3ead9b7a4c3d7a35d",
+                "reference": "a8dc1b6fcdd06afc1ab084c3ead9b7a4c3d7a35d",
+                "shasum": ""
+            },
+            "require": {
+                "ext-json": "*",
+                "php": "^8.0"
+            },
+            "require-dev": {
+                "friendsofphp/php-cs-fixer": "^3.6",
+                "graham-campbell/analyzer": "^4.2",
+                "illuminate/collections": "^8.62.0",
+                "league/flysystem": "^2.3.0 || ^3.0",
+                "pestphp/pest": "^1.21",
+                "symfony/console": "^5.3.7 || 6.0",
+                "twig/twig": "^3.3.3"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Spatie\\SchemaOrg\\": "src"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Sebastian De Deyne",
+                    "email": "sebastian@spatie.be",
+                    "homepage": "https://spatie.be",
+                    "role": "Developer"
+                },
+                {
+                    "name": "Tom Witkowski",
+                    "email": "dev.gummibeer@gmail.com",
+                    "homepage": "https://gummibeer.de",
+                    "role": "Developer"
+                }
+            ],
+            "description": "A fluent builder Schema.org types and ld+json generator",
+            "homepage": "https://github.com/spatie/schema-org",
+            "keywords": [
+                "schema-org",
+                "spatie"
+            ],
+            "support": {
+                "issues": "https://github.com/spatie/schema-org/issues",
+                "source": "https://github.com/spatie/schema-org/tree/3.23.1"
+            },
+            "funding": [
+                {
+                    "url": "https://spatie.be/open-source/support-us",
+                    "type": "custom"
+                },
+                {
+                    "url": "https://github.com/spatie",
+                    "type": "github"
+                }
+            ],
+            "time": "2025-01-31T14:54:12+00:00"
+        },
         {
             "name": "spatie/shiki-php",
             "version": "2.3.3",
diff --git a/routes/console.php b/routes/console.php
index 55fd9db..31c5ba7 100644
--- a/routes/console.php
+++ b/routes/console.php
@@ -3,3 +3,4 @@
 use Illuminate\Support\Facades\Schedule;
 
 Schedule::command('cooperation:update-status')->daily();
+Schedule::command('monitoring:crawl')->dailyAt('12:00');
diff --git a/scripts/crawler.py b/scripts/crawler.py
new file mode 100644
index 0000000..1c329ac
--- /dev/null
+++ b/scripts/crawler.py
@@ -0,0 +1,135 @@
+import requests
+from bs4 import BeautifulSoup
+import xml.etree.ElementTree as ET
+import json
+import time
+import sys
+import argparse
+from googlenewsdecoder import gnewsdecoder
+
+HEADERS = {
+    "User-Agent": "Mozilla/5.0"
+}
+
+# ========================
+# GENERIC PARSER
+# ========================
+
+def parse_content(html):
+    soup = BeautifulSoup(html, "html.parser")
+
+    decompose_tags = [
+        "script", "style", "nav", "footer", "header", "aside", 
+        "form", "iframe", "noscript", "svg", "button", "menu",
+        "figure", "ins", "dialog", ".sidebar", "#sidebar"
+    ]
+    for tag in soup(decompose_tags):
+        tag.decompose()
+
+    content_tags = ["p", "div", "article", "section", "span", "blockquote", "li", "td", "h1", "h2", "h3", "h4", "h5", "h6"]
+    block_tags = ["p", "div", "article", "section", "ul", "ol", "table", "blockquote"]
+    
+    paragraphs = []
+    for tag in soup.find_all(content_tags):
+        if not tag.find(block_tags):
+            text = tag.get_text(separator=" ", strip=True)
+            if len(text) > 50:
+                paragraphs.append(text)
+
+    if not paragraphs:
+        text_blocks = soup.get_text(separator='\n').split('\n')
+        paragraphs = [t.strip() for t in text_blocks if len(t.strip()) > 50]
+
+    seen = set()
+    unique_paragraphs = []
+    for p_text in paragraphs:
+        if p_text not in seen:
+            seen.add(p_text)
+            unique_paragraphs.append(p_text)
+
+    return " ".join(unique_paragraphs)
+
+# ========================
+# RSS FETCH
+# ========================
+
+def get_rss(keyword):
+    url = f"https://news.google.com/rss/search?q={keyword}&hl=id&gl=ID&ceid=ID:id"
+    
+    res = requests.get(url)
+    root = ET.fromstring(res.content)
+
+    return [
+        {
+            "title": item.find("title").text,
+            "link": item.find("link").text,
+            "pubDate": item.find("pubDate").text,
+            "source": item.find("source").text if item.find("source") is not None else "-"
+        }
+        for item in root.findall(".//item")
+    ]
+
+# ========================
+# RESOLVE URL
+# ========================
+
+def resolve_url(url):
+    try:
+        res = requests.get(url, headers=HEADERS, timeout=10)
+        decoded = gnewsdecoder(res.url)
+
+        if decoded and decoded.get("status"):
+            final_url = decoded.get("decoded_url")
+            if final_url and "news.google.com" not in final_url:
+                return final_url
+
+        return None
+    except:
+        return None
+
+# ========================
+# CRAWLER
+# ========================
+
+def crawl(keyword="purwakarta", limit=5, offset=0):
+    results = []
+
+    for item in get_rss(keyword)[offset:offset+limit]:
+        real_url = resolve_url(item["link"]) or item["link"]
+
+        try:
+            res = requests.get(real_url, headers=HEADERS)
+            content = parse_content(res.text)
+
+            results.append({
+                "title": item["title"],
+                "url": real_url,
+                "published_at": item["pubDate"],
+                "content": content,
+                "keyword": keyword,
+                "source": item["source"]
+            })
+
+            print(f"✔ {item['title']}", file=sys.stderr, flush=True)
+            time.sleep(1)
+
+        except Exception as e:
+            print(f"❌ error: {e}", file=sys.stderr, flush=True)
+
+    return results
+
+# ========================
+# MAIN
+# ========================
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--keyword", type=str, default="purwakarta")
+    parser.add_argument("--limit", type=int, default=1)
+    parser.add_argument("--offset", type=int, default=0)
+    args = parser.parse_args()
+
+    print(f"🔎 Crawling for keyword: {args.keyword} (Limit: {args.limit}, Offset: {args.offset})", file=sys.stderr, flush=True)
+    data = crawl(args.keyword, args.limit, args.offset)
+
+    print(json.dumps(data, ensure_ascii=False))
\ No newline at end of file