refactor: optimize news migration with chunking, media caching, and detailed CLI progress reporting
This commit is contained in:
Yoga Pangestu
parent
a331be8dec
commit
28e01c6148
@ -5,7 +5,9 @@
|
|||||||
use App\Enums\NewsStatus;
|
use App\Enums\NewsStatus;
|
||||||
use App\Models\News;
|
use App\Models\News;
|
||||||
use Illuminate\Console\Command;
|
use Illuminate\Console\Command;
|
||||||
|
use Illuminate\Support\Carbon;
|
||||||
use Illuminate\Support\Facades\DB;
|
use Illuminate\Support\Facades\DB;
|
||||||
|
use Illuminate\Support\Facades\Storage;
|
||||||
use Illuminate\Support\Str;
|
use Illuminate\Support\Str;
|
||||||
|
|
||||||
class MigrateNewsCommand extends Command
|
class MigrateNewsCommand extends Command
|
||||||
@ -22,63 +24,154 @@ class MigrateNewsCommand extends Command
|
|||||||
*
|
*
|
||||||
* @var string
|
* @var string
|
||||||
*/
|
*/
|
||||||
protected $description = 'Migrate news data from legacy table where category = 1';
|
protected $description = 'Migrate news data from legacy table with optimized processing and detailed output';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Store migrated news IDs to optimize processing.
|
||||||
|
*/
|
||||||
|
private array $migratedIds = [];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Execute the console command.
|
* Execute the console command.
|
||||||
*/
|
*/
|
||||||
public function handle()
|
public function handle()
|
||||||
{
|
{
|
||||||
$this->info('Starting news migration...');
|
|
||||||
|
|
||||||
$legacyConn = DB::connection('mysql_second');
|
$legacyConn = DB::connection('mysql_second');
|
||||||
|
$query = $legacyConn->table('news')->where('category', '0');
|
||||||
|
|
||||||
// Get legacy news with category 1
|
$totalCount = $query->count();
|
||||||
$legacyNews = $legacyConn->table('news')
|
if ($totalCount === 0) {
|
||||||
->where('category', '0')
|
$this->warn('No news found with category 0.');
|
||||||
->get();
|
|
||||||
|
|
||||||
if ($legacyNews->isEmpty()) {
|
|
||||||
$this->warn('No news found with category 1.');
|
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get a default author ID (prefer Developer or Admin)
|
$this->info("Found {$totalCount} legacy news records.");
|
||||||
$defaultAuthorId = DB::table('users')->first()?->id ?? 1;
|
|
||||||
$this->withProgressBar($legacyNews, function ($legacy) use ($defaultAuthorId) {
|
|
||||||
// Use updateOrInsert for the basic record
|
|
||||||
DB::table('news')->updateOrInsert(
|
|
||||||
['id' => $legacy->id],
|
|
||||||
[
|
|
||||||
'author_id' => $defaultAuthorId,
|
|
||||||
'title' => $legacy->title,
|
|
||||||
'slug' => Str::slug($legacy->title).'-'.$legacy->id,
|
|
||||||
'content' => $legacy->content,
|
|
||||||
'excerpt' => Str::limit(strip_tags($legacy->content), 160),
|
|
||||||
'link' => $legacy->link ? Str::limit($legacy->link, 50, '') : null,
|
|
||||||
'views' => 0,
|
|
||||||
'status' => NewsStatus::PUBLISHED->value,
|
|
||||||
'published_at' => $legacy->created_at,
|
|
||||||
'created_at' => $legacy->created_at ?? now(),
|
|
||||||
'updated_at' => $legacy->updated_at ?? now(),
|
|
||||||
'deleted_at' => $legacy->deleted_at,
|
|
||||||
]
|
|
||||||
);
|
|
||||||
|
|
||||||
// Handle Tags using the Model (if tag exists in legacy)
|
// Cache existing news IDs with media to avoid N+1 media queries
|
||||||
if (! empty($legacy->tag)) {
|
$this->migratedIds = DB::table('media')
|
||||||
$news = News::find($legacy->id);
|
->where('model_type', News::class)
|
||||||
if ($news) {
|
->where('collection_name', 'news')
|
||||||
// split by comma or other delimiter if multiple tags,
|
->pluck('model_id')
|
||||||
// legacy 'tag' column is varchar(20), likely a single tag or comma separated
|
->flip()
|
||||||
$tags = array_filter(array_map('trim', explode(',', $legacy->tag)));
|
->all();
|
||||||
$news->syncTags($tags);
|
|
||||||
}
|
// Process in chunks for memory efficiency
|
||||||
|
$query->orderBy('id')->chunk(50, function ($chunk) {
|
||||||
|
foreach ($chunk as $legacy) {
|
||||||
|
$this->processNews($legacy);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
$this->newLine();
|
|
||||||
|
|
||||||
|
$this->newLine();
|
||||||
$this->info('News migration completed successfully!');
|
$this->info('News migration completed successfully!');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process a single legacy news record.
|
||||||
|
*/
|
||||||
|
private function processNews($legacy): void
|
||||||
|
{
|
||||||
|
$idText = "[#{$legacy->id}]";
|
||||||
|
$idOutput = $legacy->deleted_at ? "<fg=red>{$idText}</>" : "<info>{$idText}</info>";
|
||||||
|
|
||||||
|
$this->output->write("{$idOutput} Processing: <comment>".Str::limit($legacy->title, 40).'</comment>... ');
|
||||||
|
|
||||||
|
// Step 1: Basic data update (Always sync basic info)
|
||||||
|
DB::table('news')->updateOrInsert(
|
||||||
|
['id' => $legacy->id],
|
||||||
|
[
|
||||||
|
'author_id' => $legacy->enhancer,
|
||||||
|
'title' => $legacy->title,
|
||||||
|
'slug' => Str::slug($legacy->title).'-'.$legacy->id,
|
||||||
|
'content' => $legacy->content,
|
||||||
|
'excerpt' => Str::limit(strip_tags($legacy->content), 160),
|
||||||
|
'link' => $legacy->link ? Str::limit($legacy->link, 255, '') : null,
|
||||||
|
'views' => 0,
|
||||||
|
'status' => NewsStatus::PUBLISHED->value,
|
||||||
|
'published_at' => $legacy->created_at,
|
||||||
|
'created_at' => $legacy->created_at ?? now(),
|
||||||
|
'updated_at' => $legacy->updated_at ?? now(),
|
||||||
|
'deleted_at' => $legacy->deleted_at,
|
||||||
|
]
|
||||||
|
);
|
||||||
|
|
||||||
|
// Step 2: Skip heavy operations (tags & media) if record already has media
|
||||||
|
if (isset($this->migratedIds[$legacy->id])) {
|
||||||
|
$this->output->writeln('<info>SKIP</info>');
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$news = News::withTrashed()->find($legacy->id);
|
||||||
|
if (! $news) {
|
||||||
|
$this->output->writeln('<error>NOT FOUND</error>');
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 3: Handle Tags
|
||||||
|
if (! empty($legacy->tag)) {
|
||||||
|
$tags = array_filter(array_map('trim', explode(',', $legacy->tag)));
|
||||||
|
$news->syncTags($tags);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4: Handle Media
|
||||||
|
if (! empty($legacy->image)) {
|
||||||
|
$this->migrateMedia($news, $legacy);
|
||||||
|
} else {
|
||||||
|
$this->output->writeln('<info>DONE</info>');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrate media for a news record.
|
||||||
|
*/
|
||||||
|
private function migrateMedia(News $news, $legacy): void
|
||||||
|
{
|
||||||
|
$filename = trim(basename($legacy->image));
|
||||||
|
$sourcePath = 'old data/news/'.$filename;
|
||||||
|
|
||||||
|
$exists = false;
|
||||||
|
$attempts = 0;
|
||||||
|
$maxAttempts = 3;
|
||||||
|
|
||||||
|
while ($attempts < $maxAttempts) {
|
||||||
|
try {
|
||||||
|
$exists = Storage::disk('s3')->exists($sourcePath);
|
||||||
|
break;
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
$attempts++;
|
||||||
|
if ($attempts >= $maxAttempts) {
|
||||||
|
$actualError = $e->getPrevious() ? $e->getPrevious()->getMessage() : $e->getMessage();
|
||||||
|
$this->output->writeln("<error>S3 ERROR ({$attempts}x): ".$actualError.'</error>');
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
usleep(200000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (! $exists) {
|
||||||
|
$this->output->writeln('<error>IMAGE MISSING</error>');
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
$date = Carbon::parse($legacy->created_at ?? now())->toDateString();
|
||||||
|
|
||||||
|
$news->addMediaFromDisk($sourcePath, 's3')
|
||||||
|
->withCustomProperties([
|
||||||
|
'feature' => 'news',
|
||||||
|
'date' => $date,
|
||||||
|
])
|
||||||
|
->preservingOriginal()
|
||||||
|
->toMediaCollection('news', 's3');
|
||||||
|
|
||||||
|
$this->output->writeln('<info>DONE + IMG</info>');
|
||||||
|
} catch (\Exception $e) {
|
||||||
|
$this->output->writeln("<error>FAILED: {$e->getMessage()}</error>");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user