option('dry-run'); $log = Log::channel('jobs_deduplicate'); $log->info('Starting deduplication', ['dry_run' => $dryRun]); $this->info('Searching for duplicate job listings...'); $this->newLine(); // Find duplicates by fingerprint $duplicates = $this->findDuplicates(); if ($duplicates->isEmpty()) { $this->info('No duplicates found!'); $log->info('No duplicates found'); return self::SUCCESS; } $this->warn("Found {$duplicates->count()} sets of duplicates."); $this->newLine(); $removedCount = 0; foreach ($duplicates as $duplicate) { $fingerprint = $duplicate->fingerprint; $count = $duplicate->count; // Get all jobs with this fingerprint, ordered by created_at (keep oldest) $jobs = Job::where('fingerprint', $fingerprint) ->orderBy('created_at', 'asc') ->get(); // Keep the first one, mark others for removal $toKeep = $jobs->first(); $toRemove = $jobs->skip(1); $this->line("Fingerprint: {$fingerprint}"); $this->line(" Keeping: #{$toKeep->id} - {$toKeep->title} ({$toKeep->source->name})"); foreach ($toRemove as $job) { $this->line(" Removing: #{$job->id} - {$job->title} ({$job->source->name})"); if (!$dryRun) { $job->delete(); $removedCount++; } } $this->newLine(); } if ($dryRun) { $wouldRemove = $this->countDuplicatesToRemove($duplicates); $this->info("Dry run completed. Would remove {$wouldRemove} duplicate jobs."); $this->line('Run without --dry-run to actually remove them.'); $log->info('Dry run complete', ['would_remove' => $wouldRemove]); } else { $this->info("Removed {$removedCount} duplicate jobs."); $log->info('Deduplication complete', ['removed' => $removedCount]); } return self::SUCCESS; } /** * Find duplicate fingerprints */ private function findDuplicates() { return DB::table('jobs') ->select('fingerprint', DB::raw('COUNT(*) as count')) ->whereNull('deleted_at') ->whereNotNull('fingerprint') ->groupBy('fingerprint') ->having('count', '>', 1) ->orderBy('count', 'desc') ->get(); } /** * Count total duplicates to remove */ private function countDuplicatesToRemove($duplicates): int { $total = 0; foreach ($duplicates as $duplicate) { $total += ($duplicate->count - 1); // Keep 1, remove the rest } return $total; } }