jobifygarden/app/Console/Commands/DeduplicateJobs.php

<?php

namespace App\Console\Commands;

use App\Models\Job;
use Illuminate\Console\Attributes\Description;
use Illuminate\Console\Attributes\Signature;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;

#[Signature('jobs:deduplicate {--dry-run}')]
#[Description('Find and remove duplicate job listings based on fingerprint')]
class DeduplicateJobs extends Command
{
    protected $signature = 'jobs:deduplicate {--dry-run}';
    protected $description = 'Find and remove duplicate job listings based on fingerprint';

    /**
     * Execute the console command.
     */
    public function handle(): int
    {
        $dryRun = $this->option('dry-run');

        $this->info('Searching for duplicate job listings...');
        $this->newLine();

        // Find duplicates by fingerprint
        $duplicates = $this->findDuplicates();

        if ($duplicates->isEmpty()) {
            $this->info('No duplicates found!');
            return self::SUCCESS;
        }

        $this->warn("Found {$duplicates->count()} sets of duplicates.");
        $this->newLine();

        $removedCount = 0;

        foreach ($duplicates as $duplicate) {
            $fingerprint = $duplicate->fingerprint;
            $count = $duplicate->count;

            // Get all jobs with this fingerprint, ordered by created_at (keep oldest)
            $jobs = Job::where('fingerprint', $fingerprint)
                ->orderBy('created_at', 'asc')
                ->get();

            // Keep the first one, mark others for removal
            $toKeep = $jobs->first();
            $toRemove = $jobs->skip(1);

            $this->line("Fingerprint: {$fingerprint}");
            $this->line("  Keeping: #{$toKeep->id} - {$toKeep->title} ({$toKeep->source->name})");

            foreach ($toRemove as $job) {
                $this->line("  Removing: #{$job->id} - {$job->title} ({$job->source->name})");

                if (!$dryRun) {
                    $job->delete();
                    $removedCount++;
                }
            }

            $this->newLine();
        }

        if ($dryRun) {
            $this->info("Dry run completed. Would remove {$this->countDuplicatesToRemove($duplicates)} duplicate jobs.");
            $this->line('Run without --dry-run to actually remove them.');
        } else {
            $this->info("Removed {$removedCount} duplicate jobs.");
        }

        return self::SUCCESS;
    }

    /**
     * Find duplicate fingerprints
     */
    private function findDuplicates()
    {
        return DB::table('jobs')
            ->select('fingerprint', DB::raw('COUNT(*) as count'))
            ->whereNull('deleted_at')
            ->whereNotNull('fingerprint')
            ->groupBy('fingerprint')
            ->having('count', '>', 1)
            ->orderBy('count', 'desc')
            ->get();
    }

    /**
     * Count total duplicates to remove
     */
    private function countDuplicatesToRemove($duplicates): int
    {
        $total = 0;
        foreach ($duplicates as $duplicate) {
            $total += ($duplicate->count - 1); // Keep 1, remove the rest
        }
        return $total;
    }
}