phase 3

2026-04-03 00:29:05 +03:00
parent 9dcf3c36d7
commit 4e6b636787
6 changed files with 525 additions and 0 deletions
@@ -0,0 +1,102 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Job;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+#[Signature('jobs:deduplicate {--dry-run}')]
+#[Description('Find and remove duplicate job listings based on fingerprint')]
+class DeduplicateJobs extends Command
+{
+    /**
+     * Execute the console command.
+     */
+    public function handle(): int
+    {
+        $dryRun = $this->option('dry-run');
+
+        $this->info('Searching for duplicate job listings...');
+        $this->newLine();
+
+        // Find duplicates by fingerprint
+        $duplicates = $this->findDuplicates();
+
+        if ($duplicates->isEmpty()) {
+            $this->info('No duplicates found!');
+            return self::SUCCESS;
+        }
+
+        $this->warn("Found {$duplicates->count()} sets of duplicates.");
+        $this->newLine();
+
+        $removedCount = 0;
+
+        foreach ($duplicates as $duplicate) {
+            $fingerprint = $duplicate->fingerprint;
+            $count = $duplicate->count;
+
+            // Get all jobs with this fingerprint, ordered by created_at (keep oldest)
+            $jobs = Job::where('fingerprint', $fingerprint)
+                ->orderBy('created_at', 'asc')
+                ->get();
+
+            // Keep the first one, mark others for removal
+            $toKeep = $jobs->first();
+            $toRemove = $jobs->skip(1);
+
+            $this->line("Fingerprint: {$fingerprint}");
+            $this->line("  Keeping: #{$toKeep->id} - {$toKeep->title} ({$toKeep->source->name})");
+
+            foreach ($toRemove as $job) {
+                $this->line("  Removing: #{$job->id} - {$job->title} ({$job->source->name})");
+
+                if (!$dryRun) {
+                    $job->delete();
+                    $removedCount++;
+                }
+            }
+
+            $this->newLine();
+        }
+
+        if ($dryRun) {
+            $this->info("Dry run completed. Would remove {$this->countDuplicatesToRemove($duplicates)} duplicate jobs.");
+            $this->line('Run without --dry-run to actually remove them.');
+        } else {
+            $this->info("Removed {$removedCount} duplicate jobs.");
+        }
+
+        return self::SUCCESS;
+    }
+
+    /**
+     * Find duplicate fingerprints
+     */
+    private function findDuplicates()
+    {
+        return DB::table('jobs')
+            ->select('fingerprint', DB::raw('COUNT(*) as count'))
+            ->whereNull('deleted_at')
+            ->whereNotNull('fingerprint')
+            ->groupBy('fingerprint')
+            ->having('count', '>', 1)
+            ->orderBy('count', 'desc')
+            ->get();
+    }
+
+    /**
+     * Count total duplicates to remove
+     */
+    private function countDuplicatesToRemove($duplicates): int
+    {
+        $total = 0;
+        foreach ($duplicates as $duplicate) {
+            $total += ($duplicate->count - 1); // Keep 1, remove the rest
+        }
+        return $total;
+    }
+}