phase 3

app/Console/Commands/CleanJobs.php

@@ -0,0 +1,114 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Job;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+use Illuminate\Console\Command;
+
+#[Signature('jobs:clean {--days=30} {--force}')]
+#[Description('Remove expired and old job listings from the database')]
+class CleanJobs extends Command
+{
+    /**
+     * Execute the console command.
+     */
+    public function handle(): int
+    {
+        $days = (int) $this->option('days');
+        $force = $this->option('force');
+
+        $this->info("Cleaning job listings...");
+        $this->newLine();
+
+        // Remove expired jobs
+        $expiredCount = $this->removeExpiredJobs($force);
+
+        // Remove old jobs (not posted in the last X days)
+        $oldCount = $this->removeOldJobs($days, $force);
+
+        // Clean up soft-deleted jobs
+        $trashedCount = $this->cleanTrashedJobs($force);
+
+        $this->newLine();
+        $this->info("Cleanup Summary:");
+        $this->line("  • Expired jobs removed: {$expiredCount}");
+        $this->line("  • Old jobs removed (>{$days} days): {$oldCount}");
+        $this->line("  • Trashed jobs cleaned: {$trashedCount}");
+        $this->line("  • Total removed: " . ($expiredCount + $oldCount + $trashedCount));
+
+        return self::SUCCESS;
+    }
+
+    private function removeExpiredJobs(bool $force): int
+    {
+        $query = Job::whereNotNull('expires_at')
+            ->where('expires_at', '<', now());
+
+        $count = $query->count();
+
+        if ($count === 0) {
+            return 0;
+        }
+
+        if (!$force) {
+            $this->line("Found {$count} expired jobs (soft deleting)...");
+            $query->delete(); // Soft delete
+        } else {
+            $this->line("Found {$count} expired jobs (force deleting)...");
+            $query->forceDelete(); // Permanent delete
+        }
+
+        return $count;
+    }
+
+    private function removeOldJobs(int $days, bool $force): int
+    {
+        $cutoffDate = now()->subDays($days);
+
+        $query = Job::where(function ($q) use ($cutoffDate) {
+            $q->where('posted_at', '<', $cutoffDate)
+              ->orWhere(function ($q2) use ($cutoffDate) {
+                  $q2->whereNull('posted_at')
+                     ->where('created_at', '<', $cutoffDate);
+              });
+        });
+
+        $count = $query->count();
+
+        if ($count === 0) {
+            return 0;
+        }
+
+        if (!$force) {
+            $this->line("Found {$count} old jobs (soft deleting)...");
+            $query->delete(); // Soft delete
+        } else {
+            $this->line("Found {$count} old jobs (force deleting)...");
+            $query->forceDelete(); // Permanent delete
+        }
+
+        return $count;
+    }
+
+    private function cleanTrashedJobs(bool $force): int
+    {
+        if (!$force) {
+            return 0;
+        }
+
+        // Permanently delete jobs that have been soft-deleted for more than 30 days
+        $query = Job::onlyTrashed()
+            ->where('deleted_at', '<', now()->subDays(30));
+
+        $count = $query->count();
+
+        if ($count > 0) {
+            $this->line("Found {$count} trashed jobs older than 30 days (permanently deleting)...");
+            $query->forceDelete();
+        }
+
+        return $count;
+    }
+}

app/Console/Commands/DeduplicateJobs.php

@@ -0,0 +1,102 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Job;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+#[Signature('jobs:deduplicate {--dry-run}')]
+#[Description('Find and remove duplicate job listings based on fingerprint')]
+class DeduplicateJobs extends Command
+{
+    /**
+     * Execute the console command.
+     */
+    public function handle(): int
+    {
+        $dryRun = $this->option('dry-run');
+
+        $this->info('Searching for duplicate job listings...');
+        $this->newLine();
+
+        // Find duplicates by fingerprint
+        $duplicates = $this->findDuplicates();
+
+        if ($duplicates->isEmpty()) {
+            $this->info('No duplicates found!');
+            return self::SUCCESS;
+        }
+
+        $this->warn("Found {$duplicates->count()} sets of duplicates.");
+        $this->newLine();
+
+        $removedCount = 0;
+
+        foreach ($duplicates as $duplicate) {
+            $fingerprint = $duplicate->fingerprint;
+            $count = $duplicate->count;
+
+            // Get all jobs with this fingerprint, ordered by created_at (keep oldest)
+            $jobs = Job::where('fingerprint', $fingerprint)
+                ->orderBy('created_at', 'asc')
+                ->get();
+
+            // Keep the first one, mark others for removal
+            $toKeep = $jobs->first();
+            $toRemove = $jobs->skip(1);
+
+            $this->line("Fingerprint: {$fingerprint}");
+            $this->line("  Keeping: #{$toKeep->id} - {$toKeep->title} ({$toKeep->source->name})");
+
+            foreach ($toRemove as $job) {
+                $this->line("  Removing: #{$job->id} - {$job->title} ({$job->source->name})");
+
+                if (!$dryRun) {
+                    $job->delete();
+                    $removedCount++;
+                }
+            }
+
+            $this->newLine();
+        }
+
+        if ($dryRun) {
+            $this->info("Dry run completed. Would remove {$this->countDuplicatesToRemove($duplicates)} duplicate jobs.");
+            $this->line('Run without --dry-run to actually remove them.');
+        } else {
+            $this->info("Removed {$removedCount} duplicate jobs.");
+        }
+
+        return self::SUCCESS;
+    }
+
+    /**
+     * Find duplicate fingerprints
+     */
+    private function findDuplicates()
+    {
+        return DB::table('jobs')
+            ->select('fingerprint', DB::raw('COUNT(*) as count'))
+            ->whereNull('deleted_at')
+            ->whereNotNull('fingerprint')
+            ->groupBy('fingerprint')
+            ->having('count', '>', 1)
+            ->orderBy('count', 'desc')
+            ->get();
+    }
+
+    /**
+     * Count total duplicates to remove
+     */
+    private function countDuplicatesToRemove($duplicates): int
+    {
+        $total = 0;
+        foreach ($duplicates as $duplicate) {
+            $total += ($duplicate->count - 1); // Keep 1, remove the rest
+        }
+        return $total;
+    }
+}

app/Console/Commands/FetchJobs.php

@@ -0,0 +1,103 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Jobs\FetchJobsFromSource;
+use App\Models\JobSource;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+use Illuminate\Console\Command;
+
+#[Signature('jobs:fetch {source?} {--limit=50} {--sync}')]
+#[Description('Fetch jobs from API sources and queue them for processing')]
+class FetchJobs extends Command
+{
+    /**
+     * Execute the console command.
+     */
+    public function handle(): int
+    {
+        $sourceName = $this->argument('source');
+        $limit = (int) $this->option('limit');
+        $sync = $this->option('sync');
+
+        $this->info('Starting job fetch process...');
+        $this->newLine();
+
+        try {
+            if ($sourceName) {
+                // Fetch from specific source
+                $this->fetchFromSource($sourceName, $limit, $sync);
+            } else {
+                // Fetch from all active sources
+                $this->fetchFromAllSources($limit, $sync);
+            }
+
+            $this->newLine();
+            $this->info('Job fetch process completed!');
+
+            return self::SUCCESS;
+
+        } catch (\Exception $e) {
+            $this->error("Failed to fetch jobs: {$e->getMessage()}");
+            return self::FAILURE;
+        }
+    }
+
+    private function fetchFromAllSources(int $limit, bool $sync): void
+    {
+        $sources = JobSource::active()->get();
+
+        if ($sources->isEmpty()) {
+            $this->warn('No active job sources found.');
+            $this->line('Run: php artisan db:seed --class=JobSourceSeeder');
+            return;
+        }
+
+        $this->info("Fetching jobs from {$sources->count()} active sources...");
+
+        foreach ($sources as $source) {
+            $this->line("  • Queuing fetch for: {$source->name}");
+
+            $params = ['limit' => $limit];
+
+            if ($sync) {
+                FetchJobsFromSource::dispatchSync($source, $params);
+            } else {
+                FetchJobsFromSource::dispatch($source, $params);
+            }
+        }
+
+        if (!$sync) {
+            $this->info("\nJobs have been queued for processing.");
+            $this->line('Run: php artisan queue:work to process them.');
+        }
+    }
+
+    private function fetchFromSource(string $sourceName, int $limit, bool $sync): void
+    {
+        $source = JobSource::where('name', 'LIKE', "%{$sourceName}%")
+            ->where('is_active', true)
+            ->first();
+
+        if (!$source) {
+            $this->error("Active source not found: {$sourceName}");
+            $this->line("\nAvailable sources:");
+            JobSource::active()->get()->each(fn($s) => $this->line("  • {$s->name}"));
+            return;
+        }
+
+        $this->info("Fetching jobs from: {$source->name}");
+
+        $params = ['limit' => $limit];
+
+        if ($sync) {
+            FetchJobsFromSource::dispatchSync($source, $params);
+            $this->info('Fetch completed synchronously.');
+        } else {
+            FetchJobsFromSource::dispatch($source, $params);
+            $this->info('Fetch job has been queued.');
+            $this->line('Run: php artisan queue:work to process it.');
+        }
+    }
+}

app/Jobs/FetchJobsFromSource.php

@@ -0,0 +1,94 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\JobSource;
+use App\Services\JobApi\Exceptions\ApiException;
+use App\Services\JobApi\JobApiClientFactory;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class FetchJobsFromSource implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $tries = 3;
+    public int $timeout = 300; // 5 minutes
+    public int $backoff = 60; // Wait 60 seconds between retries
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(
+        public JobSource $source,
+        public array $params = []
+    ) {}
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        Log::info("Fetching jobs from source: {$this->source->name}");
+
+        try {
+            // Create API client for this source
+            $client = JobApiClientFactory::make($this->source);
+
+            // Fetch jobs from the API
+            $jobsData = $client->fetchJobs($this->params);
+
+            Log::info("Fetched {count} jobs from {source}", [
+                'count' => count($jobsData),
+                'source' => $this->source->name,
+            ]);
+
+            // Dispatch individual jobs to process each job listing
+            foreach ($jobsData as $jobData) {
+                ProcessJobListing::dispatch($this->source, $jobData);
+            }
+
+        } catch (ApiException $e) {
+            Log::error("API error fetching jobs from {source}", [
+                'source' => $this->source->name,
+                'error' => $e->getMessage(),
+                'context' => $e->getContext(),
+            ]);
+
+            // Don't retry if it's a rate limit error with a long wait time
+            if ($e instanceof \App\Services\JobApi\Exceptions\RateLimitException) {
+                if ($e->retryAfter && $e->retryAfter > 3600) {
+                    $this->fail($e);
+                    return;
+                }
+            }
+
+            throw $e;
+
+        } catch (\Exception $e) {
+            Log::error("Unexpected error fetching jobs from {source}", [
+                'source' => $this->source->name,
+                'error' => $e->getMessage(),
+                'trace' => $e->getTraceAsString(),
+            ]);
+
+            throw $e;
+        }
+    }
+
+    /**
+     * Handle a job failure.
+     */
+    public function failed(\Throwable $exception): void
+    {
+        Log::error("Failed to fetch jobs from {source} after {tries} attempts", [
+            'source' => $this->source->name,
+            'tries' => $this->tries,
+            'error' => $exception->getMessage(),
+        ]);
+    }
+}

app/Jobs/ProcessJobListing.php

@@ -0,0 +1,94 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Job;
+use App\Models\JobSource;
+use App\Services\JobApi\DTO\JobData;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class ProcessJobListing implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public int $tries = 3;
+    public int $timeout = 60;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(
+        public JobSource $source,
+        public JobData $jobData
+    ) {}
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        try {
+            $fingerprint = $this->jobData->generateFingerprint();
+
+            // Check if job already exists by fingerprint or external_id
+            $existingJob = Job::where('fingerprint', $fingerprint)
+                ->orWhere(function ($query) {
+                    $query->where('job_source_id', $this->source->id)
+                          ->where('external_id', $this->jobData->externalId);
+                })
+                ->first();
+
+            if ($existingJob) {
+                // Update existing job with fresh data
+                $existingJob->update(array_merge(
+                    $this->jobData->toArray(),
+                    [
+                        'job_source_id' => $this->source->id,
+                        'fingerprint' => $fingerprint,
+                    ]
+                ));
+
+                Log::debug("Updated existing job: {$this->jobData->title}");
+            } else {
+                // Create new job
+                Job::create(array_merge(
+                    $this->jobData->toArray(),
+                    [
+                        'job_source_id' => $this->source->id,
+                        'fingerprint' => $fingerprint,
+                    ]
+                ));
+
+                Log::debug("Created new job: {$this->jobData->title}");
+            }
+
+        } catch (\Exception $e) {
+            Log::error("Failed to process job listing", [
+                'source' => $this->source->name,
+                'external_id' => $this->jobData->externalId,
+                'title' => $this->jobData->title,
+                'error' => $e->getMessage(),
+            ]);
+
+            throw $e;
+        }
+    }
+
+    /**
+     * Handle a job failure.
+     */
+    public function failed(\Throwable $exception): void
+    {
+        Log::error("Failed to process job listing after {tries} attempts", [
+            'source' => $this->source->name,
+            'external_id' => $this->jobData->externalId,
+            'tries' => $this->tries,
+            'error' => $exception->getMessage(),
+        ]);
+    }
+}

bootstrap/app.php

@@ -3,6 +3,7 @@
 use Illuminate\Foundation\Application;
 use Illuminate\Foundation\Configuration\Exceptions;
 use Illuminate\Foundation\Configuration\Middleware;
+use Illuminate\Console\Scheduling\Schedule;
 
 return Application::configure(basePath: dirname(__DIR__))
     ->withRouting(
@@ -13,6 +14,23 @@ return Application::configure(basePath: dirname(__DIR__))
     ->withMiddleware(function (Middleware $middleware): void {
         //
     })
+    ->withSchedule(function (Schedule $schedule): void {
+        // Fetch jobs from all sources hourly
+        $schedule->command('jobs:fetch --limit=100')
+            ->hourly()
+            ->withoutOverlapping()
+            ->onOneServer();
+
+        // Clean up old jobs daily at 2 AM
+        $schedule->command('jobs:clean --days=30')
+            ->dailyAt('02:00')
+            ->onOneServer();
+
+        // Deduplicate jobs weekly on Sunday at 3 AM
+        $schedule->command('jobs:deduplicate')
+            ->weeklyOn(0, '03:00')
+            ->onOneServer();
+    })
     ->withExceptions(function (Exceptions $exceptions): void {
         //
     })->create();