diff --git a/app/Console/Commands/CleanJobs.php b/app/Console/Commands/CleanJobs.php new file mode 100644 index 0000000..e787228 --- /dev/null +++ b/app/Console/Commands/CleanJobs.php @@ -0,0 +1,114 @@ +option('days'); + $force = $this->option('force'); + + $this->info("Cleaning job listings..."); + $this->newLine(); + + // Remove expired jobs + $expiredCount = $this->removeExpiredJobs($force); + + // Remove old jobs (not posted in the last X days) + $oldCount = $this->removeOldJobs($days, $force); + + // Clean up soft-deleted jobs + $trashedCount = $this->cleanTrashedJobs($force); + + $this->newLine(); + $this->info("Cleanup Summary:"); + $this->line(" • Expired jobs removed: {$expiredCount}"); + $this->line(" • Old jobs removed (>{$days} days): {$oldCount}"); + $this->line(" • Trashed jobs cleaned: {$trashedCount}"); + $this->line(" • Total removed: " . ($expiredCount + $oldCount + $trashedCount)); + + return self::SUCCESS; + } + + private function removeExpiredJobs(bool $force): int + { + $query = Job::whereNotNull('expires_at') + ->where('expires_at', '<', now()); + + $count = $query->count(); + + if ($count === 0) { + return 0; + } + + if (!$force) { + $this->line("Found {$count} expired jobs (soft deleting)..."); + $query->delete(); // Soft delete + } else { + $this->line("Found {$count} expired jobs (force deleting)..."); + $query->forceDelete(); // Permanent delete + } + + return $count; + } + + private function removeOldJobs(int $days, bool $force): int + { + $cutoffDate = now()->subDays($days); + + $query = Job::where(function ($q) use ($cutoffDate) { + $q->where('posted_at', '<', $cutoffDate) + ->orWhere(function ($q2) use ($cutoffDate) { + $q2->whereNull('posted_at') + ->where('created_at', '<', $cutoffDate); + }); + }); + + $count = $query->count(); + + if ($count === 0) { + return 0; + } + + if (!$force) { + $this->line("Found {$count} old jobs (soft deleting)..."); + $query->delete(); // Soft delete + } else { + $this->line("Found {$count} old jobs (force deleting)..."); + $query->forceDelete(); // Permanent delete + } + + return $count; + } + + private function cleanTrashedJobs(bool $force): int + { + if (!$force) { + return 0; + } + + // Permanently delete jobs that have been soft-deleted for more than 30 days + $query = Job::onlyTrashed() + ->where('deleted_at', '<', now()->subDays(30)); + + $count = $query->count(); + + if ($count > 0) { + $this->line("Found {$count} trashed jobs older than 30 days (permanently deleting)..."); + $query->forceDelete(); + } + + return $count; + } +} diff --git a/app/Console/Commands/DeduplicateJobs.php b/app/Console/Commands/DeduplicateJobs.php new file mode 100644 index 0000000..a06756d --- /dev/null +++ b/app/Console/Commands/DeduplicateJobs.php @@ -0,0 +1,102 @@ +option('dry-run'); + + $this->info('Searching for duplicate job listings...'); + $this->newLine(); + + // Find duplicates by fingerprint + $duplicates = $this->findDuplicates(); + + if ($duplicates->isEmpty()) { + $this->info('No duplicates found!'); + return self::SUCCESS; + } + + $this->warn("Found {$duplicates->count()} sets of duplicates."); + $this->newLine(); + + $removedCount = 0; + + foreach ($duplicates as $duplicate) { + $fingerprint = $duplicate->fingerprint; + $count = $duplicate->count; + + // Get all jobs with this fingerprint, ordered by created_at (keep oldest) + $jobs = Job::where('fingerprint', $fingerprint) + ->orderBy('created_at', 'asc') + ->get(); + + // Keep the first one, mark others for removal + $toKeep = $jobs->first(); + $toRemove = $jobs->skip(1); + + $this->line("Fingerprint: {$fingerprint}"); + $this->line(" Keeping: #{$toKeep->id} - {$toKeep->title} ({$toKeep->source->name})"); + + foreach ($toRemove as $job) { + $this->line(" Removing: #{$job->id} - {$job->title} ({$job->source->name})"); + + if (!$dryRun) { + $job->delete(); + $removedCount++; + } + } + + $this->newLine(); + } + + if ($dryRun) { + $this->info("Dry run completed. Would remove {$this->countDuplicatesToRemove($duplicates)} duplicate jobs."); + $this->line('Run without --dry-run to actually remove them.'); + } else { + $this->info("Removed {$removedCount} duplicate jobs."); + } + + return self::SUCCESS; + } + + /** + * Find duplicate fingerprints + */ + private function findDuplicates() + { + return DB::table('jobs') + ->select('fingerprint', DB::raw('COUNT(*) as count')) + ->whereNull('deleted_at') + ->whereNotNull('fingerprint') + ->groupBy('fingerprint') + ->having('count', '>', 1) + ->orderBy('count', 'desc') + ->get(); + } + + /** + * Count total duplicates to remove + */ + private function countDuplicatesToRemove($duplicates): int + { + $total = 0; + foreach ($duplicates as $duplicate) { + $total += ($duplicate->count - 1); // Keep 1, remove the rest + } + return $total; + } +} diff --git a/app/Console/Commands/FetchJobs.php b/app/Console/Commands/FetchJobs.php new file mode 100644 index 0000000..66f025f --- /dev/null +++ b/app/Console/Commands/FetchJobs.php @@ -0,0 +1,103 @@ +argument('source'); + $limit = (int) $this->option('limit'); + $sync = $this->option('sync'); + + $this->info('Starting job fetch process...'); + $this->newLine(); + + try { + if ($sourceName) { + // Fetch from specific source + $this->fetchFromSource($sourceName, $limit, $sync); + } else { + // Fetch from all active sources + $this->fetchFromAllSources($limit, $sync); + } + + $this->newLine(); + $this->info('Job fetch process completed!'); + + return self::SUCCESS; + + } catch (\Exception $e) { + $this->error("Failed to fetch jobs: {$e->getMessage()}"); + return self::FAILURE; + } + } + + private function fetchFromAllSources(int $limit, bool $sync): void + { + $sources = JobSource::active()->get(); + + if ($sources->isEmpty()) { + $this->warn('No active job sources found.'); + $this->line('Run: php artisan db:seed --class=JobSourceSeeder'); + return; + } + + $this->info("Fetching jobs from {$sources->count()} active sources..."); + + foreach ($sources as $source) { + $this->line(" • Queuing fetch for: {$source->name}"); + + $params = ['limit' => $limit]; + + if ($sync) { + FetchJobsFromSource::dispatchSync($source, $params); + } else { + FetchJobsFromSource::dispatch($source, $params); + } + } + + if (!$sync) { + $this->info("\nJobs have been queued for processing."); + $this->line('Run: php artisan queue:work to process them.'); + } + } + + private function fetchFromSource(string $sourceName, int $limit, bool $sync): void + { + $source = JobSource::where('name', 'LIKE', "%{$sourceName}%") + ->where('is_active', true) + ->first(); + + if (!$source) { + $this->error("Active source not found: {$sourceName}"); + $this->line("\nAvailable sources:"); + JobSource::active()->get()->each(fn($s) => $this->line(" • {$s->name}")); + return; + } + + $this->info("Fetching jobs from: {$source->name}"); + + $params = ['limit' => $limit]; + + if ($sync) { + FetchJobsFromSource::dispatchSync($source, $params); + $this->info('Fetch completed synchronously.'); + } else { + FetchJobsFromSource::dispatch($source, $params); + $this->info('Fetch job has been queued.'); + $this->line('Run: php artisan queue:work to process it.'); + } + } +} diff --git a/app/Jobs/FetchJobsFromSource.php b/app/Jobs/FetchJobsFromSource.php new file mode 100644 index 0000000..81421cc --- /dev/null +++ b/app/Jobs/FetchJobsFromSource.php @@ -0,0 +1,94 @@ +source->name}"); + + try { + // Create API client for this source + $client = JobApiClientFactory::make($this->source); + + // Fetch jobs from the API + $jobsData = $client->fetchJobs($this->params); + + Log::info("Fetched {count} jobs from {source}", [ + 'count' => count($jobsData), + 'source' => $this->source->name, + ]); + + // Dispatch individual jobs to process each job listing + foreach ($jobsData as $jobData) { + ProcessJobListing::dispatch($this->source, $jobData); + } + + } catch (ApiException $e) { + Log::error("API error fetching jobs from {source}", [ + 'source' => $this->source->name, + 'error' => $e->getMessage(), + 'context' => $e->getContext(), + ]); + + // Don't retry if it's a rate limit error with a long wait time + if ($e instanceof \App\Services\JobApi\Exceptions\RateLimitException) { + if ($e->retryAfter && $e->retryAfter > 3600) { + $this->fail($e); + return; + } + } + + throw $e; + + } catch (\Exception $e) { + Log::error("Unexpected error fetching jobs from {source}", [ + 'source' => $this->source->name, + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + ]); + + throw $e; + } + } + + /** + * Handle a job failure. + */ + public function failed(\Throwable $exception): void + { + Log::error("Failed to fetch jobs from {source} after {tries} attempts", [ + 'source' => $this->source->name, + 'tries' => $this->tries, + 'error' => $exception->getMessage(), + ]); + } +} diff --git a/app/Jobs/ProcessJobListing.php b/app/Jobs/ProcessJobListing.php new file mode 100644 index 0000000..2e27665 --- /dev/null +++ b/app/Jobs/ProcessJobListing.php @@ -0,0 +1,94 @@ +jobData->generateFingerprint(); + + // Check if job already exists by fingerprint or external_id + $existingJob = Job::where('fingerprint', $fingerprint) + ->orWhere(function ($query) { + $query->where('job_source_id', $this->source->id) + ->where('external_id', $this->jobData->externalId); + }) + ->first(); + + if ($existingJob) { + // Update existing job with fresh data + $existingJob->update(array_merge( + $this->jobData->toArray(), + [ + 'job_source_id' => $this->source->id, + 'fingerprint' => $fingerprint, + ] + )); + + Log::debug("Updated existing job: {$this->jobData->title}"); + } else { + // Create new job + Job::create(array_merge( + $this->jobData->toArray(), + [ + 'job_source_id' => $this->source->id, + 'fingerprint' => $fingerprint, + ] + )); + + Log::debug("Created new job: {$this->jobData->title}"); + } + + } catch (\Exception $e) { + Log::error("Failed to process job listing", [ + 'source' => $this->source->name, + 'external_id' => $this->jobData->externalId, + 'title' => $this->jobData->title, + 'error' => $e->getMessage(), + ]); + + throw $e; + } + } + + /** + * Handle a job failure. + */ + public function failed(\Throwable $exception): void + { + Log::error("Failed to process job listing after {tries} attempts", [ + 'source' => $this->source->name, + 'external_id' => $this->jobData->externalId, + 'tries' => $this->tries, + 'error' => $exception->getMessage(), + ]); + } +} diff --git a/bootstrap/app.php b/bootstrap/app.php index c183276..232de5e 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -3,6 +3,7 @@ use Illuminate\Foundation\Application; use Illuminate\Foundation\Configuration\Exceptions; use Illuminate\Foundation\Configuration\Middleware; +use Illuminate\Console\Scheduling\Schedule; return Application::configure(basePath: dirname(__DIR__)) ->withRouting( @@ -13,6 +14,23 @@ return Application::configure(basePath: dirname(__DIR__)) ->withMiddleware(function (Middleware $middleware): void { // }) + ->withSchedule(function (Schedule $schedule): void { + // Fetch jobs from all sources hourly + $schedule->command('jobs:fetch --limit=100') + ->hourly() + ->withoutOverlapping() + ->onOneServer(); + + // Clean up old jobs daily at 2 AM + $schedule->command('jobs:clean --days=30') + ->dailyAt('02:00') + ->onOneServer(); + + // Deduplicate jobs weekly on Sunday at 3 AM + $schedule->command('jobs:deduplicate') + ->weeklyOn(0, '03:00') + ->onOneServer(); + }) ->withExceptions(function (Exceptions $exceptions): void { // })->create();