phase 3
This commit is contained in:
Doar Mihai
@@ -0,0 +1,114 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Job;
|
||||
use Illuminate\Console\Attributes\Description;
|
||||
use Illuminate\Console\Attributes\Signature;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
#[Signature('jobs:clean {--days=30} {--force}')]
|
||||
#[Description('Remove expired and old job listings from the database')]
|
||||
class CleanJobs extends Command
|
||||
{
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(): int
|
||||
{
|
||||
$days = (int) $this->option('days');
|
||||
$force = $this->option('force');
|
||||
|
||||
$this->info("Cleaning job listings...");
|
||||
$this->newLine();
|
||||
|
||||
// Remove expired jobs
|
||||
$expiredCount = $this->removeExpiredJobs($force);
|
||||
|
||||
// Remove old jobs (not posted in the last X days)
|
||||
$oldCount = $this->removeOldJobs($days, $force);
|
||||
|
||||
// Clean up soft-deleted jobs
|
||||
$trashedCount = $this->cleanTrashedJobs($force);
|
||||
|
||||
$this->newLine();
|
||||
$this->info("Cleanup Summary:");
|
||||
$this->line(" • Expired jobs removed: {$expiredCount}");
|
||||
$this->line(" • Old jobs removed (>{$days} days): {$oldCount}");
|
||||
$this->line(" • Trashed jobs cleaned: {$trashedCount}");
|
||||
$this->line(" • Total removed: " . ($expiredCount + $oldCount + $trashedCount));
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
private function removeExpiredJobs(bool $force): int
|
||||
{
|
||||
$query = Job::whereNotNull('expires_at')
|
||||
->where('expires_at', '<', now());
|
||||
|
||||
$count = $query->count();
|
||||
|
||||
if ($count === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!$force) {
|
||||
$this->line("Found {$count} expired jobs (soft deleting)...");
|
||||
$query->delete(); // Soft delete
|
||||
} else {
|
||||
$this->line("Found {$count} expired jobs (force deleting)...");
|
||||
$query->forceDelete(); // Permanent delete
|
||||
}
|
||||
|
||||
return $count;
|
||||
}
|
||||
|
||||
private function removeOldJobs(int $days, bool $force): int
|
||||
{
|
||||
$cutoffDate = now()->subDays($days);
|
||||
|
||||
$query = Job::where(function ($q) use ($cutoffDate) {
|
||||
$q->where('posted_at', '<', $cutoffDate)
|
||||
->orWhere(function ($q2) use ($cutoffDate) {
|
||||
$q2->whereNull('posted_at')
|
||||
->where('created_at', '<', $cutoffDate);
|
||||
});
|
||||
});
|
||||
|
||||
$count = $query->count();
|
||||
|
||||
if ($count === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!$force) {
|
||||
$this->line("Found {$count} old jobs (soft deleting)...");
|
||||
$query->delete(); // Soft delete
|
||||
} else {
|
||||
$this->line("Found {$count} old jobs (force deleting)...");
|
||||
$query->forceDelete(); // Permanent delete
|
||||
}
|
||||
|
||||
return $count;
|
||||
}
|
||||
|
||||
private function cleanTrashedJobs(bool $force): int
|
||||
{
|
||||
if (!$force) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Permanently delete jobs that have been soft-deleted for more than 30 days
|
||||
$query = Job::onlyTrashed()
|
||||
->where('deleted_at', '<', now()->subDays(30));
|
||||
|
||||
$count = $query->count();
|
||||
|
||||
if ($count > 0) {
|
||||
$this->line("Found {$count} trashed jobs older than 30 days (permanently deleting)...");
|
||||
$query->forceDelete();
|
||||
}
|
||||
|
||||
return $count;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Job;
|
||||
use Illuminate\Console\Attributes\Description;
|
||||
use Illuminate\Console\Attributes\Signature;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
#[Signature('jobs:deduplicate {--dry-run}')]
|
||||
#[Description('Find and remove duplicate job listings based on fingerprint')]
|
||||
class DeduplicateJobs extends Command
|
||||
{
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(): int
|
||||
{
|
||||
$dryRun = $this->option('dry-run');
|
||||
|
||||
$this->info('Searching for duplicate job listings...');
|
||||
$this->newLine();
|
||||
|
||||
// Find duplicates by fingerprint
|
||||
$duplicates = $this->findDuplicates();
|
||||
|
||||
if ($duplicates->isEmpty()) {
|
||||
$this->info('No duplicates found!');
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$this->warn("Found {$duplicates->count()} sets of duplicates.");
|
||||
$this->newLine();
|
||||
|
||||
$removedCount = 0;
|
||||
|
||||
foreach ($duplicates as $duplicate) {
|
||||
$fingerprint = $duplicate->fingerprint;
|
||||
$count = $duplicate->count;
|
||||
|
||||
// Get all jobs with this fingerprint, ordered by created_at (keep oldest)
|
||||
$jobs = Job::where('fingerprint', $fingerprint)
|
||||
->orderBy('created_at', 'asc')
|
||||
->get();
|
||||
|
||||
// Keep the first one, mark others for removal
|
||||
$toKeep = $jobs->first();
|
||||
$toRemove = $jobs->skip(1);
|
||||
|
||||
$this->line("Fingerprint: {$fingerprint}");
|
||||
$this->line(" Keeping: #{$toKeep->id} - {$toKeep->title} ({$toKeep->source->name})");
|
||||
|
||||
foreach ($toRemove as $job) {
|
||||
$this->line(" Removing: #{$job->id} - {$job->title} ({$job->source->name})");
|
||||
|
||||
if (!$dryRun) {
|
||||
$job->delete();
|
||||
$removedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
}
|
||||
|
||||
if ($dryRun) {
|
||||
$this->info("Dry run completed. Would remove {$this->countDuplicatesToRemove($duplicates)} duplicate jobs.");
|
||||
$this->line('Run without --dry-run to actually remove them.');
|
||||
} else {
|
||||
$this->info("Removed {$removedCount} duplicate jobs.");
|
||||
}
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find duplicate fingerprints
|
||||
*/
|
||||
private function findDuplicates()
|
||||
{
|
||||
return DB::table('jobs')
|
||||
->select('fingerprint', DB::raw('COUNT(*) as count'))
|
||||
->whereNull('deleted_at')
|
||||
->whereNotNull('fingerprint')
|
||||
->groupBy('fingerprint')
|
||||
->having('count', '>', 1)
|
||||
->orderBy('count', 'desc')
|
||||
->get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Count total duplicates to remove
|
||||
*/
|
||||
private function countDuplicatesToRemove($duplicates): int
|
||||
{
|
||||
$total = 0;
|
||||
foreach ($duplicates as $duplicate) {
|
||||
$total += ($duplicate->count - 1); // Keep 1, remove the rest
|
||||
}
|
||||
return $total;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Jobs\FetchJobsFromSource;
|
||||
use App\Models\JobSource;
|
||||
use Illuminate\Console\Attributes\Description;
|
||||
use Illuminate\Console\Attributes\Signature;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
#[Signature('jobs:fetch {source?} {--limit=50} {--sync}')]
|
||||
#[Description('Fetch jobs from API sources and queue them for processing')]
|
||||
class FetchJobs extends Command
|
||||
{
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(): int
|
||||
{
|
||||
$sourceName = $this->argument('source');
|
||||
$limit = (int) $this->option('limit');
|
||||
$sync = $this->option('sync');
|
||||
|
||||
$this->info('Starting job fetch process...');
|
||||
$this->newLine();
|
||||
|
||||
try {
|
||||
if ($sourceName) {
|
||||
// Fetch from specific source
|
||||
$this->fetchFromSource($sourceName, $limit, $sync);
|
||||
} else {
|
||||
// Fetch from all active sources
|
||||
$this->fetchFromAllSources($limit, $sync);
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
$this->info('Job fetch process completed!');
|
||||
|
||||
return self::SUCCESS;
|
||||
|
||||
} catch (\Exception $e) {
|
||||
$this->error("Failed to fetch jobs: {$e->getMessage()}");
|
||||
return self::FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
private function fetchFromAllSources(int $limit, bool $sync): void
|
||||
{
|
||||
$sources = JobSource::active()->get();
|
||||
|
||||
if ($sources->isEmpty()) {
|
||||
$this->warn('No active job sources found.');
|
||||
$this->line('Run: php artisan db:seed --class=JobSourceSeeder');
|
||||
return;
|
||||
}
|
||||
|
||||
$this->info("Fetching jobs from {$sources->count()} active sources...");
|
||||
|
||||
foreach ($sources as $source) {
|
||||
$this->line(" • Queuing fetch for: {$source->name}");
|
||||
|
||||
$params = ['limit' => $limit];
|
||||
|
||||
if ($sync) {
|
||||
FetchJobsFromSource::dispatchSync($source, $params);
|
||||
} else {
|
||||
FetchJobsFromSource::dispatch($source, $params);
|
||||
}
|
||||
}
|
||||
|
||||
if (!$sync) {
|
||||
$this->info("\nJobs have been queued for processing.");
|
||||
$this->line('Run: php artisan queue:work to process them.');
|
||||
}
|
||||
}
|
||||
|
||||
private function fetchFromSource(string $sourceName, int $limit, bool $sync): void
|
||||
{
|
||||
$source = JobSource::where('name', 'LIKE', "%{$sourceName}%")
|
||||
->where('is_active', true)
|
||||
->first();
|
||||
|
||||
if (!$source) {
|
||||
$this->error("Active source not found: {$sourceName}");
|
||||
$this->line("\nAvailable sources:");
|
||||
JobSource::active()->get()->each(fn($s) => $this->line(" • {$s->name}"));
|
||||
return;
|
||||
}
|
||||
|
||||
$this->info("Fetching jobs from: {$source->name}");
|
||||
|
||||
$params = ['limit' => $limit];
|
||||
|
||||
if ($sync) {
|
||||
FetchJobsFromSource::dispatchSync($source, $params);
|
||||
$this->info('Fetch completed synchronously.');
|
||||
} else {
|
||||
FetchJobsFromSource::dispatch($source, $params);
|
||||
$this->info('Fetch job has been queued.');
|
||||
$this->line('Run: php artisan queue:work to process it.');
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user