specific log files for commands and jobs

This commit is contained in:
Doar Mihai
2026-05-02 01:40:46 +03:00
parent 792a598eda
commit 462eb2fc61
10 changed files with 138 additions and 31 deletions
+14 -1
View File
@@ -6,6 +6,7 @@ use App\Models\Job;
use Illuminate\Console\Attributes\Description; use Illuminate\Console\Attributes\Description;
use Illuminate\Console\Attributes\Signature; use Illuminate\Console\Attributes\Signature;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
#[Signature('jobs:clean {--days=30} {--force}')] #[Signature('jobs:clean {--days=30} {--force}')]
#[Description('Remove expired and old job listings from the database')] #[Description('Remove expired and old job listings from the database')]
@@ -22,6 +23,9 @@ class CleanJobs extends Command
$days = (int) $this->option('days'); $days = (int) $this->option('days');
$force = $this->option('force'); $force = $this->option('force');
$log = Log::channel('jobs_clean');
$log->info('Starting job cleanup', ['days' => $days, 'force' => $force]);
$this->info("Cleaning job listings..."); $this->info("Cleaning job listings...");
$this->newLine(); $this->newLine();
@@ -35,11 +39,20 @@ class CleanJobs extends Command
$trashedCount = $this->cleanTrashedJobs($force); $trashedCount = $this->cleanTrashedJobs($force);
$this->newLine(); $this->newLine();
$total = $expiredCount + $oldCount + $trashedCount;
$this->info("Cleanup Summary:"); $this->info("Cleanup Summary:");
$this->line(" • Expired jobs removed: {$expiredCount}"); $this->line(" • Expired jobs removed: {$expiredCount}");
$this->line(" • Old jobs removed (>{$days} days): {$oldCount}"); $this->line(" • Old jobs removed (>{$days} days): {$oldCount}");
$this->line(" • Trashed jobs cleaned: {$trashedCount}"); $this->line(" • Trashed jobs cleaned: {$trashedCount}");
$this->line(" • Total removed: " . ($expiredCount + $oldCount + $trashedCount)); $this->line(" • Total removed: {$total}");
$log->info('Cleanup complete', [
'expired' => $expiredCount,
'old' => $oldCount,
'trashed' => $trashedCount,
'total' => $total,
]);
return self::SUCCESS; return self::SUCCESS;
} }
+10 -3
View File
@@ -31,7 +31,10 @@ class CurateJobs extends Command
return self::FAILURE; return self::FAILURE;
} }
$log = Log::channel('jobs_curate');
$this->info('Starting job curation...'); $this->info('Starting job curation...');
$log->info('Starting job curation batch', ['batch_size' => $batchSize]);
$this->newLine(); $this->newLine();
// Fetch uncurated jobs // Fetch uncurated jobs
@@ -72,11 +75,14 @@ class CurateJobs extends Command
'stream' => false, 'stream' => false,
'prompt' => $body, 'prompt' => $body,
]); ]);
var_dump($response->json());
if ($response->successful()) { if ($response->successful()) {
$responseArray = $response->json(); $responseArray = $response->json();
$responseTagsJson = $responseArray['response']; $responseTagsJson = $responseArray['response'];
Log::info("Curated job id: {$responseArray['job_id']} Result: {$responseTagsJson}"); $log->info("Curated job id: {$job->id} Result: {$responseTagsJson}");
$tags = json_decode($responseTagsJson, true); $tags = json_decode($responseTagsJson, true);
if (empty($tags) || $tags === null) { if (empty($tags) || $tags === null) {
@@ -111,11 +117,11 @@ class CurateJobs extends Command
} }
} else { } else {
$failedCount++; $failedCount++;
Log::warning("Failed to curate job #{$job->id}: API returned {$response->status()}"); $log->warning("Failed to curate job #{$job->id}: API returned {$response->status()} -> {$response->body()}");
} }
} catch (\Exception $e) { } catch (\Exception $e) {
$failedCount++; $failedCount++;
Log::error("Error curating job #{$job->id}: {$e->getMessage()}"); $log->error("Error curating job #{$job->id}: {$e->getMessage()}");
} }
$progressBar->advance(); $progressBar->advance();
@@ -125,6 +131,7 @@ class CurateJobs extends Command
$this->newLine(2); $this->newLine(2);
$this->info("Successfully curated {$curatedCount} jobs."); $this->info("Successfully curated {$curatedCount} jobs.");
$log->info("Curation complete", ['curated' => $curatedCount, 'failed' => $failedCount]);
if ($failedCount > 0) { if ($failedCount > 0) {
$this->warn("Failed to curate {$failedCount} jobs. Check logs for details."); $this->warn("Failed to curate {$failedCount} jobs. Check logs for details.");
+8 -1
View File
@@ -7,6 +7,7 @@ use Illuminate\Console\Attributes\Description;
use Illuminate\Console\Attributes\Signature; use Illuminate\Console\Attributes\Signature;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
#[Signature('jobs:deduplicate {--dry-run}')] #[Signature('jobs:deduplicate {--dry-run}')]
#[Description('Find and remove duplicate job listings based on fingerprint')] #[Description('Find and remove duplicate job listings based on fingerprint')]
@@ -21,6 +22,8 @@ class DeduplicateJobs extends Command
public function handle(): int public function handle(): int
{ {
$dryRun = $this->option('dry-run'); $dryRun = $this->option('dry-run');
$log = Log::channel('jobs_deduplicate');
$log->info('Starting deduplication', ['dry_run' => $dryRun]);
$this->info('Searching for duplicate job listings...'); $this->info('Searching for duplicate job listings...');
$this->newLine(); $this->newLine();
@@ -30,6 +33,7 @@ class DeduplicateJobs extends Command
if ($duplicates->isEmpty()) { if ($duplicates->isEmpty()) {
$this->info('No duplicates found!'); $this->info('No duplicates found!');
$log->info('No duplicates found');
return self::SUCCESS; return self::SUCCESS;
} }
@@ -67,10 +71,13 @@ class DeduplicateJobs extends Command
} }
if ($dryRun) { if ($dryRun) {
$this->info("Dry run completed. Would remove {$this->countDuplicatesToRemove($duplicates)} duplicate jobs."); $wouldRemove = $this->countDuplicatesToRemove($duplicates);
$this->info("Dry run completed. Would remove {$wouldRemove} duplicate jobs.");
$this->line('Run without --dry-run to actually remove them.'); $this->line('Run without --dry-run to actually remove them.');
$log->info('Dry run complete', ['would_remove' => $wouldRemove]);
} else { } else {
$this->info("Removed {$removedCount} duplicate jobs."); $this->info("Removed {$removedCount} duplicate jobs.");
$log->info('Deduplication complete', ['removed' => $removedCount]);
} }
return self::SUCCESS; return self::SUCCESS;
+5
View File
@@ -6,6 +6,7 @@ use App\Services\JobEnrichmentService;
use Illuminate\Console\Attributes\Description; use Illuminate\Console\Attributes\Description;
use Illuminate\Console\Attributes\Signature; use Illuminate\Console\Attributes\Signature;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
#[Signature('jobs:enrich {--batch=100}')] #[Signature('jobs:enrich {--batch=100}')]
#[Description('Enrich job listings with additional data (categories, experience levels, etc.)')] #[Description('Enrich job listings with additional data (categories, experience levels, etc.)')]
@@ -27,6 +28,9 @@ class EnrichJobs extends Command
{ {
$batchSize = (int) $this->option('batch'); $batchSize = (int) $this->option('batch');
$log = Log::channel('jobs_enrich');
$log->info('Starting job enrichment', ['batch_size' => $batchSize]);
$this->info('Enriching job listings...'); $this->info('Enriching job listings...');
$this->newLine(); $this->newLine();
@@ -34,6 +38,7 @@ class EnrichJobs extends Command
$this->newLine(); $this->newLine();
$this->info("Enriched {$enrichedCount} jobs successfully!"); $this->info("Enriched {$enrichedCount} jobs successfully!");
$log->info('Enrichment complete', ['enriched' => $enrichedCount]);
return self::SUCCESS; return self::SUCCESS;
} }
+6 -2
View File
@@ -7,6 +7,7 @@ use App\Models\JobSource;
use Illuminate\Console\Attributes\Description; use Illuminate\Console\Attributes\Description;
use Illuminate\Console\Attributes\Signature; use Illuminate\Console\Attributes\Signature;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
#[Signature('jobs:fetch {source?} {--limit=50} {--sync}')] #[Signature('jobs:fetch {source?} {--limit=50} {--sync}')]
#[Description('Fetch jobs from API sources and queue them for processing')] #[Description('Fetch jobs from API sources and queue them for processing')]
@@ -24,25 +25,28 @@ class FetchJobs extends Command
$limit = (int) $this->option('limit'); $limit = (int) $this->option('limit');
$sync = $this->option('sync'); $sync = $this->option('sync');
$log = Log::channel('jobs_fetch');
$log->info('Starting job fetch command', ['source' => $sourceName, 'limit' => $limit, 'sync' => $sync]);
$this->info('Starting job fetch process...'); $this->info('Starting job fetch process...');
$this->newLine(); $this->newLine();
try { try {
if ($sourceName) { if ($sourceName) {
// Fetch from specific source
$this->fetchFromSource($sourceName, $limit, $sync); $this->fetchFromSource($sourceName, $limit, $sync);
} else { } else {
// Fetch from all active sources
$this->fetchFromAllSources($limit, $sync); $this->fetchFromAllSources($limit, $sync);
} }
$this->newLine(); $this->newLine();
$this->info('Job fetch process completed!'); $this->info('Job fetch process completed!');
$log->info('Job fetch command completed');
return self::SUCCESS; return self::SUCCESS;
} catch (\Exception $e) { } catch (\Exception $e) {
$this->error("Failed to fetch jobs: {$e->getMessage()}"); $this->error("Failed to fetch jobs: {$e->getMessage()}");
$log->error('Job fetch command failed', ['error' => $e->getMessage()]);
return self::FAILURE; return self::FAILURE;
} }
} }
@@ -7,6 +7,7 @@ use Illuminate\Console\Attributes\Description;
use Illuminate\Console\Attributes\Signature; use Illuminate\Console\Attributes\Signature;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
#[Signature('jobs:summary {--batch=50}')] #[Signature('jobs:summary {--batch=50}')]
#[Description('Command description')] #[Description('Command description')]
@@ -18,12 +19,24 @@ class GenerateJobSummaryCommand extends Command
public function handle() public function handle()
{ {
$batchSize = (int) $this->option('batch'); $batchSize = (int) $this->option('batch');
$log = Log::channel('jobs_summary');
$log->info('Starting summary generation', ['batch_size' => $batchSize]);
$jobs = $this->getJobs($batchSize); $jobs = $this->getJobs($batchSize);
$processed = 0;
$failed = 0;
foreach ($jobs as $job) { foreach ($jobs as $job) {
$this->processJob($job); try {
$this->processJob($job);
$processed++;
} catch (\Exception $e) {
$failed++;
$log->error("Failed to generate summary for job #{$job->id}", ['error' => $e->getMessage()]);
}
} }
$log->info('Summary generation complete', ['processed' => $processed, 'failed' => $failed]);
} }
private function getJobs(int $batchSize) private function getJobs(int $batchSize)
+6 -13
View File
@@ -33,33 +33,29 @@ class FetchJobsFromSource implements ShouldQueue
*/ */
public function handle(): void public function handle(): void
{ {
Log::info("Fetching jobs from source: {$this->source->name}"); $log = Log::channel('jobs_fetch');
$log->info("Fetching jobs from source: {$this->source->name}");
try { try {
// Create API client for this source
$client = JobApiClientFactory::make($this->source); $client = JobApiClientFactory::make($this->source);
// Fetch jobs from the API
$jobsData = $client->fetchJobs($this->params); $jobsData = $client->fetchJobs($this->params);
Log::info("Fetched {count} jobs from {source}", [ $log->info("Fetched {count} jobs from {source}", [
'count' => count($jobsData), 'count' => count($jobsData),
'source' => $this->source->name, 'source' => $this->source->name,
]); ]);
// Dispatch individual jobs to process each job listing
foreach ($jobsData as $jobData) { foreach ($jobsData as $jobData) {
ProcessJobListing::dispatch($this->source, $jobData); ProcessJobListing::dispatch($this->source, $jobData);
} }
} catch (ApiException $e) { } catch (ApiException $e) {
Log::error("API error fetching jobs from {source}", [ $log->error("API error fetching jobs from {source}", [
'source' => $this->source->name, 'source' => $this->source->name,
'error' => $e->getMessage(), 'error' => $e->getMessage(),
'context' => $e->getContext(), 'context' => $e->getContext(),
]); ]);
// Don't retry if it's a rate limit error with a long wait time
if ($e instanceof \App\Services\JobApi\Exceptions\RateLimitException) { if ($e instanceof \App\Services\JobApi\Exceptions\RateLimitException) {
if ($e->retryAfter && $e->retryAfter > 3600) { if ($e->retryAfter && $e->retryAfter > 3600) {
$this->fail($e); $this->fail($e);
@@ -70,7 +66,7 @@ class FetchJobsFromSource implements ShouldQueue
throw $e; throw $e;
} catch (\Exception $e) { } catch (\Exception $e) {
Log::error("Unexpected error fetching jobs from {source}", [ $log->error("Unexpected error fetching jobs from {source}", [
'source' => $this->source->name, 'source' => $this->source->name,
'error' => $e->getMessage(), 'error' => $e->getMessage(),
'trace' => $e->getTraceAsString(), 'trace' => $e->getTraceAsString(),
@@ -80,12 +76,9 @@ class FetchJobsFromSource implements ShouldQueue
} }
} }
/**
* Handle a job failure.
*/
public function failed(\Throwable $exception): void public function failed(\Throwable $exception): void
{ {
Log::error("Failed to fetch jobs from {source} after {tries} attempts", [ Log::channel('jobs_fetch')->error("Failed to fetch jobs from {source} after {tries} attempts", [
'source' => $this->source->name, 'source' => $this->source->name,
'tries' => $this->tries, 'tries' => $this->tries,
'error' => $exception->getMessage(), 'error' => $exception->getMessage(),
+6 -10
View File
@@ -32,10 +32,11 @@ class ProcessJobListing implements ShouldQueue
*/ */
public function handle(): void public function handle(): void
{ {
$log = Log::channel('jobs_process');
try { try {
$fingerprint = $this->jobData->generateFingerprint(); $fingerprint = $this->jobData->generateFingerprint();
// Check if job already exists by fingerprint or external_id
$existingJob = Job::where('fingerprint', $fingerprint) $existingJob = Job::where('fingerprint', $fingerprint)
->orWhere(function ($query) { ->orWhere(function ($query) {
$query->where('job_source_id', $this->source->id) $query->where('job_source_id', $this->source->id)
@@ -44,7 +45,6 @@ class ProcessJobListing implements ShouldQueue
->first(); ->first();
if ($existingJob) { if ($existingJob) {
// Update existing job with fresh data
$existingJob->update(array_merge( $existingJob->update(array_merge(
$this->jobData->toArray(), $this->jobData->toArray(),
[ [
@@ -53,9 +53,8 @@ class ProcessJobListing implements ShouldQueue
] ]
)); ));
Log::debug("Updated existing job: {$this->jobData->title}"); $log->debug("Updated existing job: {$this->jobData->title}");
} else { } else {
// Create new job
Job::create(array_merge( Job::create(array_merge(
$this->jobData->toArray(), $this->jobData->toArray(),
[ [
@@ -64,11 +63,11 @@ class ProcessJobListing implements ShouldQueue
] ]
)); ));
Log::debug("Created new job: {$this->jobData->title}"); $log->debug("Created new job: {$this->jobData->title}");
} }
} catch (\Exception $e) { } catch (\Exception $e) {
Log::error("Failed to process job listing", [ $log->error("Failed to process job listing", [
'source' => $this->source->name, 'source' => $this->source->name,
'external_id' => $this->jobData->externalId, 'external_id' => $this->jobData->externalId,
'title' => $this->jobData->title, 'title' => $this->jobData->title,
@@ -79,12 +78,9 @@ class ProcessJobListing implements ShouldQueue
} }
} }
/**
* Handle a job failure.
*/
public function failed(\Throwable $exception): void public function failed(\Throwable $exception): void
{ {
Log::error("Failed to process job listing after {tries} attempts", [ Log::channel('jobs_process')->error("Failed to process job listing after {tries} attempts", [
'source' => $this->source->name, 'source' => $this->source->name,
'external_id' => $this->jobData->externalId, 'external_id' => $this->jobData->externalId,
'tries' => $this->tries, 'tries' => $this->tries,
+60
View File
@@ -127,6 +127,66 @@ return [
'path' => storage_path('logs/laravel.log'), 'path' => storage_path('logs/laravel.log'),
], ],
// ---------------------------------------------------------------
// Per-command / per-job channels
// ---------------------------------------------------------------
'jobs_fetch' => [
'driver' => 'daily',
'path' => storage_path('logs/commands/jobs-fetch.log'),
'level' => 'debug',
'days' => 30,
'replace_placeholders' => true,
],
'jobs_process' => [
'driver' => 'daily',
'path' => storage_path('logs/commands/jobs-process.log'),
'level' => 'debug',
'days' => 30,
'replace_placeholders' => true,
],
'jobs_enrich' => [
'driver' => 'daily',
'path' => storage_path('logs/commands/jobs-enrich.log'),
'level' => 'debug',
'days' => 30,
'replace_placeholders' => true,
],
'jobs_curate' => [
'driver' => 'daily',
'path' => storage_path('logs/commands/jobs-curate.log'),
'level' => 'debug',
'days' => 30,
'replace_placeholders' => true,
],
'jobs_summary' => [
'driver' => 'daily',
'path' => storage_path('logs/commands/jobs-summary.log'),
'level' => 'debug',
'days' => 30,
'replace_placeholders' => true,
],
'jobs_deduplicate' => [
'driver' => 'daily',
'path' => storage_path('logs/commands/jobs-deduplicate.log'),
'level' => 'debug',
'days' => 30,
'replace_placeholders' => true,
],
'jobs_clean' => [
'driver' => 'daily',
'path' => storage_path('logs/commands/jobs-clean.log'),
'level' => 'debug',
'days' => 30,
'replace_placeholders' => true,
],
], ],
]; ];
+9
View File
@@ -112,6 +112,15 @@
<!-- Job Description --> <!-- Job Description -->
<div class="mb-8"> <div class="mb-8">
@if($job->summary)
<h3 class="text-xl font-semibold text-gray-900 mb-4">Job Summary</h3>
<div class="prose prose-emerald prose-lg max-w-none text-gray-700">
{{ $job->summary }}
</div>
<br>
<hr>
<br>
@endif
<h3 class="text-xl font-semibold text-gray-900 mb-4">Job Description</h3> <h3 class="text-xl font-semibold text-gray-900 mb-4">Job Description</h3>
<div class="prose prose-emerald prose-lg max-w-none text-gray-700"> <div class="prose prose-emerald prose-lg max-w-none text-gray-700">
{!! Str::of($job->description) {!! Str::of($job->description)