gutendex integration, phase 4
This commit is contained in:
Doar Mihai
@@ -0,0 +1,210 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\Book;
|
||||
use App\Services\GutendexClient;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class BatchImportFromGutendex implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public array $filters;
|
||||
public int $startPage;
|
||||
public int $endPage;
|
||||
public int $limit;
|
||||
public bool $generateSummaries;
|
||||
|
||||
/**
|
||||
* The number of times the job may be attempted.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $tries = 3;
|
||||
|
||||
/**
|
||||
* The number of seconds to wait before retrying the job.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $backoff = 120;
|
||||
|
||||
/**
|
||||
* The number of seconds the job can run before timing out.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $timeout = 3600; // 1 hour for batch operations
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(
|
||||
array $filters = [],
|
||||
int $startPage = 1,
|
||||
int $endPage = 1,
|
||||
int $limit = 50,
|
||||
bool $generateSummaries = true
|
||||
) {
|
||||
$this->filters = $filters;
|
||||
$this->startPage = $startPage;
|
||||
$this->endPage = $endPage;
|
||||
$this->limit = $limit;
|
||||
$this->generateSummaries = $generateSummaries;
|
||||
$this->onQueue('gutendex-import');
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(GutendexClient $gutendexClient): void
|
||||
{
|
||||
Log::info('BatchImportFromGutendex: Starting batch import', [
|
||||
'filters' => $this->filters,
|
||||
'start_page' => $this->startPage,
|
||||
'end_page' => $this->endPage,
|
||||
'limit' => $this->limit,
|
||||
]);
|
||||
|
||||
$totalImported = 0;
|
||||
$totalSkipped = 0;
|
||||
$totalErrors = 0;
|
||||
|
||||
try {
|
||||
for ($page = $this->startPage; $page <= $this->endPage; $page++) {
|
||||
Log::info('BatchImportFromGutendex: Processing page', [
|
||||
'page' => $page,
|
||||
'of' => $this->endPage,
|
||||
]);
|
||||
|
||||
// Fetch books from Gutendex
|
||||
$response = $gutendexClient->searchBooks(null, $page, $this->filters);
|
||||
|
||||
if (empty($response['results'])) {
|
||||
Log::warning('BatchImportFromGutendex: No results on page', [
|
||||
'page' => $page,
|
||||
]);
|
||||
break;
|
||||
}
|
||||
|
||||
$booksOnPage = count($response['results']);
|
||||
$imported = 0;
|
||||
$skipped = 0;
|
||||
|
||||
foreach ($response['results'] as $bookData) {
|
||||
// Check if we've reached the limit
|
||||
if ($totalImported >= $this->limit) {
|
||||
Log::info('BatchImportFromGutendex: Limit reached', [
|
||||
'limit' => $this->limit,
|
||||
'imported' => $totalImported,
|
||||
]);
|
||||
break 2; // Break out of both loops
|
||||
}
|
||||
|
||||
$gutenbergId = $bookData['id'] ?? null;
|
||||
|
||||
if (!$gutenbergId) {
|
||||
Log::warning('BatchImportFromGutendex: Book missing ID', [
|
||||
'book_data' => $bookData,
|
||||
]);
|
||||
$totalErrors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if book already exists
|
||||
if (Book::where('gutenberg_id', $gutenbergId)->exists()) {
|
||||
Log::debug('BatchImportFromGutendex: Book already exists', [
|
||||
'gutenberg_id' => $gutenbergId,
|
||||
]);
|
||||
$skipped++;
|
||||
$totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if book has plain text available
|
||||
$formats = $bookData['formats'] ?? [];
|
||||
$hasPlainText = false;
|
||||
foreach ($formats as $format => $url) {
|
||||
if (str_contains($format, 'text/plain')) {
|
||||
$hasPlainText = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!$hasPlainText) {
|
||||
Log::debug('BatchImportFromGutendex: No plain text available', [
|
||||
'gutenberg_id' => $gutenbergId,
|
||||
]);
|
||||
$skipped++;
|
||||
$totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Dispatch individual import job
|
||||
ImportBookFromGutendex::dispatch($gutenbergId, $this->generateSummaries)
|
||||
->delay(now()->addSeconds($totalImported * 2)); // Stagger dispatching
|
||||
|
||||
$imported++;
|
||||
$totalImported++;
|
||||
|
||||
// Rate limiting: add delay after dispatching
|
||||
$rateLimit = config('gutendex.rate_limit_delay', 1000);
|
||||
if ($rateLimit > 0) {
|
||||
usleep($rateLimit * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
Log::info('BatchImportFromGutendex: Page processed', [
|
||||
'page' => $page,
|
||||
'books_on_page' => $booksOnPage,
|
||||
'imported' => $imported,
|
||||
'skipped' => $skipped,
|
||||
]);
|
||||
|
||||
// Stop if we've reached the limit
|
||||
if ($totalImported >= $this->limit) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Stop if there are no more results
|
||||
if (empty($response['next'])) {
|
||||
Log::info('BatchImportFromGutendex: No more pages available');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Log::info('BatchImportFromGutendex: Batch import completed', [
|
||||
'total_imported' => $totalImported,
|
||||
'total_skipped' => $totalSkipped,
|
||||
'total_errors' => $totalErrors,
|
||||
'pages_processed' => $page - $this->startPage + 1,
|
||||
]);
|
||||
} catch (\Exception $e) {
|
||||
Log::error('BatchImportFromGutendex: Batch import failed', [
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString(),
|
||||
'imported_before_error' => $totalImported,
|
||||
]);
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a job failure.
|
||||
*/
|
||||
public function failed(\Throwable $exception): void
|
||||
{
|
||||
Log::error('BatchImportFromGutendex: Job failed permanently', [
|
||||
'filters' => $this->filters,
|
||||
'start_page' => $this->startPage,
|
||||
'end_page' => $this->endPage,
|
||||
'error' => $exception->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user