filters = $filters; $this->startPage = $startPage; $this->endPage = $endPage; $this->limit = $limit; $this->generateSummaries = $generateSummaries; $this->onQueue('gutendex-import'); } /** * Execute the job. */ public function handle(GutendexClient $gutendexClient): void { Log::info('BatchImportFromGutendex: Starting batch import', [ 'filters' => $this->filters, 'start_page' => $this->startPage, 'end_page' => $this->endPage, 'limit' => $this->limit, ]); $totalImported = 0; $totalSkipped = 0; $totalErrors = 0; try { for ($page = $this->startPage; $page <= $this->endPage; $page++) { Log::info('BatchImportFromGutendex: Processing page', [ 'page' => $page, 'of' => $this->endPage, ]); // Fetch books from Gutendex $response = $gutendexClient->searchBooks(null, $page, $this->filters); if (empty($response['results'])) { Log::warning('BatchImportFromGutendex: No results on page', [ 'page' => $page, ]); break; } $booksOnPage = count($response['results']); $imported = 0; $skipped = 0; foreach ($response['results'] as $bookData) { // Check if we've reached the limit if ($totalImported >= $this->limit) { Log::info('BatchImportFromGutendex: Limit reached', [ 'limit' => $this->limit, 'imported' => $totalImported, ]); break 2; // Break out of both loops } $gutenbergId = $bookData['id'] ?? null; if (!$gutenbergId) { Log::warning('BatchImportFromGutendex: Book missing ID', [ 'book_data' => $bookData, ]); $totalErrors++; continue; } // Check if book already exists if (Book::where('gutenberg_id', $gutenbergId)->exists()) { Log::debug('BatchImportFromGutendex: Book already exists', [ 'gutenberg_id' => $gutenbergId, ]); $skipped++; $totalSkipped++; continue; } // Check if book has plain text available $formats = $bookData['formats'] ?? []; $hasPlainText = false; foreach ($formats as $format => $url) { if (str_contains($format, 'text/plain')) { $hasPlainText = true; break; } } if (!$hasPlainText) { Log::debug('BatchImportFromGutendex: No plain text available', [ 'gutenberg_id' => $gutenbergId, ]); $skipped++; $totalSkipped++; continue; } // Dispatch individual import job ImportBookFromGutendex::dispatch($gutenbergId, $this->generateSummaries) ->delay(now()->addSeconds($totalImported * 2)); // Stagger dispatching $imported++; $totalImported++; // Rate limiting: add delay after dispatching $rateLimit = config('gutendex.rate_limit_delay', 1000); if ($rateLimit > 0) { usleep($rateLimit * 1000); } } Log::info('BatchImportFromGutendex: Page processed', [ 'page' => $page, 'books_on_page' => $booksOnPage, 'imported' => $imported, 'skipped' => $skipped, ]); // Stop if we've reached the limit if ($totalImported >= $this->limit) { break; } // Stop if there are no more results if (empty($response['next'])) { Log::info('BatchImportFromGutendex: No more pages available'); break; } } Log::info('BatchImportFromGutendex: Batch import completed', [ 'total_imported' => $totalImported, 'total_skipped' => $totalSkipped, 'total_errors' => $totalErrors, 'pages_processed' => $page - $this->startPage + 1, ]); } catch (\Exception $e) { Log::error('BatchImportFromGutendex: Batch import failed', [ 'error' => $e->getMessage(), 'trace' => $e->getTraceAsString(), 'imported_before_error' => $totalImported, ]); throw $e; } } /** * Handle a job failure. */ public function failed(\Throwable $exception): void { Log::error('BatchImportFromGutendex: Job failed permanently', [ 'filters' => $this->filters, 'start_page' => $this->startPage, 'end_page' => $this->endPage, 'error' => $exception->getMessage(), ]); } }