From 0b26cde74ee2dcab7e4eec3bff3fac6c546a6ca4 Mon Sep 17 00:00:00 2001 From: Doar Mihai Date: Sun, 29 Mar 2026 15:35:08 +0300 Subject: [PATCH] gutendex integration, phase 4 --- app/Jobs/BatchImportFromGutendex.php | 210 ++++++++++++++++++++++++ app/Jobs/GenerateSummary.php | 206 ++++++++++++++++++++++++ app/Jobs/ImportBookFromGutendex.php | 228 +++++++++++++++++++++++++++ 3 files changed, 644 insertions(+) create mode 100644 app/Jobs/BatchImportFromGutendex.php create mode 100644 app/Jobs/GenerateSummary.php create mode 100644 app/Jobs/ImportBookFromGutendex.php diff --git a/app/Jobs/BatchImportFromGutendex.php b/app/Jobs/BatchImportFromGutendex.php new file mode 100644 index 0000000..e9ed598 --- /dev/null +++ b/app/Jobs/BatchImportFromGutendex.php @@ -0,0 +1,210 @@ +filters = $filters; + $this->startPage = $startPage; + $this->endPage = $endPage; + $this->limit = $limit; + $this->generateSummaries = $generateSummaries; + $this->onQueue('gutendex-import'); + } + + /** + * Execute the job. + */ + public function handle(GutendexClient $gutendexClient): void + { + Log::info('BatchImportFromGutendex: Starting batch import', [ + 'filters' => $this->filters, + 'start_page' => $this->startPage, + 'end_page' => $this->endPage, + 'limit' => $this->limit, + ]); + + $totalImported = 0; + $totalSkipped = 0; + $totalErrors = 0; + + try { + for ($page = $this->startPage; $page <= $this->endPage; $page++) { + Log::info('BatchImportFromGutendex: Processing page', [ + 'page' => $page, + 'of' => $this->endPage, + ]); + + // Fetch books from Gutendex + $response = $gutendexClient->searchBooks(null, $page, $this->filters); + + if (empty($response['results'])) { + Log::warning('BatchImportFromGutendex: No results on page', [ + 'page' => $page, + ]); + break; + } + + $booksOnPage = count($response['results']); + $imported = 0; + $skipped = 0; + + foreach ($response['results'] as $bookData) { + // Check if we've reached the limit + if ($totalImported >= $this->limit) { + Log::info('BatchImportFromGutendex: Limit reached', [ + 'limit' => $this->limit, + 'imported' => $totalImported, + ]); + break 2; // Break out of both loops + } + + $gutenbergId = $bookData['id'] ?? null; + + if (!$gutenbergId) { + Log::warning('BatchImportFromGutendex: Book missing ID', [ + 'book_data' => $bookData, + ]); + $totalErrors++; + continue; + } + + // Check if book already exists + if (Book::where('gutenberg_id', $gutenbergId)->exists()) { + Log::debug('BatchImportFromGutendex: Book already exists', [ + 'gutenberg_id' => $gutenbergId, + ]); + $skipped++; + $totalSkipped++; + continue; + } + + // Check if book has plain text available + $formats = $bookData['formats'] ?? []; + $hasPlainText = false; + foreach ($formats as $format => $url) { + if (str_contains($format, 'text/plain')) { + $hasPlainText = true; + break; + } + } + + if (!$hasPlainText) { + Log::debug('BatchImportFromGutendex: No plain text available', [ + 'gutenberg_id' => $gutenbergId, + ]); + $skipped++; + $totalSkipped++; + continue; + } + + // Dispatch individual import job + ImportBookFromGutendex::dispatch($gutenbergId, $this->generateSummaries) + ->delay(now()->addSeconds($totalImported * 2)); // Stagger dispatching + + $imported++; + $totalImported++; + + // Rate limiting: add delay after dispatching + $rateLimit = config('gutendex.rate_limit_delay', 1000); + if ($rateLimit > 0) { + usleep($rateLimit * 1000); + } + } + + Log::info('BatchImportFromGutendex: Page processed', [ + 'page' => $page, + 'books_on_page' => $booksOnPage, + 'imported' => $imported, + 'skipped' => $skipped, + ]); + + // Stop if we've reached the limit + if ($totalImported >= $this->limit) { + break; + } + + // Stop if there are no more results + if (empty($response['next'])) { + Log::info('BatchImportFromGutendex: No more pages available'); + break; + } + } + + Log::info('BatchImportFromGutendex: Batch import completed', [ + 'total_imported' => $totalImported, + 'total_skipped' => $totalSkipped, + 'total_errors' => $totalErrors, + 'pages_processed' => $page - $this->startPage + 1, + ]); + } catch (\Exception $e) { + Log::error('BatchImportFromGutendex: Batch import failed', [ + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + 'imported_before_error' => $totalImported, + ]); + throw $e; + } + } + + /** + * Handle a job failure. + */ + public function failed(\Throwable $exception): void + { + Log::error('BatchImportFromGutendex: Job failed permanently', [ + 'filters' => $this->filters, + 'start_page' => $this->startPage, + 'end_page' => $this->endPage, + 'error' => $exception->getMessage(), + ]); + } +} diff --git a/app/Jobs/GenerateSummary.php b/app/Jobs/GenerateSummary.php new file mode 100644 index 0000000..ab989d6 --- /dev/null +++ b/app/Jobs/GenerateSummary.php @@ -0,0 +1,206 @@ +bookId = $bookId; + $this->onQueue('summary-generation'); + } + + /** + * Execute the job. + */ + public function handle(GutendexClient $gutendexClient, OllamaService $ollamaService): void + { + Log::info('GenerateSummary: Starting summary generation', [ + 'book_id' => $this->bookId, + ]); + + try { + // Load the book + $book = Book::find($this->bookId); + + if (!$book) { + Log::error('GenerateSummary: Book not found', [ + 'book_id' => $this->bookId, + ]); + return; + } + + // Check if summary already exists and is completed + $existingSummary = $book->summary; + if ($existingSummary && $existingSummary->processing_status === 'completed') { + Log::info('GenerateSummary: Summary already exists and is completed', [ + 'book_id' => $this->bookId, + 'summary_id' => $existingSummary->id, + ]); + return; + } + + // Create or get summary record + $summary = $existingSummary ?? Summary::create([ + 'book_id' => $book->id, + 'content' => '', + 'processing_status' => 'pending', + ]); + + // Mark as processing + $summary->markAsProcessing(); + + Log::info('GenerateSummary: Fetching full text', [ + 'book_id' => $this->bookId, + 'url' => $book->full_text_url, + ]); + + // Fetch full text from URL + $fullText = $gutendexClient->fetchFullText($book->full_text_url); + + if (!$fullText) { + $errorMessage = 'Failed to fetch full text from URL'; + Log::error('GenerateSummary: ' . $errorMessage, [ + 'book_id' => $this->bookId, + 'url' => $book->full_text_url, + ]); + $summary->markAsFailed($errorMessage); + return; + } + + // Clean and extract excerpt from the text + $cleanedText = TextProcessor::cleanText($fullText); + $maxLength = config('ollama.text_extraction.max_length', 50000); + $strategy = config('ollama.text_extraction.strategy', 'beginning'); + $excerpt = TextProcessor::extractExcerpt($cleanedText, $maxLength, $strategy); + + Log::info('GenerateSummary: Text processed', [ + 'book_id' => $this->bookId, + 'original_length' => strlen($fullText), + 'cleaned_length' => strlen($cleanedText), + 'excerpt_length' => strlen($excerpt), + ]); + + // Generate summary using Ollama + Log::info('GenerateSummary: Calling Ollama API for summary', [ + 'book_id' => $this->bookId, + 'title' => $book->title, + 'author' => $book->author, + ]); + + $summaryContent = $ollamaService->generateSummary($excerpt, $book->title, $book->author); + + if (!$summaryContent) { + $errorMessage = 'Failed to generate summary from Ollama API'; + Log::error('GenerateSummary: ' . $errorMessage, [ + 'book_id' => $this->bookId, + ]); + $summary->markAsFailed($errorMessage); + return; + } + + // Generate key insights using Ollama + Log::info('GenerateSummary: Calling Ollama API for key insights', [ + 'book_id' => $this->bookId, + ]); + + $keyInsights = $ollamaService->generateKeyInsights($summaryContent, $book->title, $book->author); + + if (!$keyInsights) { + Log::warning('GenerateSummary: Failed to generate key insights, proceeding without them', [ + 'book_id' => $this->bookId, + ]); + $keyInsights = null; + } + + // Store the summary + $summary->update([ + 'content' => $summaryContent, + 'key_insights' => $keyInsights, + ]); + + // Mark as completed + $summary->markAsCompleted($ollamaService->getModel()); + + Log::info('GenerateSummary: Summary generated successfully', [ + 'book_id' => $this->bookId, + 'summary_id' => $summary->id, + 'summary_length' => strlen($summaryContent), + 'has_insights' => !empty($keyInsights), + ]); + } catch (\Exception $e) { + Log::error('GenerateSummary: Exception occurred', [ + 'book_id' => $this->bookId, + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + ]); + + // Try to mark summary as failed if it exists + $book = Book::find($this->bookId); + if ($book && $book->summary) { + $book->summary->markAsFailed($e->getMessage()); + } + + throw $e; + } + } + + /** + * Handle a job failure. + */ + public function failed(\Throwable $exception): void + { + Log::error('GenerateSummary: Job failed permanently', [ + 'book_id' => $this->bookId, + 'error' => $exception->getMessage(), + ]); + + // Mark summary as failed + $book = Book::find($this->bookId); + if ($book && $book->summary) { + $book->summary->markAsFailed( + 'Job failed after ' . $this->tries . ' attempts: ' . $exception->getMessage() + ); + } + } +} diff --git a/app/Jobs/ImportBookFromGutendex.php b/app/Jobs/ImportBookFromGutendex.php new file mode 100644 index 0000000..f61a57d --- /dev/null +++ b/app/Jobs/ImportBookFromGutendex.php @@ -0,0 +1,228 @@ +gutenbergId = $gutenbergId; + $this->generateSummary = $generateSummary; + $this->onQueue('gutendex-import'); + } + + /** + * Execute the job. + */ + public function handle(GutendexClient $gutendexClient): void + { + Log::info('ImportBookFromGutendex: Starting import', [ + 'gutenberg_id' => $this->gutenbergId, + ]); + + try { + // Check if book already exists + $existingBook = Book::where('gutenberg_id', $this->gutenbergId)->first(); + if ($existingBook) { + Log::info('ImportBookFromGutendex: Book already exists, skipping', [ + 'gutenberg_id' => $this->gutenbergId, + 'book_id' => $existingBook->id, + ]); + return; + } + + // Fetch book metadata from Gutendex + $gutendexBook = $gutendexClient->getBook($this->gutenbergId); + + if (!$gutendexBook) { + Log::error('ImportBookFromGutendex: Failed to fetch book from Gutendex', [ + 'gutenberg_id' => $this->gutenbergId, + ]); + $this->fail(new \Exception("Failed to fetch book {$this->gutenbergId} from Gutendex")); + return; + } + + // Only import public domain books + if (!$gutendexBook->isPublicDomain()) { + Log::warning('ImportBookFromGutendex: Book is not public domain, skipping', [ + 'gutenberg_id' => $this->gutenbergId, + 'copyright' => $gutendexBook->copyright, + ]); + return; + } + + // Check if full text is available + if (!$gutendexBook->getPlainTextUrl()) { + Log::warning('ImportBookFromGutendex: No plain text URL available, skipping', [ + 'gutenberg_id' => $this->gutenbergId, + ]); + return; + } + + DB::beginTransaction(); + + try { + // Find or create category + $category = $this->findOrCreateCategory($gutendexBook->suggestCategory()); + + // Create book record + $bookData = $gutendexBook->toBookArray(); + $bookData['category_id'] = $category->id; + $bookData['slug'] = $this->generateUniqueSlug($gutendexBook->title); + + $book = Book::create($bookData); + + // Map authors to the authors table + $this->syncAuthors($book, $gutendexBook->getAuthorNames()); + + DB::commit(); + + Log::info('ImportBookFromGutendex: Book imported successfully', [ + 'gutenberg_id' => $this->gutenbergId, + 'book_id' => $book->id, + 'title' => $book->title, + ]); + + // Dispatch summary generation job + if ($this->generateSummary) { + GenerateSummary::dispatch($book->id) + ->delay(now()->addSeconds(5)); // Small delay to spread load + } + } catch (\Exception $e) { + DB::rollBack(); + throw $e; + } + } catch (\Exception $e) { + Log::error('ImportBookFromGutendex: Import failed', [ + 'gutenberg_id' => $this->gutenbergId, + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + ]); + throw $e; + } + } + + /** + * Find or create a category. + * + * @param string|null $categorySlug + * @return Category + */ + protected function findOrCreateCategory(?string $categorySlug): Category + { + if (!$categorySlug) { + $categorySlug = 'uncategorized'; + } + + $category = Category::where('slug', $categorySlug)->first(); + + if (!$category) { + $categoryName = Str::title(str_replace('-', ' ', $categorySlug)); + $category = Category::create([ + 'name' => $categoryName, + 'slug' => $categorySlug, + 'description' => "Books in the {$categoryName} category", + ]); + + Log::info('ImportBookFromGutendex: Created new category', [ + 'category_id' => $category->id, + 'slug' => $categorySlug, + ]); + } + + return $category; + } + + /** + * Generate a unique slug for the book. + * + * @param string $title + * @return string + */ + protected function generateUniqueSlug(string $title): string + { + $slug = Str::slug($title); + $originalSlug = $slug; + $counter = 1; + + while (Book::where('slug', $slug)->exists()) { + $slug = $originalSlug . '-' . $counter; + $counter++; + } + + return $slug; + } + + /** + * Sync authors to the book. + * + * @param Book $book + * @param array $authorNames + * @return void + */ + protected function syncAuthors(Book $book, array $authorNames): void + { + $authorIds = []; + + foreach ($authorNames as $authorName) { + $author = Author::firstOrCreate( + ['name' => $authorName], + ['slug' => Str::slug($authorName)] + ); + $authorIds[] = $author->id; + } + + $book->authors()->sync($authorIds); + + Log::info('ImportBookFromGutendex: Authors synced', [ + 'book_id' => $book->id, + 'author_count' => count($authorIds), + ]); + } + + /** + * Handle a job failure. + */ + public function failed(\Throwable $exception): void + { + Log::error('ImportBookFromGutendex: Job failed permanently', [ + 'gutenberg_id' => $this->gutenbergId, + 'error' => $exception->getMessage(), + ]); + } +}