gutendex integration, phase 4
This commit is contained in:
Doar Mihai
@@ -0,0 +1,210 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\Book;
|
||||
use App\Services\GutendexClient;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class BatchImportFromGutendex implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public array $filters;
|
||||
public int $startPage;
|
||||
public int $endPage;
|
||||
public int $limit;
|
||||
public bool $generateSummaries;
|
||||
|
||||
/**
|
||||
* The number of times the job may be attempted.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $tries = 3;
|
||||
|
||||
/**
|
||||
* The number of seconds to wait before retrying the job.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $backoff = 120;
|
||||
|
||||
/**
|
||||
* The number of seconds the job can run before timing out.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $timeout = 3600; // 1 hour for batch operations
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(
|
||||
array $filters = [],
|
||||
int $startPage = 1,
|
||||
int $endPage = 1,
|
||||
int $limit = 50,
|
||||
bool $generateSummaries = true
|
||||
) {
|
||||
$this->filters = $filters;
|
||||
$this->startPage = $startPage;
|
||||
$this->endPage = $endPage;
|
||||
$this->limit = $limit;
|
||||
$this->generateSummaries = $generateSummaries;
|
||||
$this->onQueue('gutendex-import');
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(GutendexClient $gutendexClient): void
|
||||
{
|
||||
Log::info('BatchImportFromGutendex: Starting batch import', [
|
||||
'filters' => $this->filters,
|
||||
'start_page' => $this->startPage,
|
||||
'end_page' => $this->endPage,
|
||||
'limit' => $this->limit,
|
||||
]);
|
||||
|
||||
$totalImported = 0;
|
||||
$totalSkipped = 0;
|
||||
$totalErrors = 0;
|
||||
|
||||
try {
|
||||
for ($page = $this->startPage; $page <= $this->endPage; $page++) {
|
||||
Log::info('BatchImportFromGutendex: Processing page', [
|
||||
'page' => $page,
|
||||
'of' => $this->endPage,
|
||||
]);
|
||||
|
||||
// Fetch books from Gutendex
|
||||
$response = $gutendexClient->searchBooks(null, $page, $this->filters);
|
||||
|
||||
if (empty($response['results'])) {
|
||||
Log::warning('BatchImportFromGutendex: No results on page', [
|
||||
'page' => $page,
|
||||
]);
|
||||
break;
|
||||
}
|
||||
|
||||
$booksOnPage = count($response['results']);
|
||||
$imported = 0;
|
||||
$skipped = 0;
|
||||
|
||||
foreach ($response['results'] as $bookData) {
|
||||
// Check if we've reached the limit
|
||||
if ($totalImported >= $this->limit) {
|
||||
Log::info('BatchImportFromGutendex: Limit reached', [
|
||||
'limit' => $this->limit,
|
||||
'imported' => $totalImported,
|
||||
]);
|
||||
break 2; // Break out of both loops
|
||||
}
|
||||
|
||||
$gutenbergId = $bookData['id'] ?? null;
|
||||
|
||||
if (!$gutenbergId) {
|
||||
Log::warning('BatchImportFromGutendex: Book missing ID', [
|
||||
'book_data' => $bookData,
|
||||
]);
|
||||
$totalErrors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if book already exists
|
||||
if (Book::where('gutenberg_id', $gutenbergId)->exists()) {
|
||||
Log::debug('BatchImportFromGutendex: Book already exists', [
|
||||
'gutenberg_id' => $gutenbergId,
|
||||
]);
|
||||
$skipped++;
|
||||
$totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if book has plain text available
|
||||
$formats = $bookData['formats'] ?? [];
|
||||
$hasPlainText = false;
|
||||
foreach ($formats as $format => $url) {
|
||||
if (str_contains($format, 'text/plain')) {
|
||||
$hasPlainText = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!$hasPlainText) {
|
||||
Log::debug('BatchImportFromGutendex: No plain text available', [
|
||||
'gutenberg_id' => $gutenbergId,
|
||||
]);
|
||||
$skipped++;
|
||||
$totalSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Dispatch individual import job
|
||||
ImportBookFromGutendex::dispatch($gutenbergId, $this->generateSummaries)
|
||||
->delay(now()->addSeconds($totalImported * 2)); // Stagger dispatching
|
||||
|
||||
$imported++;
|
||||
$totalImported++;
|
||||
|
||||
// Rate limiting: add delay after dispatching
|
||||
$rateLimit = config('gutendex.rate_limit_delay', 1000);
|
||||
if ($rateLimit > 0) {
|
||||
usleep($rateLimit * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
Log::info('BatchImportFromGutendex: Page processed', [
|
||||
'page' => $page,
|
||||
'books_on_page' => $booksOnPage,
|
||||
'imported' => $imported,
|
||||
'skipped' => $skipped,
|
||||
]);
|
||||
|
||||
// Stop if we've reached the limit
|
||||
if ($totalImported >= $this->limit) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Stop if there are no more results
|
||||
if (empty($response['next'])) {
|
||||
Log::info('BatchImportFromGutendex: No more pages available');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Log::info('BatchImportFromGutendex: Batch import completed', [
|
||||
'total_imported' => $totalImported,
|
||||
'total_skipped' => $totalSkipped,
|
||||
'total_errors' => $totalErrors,
|
||||
'pages_processed' => $page - $this->startPage + 1,
|
||||
]);
|
||||
} catch (\Exception $e) {
|
||||
Log::error('BatchImportFromGutendex: Batch import failed', [
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString(),
|
||||
'imported_before_error' => $totalImported,
|
||||
]);
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a job failure.
|
||||
*/
|
||||
public function failed(\Throwable $exception): void
|
||||
{
|
||||
Log::error('BatchImportFromGutendex: Job failed permanently', [
|
||||
'filters' => $this->filters,
|
||||
'start_page' => $this->startPage,
|
||||
'end_page' => $this->endPage,
|
||||
'error' => $exception->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Helpers\TextProcessor;
|
||||
use App\Models\Book;
|
||||
use App\Models\Summary;
|
||||
use App\Services\GutendexClient;
|
||||
use App\Services\OllamaService;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class GenerateSummary implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public int $bookId;
|
||||
|
||||
/**
|
||||
* The number of times the job may be attempted.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $tries = 3;
|
||||
|
||||
/**
|
||||
* The number of seconds to wait before retrying the job.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $backoff = 300; // 5 minutes between retries
|
||||
|
||||
/**
|
||||
* The number of seconds the job can run before timing out.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $timeout = 600; // 10 minutes
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(int $bookId)
|
||||
{
|
||||
$this->bookId = $bookId;
|
||||
$this->onQueue('summary-generation');
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(GutendexClient $gutendexClient, OllamaService $ollamaService): void
|
||||
{
|
||||
Log::info('GenerateSummary: Starting summary generation', [
|
||||
'book_id' => $this->bookId,
|
||||
]);
|
||||
|
||||
try {
|
||||
// Load the book
|
||||
$book = Book::find($this->bookId);
|
||||
|
||||
if (!$book) {
|
||||
Log::error('GenerateSummary: Book not found', [
|
||||
'book_id' => $this->bookId,
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if summary already exists and is completed
|
||||
$existingSummary = $book->summary;
|
||||
if ($existingSummary && $existingSummary->processing_status === 'completed') {
|
||||
Log::info('GenerateSummary: Summary already exists and is completed', [
|
||||
'book_id' => $this->bookId,
|
||||
'summary_id' => $existingSummary->id,
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Create or get summary record
|
||||
$summary = $existingSummary ?? Summary::create([
|
||||
'book_id' => $book->id,
|
||||
'content' => '',
|
||||
'processing_status' => 'pending',
|
||||
]);
|
||||
|
||||
// Mark as processing
|
||||
$summary->markAsProcessing();
|
||||
|
||||
Log::info('GenerateSummary: Fetching full text', [
|
||||
'book_id' => $this->bookId,
|
||||
'url' => $book->full_text_url,
|
||||
]);
|
||||
|
||||
// Fetch full text from URL
|
||||
$fullText = $gutendexClient->fetchFullText($book->full_text_url);
|
||||
|
||||
if (!$fullText) {
|
||||
$errorMessage = 'Failed to fetch full text from URL';
|
||||
Log::error('GenerateSummary: ' . $errorMessage, [
|
||||
'book_id' => $this->bookId,
|
||||
'url' => $book->full_text_url,
|
||||
]);
|
||||
$summary->markAsFailed($errorMessage);
|
||||
return;
|
||||
}
|
||||
|
||||
// Clean and extract excerpt from the text
|
||||
$cleanedText = TextProcessor::cleanText($fullText);
|
||||
$maxLength = config('ollama.text_extraction.max_length', 50000);
|
||||
$strategy = config('ollama.text_extraction.strategy', 'beginning');
|
||||
$excerpt = TextProcessor::extractExcerpt($cleanedText, $maxLength, $strategy);
|
||||
|
||||
Log::info('GenerateSummary: Text processed', [
|
||||
'book_id' => $this->bookId,
|
||||
'original_length' => strlen($fullText),
|
||||
'cleaned_length' => strlen($cleanedText),
|
||||
'excerpt_length' => strlen($excerpt),
|
||||
]);
|
||||
|
||||
// Generate summary using Ollama
|
||||
Log::info('GenerateSummary: Calling Ollama API for summary', [
|
||||
'book_id' => $this->bookId,
|
||||
'title' => $book->title,
|
||||
'author' => $book->author,
|
||||
]);
|
||||
|
||||
$summaryContent = $ollamaService->generateSummary($excerpt, $book->title, $book->author);
|
||||
|
||||
if (!$summaryContent) {
|
||||
$errorMessage = 'Failed to generate summary from Ollama API';
|
||||
Log::error('GenerateSummary: ' . $errorMessage, [
|
||||
'book_id' => $this->bookId,
|
||||
]);
|
||||
$summary->markAsFailed($errorMessage);
|
||||
return;
|
||||
}
|
||||
|
||||
// Generate key insights using Ollama
|
||||
Log::info('GenerateSummary: Calling Ollama API for key insights', [
|
||||
'book_id' => $this->bookId,
|
||||
]);
|
||||
|
||||
$keyInsights = $ollamaService->generateKeyInsights($summaryContent, $book->title, $book->author);
|
||||
|
||||
if (!$keyInsights) {
|
||||
Log::warning('GenerateSummary: Failed to generate key insights, proceeding without them', [
|
||||
'book_id' => $this->bookId,
|
||||
]);
|
||||
$keyInsights = null;
|
||||
}
|
||||
|
||||
// Store the summary
|
||||
$summary->update([
|
||||
'content' => $summaryContent,
|
||||
'key_insights' => $keyInsights,
|
||||
]);
|
||||
|
||||
// Mark as completed
|
||||
$summary->markAsCompleted($ollamaService->getModel());
|
||||
|
||||
Log::info('GenerateSummary: Summary generated successfully', [
|
||||
'book_id' => $this->bookId,
|
||||
'summary_id' => $summary->id,
|
||||
'summary_length' => strlen($summaryContent),
|
||||
'has_insights' => !empty($keyInsights),
|
||||
]);
|
||||
} catch (\Exception $e) {
|
||||
Log::error('GenerateSummary: Exception occurred', [
|
||||
'book_id' => $this->bookId,
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString(),
|
||||
]);
|
||||
|
||||
// Try to mark summary as failed if it exists
|
||||
$book = Book::find($this->bookId);
|
||||
if ($book && $book->summary) {
|
||||
$book->summary->markAsFailed($e->getMessage());
|
||||
}
|
||||
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a job failure.
|
||||
*/
|
||||
public function failed(\Throwable $exception): void
|
||||
{
|
||||
Log::error('GenerateSummary: Job failed permanently', [
|
||||
'book_id' => $this->bookId,
|
||||
'error' => $exception->getMessage(),
|
||||
]);
|
||||
|
||||
// Mark summary as failed
|
||||
$book = Book::find($this->bookId);
|
||||
if ($book && $book->summary) {
|
||||
$book->summary->markAsFailed(
|
||||
'Job failed after ' . $this->tries . ' attempts: ' . $exception->getMessage()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Models\Author;
|
||||
use App\Models\Book;
|
||||
use App\Models\Category;
|
||||
use App\Services\GutendexClient;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class ImportBookFromGutendex implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public int $gutenbergId;
|
||||
public bool $generateSummary;
|
||||
|
||||
/**
|
||||
* The number of times the job may be attempted.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $tries = 3;
|
||||
|
||||
/**
|
||||
* The number of seconds to wait before retrying the job.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
public $backoff = 60;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(int $gutenbergId, bool $generateSummary = true)
|
||||
{
|
||||
$this->gutenbergId = $gutenbergId;
|
||||
$this->generateSummary = $generateSummary;
|
||||
$this->onQueue('gutendex-import');
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(GutendexClient $gutendexClient): void
|
||||
{
|
||||
Log::info('ImportBookFromGutendex: Starting import', [
|
||||
'gutenberg_id' => $this->gutenbergId,
|
||||
]);
|
||||
|
||||
try {
|
||||
// Check if book already exists
|
||||
$existingBook = Book::where('gutenberg_id', $this->gutenbergId)->first();
|
||||
if ($existingBook) {
|
||||
Log::info('ImportBookFromGutendex: Book already exists, skipping', [
|
||||
'gutenberg_id' => $this->gutenbergId,
|
||||
'book_id' => $existingBook->id,
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Fetch book metadata from Gutendex
|
||||
$gutendexBook = $gutendexClient->getBook($this->gutenbergId);
|
||||
|
||||
if (!$gutendexBook) {
|
||||
Log::error('ImportBookFromGutendex: Failed to fetch book from Gutendex', [
|
||||
'gutenberg_id' => $this->gutenbergId,
|
||||
]);
|
||||
$this->fail(new \Exception("Failed to fetch book {$this->gutenbergId} from Gutendex"));
|
||||
return;
|
||||
}
|
||||
|
||||
// Only import public domain books
|
||||
if (!$gutendexBook->isPublicDomain()) {
|
||||
Log::warning('ImportBookFromGutendex: Book is not public domain, skipping', [
|
||||
'gutenberg_id' => $this->gutenbergId,
|
||||
'copyright' => $gutendexBook->copyright,
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if full text is available
|
||||
if (!$gutendexBook->getPlainTextUrl()) {
|
||||
Log::warning('ImportBookFromGutendex: No plain text URL available, skipping', [
|
||||
'gutenberg_id' => $this->gutenbergId,
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
DB::beginTransaction();
|
||||
|
||||
try {
|
||||
// Find or create category
|
||||
$category = $this->findOrCreateCategory($gutendexBook->suggestCategory());
|
||||
|
||||
// Create book record
|
||||
$bookData = $gutendexBook->toBookArray();
|
||||
$bookData['category_id'] = $category->id;
|
||||
$bookData['slug'] = $this->generateUniqueSlug($gutendexBook->title);
|
||||
|
||||
$book = Book::create($bookData);
|
||||
|
||||
// Map authors to the authors table
|
||||
$this->syncAuthors($book, $gutendexBook->getAuthorNames());
|
||||
|
||||
DB::commit();
|
||||
|
||||
Log::info('ImportBookFromGutendex: Book imported successfully', [
|
||||
'gutenberg_id' => $this->gutenbergId,
|
||||
'book_id' => $book->id,
|
||||
'title' => $book->title,
|
||||
]);
|
||||
|
||||
// Dispatch summary generation job
|
||||
if ($this->generateSummary) {
|
||||
GenerateSummary::dispatch($book->id)
|
||||
->delay(now()->addSeconds(5)); // Small delay to spread load
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
DB::rollBack();
|
||||
throw $e;
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
Log::error('ImportBookFromGutendex: Import failed', [
|
||||
'gutenberg_id' => $this->gutenbergId,
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString(),
|
||||
]);
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find or create a category.
|
||||
*
|
||||
* @param string|null $categorySlug
|
||||
* @return Category
|
||||
*/
|
||||
protected function findOrCreateCategory(?string $categorySlug): Category
|
||||
{
|
||||
if (!$categorySlug) {
|
||||
$categorySlug = 'uncategorized';
|
||||
}
|
||||
|
||||
$category = Category::where('slug', $categorySlug)->first();
|
||||
|
||||
if (!$category) {
|
||||
$categoryName = Str::title(str_replace('-', ' ', $categorySlug));
|
||||
$category = Category::create([
|
||||
'name' => $categoryName,
|
||||
'slug' => $categorySlug,
|
||||
'description' => "Books in the {$categoryName} category",
|
||||
]);
|
||||
|
||||
Log::info('ImportBookFromGutendex: Created new category', [
|
||||
'category_id' => $category->id,
|
||||
'slug' => $categorySlug,
|
||||
]);
|
||||
}
|
||||
|
||||
return $category;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a unique slug for the book.
|
||||
*
|
||||
* @param string $title
|
||||
* @return string
|
||||
*/
|
||||
protected function generateUniqueSlug(string $title): string
|
||||
{
|
||||
$slug = Str::slug($title);
|
||||
$originalSlug = $slug;
|
||||
$counter = 1;
|
||||
|
||||
while (Book::where('slug', $slug)->exists()) {
|
||||
$slug = $originalSlug . '-' . $counter;
|
||||
$counter++;
|
||||
}
|
||||
|
||||
return $slug;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync authors to the book.
|
||||
*
|
||||
* @param Book $book
|
||||
* @param array $authorNames
|
||||
* @return void
|
||||
*/
|
||||
protected function syncAuthors(Book $book, array $authorNames): void
|
||||
{
|
||||
$authorIds = [];
|
||||
|
||||
foreach ($authorNames as $authorName) {
|
||||
$author = Author::firstOrCreate(
|
||||
['name' => $authorName],
|
||||
['slug' => Str::slug($authorName)]
|
||||
);
|
||||
$authorIds[] = $author->id;
|
||||
}
|
||||
|
||||
$book->authors()->sync($authorIds);
|
||||
|
||||
Log::info('ImportBookFromGutendex: Authors synced', [
|
||||
'book_id' => $book->id,
|
||||
'author_count' => count($authorIds),
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a job failure.
|
||||
*/
|
||||
public function failed(\Throwable $exception): void
|
||||
{
|
||||
Log::error('ImportBookFromGutendex: Job failed permanently', [
|
||||
'gutenberg_id' => $this->gutenbergId,
|
||||
'error' => $exception->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user