From 93fef1512a588973c9561cdcd23a2d8370e2aa7a Mon Sep 17 00:00:00 2001 From: Doar Mihai Date: Sun, 29 Mar 2026 15:23:02 +0300 Subject: [PATCH] gutendex processing implementation --- .env.example | 30 +++ app/Models/Book.php | 47 +++++ app/Models/Summary.php | 89 +++++++++ config/gutendex.php | 139 ++++++++++++++ config/ollama.php | 173 ++++++++++++++++++ ...710_add_gutendex_fields_to_books_table.php | 31 ++++ ...d_processing_fields_to_summaries_table.php | 33 ++++ 7 files changed, 542 insertions(+) create mode 100644 config/gutendex.php create mode 100644 config/ollama.php create mode 100644 database/migrations/2026_03_29_121710_add_gutendex_fields_to_books_table.php create mode 100644 database/migrations/2026_03_29_121740_add_processing_fields_to_summaries_table.php diff --git a/.env.example b/.env.example index a6ffcc3..8a8833a 100644 --- a/.env.example +++ b/.env.example @@ -63,3 +63,33 @@ AWS_BUCKET= AWS_USE_PATH_STYLE_ENDPOINT=false VITE_APP_NAME="${APP_NAME}" + +# Gutendex Configuration +GUTENDEX_API_URL=https://gutendex.com +GUTENDEX_DEFAULT_LANGUAGE=en +GUTENDEX_BATCH_SIZE=50 +GUTENDEX_RATE_LIMIT_DELAY=1000 +GUTENDEX_TIMEOUT=30 +GUTENDEX_RETRY_MAX_ATTEMPTS=3 +GUTENDEX_RETRY_BACKOFF=2 +GUTENDEX_COPYRIGHT_FILTER=false +GUTENDEX_CACHE_ENABLED=true +GUTENDEX_CACHE_TTL=86400 + +# Ollama Configuration +OLLAMA_API_URL=http://localhost:11434 +OLLAMA_MODEL=llama3 +OLLAMA_TIMEOUT=300 +OLLAMA_MAX_TOKENS=4000 +OLLAMA_TEMPERATURE=0.3 +OLLAMA_RETRY_MAX_ATTEMPTS=3 +OLLAMA_RETRY_BACKOFF=5 +OLLAMA_TEXT_MAX_LENGTH=50000 +OLLAMA_TEXT_STRATEGY=beginning +OLLAMA_MIN_WORD_COUNT=100 +OLLAMA_MAX_WORD_COUNT=1000 +OLLAMA_MIN_SENTENCE_COUNT=5 +OLLAMA_NUM_PREDICT=1000 +OLLAMA_TOP_K=40 +OLLAMA_TOP_P=0.9 +OLLAMA_REPEAT_PENALTY=1.1 diff --git a/app/Models/Book.php b/app/Models/Book.php index e5ac91a..15f6f05 100644 --- a/app/Models/Book.php +++ b/app/Models/Book.php @@ -14,8 +14,23 @@ class Book extends Model 'publication_date', 'description', 'category_id', + 'gutenberg_id', + 'source', + 'full_text_url', + 'language', ]; + /** + * Get the attributes that should be cast. + */ + protected function casts(): array + { + return [ + 'publication_date' => 'date', + 'gutenberg_id' => 'integer', + ]; + } + /** * Use slug for route model binding and URL generation. */ @@ -24,6 +39,38 @@ class Book extends Model return 'slug'; } + /** + * Scope a query to only include books from a specific source. + */ + public function scopeFromSource($query, string $source) + { + return $query->where('source', $source); + } + + /** + * Scope a query to only include manually added books. + */ + public function scopeManual($query) + { + return $query->where('source', 'manual'); + } + + /** + * Scope a query to only include Gutendex imported books. + */ + public function scopeGutendex($query) + { + return $query->where('source', 'gutendex'); + } + + /** + * Scope a query to only include books in a specific language. + */ + public function scopeInLanguage($query, string $language) + { + return $query->where('language', $language); + } + /** * Get the category that owns the book. */ diff --git a/app/Models/Summary.php b/app/Models/Summary.php index 09ef208..21fac6f 100644 --- a/app/Models/Summary.php +++ b/app/Models/Summary.php @@ -10,8 +10,22 @@ class Summary extends Model 'content', 'key_insights', 'book_id', + 'processing_status', + 'processing_error', + 'processed_at', + 'model_used', ]; + /** + * Get the attributes that should be cast. + */ + protected function casts(): array + { + return [ + 'processed_at' => 'datetime', + ]; + } + /** * Get the book that owns the summary. */ @@ -19,4 +33,79 @@ class Summary extends Model { return $this->belongsTo(Book::class); } + + /** + * Scope a query to only include summaries with a specific status. + */ + public function scopeWithStatus($query, string $status) + { + return $query->where('processing_status', $status); + } + + /** + * Scope a query to only include pending summaries. + */ + public function scopePending($query) + { + return $query->where('processing_status', 'pending'); + } + + /** + * Scope a query to only include processing summaries. + */ + public function scopeProcessing($query) + { + return $query->where('processing_status', 'processing'); + } + + /** + * Scope a query to only include completed summaries. + */ + public function scopeCompleted($query) + { + return $query->where('processing_status', 'completed'); + } + + /** + * Scope a query to only include failed summaries. + */ + public function scopeFailed($query) + { + return $query->where('processing_status', 'failed'); + } + + /** + * Mark the summary as processing. + */ + public function markAsProcessing(): void + { + $this->update([ + 'processing_status' => 'processing', + 'processing_error' => null, + ]); + } + + /** + * Mark the summary as completed. + */ + public function markAsCompleted(string $modelUsed): void + { + $this->update([ + 'processing_status' => 'completed', + 'processing_error' => null, + 'processed_at' => now(), + 'model_used' => $modelUsed, + ]); + } + + /** + * Mark the summary as failed with an error message. + */ + public function markAsFailed(string $error): void + { + $this->update([ + 'processing_status' => 'failed', + 'processing_error' => $error, + ]); + } } diff --git a/config/gutendex.php b/config/gutendex.php new file mode 100644 index 0000000..d357d88 --- /dev/null +++ b/config/gutendex.php @@ -0,0 +1,139 @@ + env('GUTENDEX_API_URL', 'https://gutendex.com'), + + /* + |-------------------------------------------------------------------------- + | Default Language Filter + |-------------------------------------------------------------------------- + | + | The default language code to use when fetching books from Gutendex. + | Common values: 'en' (English), 'fr' (French), 'de' (German), etc. + | + */ + + 'default_language' => env('GUTENDEX_DEFAULT_LANGUAGE', 'en'), + + /* + |-------------------------------------------------------------------------- + | Batch Size + |-------------------------------------------------------------------------- + | + | The number of books to process in a single batch import operation. + | Adjust based on your server capacity and Ollama processing speed. + | + */ + + 'batch_size' => env('GUTENDEX_BATCH_SIZE', 50), + + /* + |-------------------------------------------------------------------------- + | Rate Limit Delay + |-------------------------------------------------------------------------- + | + | Delay in milliseconds between API requests to respect rate limits. + | Increase if you encounter 429 (Too Many Requests) errors. + | + */ + + 'rate_limit_delay' => env('GUTENDEX_RATE_LIMIT_DELAY', 1000), + + /* + |-------------------------------------------------------------------------- + | Request Timeout + |-------------------------------------------------------------------------- + | + | Maximum time in seconds to wait for a response from the Gutendex API. + | + */ + + 'timeout' => env('GUTENDEX_TIMEOUT', 30), + + /* + |-------------------------------------------------------------------------- + | Retry Configuration + |-------------------------------------------------------------------------- + | + | Configuration for retrying failed API requests. + | - max_attempts: Maximum number of retry attempts + | - backoff_multiplier: Multiplier for exponential backoff (in seconds) + | + */ + + 'retry' => [ + 'max_attempts' => env('GUTENDEX_RETRY_MAX_ATTEMPTS', 3), + 'backoff_multiplier' => env('GUTENDEX_RETRY_BACKOFF', 2), + ], + + /* + |-------------------------------------------------------------------------- + | Copyright Filter + |-------------------------------------------------------------------------- + | + | Filter books by copyright status. Setting to 'false' will only fetch + | public domain books. Setting to 'true' includes copyrighted works. + | It's recommended to keep this as 'false' for legal compliance. + | + */ + + 'copyright' => env('GUTENDEX_COPYRIGHT_FILTER', 'false'), + + /* + |-------------------------------------------------------------------------- + | Cache Configuration + |-------------------------------------------------------------------------- + | + | Enable caching of Gutendex API responses to reduce API calls. + | - enabled: Whether to cache responses + | - ttl: Time to live in seconds (default: 24 hours) + | + */ + + 'cache' => [ + 'enabled' => env('GUTENDEX_CACHE_ENABLED', true), + 'ttl' => env('GUTENDEX_CACHE_TTL', 86400), + ], + + /* + |-------------------------------------------------------------------------- + | Category Mapping + |-------------------------------------------------------------------------- + | + | Map Gutendex subjects/bookshelves to your application's categories. + | This helps in auto-categorizing imported books. + | + */ + + 'category_mapping' => [ + 'Fiction' => 'fiction', + 'Science Fiction' => 'science-fiction', + 'Fantasy' => 'fantasy', + 'Mystery' => 'mystery', + 'Romance' => 'romance', + 'Historical Fiction' => 'historical-fiction', + 'Biography' => 'biography', + 'History' => 'history', + 'Philosophy' => 'philosophy', + 'Science' => 'science', + 'Technology' => 'technology', + 'Business' => 'business', + 'Self-Help' => 'self-help', + 'Poetry' => 'poetry', + 'Drama' => 'drama', + 'Children' => 'children', + // Add more mappings as needed + ], + +]; diff --git a/config/ollama.php b/config/ollama.php new file mode 100644 index 0000000..c01825c --- /dev/null +++ b/config/ollama.php @@ -0,0 +1,173 @@ + env('OLLAMA_API_URL', 'http://localhost:11434'), + + /* + |-------------------------------------------------------------------------- + | Default Model + |-------------------------------------------------------------------------- + | + | The default Ollama model to use for generating summaries. Popular + | options include: llama3, llama2, mistral, mixtral, phi, gemma. + | Make sure the model is pulled: ollama pull + | + */ + + 'model' => env('OLLAMA_MODEL', 'llama3'), + + /* + |-------------------------------------------------------------------------- + | Request Timeout + |-------------------------------------------------------------------------- + | + | Maximum time in seconds to wait for a response from Ollama. + | Summary generation can take several minutes for long texts. + | + */ + + 'timeout' => env('OLLAMA_TIMEOUT', 300), + + /* + |-------------------------------------------------------------------------- + | Maximum Tokens + |-------------------------------------------------------------------------- + | + | Maximum number of tokens to send to the model. Most models have + | context limits (e.g., 4096, 8192, 32768). Adjust based on your model. + | + */ + + 'max_tokens' => env('OLLAMA_MAX_TOKENS', 4000), + + /* + |-------------------------------------------------------------------------- + | Temperature + |-------------------------------------------------------------------------- + | + | Controls randomness in generation. Lower values (0.1-0.5) are more + | deterministic and factual. Higher values (0.7-1.0) are more creative. + | For summaries, a lower temperature is recommended. + | + */ + + 'temperature' => env('OLLAMA_TEMPERATURE', 0.3), + + /* + |-------------------------------------------------------------------------- + | Retry Configuration + |-------------------------------------------------------------------------- + | + | Configuration for retrying failed Ollama API requests. + | - max_attempts: Maximum number of retry attempts + | - backoff_multiplier: Multiplier for exponential backoff (in seconds) + | + */ + + 'retry' => [ + 'max_attempts' => env('OLLAMA_RETRY_MAX_ATTEMPTS', 3), + 'backoff_multiplier' => env('OLLAMA_RETRY_BACKOFF', 5), + ], + + /* + |-------------------------------------------------------------------------- + | Text Extraction + |-------------------------------------------------------------------------- + | + | Configuration for extracting representative excerpts from books. + | - max_length: Maximum character length of text to send to Ollama + | - strategy: 'beginning', 'middle', 'end', or 'distributed' + | + */ + + 'text_extraction' => [ + 'max_length' => env('OLLAMA_TEXT_MAX_LENGTH', 50000), + 'strategy' => env('OLLAMA_TEXT_STRATEGY', 'beginning'), + ], + + /* + |-------------------------------------------------------------------------- + | Prompt Templates + |-------------------------------------------------------------------------- + | + | Templates for prompts sent to Ollama. Available placeholders: + | {title}, {author}, {text}, {summary} + | + */ + + 'prompts' => [ + 'summary' => env('OLLAMA_PROMPT_SUMMARY', + "You are a literary expert tasked with creating comprehensive book summaries. " . + "Analyze the following excerpt from \"{title}\" by {author} and provide a detailed, " . + "engaging summary that captures the main themes, key points, and overall significance " . + "of the work. Focus on what makes this book important and valuable to readers. " . + "Keep the summary between 400-600 words.\n\n" . + "Book excerpt:\n{text}\n\n" . + "Summary:" + ), + + 'key_insights' => env('OLLAMA_PROMPT_INSIGHTS', + "Based on the following book summary, extract 5-7 key insights, lessons, or takeaways " . + "that readers should remember. Format your response as a concise bullet list with each " . + "insight on its own line, starting with a dash (-).\n\n" . + "Book: \"{title}\" by {author}\n\n" . + "Summary:\n{summary}\n\n" . + "Key Insights:" + ), + + 'categorize' => env('OLLAMA_PROMPT_CATEGORIZE', + "Based on the following book information, suggest the most appropriate category " . + "from this list: Fiction, Non-Fiction, Science Fiction, Fantasy, Mystery, Romance, " . + "Historical Fiction, Biography, History, Philosophy, Science, Technology, Business, " . + "Self-Help, Poetry, Drama, Children. Respond with ONLY the category name.\n\n" . + "Book: \"{title}\" by {author}\n" . + "Description: {description}\n\n" . + "Category:" + ), + ], + + /* + |-------------------------------------------------------------------------- + | Summary Quality Checks + |-------------------------------------------------------------------------- + | + | Minimum quality thresholds for generated summaries. + | Summaries below these thresholds will be marked for review. + | + */ + + 'quality_checks' => [ + 'min_word_count' => env('OLLAMA_MIN_WORD_COUNT', 100), + 'max_word_count' => env('OLLAMA_MAX_WORD_COUNT', 1000), + 'min_sentence_count' => env('OLLAMA_MIN_SENTENCE_COUNT', 5), + ], + + /* + |-------------------------------------------------------------------------- + | Model Options + |-------------------------------------------------------------------------- + | + | Additional options to pass to the Ollama API. + | See: https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values + | + */ + + 'options' => [ + 'num_predict' => env('OLLAMA_NUM_PREDICT', 1000), // Max tokens to generate + 'top_k' => env('OLLAMA_TOP_K', 40), + 'top_p' => env('OLLAMA_TOP_P', 0.9), + 'repeat_penalty' => env('OLLAMA_REPEAT_PENALTY', 1.1), + ], + +]; diff --git a/database/migrations/2026_03_29_121710_add_gutendex_fields_to_books_table.php b/database/migrations/2026_03_29_121710_add_gutendex_fields_to_books_table.php new file mode 100644 index 0000000..a3f2dc9 --- /dev/null +++ b/database/migrations/2026_03_29_121710_add_gutendex_fields_to_books_table.php @@ -0,0 +1,31 @@ +integer('gutenberg_id')->unique()->nullable()->after('id'); + $table->string('source', 50)->default('manual')->after('gutenberg_id'); + $table->text('full_text_url')->nullable()->after('cover_image'); + $table->string('language', 10)->nullable()->after('full_text_url'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('books', function (Blueprint $table) { + $table->dropColumn(['gutenberg_id', 'source', 'full_text_url', 'language']); + }); + } +}; diff --git a/database/migrations/2026_03_29_121740_add_processing_fields_to_summaries_table.php b/database/migrations/2026_03_29_121740_add_processing_fields_to_summaries_table.php new file mode 100644 index 0000000..ff9c1cb --- /dev/null +++ b/database/migrations/2026_03_29_121740_add_processing_fields_to_summaries_table.php @@ -0,0 +1,33 @@ +enum('processing_status', ['pending', 'processing', 'completed', 'failed']) + ->default('pending') + ->after('book_id'); + $table->text('processing_error')->nullable()->after('processing_status'); + $table->timestamp('processed_at')->nullable()->after('processing_error'); + $table->string('model_used', 100)->nullable()->after('processed_at'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('summaries', function (Blueprint $table) { + $table->dropColumn(['processing_status', 'processing_error', 'processed_at', 'model_used']); + }); + } +};