root/opentextsummaries
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

gutendex processing implementation

Browse Source
This commit is contained in:
Doar Mihai
2026-03-29 15:23:02 +03:00
parent 7be6af781b
commit 93fef1512a
7 changed files with 542 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+30
View File
@@ -63,3 +63,33 @@ AWS_BUCKET=
AWS_USE_PATH_STYLE_ENDPOINT=false AWS_USE_PATH_STYLE_ENDPOINT=false
VITE_APP_NAME="${APP_NAME}" VITE_APP_NAME="${APP_NAME}"
# Gutendex Configuration
GUTENDEX_API_URL=https://gutendex.com
GUTENDEX_DEFAULT_LANGUAGE=en
GUTENDEX_BATCH_SIZE=50
GUTENDEX_RATE_LIMIT_DELAY=1000
GUTENDEX_TIMEOUT=30
GUTENDEX_RETRY_MAX_ATTEMPTS=3
GUTENDEX_RETRY_BACKOFF=2
GUTENDEX_COPYRIGHT_FILTER=false
GUTENDEX_CACHE_ENABLED=true
GUTENDEX_CACHE_TTL=86400
# Ollama Configuration
OLLAMA_API_URL=http://localhost:11434
OLLAMA_MODEL=llama3
OLLAMA_TIMEOUT=300
OLLAMA_MAX_TOKENS=4000
OLLAMA_TEMPERATURE=0.3
OLLAMA_RETRY_MAX_ATTEMPTS=3
OLLAMA_RETRY_BACKOFF=5
OLLAMA_TEXT_MAX_LENGTH=50000
OLLAMA_TEXT_STRATEGY=beginning
OLLAMA_MIN_WORD_COUNT=100
OLLAMA_MAX_WORD_COUNT=1000
OLLAMA_MIN_SENTENCE_COUNT=5
OLLAMA_NUM_PREDICT=1000
OLLAMA_TOP_K=40
OLLAMA_TOP_P=0.9
OLLAMA_REPEAT_PENALTY=1.1
app/Models/Book.php
+47
View File
@@ -14,8 +14,23 @@ class Book extends Model
'publication_date', 'publication_date',
'description', 'description',
'category_id', 'category_id',
'gutenberg_id',
'source',
'full_text_url',
'language',
]; ];
/**
* Get the attributes that should be cast.
*/
protected function casts(): array
{
return [
'publication_date' => 'date',
'gutenberg_id' => 'integer',
];
}
/** /**
* Use slug for route model binding and URL generation. * Use slug for route model binding and URL generation.
*/ */
@@ -24,6 +39,38 @@ class Book extends Model
return 'slug'; return 'slug';
} }
/**
* Scope a query to only include books from a specific source.
*/
public function scopeFromSource($query, string $source)
{
return $query->where('source', $source);
}
/**
* Scope a query to only include manually added books.
*/
public function scopeManual($query)
{
return $query->where('source', 'manual');
}
/**
* Scope a query to only include Gutendex imported books.
*/
public function scopeGutendex($query)
{
return $query->where('source', 'gutendex');
}
/**
* Scope a query to only include books in a specific language.
*/
public function scopeInLanguage($query, string $language)
{
return $query->where('language', $language);
}
/** /**
* Get the category that owns the book. * Get the category that owns the book.
*/ */
app/Models/Summary.php
+89
View File
@@ -10,8 +10,22 @@ class Summary extends Model
'content', 'content',
'key_insights', 'key_insights',
'book_id', 'book_id',
'processing_status',
'processing_error',
'processed_at',
'model_used',
]; ];
/**
* Get the attributes that should be cast.
*/
protected function casts(): array
{
return [
'processed_at' => 'datetime',
];
}
/** /**
* Get the book that owns the summary. * Get the book that owns the summary.
*/ */
@@ -19,4 +33,79 @@ class Summary extends Model
{ {
return $this->belongsTo(Book::class); return $this->belongsTo(Book::class);
} }
/**
* Scope a query to only include summaries with a specific status.
*/
public function scopeWithStatus($query, string $status)
{
return $query->where('processing_status', $status);
}
/**
* Scope a query to only include pending summaries.
*/
public function scopePending($query)
{
return $query->where('processing_status', 'pending');
}
/**
* Scope a query to only include processing summaries.
*/
public function scopeProcessing($query)
{
return $query->where('processing_status', 'processing');
}
/**
* Scope a query to only include completed summaries.
*/
public function scopeCompleted($query)
{
return $query->where('processing_status', 'completed');
}
/**
* Scope a query to only include failed summaries.
*/
public function scopeFailed($query)
{
return $query->where('processing_status', 'failed');
}
/**
* Mark the summary as processing.
*/
public function markAsProcessing(): void
{
$this->update([
'processing_status' => 'processing',
'processing_error' => null,
]);
}
/**
* Mark the summary as completed.
*/
public function markAsCompleted(string $modelUsed): void
{
$this->update([
'processing_status' => 'completed',
'processing_error' => null,
'processed_at' => now(),
'model_used' => $modelUsed,
]);
}
/**
* Mark the summary as failed with an error message.
*/
public function markAsFailed(string $error): void
{
$this->update([
'processing_status' => 'failed',
'processing_error' => $error,
]);
}
} }
config/gutendex.php
+139
View File
@@ -0,0 +1,139 @@
<?php
return [
/*
|--------------------------------------------------------------------------
| Gutendex API Base URL
|--------------------------------------------------------------------------
|
| The base URL for the Gutendex API. This is the Project Gutenberg
| catalog API that provides metadata about public domain books.
|
*/
'api_url' => env('GUTENDEX_API_URL', 'https://gutendex.com'),
/*
|--------------------------------------------------------------------------
| Default Language Filter
|--------------------------------------------------------------------------
|
| The default language code to use when fetching books from Gutendex.
| Common values: 'en' (English), 'fr' (French), 'de' (German), etc.
|
*/
'default_language' => env('GUTENDEX_DEFAULT_LANGUAGE', 'en'),
/*
|--------------------------------------------------------------------------
| Batch Size
|--------------------------------------------------------------------------
|
| The number of books to process in a single batch import operation.
| Adjust based on your server capacity and Ollama processing speed.
|
*/
'batch_size' => env('GUTENDEX_BATCH_SIZE', 50),
/*
|--------------------------------------------------------------------------
| Rate Limit Delay
|--------------------------------------------------------------------------
|
| Delay in milliseconds between API requests to respect rate limits.
| Increase if you encounter 429 (Too Many Requests) errors.
|
*/
'rate_limit_delay' => env('GUTENDEX_RATE_LIMIT_DELAY', 1000),
/*
|--------------------------------------------------------------------------
| Request Timeout
|--------------------------------------------------------------------------
|
| Maximum time in seconds to wait for a response from the Gutendex API.
|
*/
'timeout' => env('GUTENDEX_TIMEOUT', 30),
/*
|--------------------------------------------------------------------------
| Retry Configuration
|--------------------------------------------------------------------------
|
| Configuration for retrying failed API requests.
| - max_attempts: Maximum number of retry attempts
| - backoff_multiplier: Multiplier for exponential backoff (in seconds)
|
*/
'retry' => [
'max_attempts' => env('GUTENDEX_RETRY_MAX_ATTEMPTS', 3),
'backoff_multiplier' => env('GUTENDEX_RETRY_BACKOFF', 2),
],
/*
|--------------------------------------------------------------------------
| Copyright Filter
|--------------------------------------------------------------------------
|
| Filter books by copyright status. Setting to 'false' will only fetch
| public domain books. Setting to 'true' includes copyrighted works.
| It's recommended to keep this as 'false' for legal compliance.
|
*/
'copyright' => env('GUTENDEX_COPYRIGHT_FILTER', 'false'),
/*
|--------------------------------------------------------------------------
| Cache Configuration
|--------------------------------------------------------------------------
|
| Enable caching of Gutendex API responses to reduce API calls.
| - enabled: Whether to cache responses
| - ttl: Time to live in seconds (default: 24 hours)
|
*/
'cache' => [
'enabled' => env('GUTENDEX_CACHE_ENABLED', true),
'ttl' => env('GUTENDEX_CACHE_TTL', 86400),
],
/*
|--------------------------------------------------------------------------
| Category Mapping
|--------------------------------------------------------------------------
|
| Map Gutendex subjects/bookshelves to your application's categories.
| This helps in auto-categorizing imported books.
|
*/
'category_mapping' => [
'Fiction' => 'fiction',
'Science Fiction' => 'science-fiction',
'Fantasy' => 'fantasy',
'Mystery' => 'mystery',
'Romance' => 'romance',
'Historical Fiction' => 'historical-fiction',
'Biography' => 'biography',
'History' => 'history',
'Philosophy' => 'philosophy',
'Science' => 'science',
'Technology' => 'technology',
'Business' => 'business',
'Self-Help' => 'self-help',
'Poetry' => 'poetry',
'Drama' => 'drama',
'Children' => 'children',
// Add more mappings as needed
],
];
config/ollama.php
+173
View File
@@ -0,0 +1,173 @@
<?php
return [
/*
|--------------------------------------------------------------------------
| Ollama API URL
|--------------------------------------------------------------------------
|
| The base URL for your local Ollama API instance. By default, Ollama
| runs on localhost:11434. Update this if running on a different host.
|
*/
'api_url' => env('OLLAMA_API_URL', 'http://localhost:11434'),
/*
|--------------------------------------------------------------------------
| Default Model
|--------------------------------------------------------------------------
|
| The default Ollama model to use for generating summaries. Popular
| options include: llama3, llama2, mistral, mixtral, phi, gemma.
| Make sure the model is pulled: ollama pull <model_name>
|
*/
'model' => env('OLLAMA_MODEL', 'llama3'),
/*
|--------------------------------------------------------------------------
| Request Timeout
|--------------------------------------------------------------------------
|
| Maximum time in seconds to wait for a response from Ollama.
| Summary generation can take several minutes for long texts.
|
*/
'timeout' => env('OLLAMA_TIMEOUT', 300),
/*
|--------------------------------------------------------------------------
| Maximum Tokens
|--------------------------------------------------------------------------
|
| Maximum number of tokens to send to the model. Most models have
| context limits (e.g., 4096, 8192, 32768). Adjust based on your model.
|
*/
'max_tokens' => env('OLLAMA_MAX_TOKENS', 4000),
/*
|--------------------------------------------------------------------------
| Temperature
|--------------------------------------------------------------------------
|
| Controls randomness in generation. Lower values (0.1-0.5) are more
| deterministic and factual. Higher values (0.7-1.0) are more creative.
| For summaries, a lower temperature is recommended.
|
*/
'temperature' => env('OLLAMA_TEMPERATURE', 0.3),
/*
|--------------------------------------------------------------------------
| Retry Configuration
|--------------------------------------------------------------------------
|
| Configuration for retrying failed Ollama API requests.
| - max_attempts: Maximum number of retry attempts
| - backoff_multiplier: Multiplier for exponential backoff (in seconds)
|
*/
'retry' => [
'max_attempts' => env('OLLAMA_RETRY_MAX_ATTEMPTS', 3),
'backoff_multiplier' => env('OLLAMA_RETRY_BACKOFF', 5),
],
/*
|--------------------------------------------------------------------------
| Text Extraction
|--------------------------------------------------------------------------
|
| Configuration for extracting representative excerpts from books.
| - max_length: Maximum character length of text to send to Ollama
| - strategy: 'beginning', 'middle', 'end', or 'distributed'
|
*/
'text_extraction' => [
'max_length' => env('OLLAMA_TEXT_MAX_LENGTH', 50000),
'strategy' => env('OLLAMA_TEXT_STRATEGY', 'beginning'),
],
/*
|--------------------------------------------------------------------------
| Prompt Templates
|--------------------------------------------------------------------------
|
| Templates for prompts sent to Ollama. Available placeholders:
| {title}, {author}, {text}, {summary}
|
*/
'prompts' => [
'summary' => env('OLLAMA_PROMPT_SUMMARY',
"You are a literary expert tasked with creating comprehensive book summaries. " .
"Analyze the following excerpt from \"{title}\" by {author} and provide a detailed, " .
"engaging summary that captures the main themes, key points, and overall significance " .
"of the work. Focus on what makes this book important and valuable to readers. " .
"Keep the summary between 400-600 words.\n\n" .
"Book excerpt:\n{text}\n\n" .
"Summary:"
),
'key_insights' => env('OLLAMA_PROMPT_INSIGHTS',
"Based on the following book summary, extract 5-7 key insights, lessons, or takeaways " .
"that readers should remember. Format your response as a concise bullet list with each " .
"insight on its own line, starting with a dash (-).\n\n" .
"Book: \"{title}\" by {author}\n\n" .
"Summary:\n{summary}\n\n" .
"Key Insights:"
),
'categorize' => env('OLLAMA_PROMPT_CATEGORIZE',
"Based on the following book information, suggest the most appropriate category " .
"from this list: Fiction, Non-Fiction, Science Fiction, Fantasy, Mystery, Romance, " .
"Historical Fiction, Biography, History, Philosophy, Science, Technology, Business, " .
"Self-Help, Poetry, Drama, Children. Respond with ONLY the category name.\n\n" .
"Book: \"{title}\" by {author}\n" .
"Description: {description}\n\n" .
"Category:"
),
],
/*
|--------------------------------------------------------------------------
| Summary Quality Checks
|--------------------------------------------------------------------------
|
| Minimum quality thresholds for generated summaries.
| Summaries below these thresholds will be marked for review.
|
*/
'quality_checks' => [
'min_word_count' => env('OLLAMA_MIN_WORD_COUNT', 100),
'max_word_count' => env('OLLAMA_MAX_WORD_COUNT', 1000),
'min_sentence_count' => env('OLLAMA_MIN_SENTENCE_COUNT', 5),
],
/*
|--------------------------------------------------------------------------
| Model Options
|--------------------------------------------------------------------------
|
| Additional options to pass to the Ollama API.
| See: https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values
|
*/
'options' => [
'num_predict' => env('OLLAMA_NUM_PREDICT', 1000), // Max tokens to generate
'top_k' => env('OLLAMA_TOP_K', 40),
'top_p' => env('OLLAMA_TOP_P', 0.9),
'repeat_penalty' => env('OLLAMA_REPEAT_PENALTY', 1.1),
],
];
database/migrations/2026_03_29_121710_add_gutendex_fields_to_books_table.php
+31
View File
@@ -0,0 +1,31 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('books', function (Blueprint $table) {
$table->integer('gutenberg_id')->unique()->nullable()->after('id');
$table->string('source', 50)->default('manual')->after('gutenberg_id');
$table->text('full_text_url')->nullable()->after('cover_image');
$table->string('language', 10)->nullable()->after('full_text_url');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('books', function (Blueprint $table) {
$table->dropColumn(['gutenberg_id', 'source', 'full_text_url', 'language']);
});
}
};
database/migrations/2026_03_29_121740_add_processing_fields_to_summaries_table.php
+33
View File
@@ -0,0 +1,33 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('summaries', function (Blueprint $table) {
$table->enum('processing_status', ['pending', 'processing', 'completed', 'failed'])
->default('pending')
->after('book_id');
$table->text('processing_error')->nullable()->after('processing_status');
$table->timestamp('processed_at')->nullable()->after('processing_error');
$table->string('model_used', 100)->nullable()->after('processed_at');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('summaries', function (Blueprint $table) {
$table->dropColumn(['processing_status', 'processing_error', 'processed_at', 'model_used']);
});
}
};