gutendex processing implementation
This commit is contained in:
Doar Mihai
@@ -0,0 +1,139 @@
|
||||
<?php
|
||||
|
||||
return [
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Gutendex API Base URL
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| The base URL for the Gutendex API. This is the Project Gutenberg
|
||||
| catalog API that provides metadata about public domain books.
|
||||
|
|
||||
*/
|
||||
|
||||
'api_url' => env('GUTENDEX_API_URL', 'https://gutendex.com'),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Default Language Filter
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| The default language code to use when fetching books from Gutendex.
|
||||
| Common values: 'en' (English), 'fr' (French), 'de' (German), etc.
|
||||
|
|
||||
*/
|
||||
|
||||
'default_language' => env('GUTENDEX_DEFAULT_LANGUAGE', 'en'),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Batch Size
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| The number of books to process in a single batch import operation.
|
||||
| Adjust based on your server capacity and Ollama processing speed.
|
||||
|
|
||||
*/
|
||||
|
||||
'batch_size' => env('GUTENDEX_BATCH_SIZE', 50),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Rate Limit Delay
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Delay in milliseconds between API requests to respect rate limits.
|
||||
| Increase if you encounter 429 (Too Many Requests) errors.
|
||||
|
|
||||
*/
|
||||
|
||||
'rate_limit_delay' => env('GUTENDEX_RATE_LIMIT_DELAY', 1000),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Request Timeout
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Maximum time in seconds to wait for a response from the Gutendex API.
|
||||
|
|
||||
*/
|
||||
|
||||
'timeout' => env('GUTENDEX_TIMEOUT', 30),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Retry Configuration
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Configuration for retrying failed API requests.
|
||||
| - max_attempts: Maximum number of retry attempts
|
||||
| - backoff_multiplier: Multiplier for exponential backoff (in seconds)
|
||||
|
|
||||
*/
|
||||
|
||||
'retry' => [
|
||||
'max_attempts' => env('GUTENDEX_RETRY_MAX_ATTEMPTS', 3),
|
||||
'backoff_multiplier' => env('GUTENDEX_RETRY_BACKOFF', 2),
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Copyright Filter
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Filter books by copyright status. Setting to 'false' will only fetch
|
||||
| public domain books. Setting to 'true' includes copyrighted works.
|
||||
| It's recommended to keep this as 'false' for legal compliance.
|
||||
|
|
||||
*/
|
||||
|
||||
'copyright' => env('GUTENDEX_COPYRIGHT_FILTER', 'false'),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Cache Configuration
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Enable caching of Gutendex API responses to reduce API calls.
|
||||
| - enabled: Whether to cache responses
|
||||
| - ttl: Time to live in seconds (default: 24 hours)
|
||||
|
|
||||
*/
|
||||
|
||||
'cache' => [
|
||||
'enabled' => env('GUTENDEX_CACHE_ENABLED', true),
|
||||
'ttl' => env('GUTENDEX_CACHE_TTL', 86400),
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Category Mapping
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Map Gutendex subjects/bookshelves to your application's categories.
|
||||
| This helps in auto-categorizing imported books.
|
||||
|
|
||||
*/
|
||||
|
||||
'category_mapping' => [
|
||||
'Fiction' => 'fiction',
|
||||
'Science Fiction' => 'science-fiction',
|
||||
'Fantasy' => 'fantasy',
|
||||
'Mystery' => 'mystery',
|
||||
'Romance' => 'romance',
|
||||
'Historical Fiction' => 'historical-fiction',
|
||||
'Biography' => 'biography',
|
||||
'History' => 'history',
|
||||
'Philosophy' => 'philosophy',
|
||||
'Science' => 'science',
|
||||
'Technology' => 'technology',
|
||||
'Business' => 'business',
|
||||
'Self-Help' => 'self-help',
|
||||
'Poetry' => 'poetry',
|
||||
'Drama' => 'drama',
|
||||
'Children' => 'children',
|
||||
// Add more mappings as needed
|
||||
],
|
||||
|
||||
];
|
||||
@@ -0,0 +1,173 @@
|
||||
<?php
|
||||
|
||||
return [
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Ollama API URL
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| The base URL for your local Ollama API instance. By default, Ollama
|
||||
| runs on localhost:11434. Update this if running on a different host.
|
||||
|
|
||||
*/
|
||||
|
||||
'api_url' => env('OLLAMA_API_URL', 'http://localhost:11434'),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Default Model
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| The default Ollama model to use for generating summaries. Popular
|
||||
| options include: llama3, llama2, mistral, mixtral, phi, gemma.
|
||||
| Make sure the model is pulled: ollama pull <model_name>
|
||||
|
|
||||
*/
|
||||
|
||||
'model' => env('OLLAMA_MODEL', 'llama3'),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Request Timeout
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Maximum time in seconds to wait for a response from Ollama.
|
||||
| Summary generation can take several minutes for long texts.
|
||||
|
|
||||
*/
|
||||
|
||||
'timeout' => env('OLLAMA_TIMEOUT', 300),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Maximum Tokens
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Maximum number of tokens to send to the model. Most models have
|
||||
| context limits (e.g., 4096, 8192, 32768). Adjust based on your model.
|
||||
|
|
||||
*/
|
||||
|
||||
'max_tokens' => env('OLLAMA_MAX_TOKENS', 4000),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Temperature
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Controls randomness in generation. Lower values (0.1-0.5) are more
|
||||
| deterministic and factual. Higher values (0.7-1.0) are more creative.
|
||||
| For summaries, a lower temperature is recommended.
|
||||
|
|
||||
*/
|
||||
|
||||
'temperature' => env('OLLAMA_TEMPERATURE', 0.3),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Retry Configuration
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Configuration for retrying failed Ollama API requests.
|
||||
| - max_attempts: Maximum number of retry attempts
|
||||
| - backoff_multiplier: Multiplier for exponential backoff (in seconds)
|
||||
|
|
||||
*/
|
||||
|
||||
'retry' => [
|
||||
'max_attempts' => env('OLLAMA_RETRY_MAX_ATTEMPTS', 3),
|
||||
'backoff_multiplier' => env('OLLAMA_RETRY_BACKOFF', 5),
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Text Extraction
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Configuration for extracting representative excerpts from books.
|
||||
| - max_length: Maximum character length of text to send to Ollama
|
||||
| - strategy: 'beginning', 'middle', 'end', or 'distributed'
|
||||
|
|
||||
*/
|
||||
|
||||
'text_extraction' => [
|
||||
'max_length' => env('OLLAMA_TEXT_MAX_LENGTH', 50000),
|
||||
'strategy' => env('OLLAMA_TEXT_STRATEGY', 'beginning'),
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Prompt Templates
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Templates for prompts sent to Ollama. Available placeholders:
|
||||
| {title}, {author}, {text}, {summary}
|
||||
|
|
||||
*/
|
||||
|
||||
'prompts' => [
|
||||
'summary' => env('OLLAMA_PROMPT_SUMMARY',
|
||||
"You are a literary expert tasked with creating comprehensive book summaries. " .
|
||||
"Analyze the following excerpt from \"{title}\" by {author} and provide a detailed, " .
|
||||
"engaging summary that captures the main themes, key points, and overall significance " .
|
||||
"of the work. Focus on what makes this book important and valuable to readers. " .
|
||||
"Keep the summary between 400-600 words.\n\n" .
|
||||
"Book excerpt:\n{text}\n\n" .
|
||||
"Summary:"
|
||||
),
|
||||
|
||||
'key_insights' => env('OLLAMA_PROMPT_INSIGHTS',
|
||||
"Based on the following book summary, extract 5-7 key insights, lessons, or takeaways " .
|
||||
"that readers should remember. Format your response as a concise bullet list with each " .
|
||||
"insight on its own line, starting with a dash (-).\n\n" .
|
||||
"Book: \"{title}\" by {author}\n\n" .
|
||||
"Summary:\n{summary}\n\n" .
|
||||
"Key Insights:"
|
||||
),
|
||||
|
||||
'categorize' => env('OLLAMA_PROMPT_CATEGORIZE',
|
||||
"Based on the following book information, suggest the most appropriate category " .
|
||||
"from this list: Fiction, Non-Fiction, Science Fiction, Fantasy, Mystery, Romance, " .
|
||||
"Historical Fiction, Biography, History, Philosophy, Science, Technology, Business, " .
|
||||
"Self-Help, Poetry, Drama, Children. Respond with ONLY the category name.\n\n" .
|
||||
"Book: \"{title}\" by {author}\n" .
|
||||
"Description: {description}\n\n" .
|
||||
"Category:"
|
||||
),
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Summary Quality Checks
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Minimum quality thresholds for generated summaries.
|
||||
| Summaries below these thresholds will be marked for review.
|
||||
|
|
||||
*/
|
||||
|
||||
'quality_checks' => [
|
||||
'min_word_count' => env('OLLAMA_MIN_WORD_COUNT', 100),
|
||||
'max_word_count' => env('OLLAMA_MAX_WORD_COUNT', 1000),
|
||||
'min_sentence_count' => env('OLLAMA_MIN_SENTENCE_COUNT', 5),
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Model Options
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Additional options to pass to the Ollama API.
|
||||
| See: https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values
|
||||
|
|
||||
*/
|
||||
|
||||
'options' => [
|
||||
'num_predict' => env('OLLAMA_NUM_PREDICT', 1000), // Max tokens to generate
|
||||
'top_k' => env('OLLAMA_TOP_K', 40),
|
||||
'top_p' => env('OLLAMA_TOP_P', 0.9),
|
||||
'repeat_penalty' => env('OLLAMA_REPEAT_PENALTY', 1.1),
|
||||
],
|
||||
|
||||
];
|
||||
Reference in New Issue
Block a user