From 601c7b422b8b7c665d4ff079d94158d4ab373321 Mon Sep 17 00:00:00 2001 From: Doar Mihai Date: Sun, 29 Mar 2026 15:32:00 +0300 Subject: [PATCH] gutendex integration phase 3 --- app/Helpers/TextProcessor.php | 372 +++++++++++++++++++++++++++ app/Providers/AppServiceProvider.php | 5 + app/Services/OllamaService.php | 371 ++++++++++++++++++++++++++ 3 files changed, 748 insertions(+) create mode 100644 app/Helpers/TextProcessor.php create mode 100644 app/Services/OllamaService.php diff --git a/app/Helpers/TextProcessor.php b/app/Helpers/TextProcessor.php new file mode 100644 index 0000000..6a566ff --- /dev/null +++ b/app/Helpers/TextProcessor.php @@ -0,0 +1,372 @@ + $textLength, + 'max_length' => $maxLength, + ]); + return $fullText; + } + + Log::info('TextProcessor: Extracting excerpt', [ + 'original_length' => $textLength, + 'max_length' => $maxLength, + 'strategy' => $strategy, + ]); + + $excerpt = match ($strategy) { + 'beginning' => self::extractBeginning($fullText, $maxLength), + 'middle' => self::extractMiddle($fullText, $maxLength), + 'end' => self::extractEnd($fullText, $maxLength), + 'distributed' => self::extractDistributed($fullText, $maxLength), + default => self::extractBeginning($fullText, $maxLength), + }; + + Log::info('TextProcessor: Excerpt extracted', [ + 'excerpt_length' => strlen($excerpt), + 'reduction_percent' => round((1 - strlen($excerpt) / $textLength) * 100, 2), + ]); + + return $excerpt; + } + + /** + * Extract from the beginning of the text. + * + * @param string $text Full text + * @param int $maxLength Maximum length + * @return string + */ + protected static function extractBeginning(string $text, int $maxLength): string + { + $excerpt = substr($text, 0, $maxLength); + return self::trimToLastSentence($excerpt); + } + + /** + * Extract from the middle of the text. + * + * @param string $text Full text + * @param int $maxLength Maximum length + * @return string + */ + protected static function extractMiddle(string $text, int $maxLength): string + { + $textLength = strlen($text); + $startPos = intval(($textLength - $maxLength) / 2); + $excerpt = substr($text, $startPos, $maxLength); + return self::trimToCompleteSentences($excerpt); + } + + /** + * Extract from the end of the text. + * + * @param string $text Full text + * @param int $maxLength Maximum length + * @return string + */ + protected static function extractEnd(string $text, int $maxLength): string + { + $textLength = strlen($text); + $startPos = $textLength - $maxLength; + $excerpt = substr($text, $startPos); + return self::trimToFirstSentence($excerpt); + } + + /** + * Extract distributed chunks from throughout the text. + * + * @param string $text Full text + * @param int $maxLength Maximum length + * @return string + */ + protected static function extractDistributed(string $text, int $maxLength): string + { + $textLength = strlen($text); + $numChunks = 5; // Extract 5 chunks from different parts + $chunkSize = intval($maxLength / $numChunks); + $stepSize = intval($textLength / $numChunks); + + $chunks = []; + for ($i = 0; $i < $numChunks; $i++) { + $startPos = $i * $stepSize; + $chunk = substr($text, $startPos, $chunkSize); + $chunk = self::trimToCompleteSentences($chunk); + if (!empty($chunk)) { + $chunks[] = $chunk; + } + } + + return implode("\n\n[...]\n\n", $chunks); + } + + /** + * Clean text by removing formatting artifacts and normalizing whitespace. + * + * @param string $text The text to clean + * @return string Cleaned text + */ + public static function cleanText(string $text): string + { + Log::info('TextProcessor: Cleaning text', [ + 'original_length' => strlen($text), + ]); + + // Remove Project Gutenberg header and footer + $text = self::removeGutenbergBoilerplate($text); + + // Normalize line endings to Unix style + $text = str_replace(["\r\n", "\r"], "\n", $text); + + // Remove excessive whitespace + $text = preg_replace('/\n{3,}/', "\n\n", $text); + $text = preg_replace('/[ \t]+/', ' ', $text); + + // Remove page numbers (common patterns) + $text = preg_replace('/\n\s*\d+\s*\n/', "\n", $text); + + // Remove common formatting artifacts + $text = preg_replace('/_{3,}/', '', $text); // Underscores + $text = preg_replace('/={3,}/', '', $text); // Equals signs + $text = preg_replace('/-{3,}/', '', $text); // Dashes + + // Trim + $text = trim($text); + + Log::info('TextProcessor: Text cleaned', [ + 'cleaned_length' => strlen($text), + ]); + + return $text; + } + + /** + * Remove Project Gutenberg boilerplate text. + * + * @param string $text Full text + * @return string Text without boilerplate + */ + protected static function removeGutenbergBoilerplate(string $text): string + { + // Remove header (everything before "START OF THE PROJECT GUTENBERG") + $startMarkers = [ + '*** START OF THE PROJECT GUTENBERG', + '*** START OF THIS PROJECT GUTENBERG', + '*END*THE SMALL PRINT', + ]; + + foreach ($startMarkers as $marker) { + $pos = stripos($text, $marker); + if ($pos !== false) { + // Find the end of the line after the marker + $endOfLine = strpos($text, "\n", $pos); + if ($endOfLine !== false) { + $text = substr($text, $endOfLine + 1); + break; + } + } + } + + // Remove footer (everything after "END OF THE PROJECT GUTENBERG") + $endMarkers = [ + '*** END OF THE PROJECT GUTENBERG', + '*** END OF THIS PROJECT GUTENBERG', + ]; + + foreach ($endMarkers as $marker) { + $pos = stripos($text, $marker); + if ($pos !== false) { + $text = substr($text, 0, $pos); + break; + } + } + + return $text; + } + + /** + * Truncate text to fit within a token limit (approximate). + * + * @param string $text The text to truncate + * @param int $tokenLimit Maximum number of tokens (approximate) + * @return string Truncated text + */ + public static function truncateToTokenLimit(string $text, int $tokenLimit = 4000): string + { + // Rough approximation: 1 token ≈ 4 characters for English text + $charLimit = $tokenLimit * 4; + + if (strlen($text) <= $charLimit) { + return $text; + } + + Log::info('TextProcessor: Truncating to token limit', [ + 'original_length' => strlen($text), + 'token_limit' => $tokenLimit, + 'char_limit' => $charLimit, + ]); + + $truncated = substr($text, 0, $charLimit); + return self::trimToLastSentence($truncated); + } + + /** + * Trim text to the last complete sentence. + * + * @param string $text Text to trim + * @return string Trimmed text + */ + protected static function trimToLastSentence(string $text): string + { + $sentenceEndings = ['. ', '! ', '? ', '."', '!"', '?"']; + + $lastPos = 0; + foreach ($sentenceEndings as $ending) { + $pos = strrpos($text, $ending); + if ($pos !== false && $pos > $lastPos) { + $lastPos = $pos + strlen($ending) - 1; + } + } + + if ($lastPos > 0) { + return substr($text, 0, $lastPos + 1); + } + + return $text; + } + + /** + * Trim text to the first complete sentence. + * + * @param string $text Text to trim + * @return string Trimmed text + */ + protected static function trimToFirstSentence(string $text): string + { + $sentenceEndings = ['. ', '! ', '? ']; + + $firstPos = strlen($text); + foreach ($sentenceEndings as $ending) { + $pos = strpos($text, $ending); + if ($pos !== false && $pos < $firstPos) { + $firstPos = $pos + strlen($ending); + } + } + + if ($firstPos < strlen($text)) { + return substr($text, $firstPos); + } + + return $text; + } + + /** + * Trim text to complete sentences at both ends. + * + * @param string $text Text to trim + * @return string Trimmed text + */ + protected static function trimToCompleteSentences(string $text): string + { + $text = self::trimToFirstSentence($text); + $text = self::trimToLastSentence($text); + return $text; + } + + /** + * Count approximate tokens in text. + * + * @param string $text Text to analyze + * @return int Approximate token count + */ + public static function countTokens(string $text): int + { + // Rough approximation: 1 token ≈ 4 characters for English text + return intval(strlen($text) / 4); + } + + /** + * Get text statistics. + * + * @param string $text Text to analyze + * @return array Statistics array + */ + public static function getStatistics(string $text): array + { + return [ + 'character_count' => strlen($text), + 'word_count' => str_word_count($text), + 'line_count' => substr_count($text, "\n") + 1, + 'sentence_count' => self::countSentences($text), + 'approximate_tokens' => self::countTokens($text), + ]; + } + + /** + * Count sentences in text. + * + * @param string $text Text to analyze + * @return int Number of sentences + */ + protected static function countSentences(string $text): int + { + $count = preg_match_all('/[.!?]+/', $text); + return $count ?: 1; + } + + /** + * Split text into chunks of approximately equal size. + * + * @param string $text Text to split + * @param int $chunkSize Approximate size of each chunk in characters + * @return array Array of text chunks + */ + public static function splitIntoChunks(string $text, int $chunkSize = 10000): array + { + $textLength = strlen($text); + $numChunks = ceil($textLength / $chunkSize); + $chunks = []; + + for ($i = 0; $i < $numChunks; $i++) { + $start = $i * $chunkSize; + $chunk = substr($text, $start, $chunkSize); + + // Trim to complete sentences except for the last chunk + if ($i < $numChunks - 1) { + $chunk = self::trimToLastSentence($chunk); + } + + $chunks[] = $chunk; + } + + Log::info('TextProcessor: Text split into chunks', [ + 'original_length' => $textLength, + 'chunk_size' => $chunkSize, + 'num_chunks' => count($chunks), + ]); + + return array_filter($chunks); // Remove empty chunks + } +} diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 56ad3f9..68b0c02 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -15,6 +15,11 @@ class AppServiceProvider extends ServiceProvider $this->app->singleton(\App\Services\GutendexClient::class, function ($app) { return new \App\Services\GutendexClient(); }); + + // Register OllamaService as a singleton + $this->app->singleton(\App\Services\OllamaService::class, function ($app) { + return new \App\Services\OllamaService(); + }); } /** diff --git a/app/Services/OllamaService.php b/app/Services/OllamaService.php new file mode 100644 index 0000000..1bc5b17 --- /dev/null +++ b/app/Services/OllamaService.php @@ -0,0 +1,371 @@ +apiUrl = config('ollama.api_url', 'http://localhost:11434'); + $this->model = config('ollama.model', 'llama3'); + $this->timeout = config('ollama.timeout', 300); + $this->temperature = config('ollama.temperature', 0.3); + $this->maxTokens = config('ollama.max_tokens', 4000); + $this->retryMaxAttempts = config('ollama.retry.max_attempts', 3); + $this->retryBackoff = config('ollama.retry.backoff_multiplier', 5); + $this->options = config('ollama.options', []); + } + + /** + * Generate a summary for a book. + * + * @param string $bookText The book text to summarize + * @param string $bookTitle The book title + * @param string $bookAuthor The book author + * @return string|null The generated summary or null on failure + */ + public function generateSummary(string $bookText, string $bookTitle, string $bookAuthor): ?string + { + $prompt = $this->buildPrompt('summary', [ + 'title' => $bookTitle, + 'author' => $bookAuthor, + 'text' => $bookText, + ]); + + Log::info('Ollama: Generating summary', [ + 'model' => $this->model, + 'title' => $bookTitle, + 'author' => $bookAuthor, + 'text_length' => strlen($bookText), + ]); + + $startTime = microtime(true); + + try { + $response = $this->makeRequest($prompt); + $duration = microtime(true) - $startTime; + + if ($response) { + $wordCount = str_word_count($response); + Log::info('Ollama: Summary generated successfully', [ + 'title' => $bookTitle, + 'duration' => round($duration, 2) . 's', + 'word_count' => $wordCount, + ]); + + // Quality check + $minWords = config('ollama.quality_checks.min_word_count', 100); + $maxWords = config('ollama.quality_checks.max_word_count', 1000); + + if ($wordCount < $minWords) { + Log::warning('Ollama: Summary too short', [ + 'title' => $bookTitle, + 'word_count' => $wordCount, + 'min_required' => $minWords, + ]); + } + + if ($wordCount > $maxWords) { + Log::warning('Ollama: Summary too long', [ + 'title' => $bookTitle, + 'word_count' => $wordCount, + 'max_allowed' => $maxWords, + ]); + } + + return $response; + } + + return null; + } catch (\Exception $e) { + $duration = microtime(true) - $startTime; + Log::error('Ollama: Failed to generate summary', [ + 'title' => $bookTitle, + 'error' => $e->getMessage(), + 'duration' => round($duration, 2) . 's', + ]); + return null; + } + } + + /** + * Generate key insights from a book summary. + * + * @param string $summary The book summary + * @param string $bookTitle The book title + * @param string $bookAuthor The book author + * @return string|null The generated key insights or null on failure + */ + public function generateKeyInsights(string $summary, string $bookTitle, string $bookAuthor): ?string + { + $prompt = $this->buildPrompt('key_insights', [ + 'title' => $bookTitle, + 'author' => $bookAuthor, + 'summary' => $summary, + ]); + + Log::info('Ollama: Generating key insights', [ + 'model' => $this->model, + 'title' => $bookTitle, + ]); + + $startTime = microtime(true); + + try { + $response = $this->makeRequest($prompt); + $duration = microtime(true) - $startTime; + + if ($response) { + Log::info('Ollama: Key insights generated successfully', [ + 'title' => $bookTitle, + 'duration' => round($duration, 2) . 's', + ]); + return $response; + } + + return null; + } catch (\Exception $e) { + $duration = microtime(true) - $startTime; + Log::error('Ollama: Failed to generate key insights', [ + 'title' => $bookTitle, + 'error' => $e->getMessage(), + 'duration' => round($duration, 2) . 's', + ]); + return null; + } + } + + /** + * Check if the Ollama API is accessible. + * + * @return bool + */ + public function checkConnection(): bool + { + try { + Log::info('Ollama: Checking connection', ['url' => $this->apiUrl]); + + $response = Http::timeout(10) + ->get($this->apiUrl . '/api/tags'); + + if ($response->successful()) { + Log::info('Ollama: Connection successful'); + return true; + } + + Log::warning('Ollama: Connection check failed', [ + 'status' => $response->status(), + ]); + + return false; + } catch (\Exception $e) { + Log::error('Ollama: Connection check error', [ + 'error' => $e->getMessage(), + ]); + return false; + } + } + + /** + * List available models on the Ollama instance. + * + * @return array List of model names + */ + public function listModels(): array + { + try { + Log::info('Ollama: Listing models'); + + $response = Http::timeout(10) + ->get($this->apiUrl . '/api/tags'); + + if ($response->successful()) { + $data = $response->json(); + $models = array_map( + fn($model) => $model['name'] ?? 'unknown', + $data['models'] ?? [] + ); + + Log::info('Ollama: Models retrieved', [ + 'count' => count($models), + 'models' => $models, + ]); + + return $models; + } + + Log::warning('Ollama: Failed to list models', [ + 'status' => $response->status(), + ]); + + return []; + } catch (\Exception $e) { + Log::error('Ollama: Error listing models', [ + 'error' => $e->getMessage(), + ]); + return []; + } + } + + /** + * Test summary generation with a sample text. + * + * @return array Test results with success status and details + */ + public function testGeneration(): array + { + $testText = "This is a test book about artificial intelligence and machine learning. " . + "It explores the history of AI, from its early beginnings to modern applications. " . + "The book covers neural networks, deep learning, and practical implementations."; + + $testTitle = "Test Book: Introduction to AI"; + $testAuthor = "Test Author"; + + $startTime = microtime(true); + $summary = $this->generateSummary($testText, $testTitle, $testAuthor); + $duration = microtime(true) - $startTime; + + return [ + 'success' => $summary !== null, + 'summary' => $summary, + 'duration' => round($duration, 2), + 'model' => $this->model, + 'word_count' => $summary ? str_word_count($summary) : 0, + ]; + } + + /** + * Build a prompt from a template with variable substitution. + * + * @param string $templateKey The template key from config + * @param array $variables Variables to substitute + * @return string The built prompt + */ + protected function buildPrompt(string $templateKey, array $variables): string + { + $template = config("ollama.prompts.{$templateKey}", ''); + + if (empty($template)) { + throw new \InvalidArgumentException("Prompt template '{$templateKey}' not found in config."); + } + + $prompt = $template; + + foreach ($variables as $key => $value) { + $prompt = str_replace('{' . $key . '}', $value, $prompt); + } + + return $prompt; + } + + /** + * Make a request to the Ollama API. + * + * @param string $prompt The prompt to send + * @return string|null The response text or null on failure + */ + protected function makeRequest(string $prompt): ?string + { + $url = $this->apiUrl . '/api/generate'; + + $payload = [ + 'model' => $this->model, + 'prompt' => $prompt, + 'stream' => false, + 'options' => array_merge($this->options, [ + 'temperature' => $this->temperature, + ]), + ]; + + try { + $response = Http::timeout($this->timeout) + ->retry( + $this->retryMaxAttempts, + function ($exception, $request) { + // Exponential backoff in milliseconds + return $this->retryBackoff * 1000; + }, + function ($exception, $request) { + // Retry on connection errors and 5xx server errors + if ($exception instanceof ConnectionException) { + return true; + } + + if (method_exists($exception, 'response')) { + $response = $exception->response; + return $response && $response->status() >= 500; + } + + return false; + } + ) + ->post($url, $payload); + + if ($response->successful()) { + $data = $response->json(); + return $data['response'] ?? null; + } + + Log::warning('Ollama: API request failed', [ + 'status' => $response->status(), + 'body' => $response->body(), + ]); + + return null; + } catch (ConnectionException $e) { + Log::error('Ollama: Connection error', [ + 'error' => $e->getMessage(), + 'url' => $url, + ]); + throw $e; + } catch (RequestException $e) { + Log::error('Ollama: Request exception', [ + 'error' => $e->getMessage(), + 'url' => $url, + ]); + throw $e; + } catch (\Exception $e) { + Log::error('Ollama: Unexpected error', [ + 'error' => $e->getMessage(), + 'type' => get_class($e), + ]); + throw $e; + } + } + + /** + * Get the currently configured model name. + * + * @return string + */ + public function getModel(): string + { + return $this->model; + } + + /** + * Set a different model for the next requests. + * + * @param string $model Model name + * @return void + */ + public function setModel(string $model): void + { + $this->model = $model; + Log::info('Ollama: Model changed', ['model' => $model]); + } +}