gutendex integration phase 3

app/Helpers/TextProcessor.php

@@ -0,0 +1,372 @@
+<?php
+
+namespace App\Helpers;
+
+use Illuminate\Support\Facades\Log;
+
+class TextProcessor
+{
+    /**
+     * Extract a representative excerpt from the full text.
+     *
+     * @param string $fullText The complete book text
+     * @param int $maxLength Maximum character length of the excerpt
+     * @param string $strategy Extraction strategy: 'beginning', 'middle', 'end', 'distributed'
+     * @return string The extracted excerpt
+     */
+    public static function extractExcerpt(
+        string $fullText,
+        int $maxLength = 50000,
+        string $strategy = 'beginning'
+    ): string {
+        $textLength = strlen($fullText);
+
+        // If text is already short enough, return as is
+        if ($textLength <= $maxLength) {
+            Log::info('TextProcessor: Text within limit, no extraction needed', [
+                'length' => $textLength,
+                'max_length' => $maxLength,
+            ]);
+            return $fullText;
+        }
+
+        Log::info('TextProcessor: Extracting excerpt', [
+            'original_length' => $textLength,
+            'max_length' => $maxLength,
+            'strategy' => $strategy,
+        ]);
+
+        $excerpt = match ($strategy) {
+            'beginning' => self::extractBeginning($fullText, $maxLength),
+            'middle' => self::extractMiddle($fullText, $maxLength),
+            'end' => self::extractEnd($fullText, $maxLength),
+            'distributed' => self::extractDistributed($fullText, $maxLength),
+            default => self::extractBeginning($fullText, $maxLength),
+        };
+
+        Log::info('TextProcessor: Excerpt extracted', [
+            'excerpt_length' => strlen($excerpt),
+            'reduction_percent' => round((1 - strlen($excerpt) / $textLength) * 100, 2),
+        ]);
+
+        return $excerpt;
+    }
+
+    /**
+     * Extract from the beginning of the text.
+     *
+     * @param string $text Full text
+     * @param int $maxLength Maximum length
+     * @return string
+     */
+    protected static function extractBeginning(string $text, int $maxLength): string
+    {
+        $excerpt = substr($text, 0, $maxLength);
+        return self::trimToLastSentence($excerpt);
+    }
+
+    /**
+     * Extract from the middle of the text.
+     *
+     * @param string $text Full text
+     * @param int $maxLength Maximum length
+     * @return string
+     */
+    protected static function extractMiddle(string $text, int $maxLength): string
+    {
+        $textLength = strlen($text);
+        $startPos = intval(($textLength - $maxLength) / 2);
+        $excerpt = substr($text, $startPos, $maxLength);
+        return self::trimToCompleteSentences($excerpt);
+    }
+
+    /**
+     * Extract from the end of the text.
+     *
+     * @param string $text Full text
+     * @param int $maxLength Maximum length
+     * @return string
+     */
+    protected static function extractEnd(string $text, int $maxLength): string
+    {
+        $textLength = strlen($text);
+        $startPos = $textLength - $maxLength;
+        $excerpt = substr($text, $startPos);
+        return self::trimToFirstSentence($excerpt);
+    }
+
+    /**
+     * Extract distributed chunks from throughout the text.
+     *
+     * @param string $text Full text
+     * @param int $maxLength Maximum length
+     * @return string
+     */
+    protected static function extractDistributed(string $text, int $maxLength): string
+    {
+        $textLength = strlen($text);
+        $numChunks = 5; // Extract 5 chunks from different parts
+        $chunkSize = intval($maxLength / $numChunks);
+        $stepSize = intval($textLength / $numChunks);
+
+        $chunks = [];
+        for ($i = 0; $i < $numChunks; $i++) {
+            $startPos = $i * $stepSize;
+            $chunk = substr($text, $startPos, $chunkSize);
+            $chunk = self::trimToCompleteSentences($chunk);
+            if (!empty($chunk)) {
+                $chunks[] = $chunk;
+            }
+        }
+
+        return implode("\n\n[...]\n\n", $chunks);
+    }
+
+    /**
+     * Clean text by removing formatting artifacts and normalizing whitespace.
+     *
+     * @param string $text The text to clean
+     * @return string Cleaned text
+     */
+    public static function cleanText(string $text): string
+    {
+        Log::info('TextProcessor: Cleaning text', [
+            'original_length' => strlen($text),
+        ]);
+
+        // Remove Project Gutenberg header and footer
+        $text = self::removeGutenbergBoilerplate($text);
+
+        // Normalize line endings to Unix style
+        $text = str_replace(["\r\n", "\r"], "\n", $text);
+
+        // Remove excessive whitespace
+        $text = preg_replace('/\n{3,}/', "\n\n", $text);
+        $text = preg_replace('/[ \t]+/', ' ', $text);
+
+        // Remove page numbers (common patterns)
+        $text = preg_replace('/\n\s*\d+\s*\n/', "\n", $text);
+
+        // Remove common formatting artifacts
+        $text = preg_replace('/_{3,}/', '', $text); // Underscores
+        $text = preg_replace('/={3,}/', '', $text); // Equals signs
+        $text = preg_replace('/-{3,}/', '', $text); // Dashes
+
+        // Trim
+        $text = trim($text);
+
+        Log::info('TextProcessor: Text cleaned', [
+            'cleaned_length' => strlen($text),
+        ]);
+
+        return $text;
+    }
+
+    /**
+     * Remove Project Gutenberg boilerplate text.
+     *
+     * @param string $text Full text
+     * @return string Text without boilerplate
+     */
+    protected static function removeGutenbergBoilerplate(string $text): string
+    {
+        // Remove header (everything before "START OF THE PROJECT GUTENBERG")
+        $startMarkers = [
+            '*** START OF THE PROJECT GUTENBERG',
+            '*** START OF THIS PROJECT GUTENBERG',
+            '*END*THE SMALL PRINT',
+        ];
+
+        foreach ($startMarkers as $marker) {
+            $pos = stripos($text, $marker);
+            if ($pos !== false) {
+                // Find the end of the line after the marker
+                $endOfLine = strpos($text, "\n", $pos);
+                if ($endOfLine !== false) {
+                    $text = substr($text, $endOfLine + 1);
+                    break;
+                }
+            }
+        }
+
+        // Remove footer (everything after "END OF THE PROJECT GUTENBERG")
+        $endMarkers = [
+            '*** END OF THE PROJECT GUTENBERG',
+            '*** END OF THIS PROJECT GUTENBERG',
+        ];
+
+        foreach ($endMarkers as $marker) {
+            $pos = stripos($text, $marker);
+            if ($pos !== false) {
+                $text = substr($text, 0, $pos);
+                break;
+            }
+        }
+
+        return $text;
+    }
+
+    /**
+     * Truncate text to fit within a token limit (approximate).
+     *
+     * @param string $text The text to truncate
+     * @param int $tokenLimit Maximum number of tokens (approximate)
+     * @return string Truncated text
+     */
+    public static function truncateToTokenLimit(string $text, int $tokenLimit = 4000): string
+    {
+        // Rough approximation: 1 token ≈ 4 characters for English text
+        $charLimit = $tokenLimit * 4;
+
+        if (strlen($text) <= $charLimit) {
+            return $text;
+        }
+
+        Log::info('TextProcessor: Truncating to token limit', [
+            'original_length' => strlen($text),
+            'token_limit' => $tokenLimit,
+            'char_limit' => $charLimit,
+        ]);
+
+        $truncated = substr($text, 0, $charLimit);
+        return self::trimToLastSentence($truncated);
+    }
+
+    /**
+     * Trim text to the last complete sentence.
+     *
+     * @param string $text Text to trim
+     * @return string Trimmed text
+     */
+    protected static function trimToLastSentence(string $text): string
+    {
+        $sentenceEndings = ['. ', '! ', '? ', '."', '!"', '?"'];
+
+        $lastPos = 0;
+        foreach ($sentenceEndings as $ending) {
+            $pos = strrpos($text, $ending);
+            if ($pos !== false && $pos > $lastPos) {
+                $lastPos = $pos + strlen($ending) - 1;
+            }
+        }
+
+        if ($lastPos > 0) {
+            return substr($text, 0, $lastPos + 1);
+        }
+
+        return $text;
+    }
+
+    /**
+     * Trim text to the first complete sentence.
+     *
+     * @param string $text Text to trim
+     * @return string Trimmed text
+     */
+    protected static function trimToFirstSentence(string $text): string
+    {
+        $sentenceEndings = ['. ', '! ', '? '];
+
+        $firstPos = strlen($text);
+        foreach ($sentenceEndings as $ending) {
+            $pos = strpos($text, $ending);
+            if ($pos !== false && $pos < $firstPos) {
+                $firstPos = $pos + strlen($ending);
+            }
+        }
+
+        if ($firstPos < strlen($text)) {
+            return substr($text, $firstPos);
+        }
+
+        return $text;
+    }
+
+    /**
+     * Trim text to complete sentences at both ends.
+     *
+     * @param string $text Text to trim
+     * @return string Trimmed text
+     */
+    protected static function trimToCompleteSentences(string $text): string
+    {
+        $text = self::trimToFirstSentence($text);
+        $text = self::trimToLastSentence($text);
+        return $text;
+    }
+
+    /**
+     * Count approximate tokens in text.
+     *
+     * @param string $text Text to analyze
+     * @return int Approximate token count
+     */
+    public static function countTokens(string $text): int
+    {
+        // Rough approximation: 1 token ≈ 4 characters for English text
+        return intval(strlen($text) / 4);
+    }
+
+    /**
+     * Get text statistics.
+     *
+     * @param string $text Text to analyze
+     * @return array Statistics array
+     */
+    public static function getStatistics(string $text): array
+    {
+        return [
+            'character_count' => strlen($text),
+            'word_count' => str_word_count($text),
+            'line_count' => substr_count($text, "\n") + 1,
+            'sentence_count' => self::countSentences($text),
+            'approximate_tokens' => self::countTokens($text),
+        ];
+    }
+
+    /**
+     * Count sentences in text.
+     *
+     * @param string $text Text to analyze
+     * @return int Number of sentences
+     */
+    protected static function countSentences(string $text): int
+    {
+        $count = preg_match_all('/[.!?]+/', $text);
+        return $count ?: 1;
+    }
+
+    /**
+     * Split text into chunks of approximately equal size.
+     *
+     * @param string $text Text to split
+     * @param int $chunkSize Approximate size of each chunk in characters
+     * @return array Array of text chunks
+     */
+    public static function splitIntoChunks(string $text, int $chunkSize = 10000): array
+    {
+        $textLength = strlen($text);
+        $numChunks = ceil($textLength / $chunkSize);
+        $chunks = [];
+
+        for ($i = 0; $i < $numChunks; $i++) {
+            $start = $i * $chunkSize;
+            $chunk = substr($text, $start, $chunkSize);
+
+            // Trim to complete sentences except for the last chunk
+            if ($i < $numChunks - 1) {
+                $chunk = self::trimToLastSentence($chunk);
+            }
+
+            $chunks[] = $chunk;
+        }
+
+        Log::info('TextProcessor: Text split into chunks', [
+            'original_length' => $textLength,
+            'chunk_size' => $chunkSize,
+            'num_chunks' => count($chunks),
+        ]);
+
+        return array_filter($chunks); // Remove empty chunks
+    }
+}

app/Providers/AppServiceProvider.php

@@ -15,6 +15,11 @@ class AppServiceProvider extends ServiceProvider
         $this->app->singleton(\App\Services\GutendexClient::class, function ($app) {
             return new \App\Services\GutendexClient();
         });
+
+        // Register OllamaService as a singleton
+        $this->app->singleton(\App\Services\OllamaService::class, function ($app) {
+            return new \App\Services\OllamaService();
+        });
     }
 
     /**

app/Services/OllamaService.php

@@ -0,0 +1,371 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Http\Client\ConnectionException;
+use Illuminate\Http\Client\RequestException;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Log;
+
+class OllamaService
+{
+    protected string $apiUrl;
+    protected string $model;
+    protected int $timeout;
+    protected float $temperature;
+    protected int $maxTokens;
+    protected int $retryMaxAttempts;
+    protected int $retryBackoff;
+    protected array $options;
+
+    public function __construct()
+    {
+        $this->apiUrl = config('ollama.api_url', 'http://localhost:11434');
+        $this->model = config('ollama.model', 'llama3');
+        $this->timeout = config('ollama.timeout', 300);
+        $this->temperature = config('ollama.temperature', 0.3);
+        $this->maxTokens = config('ollama.max_tokens', 4000);
+        $this->retryMaxAttempts = config('ollama.retry.max_attempts', 3);
+        $this->retryBackoff = config('ollama.retry.backoff_multiplier', 5);
+        $this->options = config('ollama.options', []);
+    }
+
+    /**
+     * Generate a summary for a book.
+     *
+     * @param string $bookText The book text to summarize
+     * @param string $bookTitle The book title
+     * @param string $bookAuthor The book author
+     * @return string|null The generated summary or null on failure
+     */
+    public function generateSummary(string $bookText, string $bookTitle, string $bookAuthor): ?string
+    {
+        $prompt = $this->buildPrompt('summary', [
+            'title' => $bookTitle,
+            'author' => $bookAuthor,
+            'text' => $bookText,
+        ]);
+
+        Log::info('Ollama: Generating summary', [
+            'model' => $this->model,
+            'title' => $bookTitle,
+            'author' => $bookAuthor,
+            'text_length' => strlen($bookText),
+        ]);
+
+        $startTime = microtime(true);
+
+        try {
+            $response = $this->makeRequest($prompt);
+            $duration = microtime(true) - $startTime;
+
+            if ($response) {
+                $wordCount = str_word_count($response);
+                Log::info('Ollama: Summary generated successfully', [
+                    'title' => $bookTitle,
+                    'duration' => round($duration, 2) . 's',
+                    'word_count' => $wordCount,
+                ]);
+
+                // Quality check
+                $minWords = config('ollama.quality_checks.min_word_count', 100);
+                $maxWords = config('ollama.quality_checks.max_word_count', 1000);
+
+                if ($wordCount < $minWords) {
+                    Log::warning('Ollama: Summary too short', [
+                        'title' => $bookTitle,
+                        'word_count' => $wordCount,
+                        'min_required' => $minWords,
+                    ]);
+                }
+
+                if ($wordCount > $maxWords) {
+                    Log::warning('Ollama: Summary too long', [
+                        'title' => $bookTitle,
+                        'word_count' => $wordCount,
+                        'max_allowed' => $maxWords,
+                    ]);
+                }
+
+                return $response;
+            }
+
+            return null;
+        } catch (\Exception $e) {
+            $duration = microtime(true) - $startTime;
+            Log::error('Ollama: Failed to generate summary', [
+                'title' => $bookTitle,
+                'error' => $e->getMessage(),
+                'duration' => round($duration, 2) . 's',
+            ]);
+            return null;
+        }
+    }
+
+    /**
+     * Generate key insights from a book summary.
+     *
+     * @param string $summary The book summary
+     * @param string $bookTitle The book title
+     * @param string $bookAuthor The book author
+     * @return string|null The generated key insights or null on failure
+     */
+    public function generateKeyInsights(string $summary, string $bookTitle, string $bookAuthor): ?string
+    {
+        $prompt = $this->buildPrompt('key_insights', [
+            'title' => $bookTitle,
+            'author' => $bookAuthor,
+            'summary' => $summary,
+        ]);
+
+        Log::info('Ollama: Generating key insights', [
+            'model' => $this->model,
+            'title' => $bookTitle,
+        ]);
+
+        $startTime = microtime(true);
+
+        try {
+            $response = $this->makeRequest($prompt);
+            $duration = microtime(true) - $startTime;
+
+            if ($response) {
+                Log::info('Ollama: Key insights generated successfully', [
+                    'title' => $bookTitle,
+                    'duration' => round($duration, 2) . 's',
+                ]);
+                return $response;
+            }
+
+            return null;
+        } catch (\Exception $e) {
+            $duration = microtime(true) - $startTime;
+            Log::error('Ollama: Failed to generate key insights', [
+                'title' => $bookTitle,
+                'error' => $e->getMessage(),
+                'duration' => round($duration, 2) . 's',
+            ]);
+            return null;
+        }
+    }
+
+    /**
+     * Check if the Ollama API is accessible.
+     *
+     * @return bool
+     */
+    public function checkConnection(): bool
+    {
+        try {
+            Log::info('Ollama: Checking connection', ['url' => $this->apiUrl]);
+
+            $response = Http::timeout(10)
+                ->get($this->apiUrl . '/api/tags');
+
+            if ($response->successful()) {
+                Log::info('Ollama: Connection successful');
+                return true;
+            }
+
+            Log::warning('Ollama: Connection check failed', [
+                'status' => $response->status(),
+            ]);
+
+            return false;
+        } catch (\Exception $e) {
+            Log::error('Ollama: Connection check error', [
+                'error' => $e->getMessage(),
+            ]);
+            return false;
+        }
+    }
+
+    /**
+     * List available models on the Ollama instance.
+     *
+     * @return array List of model names
+     */
+    public function listModels(): array
+    {
+        try {
+            Log::info('Ollama: Listing models');
+
+            $response = Http::timeout(10)
+                ->get($this->apiUrl . '/api/tags');
+
+            if ($response->successful()) {
+                $data = $response->json();
+                $models = array_map(
+                    fn($model) => $model['name'] ?? 'unknown',
+                    $data['models'] ?? []
+                );
+
+                Log::info('Ollama: Models retrieved', [
+                    'count' => count($models),
+                    'models' => $models,
+                ]);
+
+                return $models;
+            }
+
+            Log::warning('Ollama: Failed to list models', [
+                'status' => $response->status(),
+            ]);
+
+            return [];
+        } catch (\Exception $e) {
+            Log::error('Ollama: Error listing models', [
+                'error' => $e->getMessage(),
+            ]);
+            return [];
+        }
+    }
+
+    /**
+     * Test summary generation with a sample text.
+     *
+     * @return array Test results with success status and details
+     */
+    public function testGeneration(): array
+    {
+        $testText = "This is a test book about artificial intelligence and machine learning. " .
+                    "It explores the history of AI, from its early beginnings to modern applications. " .
+                    "The book covers neural networks, deep learning, and practical implementations.";
+
+        $testTitle = "Test Book: Introduction to AI";
+        $testAuthor = "Test Author";
+
+        $startTime = microtime(true);
+        $summary = $this->generateSummary($testText, $testTitle, $testAuthor);
+        $duration = microtime(true) - $startTime;
+
+        return [
+            'success' => $summary !== null,
+            'summary' => $summary,
+            'duration' => round($duration, 2),
+            'model' => $this->model,
+            'word_count' => $summary ? str_word_count($summary) : 0,
+        ];
+    }
+
+    /**
+     * Build a prompt from a template with variable substitution.
+     *
+     * @param string $templateKey The template key from config
+     * @param array $variables Variables to substitute
+     * @return string The built prompt
+     */
+    protected function buildPrompt(string $templateKey, array $variables): string
+    {
+        $template = config("ollama.prompts.{$templateKey}", '');
+
+        if (empty($template)) {
+            throw new \InvalidArgumentException("Prompt template '{$templateKey}' not found in config.");
+        }
+
+        $prompt = $template;
+
+        foreach ($variables as $key => $value) {
+            $prompt = str_replace('{' . $key . '}', $value, $prompt);
+        }
+
+        return $prompt;
+    }
+
+    /**
+     * Make a request to the Ollama API.
+     *
+     * @param string $prompt The prompt to send
+     * @return string|null The response text or null on failure
+     */
+    protected function makeRequest(string $prompt): ?string
+    {
+        $url = $this->apiUrl . '/api/generate';
+
+        $payload = [
+            'model' => $this->model,
+            'prompt' => $prompt,
+            'stream' => false,
+            'options' => array_merge($this->options, [
+                'temperature' => $this->temperature,
+            ]),
+        ];
+
+        try {
+            $response = Http::timeout($this->timeout)
+                ->retry(
+                    $this->retryMaxAttempts,
+                    function ($exception, $request) {
+                        // Exponential backoff in milliseconds
+                        return $this->retryBackoff * 1000;
+                    },
+                    function ($exception, $request) {
+                        // Retry on connection errors and 5xx server errors
+                        if ($exception instanceof ConnectionException) {
+                            return true;
+                        }
+
+                        if (method_exists($exception, 'response')) {
+                            $response = $exception->response;
+                            return $response && $response->status() >= 500;
+                        }
+
+                        return false;
+                    }
+                )
+                ->post($url, $payload);
+
+            if ($response->successful()) {
+                $data = $response->json();
+                return $data['response'] ?? null;
+            }
+
+            Log::warning('Ollama: API request failed', [
+                'status' => $response->status(),
+                'body' => $response->body(),
+            ]);
+
+            return null;
+        } catch (ConnectionException $e) {
+            Log::error('Ollama: Connection error', [
+                'error' => $e->getMessage(),
+                'url' => $url,
+            ]);
+            throw $e;
+        } catch (RequestException $e) {
+            Log::error('Ollama: Request exception', [
+                'error' => $e->getMessage(),
+                'url' => $url,
+            ]);
+            throw $e;
+        } catch (\Exception $e) {
+            Log::error('Ollama: Unexpected error', [
+                'error' => $e->getMessage(),
+                'type' => get_class($e),
+            ]);
+            throw $e;
+        }
+    }
+
+    /**
+     * Get the currently configured model name.
+     *
+     * @return string
+     */
+    public function getModel(): string
+    {
+        return $this->model;
+    }
+
+    /**
+     * Set a different model for the next requests.
+     *
+     * @param string $model Model name
+     * @return void
+     */
+    public function setModel(string $model): void
+    {
+        $this->model = $model;
+        Log::info('Ollama: Model changed', ['model' => $model]);
+    }
+}