From 7be6af781b0688e59f36b79a1b496d695062f636 Mon Sep 17 00:00:00 2001 From: Doar Mihai Date: Sun, 29 Mar 2026 15:16:37 +0300 Subject: [PATCH] progress --- GUTENDEX_INTEGRATION_PLAN.md | 461 +++++++++++++++++++++++ resources/views/books/show.blade.php | 2 + resources/views/layouts/app.blade.php | 14 + resources/views/static/about.blade.php | 2 + resources/views/static/contact.blade.php | 2 + resources/views/static/privacy.blade.php | 2 + resources/views/static/terms.blade.php | 2 + resources/views/welcome.blade.php | 1 + 8 files changed, 486 insertions(+) create mode 100644 GUTENDEX_INTEGRATION_PLAN.md diff --git a/GUTENDEX_INTEGRATION_PLAN.md b/GUTENDEX_INTEGRATION_PLAN.md new file mode 100644 index 0000000..0a23a9d --- /dev/null +++ b/GUTENDEX_INTEGRATION_PLAN.md @@ -0,0 +1,461 @@ +# Gutendex API Integration Plan + +## Overview +This document outlines the phased implementation plan for integrating the Gutendex API (https://gutendex.com/) to fetch public domain books, process their summaries using a local Ollama API, and store them in the database. + +## System Architecture + +``` +┌─────────────────┐ ┌──────────────┐ ┌─────────────────┐ +│ Gutendex API │─────▶│ Laravel │─────▶│ Ollama API │ +│ (Books Data) │ │ Queue Jobs │ │ (AI Summary) │ +└─────────────────┘ └──────────────┘ └─────────────────┘ + │ + ▼ + ┌─────────────────┐ + │ Database │ + │ (Books/Summary) │ + └─────────────────┘ +``` + +## Database Schema Considerations + +### Current Schema +- **books**: title, slug, author, cover_image, publication_date, description, category_id +- **summaries**: content, key_insights, book_id +- **authors**: Separate table with many-to-many relationship + +### Additional Fields Needed +We should track Gutenberg book metadata: + +```sql +ALTER TABLE books ADD COLUMN gutenberg_id INTEGER UNIQUE NULLABLE; +ALTER TABLE books ADD COLUMN source VARCHAR(50) DEFAULT 'manual'; +ALTER TABLE books ADD COLUMN full_text_url TEXT NULLABLE; +ALTER TABLE summaries ADD COLUMN processing_status ENUM('pending', 'processing', 'completed', 'failed') DEFAULT 'pending'; +ALTER TABLE summaries ADD COLUMN processing_error TEXT NULLABLE; +ALTER TABLE summaries ADD COLUMN processed_at TIMESTAMP NULLABLE; +``` + +--- + +## Phase 1: Foundation & Data Model + +**Goal**: Set up database schema and basic configuration for Gutendex integration + +### Tasks: +1. **Migration: Extend Books Table** + - Add `gutenberg_id` (integer, unique, nullable) - To track Project Gutenberg ID + - Add `source` (string, default: 'manual') - To differentiate between manually added and auto-imported books + - Add `full_text_url` (text, nullable) - Store URL to full book text for future reference + - Add `language` (string, nullable) - Book language code + +2. **Migration: Extend Summaries Table** + - Add `processing_status` (enum: pending/processing/completed/failed) + - Add `processing_error` (text, nullable) - Store error messages for debugging + - Add `processed_at` (timestamp, nullable) - Track when summary was generated + - Add `model_used` (string, nullable) - Track which Ollama model was used + +3. **Configuration File** + - Create `config/gutendex.php`: + - Base API URL + - Default language filter (e.g., 'en') + - Batch size for imports + - Rate limiting settings + - Create `config/ollama.php`: + - API endpoint URL + - Model name (e.g., 'llama3', 'mistral') + - Timeout settings + - Prompt templates + +4. **Update Models** + - Update `Book` model with new fillable fields and casts + - Update `Summary` model with new fillable fields and casts + - Add scopes for filtering by source, status + +### Deliverables: +- [ ] Two migration files created and run +- [ ] Two config files created +- [ ] Models updated with new fields +- [ ] `.env.example` updated with new config values + +--- + +## Phase 2: Gutendex API Client + +**Goal**: Build a robust service class to interact with Gutendex API + +### Tasks: +1. **Service Class: `GutendexClient`** + - Location: `app/Services/GutendexClient.php` + - Methods: + - `searchBooks($query, $page = 1, $filters = [])` + - `getBook($gutenbergId)` + - `getBooksByLanguage($language = 'en', $page = 1)` + - `getBooksByCopyright($copyright = 'false', $page = 1)` - Public domain only + - `fetchFullText($textUrl)` - Download book text from provided URL + - Features: + - Use Laravel HTTP client + - Implement retry logic with exponential backoff + - Handle rate limiting (respect 429 responses) + - Log all API interactions + - Cache frequently accessed data + +2. **Data Transfer Objects (DTOs)** + - Create `app/DTOs/GutendexBook.php` + - Map Gutendex API response to internal structure + - Handle author array parsing + - Extract subject/category information + - Normalize data formats + +3. **Service Provider** + - Register `GutendexClient` as singleton in `AppServiceProvider` + +### Deliverables: +- [ ] `GutendexClient` service class with all methods +- [ ] `GutendexBook` DTO class +- [ ] Unit tests for API client +- [ ] Error handling and logging + +--- + +## Phase 3: Ollama Integration + +**Goal**: Create service to generate summaries using local Ollama API + +### Tasks: +1. **Service Class: `OllamaService`** + - Location: `app/Services/OllamaService.php` + - Methods: + - `generateSummary($bookText, $bookTitle, $bookAuthor)` + - `generateKeyInsights($bookText)` + - `checkConnection()` - Verify Ollama is accessible + - `listModels()` - Get available models + - Features: + - Configurable prompts via config file + - Handle long texts (chunking if needed) + - Timeout handling + - Token limit awareness + - Retry logic + +2. **Prompt Engineering** + - Create optimized prompts in `config/ollama.php`: + ```php + 'prompts' => [ + 'summary' => 'You are a literary expert. Provide a comprehensive summary of the following book titled "{title}" by {author}. Focus on main themes, plot, and significance. Limit to 500 words.\n\nBook text:\n{text}', + 'key_insights' => 'Extract 5-7 key insights or lessons from the following book summary. Format as a bullet list.\n\nSummary:\n{summary}' + ] + ``` + +3. **Text Processing Helper** + - Create `app/Helpers/TextProcessor.php` + - Methods: + - `extractExcerpt($fullText, $maxLength = 50000)` - Get representative sample + - `cleanText($text)` - Remove formatting artifacts + - `truncateToTokenLimit($text, $limit = 4000)` + +### Deliverables: +- [ ] `OllamaService` class with all methods +- [ ] `TextProcessor` helper class +- [ ] Prompt templates in config +- [ ] Connection testing command +- [ ] Unit tests for text processing + +--- + +## Phase 4: Queue Jobs & Processing Pipeline + +**Goal**: Implement background jobs for fetching and processing books + +### Tasks: +1. **Job: `ImportBookFromGutendex`** + - Location: `app/Jobs/ImportBookFromGutendex.php` + - Inputs: Gutenberg ID + - Process: + 1. Fetch book metadata from Gutendex + 2. Map authors to `authors` table + 3. Determine category from subjects + 4. Create/update book record + 5. Dispatch `GenerateSummary` job + - Error handling: Log failures, mark as failed in database + +2. **Job: `GenerateSummary`** + - Location: `app/Jobs/GenerateSummary.php` + - Inputs: Book ID + - Process: + 1. Fetch full text from URL + 2. Extract/clean excerpt + 3. Call Ollama API for summary + 4. Call Ollama API for key insights + 5. Store in `summaries` table + 6. Update processing_status to 'completed' + - Features: + - Retry on failure (3 attempts) + - Update status to 'processing' at start + - Handle timeouts gracefully + - Store errors in `processing_error` field + +3. **Job: `BatchImportFromGutendex`** + - Location: `app/Jobs/BatchImportFromGutendex.php` + - Inputs: Search parameters (language, page range, limit) + - Process: + 1. Fetch books list from Gutendex + 2. Filter out already imported books + 3. Dispatch individual `ImportBookFromGutendex` jobs + 4. Rate limit dispatching + +4. **Queue Configuration** + - Configure queue driver in `.env` (database or Redis) + - Create queues: `gutendex-import`, `summary-generation` + - Set up queue priorities + +### Deliverables: +- [ ] Three job classes implemented +- [ ] Queue tables migrated (if using database driver) +- [ ] Job failure handling +- [ ] Queue monitoring setup + +--- + +## Phase 5: Artisan Commands + +**Goal**: Provide CLI interface for managing imports + +### Tasks: +1. **Command: `gutendex:import`** + - Location: `app/Console/Commands/ImportFromGutendex.php` + - Signature: `gutendex:import {--language=en} {--limit=100} {--page=1} {--queue}` + - Functionality: + - Import books matching criteria + - Option to queue jobs or run immediately + - Progress bar showing import status + - Summary of imported books + +2. **Command: `gutendex:sync-book`** + - Signature: `gutendex:sync-book {gutenberg_id} {--force}` + - Functionality: + - Import single book by Gutenberg ID + - `--force` flag to re-import existing book + - Show processing status + +3. **Command: `summary:generate`** + - Signature: `summary:generate {book_id?} {--all} {--pending} {--failed}` + - Functionality: + - Generate summary for specific book + - `--all` to process all books without summaries + - `--pending` to retry pending summaries + - `--failed` to retry failed summaries + +4. **Command: `summary:status`** + - Signature: `summary:status` + - Functionality: + - Display statistics: total books, summaries completed, pending, failed + - Show recent processing activity + - Estimate time remaining + +5. **Command: `ollama:test`** + - Signature: `ollama:test` + - Functionality: + - Test connection to Ollama API + - Show available models + - Run test summary generation + +### Deliverables: +- [ ] Five command classes implemented +- [ ] Commands registered in `Kernel.php` +- [ ] Help text and documentation +- [ ] Error handling for all commands + +--- + +## Phase 6: Admin Interface (Optional) + +**Goal**: Provide web interface for monitoring and managing imports + +### Tasks: +1. **Controller: `GutendexController`** + - Routes: + - `GET /admin/gutendex` - Dashboard + - `POST /admin/gutendex/import` - Trigger import + - `GET /admin/gutendex/status` - Processing status + - `POST /admin/gutendex/retry/{book}` - Retry failed book + +2. **Views** + - `resources/views/admin/gutendex/dashboard.blade.php` + - Statistics cards + - Recent imports table + - Import form + - `resources/views/admin/gutendex/status.blade.php` + - Processing queue status + - Failed jobs list with retry buttons + +3. **Middleware** + - Add authentication/authorization + - Only admin users can access + +### Deliverables: +- [ ] Controller with routes +- [ ] Blade views for dashboard +- [ ] Authentication middleware +- [ ] AJAX progress updates + +--- + +## Phase 7: Monitoring & Optimization + +**Goal**: Ensure system runs reliably and efficiently + +### Tasks: +1. **Logging & Monitoring** + - Create dedicated log channel for Gutendex operations + - Log all API calls with duration + - Track success/failure rates + - Alert on high failure rates + +2. **Performance Optimization** + - Add database indexes: + - `books.gutenberg_id` + - `summaries.processing_status` + - `books.source` + - Implement caching for Gutendex API responses + - Optimize text processing for memory usage + +3. **Scheduled Tasks** + - Add to `app/Console/Kernel.php`: + - Daily import of new books: `schedule->command('gutendex:import --limit=50')->daily()` + - Retry failed summaries: `schedule->command('summary:generate --failed')->weekly()` + - Clean up old processing jobs + +4. **Documentation** + - Update README.md with: + - Setup instructions for Ollama + - Environment variables documentation + - Command usage examples + - Create troubleshooting guide + +### Deliverables: +- [ ] Logging configuration +- [ ] Database indexes added +- [ ] Scheduled tasks configured +- [ ] Documentation updated + +--- + +## Phase 8: Testing & Quality Assurance + +**Goal**: Ensure reliability and correctness + +### Tasks: +1. **Unit Tests** + - Test `GutendexClient` with mocked HTTP responses + - Test `OllamaService` with mocked API + - Test `TextProcessor` helper methods + - Test DTOs and data transformation + +2. **Feature Tests** + - Test complete import workflow + - Test summary generation pipeline + - Test error handling scenarios + - Test queue job processing + +3. **Integration Tests** + - Test with real Gutendex API (limited) + - Test with real Ollama API (local) + - Test end-to-end workflow + +4. **Error Scenarios** + - Gutendex API down/rate limited + - Ollama API timeout + - Invalid book data + - Network failures + +### Deliverables: +- [ ] Unit test suite with >80% coverage +- [ ] Feature tests for all workflows +- [ ] Integration tests documented +- [ ] Error handling verified + +--- + +## Implementation Timeline + +### Week 1: Foundation +- Phase 1: Database & Configuration (Days 1-2) +- Phase 2: Gutendex Client (Days 3-5) + +### Week 2: Processing +- Phase 3: Ollama Integration (Days 1-3) +- Phase 4: Queue Jobs (Days 4-5) + +### Week 3: Interface & Tools +- Phase 5: Artisan Commands (Days 1-3) +- Phase 6: Admin Interface (Days 4-5) - Optional + +### Week 4: Polish +- Phase 7: Monitoring & Optimization (Days 1-2) +- Phase 8: Testing & QA (Days 3-5) + +--- + +## Environment Variables Required + +```env +# Gutendex Configuration +GUTENDEX_API_URL=https://gutendex.com +GUTENDEX_DEFAULT_LANGUAGE=en +GUTENDEX_BATCH_SIZE=50 +GUTENDEX_RATE_LIMIT_DELAY=1000 + +# Ollama Configuration +OLLAMA_API_URL=http://localhost:11434 +OLLAMA_MODEL=llama3 +OLLAMA_TIMEOUT=300 +OLLAMA_MAX_TOKENS=4000 + +# Queue Configuration +QUEUE_CONNECTION=database +``` + +--- + +## Success Metrics + +- Successfully import 100+ books from Gutendex +- Generate summaries with >90% success rate +- Average processing time per book < 5 minutes +- Zero data loss or corruption +- Comprehensive error logging and recovery + +--- + +## Future Enhancements + +1. **Multi-language Support**: Process books in multiple languages +2. **Advanced Categorization**: Use AI to auto-categorize books +3. **Quality Scoring**: Rate summary quality and retry low-quality ones +4. **User Requests**: Allow users to request specific Gutenberg books +5. **Incremental Updates**: Sync with Gutendex daily for new additions +6. **Summary Variations**: Generate different summary lengths (short/medium/long) +7. **Related Books**: Use AI to suggest related books based on content + +--- + +## Risk Mitigation + +| Risk | Impact | Mitigation | +|------|--------|------------| +| Ollama API slow/unavailable | High | Implement timeout, retry logic, queue system | +| Rate limiting from Gutendex | Medium | Respect rate limits, implement backoff, cache responses | +| Poor summary quality | Medium | Fine-tune prompts, validate output, human review process | +| Memory issues with large texts | Medium | Chunk processing, optimize text extraction | +| Database growth | Low | Regular cleanup, archival strategy | + +--- + +## Notes + +- Start with small batch (10-20 books) to test pipeline +- Monitor Ollama performance and adjust model/prompts as needed +- Consider using a faster model for initial testing +- Keep Gutenberg IDs to avoid duplicate imports +- Implement proper error notifications for production diff --git a/resources/views/books/show.blade.php b/resources/views/books/show.blade.php index 15bea1f..1f62eaa 100644 --- a/resources/views/books/show.blade.php +++ b/resources/views/books/show.blade.php @@ -32,6 +32,8 @@ {{ $desc }} @endsection +@section('canonical', route('books.show', $book)) + @section('content')
diff --git a/resources/views/layouts/app.blade.php b/resources/views/layouts/app.blade.php index 4b94489..438627e 100644 --- a/resources/views/layouts/app.blade.php +++ b/resources/views/layouts/app.blade.php @@ -20,6 +20,8 @@ @yield('page_title', config('app.name', 'OpenTextSummaries')) + + @@ -32,8 +34,20 @@ @if (file_exists(public_path('build/manifest.json')) || file_exists(public_path('hot'))) @vite(['resources/css/app.css', 'resources/js/app.js']) @endif + + + + + + +
diff --git a/resources/views/static/about.blade.php b/resources/views/static/about.blade.php index 990c454..a0962b8 100644 --- a/resources/views/static/about.blade.php +++ b/resources/views/static/about.blade.php @@ -8,6 +8,8 @@ Learn about OpenTextSummaries — our mission is to make knowledge accessible with concise, readable book summaries and key insights. @endsection +@section('canonical', url()->current()) + @section('content')

About

diff --git a/resources/views/static/contact.blade.php b/resources/views/static/contact.blade.php index 847bccf..08410d8 100644 --- a/resources/views/static/contact.blade.php +++ b/resources/views/static/contact.blade.php @@ -8,6 +8,8 @@ Get in touch with the OpenTextSummaries team — questions, feedback, or partnership inquiries. @endsection +@section('canonical', url()->current()) + @section('content')

Contact

diff --git a/resources/views/static/privacy.blade.php b/resources/views/static/privacy.blade.php index 809428c..c82da05 100644 --- a/resources/views/static/privacy.blade.php +++ b/resources/views/static/privacy.blade.php @@ -8,6 +8,8 @@ Read the OpenTextSummaries Privacy Policy — what data we collect, how we use it, and your choices. @endsection +@section('canonical', url()->current()) + @section('content')

Privacy Policy

diff --git a/resources/views/static/terms.blade.php b/resources/views/static/terms.blade.php index 7a02947..18a5590 100644 --- a/resources/views/static/terms.blade.php +++ b/resources/views/static/terms.blade.php @@ -8,6 +8,8 @@ Review the OpenTextSummaries Terms of Service — your responsibilities and the rules for using the site. @endsection +@section('canonical', url()->current()) + @section('content')

Terms of Service

diff --git a/resources/views/welcome.blade.php b/resources/views/welcome.blade.php index 36a5196..21e0c20 100644 --- a/resources/views/welcome.blade.php +++ b/resources/views/welcome.blade.php @@ -17,6 +17,7 @@ {{ config('app.name', 'Open Text Summaries') }} +