gutendex integration, phase 6

This commit is contained in:
Doar Mihai
2026-03-29 16:14:18 +03:00
parent 0b26cde74e
commit 86de669aee
12 changed files with 1820 additions and 0 deletions
@@ -0,0 +1,267 @@
<?php
namespace App\Console\Commands;
use App\Jobs\GenerateSummary;
use App\Models\Book;
use App\Models\Summary;
use Illuminate\Console\Command;
class GenerateSummaryCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'summary:generate
{book_id? : The book ID to generate summary for}
{--all : Generate summaries for all books without summaries}
{--pending : Retry pending summaries}
{--failed : Retry failed summaries}
{--limit=50 : Maximum number of summaries to generate (for batch operations)}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Generate or regenerate book summaries using Ollama';
/**
* Execute the console command.
*/
public function handle(): int
{
$bookId = $this->argument('book_id');
$all = $this->option('all');
$pending = $this->option('pending');
$failed = $this->option('failed');
$limit = (int) $this->option('limit');
// Handle single book
if ($bookId) {
return $this->generateForBook((int) $bookId);
}
// Handle batch operations
if ($all) {
return $this->generateForBooksWithoutSummaries($limit);
}
if ($pending) {
return $this->generateForPendingSummaries($limit);
}
if ($failed) {
return $this->generateForFailedSummaries($limit);
}
// No option provided
$this->error('Please specify a book ID or use one of the options: --all, --pending, --failed');
$this->newLine();
$this->comment('Examples:');
$this->comment(' php artisan summary:generate 123');
$this->comment(' php artisan summary:generate --all');
$this->comment(' php artisan summary:generate --failed --limit=10');
return self::FAILURE;
}
/**
* Generate summary for a specific book.
*/
protected function generateForBook(int $bookId): int
{
$book = Book::find($bookId);
if (!$book) {
$this->error("Book with ID {$bookId} not found.");
return self::FAILURE;
}
$this->info("Generating summary for: {$book->title}");
$this->newLine();
if (!$book->full_text_url) {
$this->error('Book does not have a full text URL. Cannot generate summary.');
return self::FAILURE;
}
// Check existing summary
if ($book->summary) {
$this->warn('Book already has a summary:');
$this->table(
['Field', 'Value'],
[
['Status', $book->summary->processing_status],
['Processed At', $book->summary->processed_at?->format('Y-m-d H:i:s') ?? 'N/A'],
['Model Used', $book->summary->model_used ?? 'N/A'],
['Has Error', $book->summary->processing_error ? 'Yes' : 'No'],
]
);
$this->newLine();
if (!$this->confirm('Do you want to regenerate the summary?', false)) {
$this->info('Operation cancelled.');
return self::SUCCESS;
}
}
$this->info('Dispatching summary generation job...');
GenerateSummary::dispatch($bookId);
$this->newLine();
$this->info('✓ Summary generation job dispatched!');
$this->comment('Run the queue worker: php artisan queue:work');
return self::SUCCESS;
}
/**
* Generate summaries for books without summaries.
*/
protected function generateForBooksWithoutSummaries(int $limit): int
{
$this->info('Finding books without summaries...');
$books = Book::doesntHave('summary')
->whereNotNull('full_text_url')
->where('source', 'gutendex')
->limit($limit)
->get();
if ($books->isEmpty()) {
$this->info('No books found without summaries.');
return self::SUCCESS;
}
$this->info("Found {$books->count()} books without summaries.");
$this->newLine();
if (!$this->confirm("Dispatch {$books->count()} summary generation jobs?", true)) {
$this->info('Operation cancelled.');
return self::SUCCESS;
}
$progressBar = $this->output->createProgressBar($books->count());
$progressBar->start();
foreach ($books as $index => $book) {
GenerateSummary::dispatch($book->id)
->delay(now()->addSeconds($index * 10)); // Stagger by 10 seconds
$progressBar->advance();
}
$progressBar->finish();
$this->newLine(2);
$this->info("✓ Dispatched {$books->count()} summary generation jobs!");
$this->comment('Jobs will be processed with 10-second intervals.');
$this->comment('Run the queue worker: php artisan queue:work');
return self::SUCCESS;
}
/**
* Generate summaries for pending summaries.
*/
protected function generateForPendingSummaries(int $limit): int
{
$this->info('Finding pending summaries...');
$summaries = Summary::pending()
->with('book')
->limit($limit)
->get();
if ($summaries->isEmpty()) {
$this->info('No pending summaries found.');
return self::SUCCESS;
}
$this->info("Found {$summaries->count()} pending summaries.");
$this->newLine();
if (!$this->confirm("Retry {$summaries->count()} pending summaries?", true)) {
$this->info('Operation cancelled.');
return self::SUCCESS;
}
$progressBar = $this->output->createProgressBar($summaries->count());
$progressBar->start();
foreach ($summaries as $index => $summary) {
if ($summary->book) {
GenerateSummary::dispatch($summary->book->id)
->delay(now()->addSeconds($index * 10));
}
$progressBar->advance();
}
$progressBar->finish();
$this->newLine(2);
$this->info("✓ Dispatched {$summaries->count()} retry jobs!");
$this->comment('Run the queue worker: php artisan queue:work');
return self::SUCCESS;
}
/**
* Generate summaries for failed summaries.
*/
protected function generateForFailedSummaries(int $limit): int
{
$this->info('Finding failed summaries...');
$summaries = Summary::failed()
->with('book')
->limit($limit)
->get();
if ($summaries->isEmpty()) {
$this->info('No failed summaries found.');
return self::SUCCESS;
}
$this->warn("Found {$summaries->count()} failed summaries.");
$this->newLine();
// Show error details
$this->table(
['Book ID', 'Title', 'Error'],
$summaries->map(fn($s) => [
$s->book_id,
$s->book?->title ?? 'Unknown',
substr($s->processing_error ?? 'Unknown error', 0, 60) . '...',
])->toArray()
);
$this->newLine();
if (!$this->confirm("Retry {$summaries->count()} failed summaries?", true)) {
$this->info('Operation cancelled.');
return self::SUCCESS;
}
$progressBar = $this->output->createProgressBar($summaries->count());
$progressBar->start();
foreach ($summaries as $index => $summary) {
if ($summary->book) {
GenerateSummary::dispatch($summary->book->id)
->delay(now()->addSeconds($index * 15)); // Longer delay for failures
}
$progressBar->advance();
}
$progressBar->finish();
$this->newLine(2);
$this->info("✓ Dispatched {$summaries->count()} retry jobs!");
$this->comment('Jobs will be processed with 15-second intervals.');
$this->comment('Run the queue worker: php artisan queue:work');
return self::SUCCESS;
}
}
+184
View File
@@ -0,0 +1,184 @@
<?php
namespace App\Console\Commands;
use App\Jobs\BatchImportFromGutendex;
use App\Jobs\ImportBookFromGutendex;
use Illuminate\Console\Command;
class ImportFromGutendex extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'gutendex:import
{--language=en : Language code to filter books}
{--limit=100 : Maximum number of books to import}
{--page=1 : Starting page number}
{--queue : Run the import via queue jobs}
{--no-summary : Skip summary generation}
{--copyright=false : Copyright filter (false for public domain)}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Import books from Gutendex API with optional filters';
/**
* Execute the console command.
*/
public function handle(): int
{
$language = $this->option('language');
$limit = (int) $this->option('limit');
$page = (int) $this->option('page');
$useQueue = $this->option('queue');
$generateSummary = !$this->option('no-summary');
$copyright = $this->option('copyright');
$this->info('Gutendex Import');
$this->info('==============');
$this->newLine();
// Build filters
$filters = [
'languages' => $language,
'copyright' => $copyright,
];
$this->table(
['Setting', 'Value'],
[
['Language', $language],
['Limit', $limit],
['Starting Page', $page],
['Mode', $useQueue ? 'Queued' : 'Immediate'],
['Generate Summaries', $generateSummary ? 'Yes' : 'No'],
['Copyright Filter', $copyright],
]
);
if (!$this->confirm('Do you want to proceed with this import?', true)) {
$this->warn('Import cancelled.');
return self::SUCCESS;
}
$this->newLine();
if ($useQueue) {
return $this->handleQueuedImport($filters, $page, $limit, $generateSummary);
} else {
return $this->handleImmediateImport($filters, $page, $limit, $generateSummary);
}
}
/**
* Handle queued import.
*/
protected function handleQueuedImport(array $filters, int $page, int $limit, bool $generateSummary): int
{
$this->info('Dispatching batch import job...');
// Calculate end page based on limit (assuming ~32 books per page)
$booksPerPage = 32;
$endPage = $page + (int) ceil($limit / $booksPerPage) - 1;
BatchImportFromGutendex::dispatch(
$filters,
$page,
$endPage,
$limit,
$generateSummary
);
$this->info('✓ Batch import job dispatched successfully!');
$this->newLine();
$this->comment('The import will be processed in the background.');
$this->comment('Run the queue worker: php artisan queue:work');
$this->comment('Monitor progress: php artisan summary:status');
return self::SUCCESS;
}
/**
* Handle immediate import (synchronous).
*/
protected function handleImmediateImport(array $filters, int $page, int $limit, bool $generateSummary): int
{
$this->warn('⚠ Running immediate import. This may take a while...');
$this->warn('⚠ For large imports, consider using --queue option.');
$this->newLine();
$gutendexClient = app(\App\Services\GutendexClient::class);
$totalImported = 0;
$totalSkipped = 0;
$currentPage = $page;
$progressBar = $this->output->createProgressBar($limit);
$progressBar->start();
while ($totalImported < $limit) {
$response = $gutendexClient->searchBooks(null, $currentPage, $filters);
if (empty($response['results'])) {
$this->newLine();
$this->warn('No more results available.');
break;
}
foreach ($response['results'] as $bookData) {
if ($totalImported >= $limit) {
break 2;
}
$gutenbergId = $bookData['id'] ?? null;
if (!$gutenbergId) {
continue;
}
// Check if already exists
if (\App\Models\Book::where('gutenberg_id', $gutenbergId)->exists()) {
$totalSkipped++;
$progressBar->advance();
continue;
}
// Dispatch job
ImportBookFromGutendex::dispatch($gutenbergId, $generateSummary);
$totalImported++;
$progressBar->advance();
}
if (empty($response['next'])) {
break;
}
$currentPage++;
}
$progressBar->finish();
$this->newLine(2);
$this->info('✓ Import jobs dispatched successfully!');
$this->newLine();
$this->table(
['Metric', 'Count'],
[
['Jobs Dispatched', $totalImported],
['Books Skipped (existing)', $totalSkipped],
['Pages Processed', $currentPage - $page + 1],
]
);
$this->newLine();
$this->comment('Jobs have been dispatched to the queue.');
$this->comment('Run the queue worker: php artisan queue:work');
return self::SUCCESS;
}
}
+229
View File
@@ -0,0 +1,229 @@
<?php
namespace App\Console\Commands;
use App\Models\Book;
use App\Models\Summary;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class SummaryStatus extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'summary:status
{--detailed : Show detailed breakdown}
{--recent=10 : Number of recent activities to show}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Display statistics and status of summary generation';
/**
* Execute the console command.
*/
public function handle(): int
{
$this->info('Summary Generation Status');
$this->info('========================');
$this->newLine();
$this->displayOverallStatistics();
$this->newLine();
if ($this->option('detailed')) {
$this->displayDetailedBreakdown();
$this->newLine();
}
$this->displayRecentActivity();
$this->newLine();
$this->displayQueueStatus();
return self::SUCCESS;
}
/**
* Display overall statistics.
*/
protected function displayOverallStatistics(): void
{
$totalBooks = Book::count();
$gutendexBooks = Book::gutendex()->count();
$booksWithText = Book::whereNotNull('full_text_url')->count();
$totalSummaries = Summary::count();
$completedSummaries = Summary::completed()->count();
$pendingSummaries = Summary::pending()->count();
$processingSummaries = Summary::processing()->count();
$failedSummaries = Summary::failed()->count();
$booksWithoutSummaries = Book::gutendex()
->whereNotNull('full_text_url')
->doesntHave('summary')
->count();
$completionRate = $totalSummaries > 0
? round(($completedSummaries / $totalSummaries) * 100, 2)
: 0;
$this->table(
['Metric', 'Count', 'Percentage'],
[
['Total Books', number_format($totalBooks), '-'],
['Gutendex Books', number_format($gutendexBooks), round(($gutendexBooks / max($totalBooks, 1)) * 100, 1) . '%'],
['Books with Full Text', number_format($booksWithText), '-'],
['Books without Summaries', number_format($booksWithoutSummaries), '-'],
['---', '---', '---'],
['Total Summaries', number_format($totalSummaries), '-'],
['✓ Completed', number_format($completedSummaries), $completionRate . '%'],
['⏳ Pending', number_format($pendingSummaries), round(($pendingSummaries / max($totalSummaries, 1)) * 100, 1) . '%'],
['⚙ Processing', number_format($processingSummaries), round(($processingSummaries / max($totalSummaries, 1)) * 100, 1) . '%'],
['✗ Failed', number_format($failedSummaries), round(($failedSummaries / max($totalSummaries, 1)) * 100, 1) . '%'],
]
);
}
/**
* Display detailed breakdown by model and category.
*/
protected function displayDetailedBreakdown(): void
{
$this->info('Detailed Breakdown');
$this->info('------------------');
$this->newLine();
// By model
$byModel = Summary::completed()
->select('model_used', DB::raw('COUNT(*) as count'))
->groupBy('model_used')
->get();
if ($byModel->isNotEmpty()) {
$this->info('Summaries by Model:');
$this->table(
['Model', 'Count'],
$byModel->map(fn($item) => [
$item->model_used ?? 'Unknown',
number_format($item->count),
])->toArray()
);
$this->newLine();
}
// By language
$byLanguage = Book::gutendex()
->select('language', DB::raw('COUNT(*) as count'))
->whereNotNull('language')
->groupBy('language')
->orderByDesc('count')
->limit(10)
->get();
if ($byLanguage->isNotEmpty()) {
$this->info('Books by Language (Top 10):');
$this->table(
['Language', 'Count'],
$byLanguage->map(fn($item) => [
$item->language,
number_format($item->count),
])->toArray()
);
$this->newLine();
}
}
/**
* Display recent activity.
*/
protected function displayRecentActivity(): void
{
$recentCount = (int) $this->option('recent');
$this->info("Recent Activity (Last {$recentCount})");
$this->info('------------------');
$this->newLine();
// Recent completed summaries
$recentCompleted = Summary::completed()
->with('book')
->orderByDesc('processed_at')
->limit($recentCount)
->get();
if ($recentCompleted->isNotEmpty()) {
$this->info('✓ Recently Completed:');
$this->table(
['Book ID', 'Title', 'Processed At', 'Model'],
$recentCompleted->map(fn($s) => [
$s->book_id,
substr($s->book?->title ?? 'Unknown', 0, 40),
$s->processed_at?->diffForHumans() ?? 'N/A',
$s->model_used ?? 'N/A',
])->toArray()
);
$this->newLine();
}
// Recent failures
$recentFailed = Summary::failed()
->with('book')
->orderByDesc('updated_at')
->limit($recentCount)
->get();
if ($recentFailed->isNotEmpty()) {
$this->warn('✗ Recent Failures:');
$this->table(
['Book ID', 'Title', 'Error'],
$recentFailed->map(fn($s) => [
$s->book_id,
substr($s->book?->title ?? 'Unknown', 0, 30),
substr($s->processing_error ?? 'Unknown error', 0, 60),
])->toArray()
);
$this->newLine();
}
}
/**
* Display queue status.
*/
protected function displayQueueStatus(): void
{
$this->info('Queue Status');
$this->info('------------');
$this->newLine();
// Count pending jobs (if using database queue)
try {
$pendingJobs = DB::table('jobs')->count();
$failedJobs = DB::table('failed_jobs')->count();
$this->table(
['Queue', 'Count'],
[
['Pending Jobs', number_format($pendingJobs)],
['Failed Jobs', number_format($failedJobs)],
]
);
if ($pendingJobs > 0) {
$this->comment('Run the queue worker: php artisan queue:work');
}
if ($failedJobs > 0) {
$this->warn("You have {$failedJobs} failed jobs. Review with: php artisan queue:failed");
}
} catch (\Exception $e) {
$this->comment('Queue status unavailable (database driver not configured).');
}
}
}
@@ -0,0 +1,134 @@
<?php
namespace App\Console\Commands;
use App\Jobs\ImportBookFromGutendex;
use App\Models\Book;
use Illuminate\Console\Command;
class SyncBookFromGutendex extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'gutendex:sync-book
{gutenberg_id : The Gutenberg book ID to import}
{--force : Force re-import even if book exists}
{--no-summary : Skip summary generation}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Import or update a specific book from Gutendex by its ID';
/**
* Execute the console command.
*/
public function handle(): int
{
$gutenbergId = (int) $this->argument('gutenberg_id');
$force = $this->option('force');
$generateSummary = !$this->option('no-summary');
if ($gutenbergId <= 0) {
$this->error('Invalid Gutenberg ID. Must be a positive integer.');
return self::FAILURE;
}
$this->info("Syncing book from Gutendex (ID: {$gutenbergId})");
$this->newLine();
// Check if book already exists
$existingBook = Book::where('gutenberg_id', $gutenbergId)->first();
if ($existingBook && !$force) {
$this->warn('Book already exists in the database!');
$this->newLine();
$this->table(
['Field', 'Value'],
[
['Database ID', $existingBook->id],
['Title', $existingBook->title],
['Author', $existingBook->author],
['Slug', $existingBook->slug],
['Category', $existingBook->category->name ?? 'N/A'],
['Has Summary', $existingBook->summary ? 'Yes' : 'No'],
['Summary Status', $existingBook->summary->processing_status ?? 'N/A'],
]
);
$this->newLine();
if (!$this->confirm('Use --force to re-import. Continue anyway?', false)) {
$this->info('Operation cancelled.');
return self::SUCCESS;
}
// Delete existing book to re-import
$this->warn('Deleting existing book...');
$existingBook->delete();
$this->info('✓ Existing book deleted.');
$this->newLine();
}
// Fetch book info from Gutendex first to show details
$this->info('Fetching book details from Gutendex...');
$gutendexClient = app(\App\Services\GutendexClient::class);
$gutendexBook = $gutendexClient->getBook($gutenbergId);
if (!$gutendexBook) {
$this->error("Failed to fetch book {$gutenbergId} from Gutendex API.");
$this->comment('The book may not exist or the API may be unavailable.');
return self::FAILURE;
}
// Display book info
$this->info('✓ Book found on Gutendex');
$this->newLine();
$this->table(
['Field', 'Value'],
[
['Gutenberg ID', $gutendexBook->id],
['Title', $gutendexBook->title],
['Author(s)', $gutendexBook->getAuthorNamesString()],
['Language', $gutendexBook->getPrimaryLanguage()],
['Public Domain', $gutendexBook->isPublicDomain() ? 'Yes' : 'No'],
['Plain Text Available', $gutendexBook->getPlainTextUrl() ? 'Yes' : 'No'],
['Download Count', number_format($gutendexBook->downloadCount)],
]
);
$this->newLine();
// Validate book
if (!$gutendexBook->isPublicDomain()) {
$this->error('This book is not in the public domain and cannot be imported.');
return self::FAILURE;
}
if (!$gutendexBook->getPlainTextUrl()) {
$this->error('This book does not have a plain text version available.');
return self::FAILURE;
}
// Dispatch import job
$this->info('Dispatching import job...');
ImportBookFromGutendex::dispatch($gutenbergId, $generateSummary);
$this->newLine();
$this->info('✓ Import job dispatched successfully!');
$this->newLine();
$this->comment('The book will be imported in the background.');
$this->comment('Run the queue worker: php artisan queue:work');
if ($generateSummary) {
$this->comment('Summary will be generated automatically after import.');
} else {
$this->comment('Summary generation was skipped (--no-summary flag).');
}
return self::SUCCESS;
}
}
+192
View File
@@ -0,0 +1,192 @@
<?php
namespace App\Console\Commands;
use App\Services\OllamaService;
use Illuminate\Console\Command;
class TestOllama extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'ollama:test
{--model= : Specific model to test (overrides config)}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Test connection to Ollama API and run a sample summary generation';
/**
* Execute the console command.
*/
public function handle(OllamaService $ollamaService): int
{
$this->info('Ollama API Test');
$this->info('===============');
$this->newLine();
// Override model if specified
$modelOption = $this->option('model');
if ($modelOption) {
$ollamaService->setModel($modelOption);
$this->comment("Using model: {$modelOption}");
$this->newLine();
}
// Display configuration
$this->displayConfiguration();
$this->newLine();
// Test connection
if (!$this->testConnection($ollamaService)) {
return self::FAILURE;
}
$this->newLine();
// List available models
$this->listAvailableModels($ollamaService);
$this->newLine();
// Run test generation
if ($this->confirm('Do you want to run a test summary generation?', true)) {
$this->newLine();
$this->testSummaryGeneration($ollamaService);
}
return self::SUCCESS;
}
/**
* Display Ollama configuration.
*/
protected function displayConfiguration(): void
{
$this->info('Configuration:');
$this->table(
['Setting', 'Value'],
[
['API URL', config('ollama.api_url')],
['Model', config('ollama.model')],
['Timeout', config('ollama.timeout') . ' seconds'],
['Temperature', config('ollama.temperature')],
['Max Tokens', config('ollama.max_tokens')],
]
);
}
/**
* Test connection to Ollama API.
*/
protected function testConnection(OllamaService $ollamaService): bool
{
$this->info('Testing connection to Ollama API...');
if ($ollamaService->checkConnection()) {
$this->info('✓ Connection successful!');
return true;
} else {
$this->error('✗ Connection failed!');
$this->newLine();
$this->warn('Possible issues:');
$this->comment(' 1. Ollama is not running (start with: ollama serve)');
$this->comment(' 2. Wrong API URL in config/ollama.php or .env');
$this->comment(' 3. Network/firewall issues');
$this->newLine();
$this->comment('Current API URL: ' . config('ollama.api_url'));
return false;
}
}
/**
* List available models.
*/
protected function listAvailableModels(OllamaService $ollamaService): void
{
$this->info('Available Models:');
$models = $ollamaService->listModels();
if (empty($models)) {
$this->warn('No models found or unable to fetch models.');
$this->newLine();
$this->comment('Pull a model with: ollama pull llama3');
return;
}
$currentModel = $ollamaService->getModel();
$rows = [];
foreach ($models as $model) {
$isCurrent = $model === $currentModel ? '✓' : '';
$rows[] = [$model, $isCurrent];
}
$this->table(['Model Name', 'Current'], $rows);
if (!in_array($currentModel, $models)) {
$this->warn("⚠ Configured model '{$currentModel}' is not available!");
$this->comment("Pull it with: ollama pull {$currentModel}");
}
}
/**
* Test summary generation.
*/
protected function testSummaryGeneration(OllamaService $ollamaService): void
{
$this->info('Running test summary generation...');
$this->newLine();
$testResult = $ollamaService->testGeneration();
if ($testResult['success']) {
$this->info('✓ Test generation successful!');
$this->newLine();
$this->table(
['Metric', 'Value'],
[
['Duration', $testResult['duration'] . ' seconds'],
['Model Used', $testResult['model']],
['Word Count', $testResult['word_count']],
]
);
$this->newLine();
$this->info('Generated Summary:');
$this->line('------------------');
$this->line($testResult['summary']);
$this->line('------------------');
$this->newLine();
// Quality assessment
$wordCount = $testResult['word_count'];
$minWords = config('ollama.quality_checks.min_word_count', 100);
$maxWords = config('ollama.quality_checks.max_word_count', 1000);
if ($wordCount < $minWords) {
$this->warn("⚠ Summary is shorter than recommended minimum ({$minWords} words)");
} elseif ($wordCount > $maxWords) {
$this->warn("⚠ Summary is longer than recommended maximum ({$maxWords} words)");
} else {
$this->info('✓ Summary length is within recommended range');
}
$avgTimePerWord = $testResult['duration'] / max($wordCount, 1);
$this->comment("Average: " . round($avgTimePerWord, 2) . " seconds per word");
} else {
$this->error('✗ Test generation failed!');
$this->newLine();
$this->warn('This could indicate:');
$this->comment(' 1. Model is not properly loaded');
$this->comment(' 2. Insufficient system resources');
$this->comment(' 3. Timeout issues (increase OLLAMA_TIMEOUT in .env)');
}
}
}
+201
View File
@@ -0,0 +1,201 @@
<?php
namespace App\Http\Controllers;
use App\Jobs\BatchImportFromGutendex;
use App\Jobs\GenerateSummary;
use App\Models\Book;
use App\Models\Summary;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\DB;
class GutendexController extends Controller
{
/**
* Display the Gutendex admin dashboard.
*/
public function dashboard()
{
$stats = $this->getStatistics();
$recentImports = $this->getRecentImports();
$recentFailures = $this->getRecentFailures();
return view('admin.gutendex.dashboard', compact('stats', 'recentImports', 'recentFailures'));
}
/**
* Display processing status.
*/
public function status()
{
$stats = $this->getStatistics();
$processing = Summary::processing()->with('book')->get();
$pending = Summary::pending()->with('book')->limit(20)->get();
$failed = Summary::failed()->with('book')->limit(20)->get();
$queueStats = $this->getQueueStatistics();
return view('admin.gutendex.status', compact('stats', 'processing', 'pending', 'failed', 'queueStats'));
}
/**
* Trigger a new import.
*/
public function import(Request $request)
{
$request->validate([
'language' => 'required|string|max:10',
'limit' => 'required|integer|min:1|max:500',
'page' => 'required|integer|min:1',
'generate_summaries' => 'boolean',
]);
$language = $request->input('language', 'en');
$limit = (int) $request->input('limit', 50);
$page = (int) $request->input('page', 1);
$generateSummaries = $request->boolean('generate_summaries', true);
$filters = [
'languages' => $language,
'copyright' => 'false',
];
// Calculate end page
$booksPerPage = 32;
$endPage = $page + (int) ceil($limit / $booksPerPage) - 1;
// Dispatch batch import job
BatchImportFromGutendex::dispatch(
$filters,
$page,
$endPage,
$limit,
$generateSummaries
);
return response()->json([
'success' => true,
'message' => "Import job dispatched for up to {$limit} books",
'details' => [
'language' => $language,
'limit' => $limit,
'start_page' => $page,
'end_page' => $endPage,
'generate_summaries' => $generateSummaries,
],
]);
}
/**
* Retry a failed book summary.
*/
public function retry(Book $book)
{
if (!$book->summary) {
return response()->json([
'success' => false,
'message' => 'Book does not have a summary record',
], 404);
}
if ($book->summary->processing_status === 'completed') {
return response()->json([
'success' => false,
'message' => 'Summary is already completed',
], 400);
}
// Dispatch job to regenerate summary
GenerateSummary::dispatch($book->id);
return response()->json([
'success' => true,
'message' => "Retry job dispatched for: {$book->title}",
'book_id' => $book->id,
]);
}
/**
* Get overall statistics.
*/
protected function getStatistics(): array
{
$totalBooks = Book::count();
$gutendexBooks = Book::gutendex()->count();
$booksWithText = Book::whereNotNull('full_text_url')->count();
$booksWithoutSummaries = Book::gutendex()
->whereNotNull('full_text_url')
->doesntHave('summary')
->count();
$totalSummaries = Summary::count();
$completedSummaries = Summary::completed()->count();
$pendingSummaries = Summary::pending()->count();
$processingSummaries = Summary::processing()->count();
$failedSummaries = Summary::failed()->count();
$completionRate = $totalSummaries > 0
? round(($completedSummaries / $totalSummaries) * 100, 2)
: 0;
return [
'total_books' => $totalBooks,
'gutendex_books' => $gutendexBooks,
'books_with_text' => $booksWithText,
'books_without_summaries' => $booksWithoutSummaries,
'total_summaries' => $totalSummaries,
'completed_summaries' => $completedSummaries,
'pending_summaries' => $pendingSummaries,
'processing_summaries' => $processingSummaries,
'failed_summaries' => $failedSummaries,
'completion_rate' => $completionRate,
];
}
/**
* Get recent imports.
*/
protected function getRecentImports(int $limit = 10): array
{
return Book::gutendex()
->with('category')
->orderByDesc('created_at')
->limit($limit)
->get()
->toArray();
}
/**
* Get recent failures.
*/
protected function getRecentFailures(int $limit = 10): array
{
return Summary::failed()
->with('book')
->orderByDesc('updated_at')
->limit($limit)
->get()
->toArray();
}
/**
* Get queue statistics.
*/
protected function getQueueStatistics(): array
{
try {
$pendingJobs = DB::table('jobs')->count();
$failedJobs = DB::table('failed_jobs')->count();
return [
'pending_jobs' => $pendingJobs,
'failed_jobs' => $failedJobs,
];
} catch (\Exception $e) {
return [
'pending_jobs' => 0,
'failed_jobs' => 0,
];
}
}
}
+30
View File
@@ -0,0 +1,30 @@
<?php
namespace App\Http\Middleware;
use Closure;
use Illuminate\Http\Request;
use Symfony\Component\HttpFoundation\Response;
class IsAdmin
{
/**
* Handle an incoming request.
*
* @param \Closure(\Illuminate\Http\Request): (\Symfony\Component\HttpFoundation\Response) $next
*/
public function handle(Request $request, Closure $next): Response
{
// Check if user is authenticated
if (!auth()->check()) {
return redirect()->route('login')->with('error', 'You must be logged in to access this page.');
}
// Check if user is admin
if (!auth()->user()->is_admin) {
abort(403, 'Unauthorized access. Admin privileges required.');
}
return $next($request);
}
}
+4
View File
@@ -1,6 +1,7 @@
<?php <?php
use App\Http\Middleware\GoatCounterMiddleware; use App\Http\Middleware\GoatCounterMiddleware;
use App\Http\Middleware\IsAdmin;
use Illuminate\Foundation\Application; use Illuminate\Foundation\Application;
use Illuminate\Foundation\Configuration\Exceptions; use Illuminate\Foundation\Configuration\Exceptions;
use Illuminate\Foundation\Configuration\Middleware; use Illuminate\Foundation\Configuration\Middleware;
@@ -14,6 +15,9 @@ return Application::configure(basePath: dirname(__DIR__))
) )
->withMiddleware(function (Middleware $middleware): void { ->withMiddleware(function (Middleware $middleware): void {
$middleware->append(GoatCounterMiddleware::class); $middleware->append(GoatCounterMiddleware::class);
$middleware->alias([
'admin' => IsAdmin::class,
]);
}) })
->withExceptions(function (Exceptions $exceptions): void { ->withExceptions(function (Exceptions $exceptions): void {
// //
@@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('users', function (Blueprint $table) {
$table->boolean('is_admin')->default(false)->after('email');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('users', function (Blueprint $table) {
$table->dropColumn('is_admin');
});
}
};
@@ -0,0 +1,273 @@
@extends('layouts.app')
@section('content')
<div class="container">
<div class="row mb-4">
<div class="col-md-12">
<h1>Gutendex Management Dashboard</h1>
<p class="text-muted">Import and manage books from Project Gutenberg</p>
</div>
</div>
<!-- Statistics Cards -->
<div class="row mb-4">
<div class="col-md-3">
<div class="card text-center">
<div class="card-body">
<h5 class="card-title text-muted">Total Books</h5>
<h2 class="mb-0">{{ number_format($stats['total_books']) }}</h2>
<small class="text-muted">{{ number_format($stats['gutendex_books']) }} from Gutendex</small>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card text-center">
<div class="card-body">
<h5 class="card-title text-muted">Completed Summaries</h5>
<h2 class="mb-0 text-success">{{ number_format($stats['completed_summaries']) }}</h2>
<small class="text-muted">{{ $stats['completion_rate'] }}% completion rate</small>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card text-center">
<div class="card-body">
<h5 class="card-title text-muted">Pending/Processing</h5>
<h2 class="mb-0 text-warning">{{ number_format($stats['pending_summaries'] + $stats['processing_summaries']) }}</h2>
<small class="text-muted">{{ number_format($stats['pending_summaries']) }} pending, {{ number_format($stats['processing_summaries']) }} processing</small>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card text-center">
<div class="card-body">
<h5 class="card-title text-muted">Failed</h5>
<h2 class="mb-0 text-danger">{{ number_format($stats['failed_summaries']) }}</h2>
<small class="text-muted">Need attention</small>
</div>
</div>
</div>
</div>
<div class="row">
<!-- Import Form -->
<div class="col-md-6">
<div class="card mb-4">
<div class="card-header">
<h5 class="mb-0">Import Books from Gutendex</h5>
</div>
<div class="card-body">
<form id="import-form">
@csrf
<div class="mb-3">
<label for="language" class="form-label">Language</label>
<select class="form-select" id="language" name="language">
<option value="en" selected>English</option>
<option value="fr">French</option>
<option value="de">German</option>
<option value="es">Spanish</option>
<option value="it">Italian</option>
</select>
</div>
<div class="mb-3">
<label for="limit" class="form-label">Number of Books</label>
<input type="number" class="form-control" id="limit" name="limit" value="50" min="1" max="500">
<small class="form-text text-muted">Maximum 500 books per import</small>
</div>
<div class="mb-3">
<label for="page" class="form-label">Starting Page</label>
<input type="number" class="form-control" id="page" name="page" value="1" min="1">
</div>
<div class="mb-3 form-check">
<input type="checkbox" class="form-check-input" id="generate_summaries" name="generate_summaries" checked>
<label class="form-check-label" for="generate_summaries">
Generate summaries automatically
</label>
</div>
<button type="submit" class="btn btn-primary" id="import-btn">
<i class="fas fa-download"></i> Start Import
</button>
</form>
<div id="import-status" class="mt-3" style="display: none;">
<div class="alert alert-info">
<strong>Import started!</strong> Jobs have been dispatched to the queue.
</div>
</div>
</div>
</div>
<!-- Quick Actions -->
<div class="card">
<div class="card-header">
<h5 class="mb-0">Quick Actions</h5>
</div>
<div class="card-body">
<a href="{{ route('admin.gutendex.status') }}" class="btn btn-info btn-sm mb-2 w-100">
<i class="fas fa-chart-line"></i> View Processing Status
</a>
<button class="btn btn-warning btn-sm mb-2 w-100" onclick="retryFailed()">
<i class="fas fa-redo"></i> Retry Failed Summaries
</button>
<a href="/horizon" class="btn btn-secondary btn-sm w-100" target="_blank">
<i class="fas fa-tachometer-alt"></i> Queue Dashboard (if Horizon installed)
</a>
</div>
</div>
</div>
<!-- Recent Activity -->
<div class="col-md-6">
<!-- Recent Imports -->
<div class="card mb-4">
<div class="card-header">
<h5 class="mb-0">Recent Imports</h5>
</div>
<div class="card-body">
@if(count($recentImports) > 0)
<div class="table-responsive">
<table class="table table-sm">
<thead>
<tr>
<th>Title</th>
<th>Author</th>
<th>Category</th>
<th>Imported</th>
</tr>
</thead>
<tbody>
@foreach($recentImports as $book)
<tr>
<td>{{ Str::limit($book['title'], 30) }}</td>
<td>{{ Str::limit($book['author'], 20) }}</td>
<td><span class="badge bg-secondary">{{ $book['category']['name'] ?? 'N/A' }}</span></td>
<td><small>{{ \Carbon\Carbon::parse($book['created_at'])->diffForHumans() }}</small></td>
</tr>
@endforeach
</tbody>
</table>
</div>
@else
<p class="text-muted">No recent imports</p>
@endif
</div>
</div>
<!-- Recent Failures -->
<div class="card">
<div class="card-header">
<h5 class="mb-0">Recent Failures</h5>
</div>
<div class="card-body">
@if(count($recentFailures) > 0)
<div class="table-responsive">
<table class="table table-sm">
<thead>
<tr>
<th>Book</th>
<th>Error</th>
<th>Action</th>
</tr>
</thead>
<tbody>
@foreach($recentFailures as $summary)
<tr>
<td>{{ Str::limit($summary['book']['title'] ?? 'Unknown', 25) }}</td>
<td><small class="text-danger">{{ Str::limit($summary['processing_error'] ?? 'Unknown error', 30) }}</small></td>
<td>
<button class="btn btn-sm btn-warning" onclick="retryBook({{ $summary['book_id'] }})">
<i class="fas fa-redo"></i>
</button>
</td>
</tr>
@endforeach
</tbody>
</table>
</div>
@else
<p class="text-muted">No recent failures</p>
@endif
</div>
</div>
</div>
</div>
</div>
<script>
document.getElementById('import-form').addEventListener('submit', async (e) => {
e.preventDefault();
const btn = document.getElementById('import-btn');
const status = document.getElementById('import-status');
btn.disabled = true;
btn.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Starting...';
const formData = new FormData(e.target);
const data = {
language: formData.get('language'),
limit: parseInt(formData.get('limit')),
page: parseInt(formData.get('page')),
generate_summaries: formData.get('generate_summaries') ? 1 : 0,
_token: formData.get('_token')
};
try {
const response = await fetch('{{ route('admin.gutendex.import') }}', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-CSRF-TOKEN': data._token
},
body: JSON.stringify(data)
});
const result = await response.json();
if (result.success) {
status.style.display = 'block';
status.innerHTML = `<div class="alert alert-success"><strong>Success!</strong> ${result.message}</div>`;
setTimeout(() => location.reload(), 2000);
} else {
status.style.display = 'block';
status.innerHTML = `<div class="alert alert-danger"><strong>Error!</strong> ${result.message || 'Import failed'}</div>`;
}
} catch (error) {
status.style.display = 'block';
status.innerHTML = `<div class="alert alert-danger"><strong>Error!</strong> ${error.message}</div>`;
} finally {
btn.disabled = false;
btn.innerHTML = '<i class="fas fa-download"></i> Start Import';
}
});
async function retryBook(bookId) {
if (!confirm('Retry summary generation for this book?')) return;
try {
const response = await fetch(`/admin/gutendex/retry/${bookId}`, {
method: 'POST',
headers: {
'X-CSRF-TOKEN': '{{ csrf_token() }}'
}
});
const result = await response.json();
if (result.success) {
alert('Retry job dispatched successfully!');
location.reload();
} else {
alert('Failed to dispatch retry job: ' + result.message);
}
} catch (error) {
alert('Error: ' + error.message);
}
}
function retryFailed() {
if (confirm('This will retry all failed summaries. Continue?')) {
alert('Feature coming soon! Use CLI: php artisan summary:generate --failed');
}
}
</script>
@endsection
@@ -0,0 +1,269 @@
@extends('layouts.app')
@section('content')
<div class="container">
<div class="row mb-4">
<div class="col-md-12">
<div class="d-flex justify-content-between align-items-center">
<div>
<h1>Processing Status</h1>
<p class="text-muted">Monitor summary generation progress</p>
</div>
<a href="{{ route('admin.gutendex.dashboard') }}" class="btn btn-secondary">
<i class="fas fa-arrow-left"></i> Back to Dashboard
</a>
</div>
</div>
</div>
<!-- Overall Statistics -->
<div class="row mb-4">
<div class="col-md-3">
<div class="card text-center">
<div class="card-body">
<h6 class="text-muted">Total Summaries</h6>
<h3>{{ number_format($stats['total_summaries']) }}</h3>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card text-center border-success">
<div class="card-body">
<h6 class="text-success">Completed</h6>
<h3 class="text-success">{{ number_format($stats['completed_summaries']) }}</h3>
<small>{{ $stats['completion_rate'] }}%</small>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card text-center border-warning">
<div class="card-body">
<h6 class="text-warning">Pending/Processing</h6>
<h3 class="text-warning">{{ number_format($stats['pending_summaries'] + $stats['processing_summaries']) }}</h3>
<small>{{ number_format($stats['pending_summaries']) }} + {{ number_format($stats['processing_summaries']) }}</small>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card text-center border-danger">
<div class="card-body">
<h6 class="text-danger">Failed</h6>
<h3 class="text-danger">{{ number_format($stats['failed_summaries']) }}</h3>
</div>
</div>
</div>
</div>
<!-- Queue Status -->
<div class="row mb-4">
<div class="col-md-12">
<div class="card">
<div class="card-header">
<h5 class="mb-0">Queue Status</h5>
</div>
<div class="card-body">
<div class="row">
<div class="col-md-6">
<h6>Pending Jobs</h6>
<h2>{{ number_format($queueStats['pending_jobs']) }}</h2>
@if($queueStats['pending_jobs'] > 0)
<small class="text-muted">Jobs waiting to be processed</small>
@else
<small class="text-success">Queue is empty</small>
@endif
</div>
<div class="col-md-6">
<h6>Failed Jobs</h6>
<h2 class="text-danger">{{ number_format($queueStats['failed_jobs']) }}</h2>
@if($queueStats['failed_jobs'] > 0)
<small class="text-danger">Requires attention</small>
@else
<small class="text-success">No failed jobs</small>
@endif
</div>
</div>
<hr>
<button class="btn btn-primary btn-sm" onclick="location.reload()">
<i class="fas fa-sync"></i> Refresh Status
</button>
</div>
</div>
</div>
</div>
<!-- Currently Processing -->
@if($processing->count() > 0)
<div class="row mb-4">
<div class="col-md-12">
<div class="card">
<div class="card-header bg-primary text-white">
<h5 class="mb-0"><i class="fas fa-spinner fa-spin"></i> Currently Processing ({{ $processing->count() }})</h5>
</div>
<div class="card-body">
<div class="table-responsive">
<table class="table">
<thead>
<tr>
<th>Book ID</th>
<th>Title</th>
<th>Author</th>
<th>Status</th>
<th>Started</th>
</tr>
</thead>
<tbody>
@foreach($processing as $summary)
<tr>
<td>{{ $summary->book_id }}</td>
<td>{{ Str::limit($summary->book->title ?? 'Unknown', 40) }}</td>
<td>{{ Str::limit($summary->book->author ?? 'Unknown', 30) }}</td>
<td><span class="badge bg-warning">Processing</span></td>
<td>{{ $summary->updated_at->diffForHumans() }}</td>
</tr>
@endforeach
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
@endif
<!-- Pending Summaries -->
@if($pending->count() > 0)
<div class="row mb-4">
<div class="col-md-12">
<div class="card">
<div class="card-header">
<h5 class="mb-0"><i class="fas fa-clock"></i> Pending Summaries ({{ $pending->count() }})</h5>
</div>
<div class="card-body">
<div class="table-responsive">
<table class="table">
<thead>
<tr>
<th>Book ID</th>
<th>Title</th>
<th>Author</th>
<th>Created</th>
<th>Action</th>
</tr>
</thead>
<tbody>
@foreach($pending as $summary)
<tr>
<td>{{ $summary->book_id }}</td>
<td>{{ Str::limit($summary->book->title ?? 'Unknown', 40) }}</td>
<td>{{ Str::limit($summary->book->author ?? 'Unknown', 30) }}</td>
<td>{{ $summary->created_at->diffForHumans() }}</td>
<td>
<button class="btn btn-sm btn-primary" onclick="retryBook({{ $summary->book_id }})">
<i class="fas fa-play"></i> Process
</button>
</td>
</tr>
@endforeach
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
@endif
<!-- Failed Summaries -->
@if($failed->count() > 0)
<div class="row mb-4">
<div class="col-md-12">
<div class="card border-danger">
<div class="card-header bg-danger text-white">
<h5 class="mb-0"><i class="fas fa-exclamation-triangle"></i> Failed Summaries ({{ $failed->count() }})</h5>
</div>
<div class="card-body">
<div class="table-responsive">
<table class="table">
<thead>
<tr>
<th>Book ID</th>
<th>Title</th>
<th>Error</th>
<th>Failed At</th>
<th>Action</th>
</tr>
</thead>
<tbody>
@foreach($failed as $summary)
<tr>
<td>{{ $summary->book_id }}</td>
<td>{{ Str::limit($summary->book->title ?? 'Unknown', 30) }}</td>
<td><small class="text-danger">{{ Str::limit($summary->processing_error ?? 'Unknown error', 60) }}</small></td>
<td>{{ $summary->updated_at->diffForHumans() }}</td>
<td>
<button class="btn btn-sm btn-warning" onclick="retryBook({{ $summary->book_id }})">
<i class="fas fa-redo"></i> Retry
</button>
</td>
</tr>
@endforeach
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
@endif
<!-- No Activity -->
@if($processing->count() == 0 && $pending->count() == 0 && $failed->count() == 0)
<div class="row">
<div class="col-md-12">
<div class="alert alert-info">
<h5><i class="fas fa-info-circle"></i> No Active Processing</h5>
<p>All summaries are either completed or no jobs are currently queued.</p>
<a href="{{ route('admin.gutendex.dashboard') }}" class="btn btn-primary">Start New Import</a>
</div>
</div>
</div>
@endif
</div>
<script>
async function retryBook(bookId) {
if (!confirm('Retry summary generation for this book?')) return;
try {
const response = await fetch(`/admin/gutendex/retry/${bookId}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-CSRF-TOKEN': '{{ csrf_token() }}'
}
});
const result = await response.json();
if (result.success) {
alert('✓ Retry job dispatched successfully!');
location.reload();
} else {
alert('✗ Failed: ' + result.message);
}
} catch (error) {
alert('✗ Error: ' + error.message);
}
}
// Auto-refresh every 30 seconds
let autoRefresh = setInterval(() => {
location.reload();
}, 30000);
// Stop auto-refresh when user interacts
document.addEventListener('click', () => {
clearInterval(autoRefresh);
});
</script>
@endsection
+9
View File
@@ -11,6 +11,7 @@ use App\Http\Controllers\SummaryReportController;
use App\Http\Controllers\SummaryRequestController; use App\Http\Controllers\SummaryRequestController;
use App\Http\Controllers\SummaryReviewController; use App\Http\Controllers\SummaryReviewController;
use App\Http\Controllers\SitemapController; use App\Http\Controllers\SitemapController;
use App\Http\Controllers\GutendexController;
use App\Models\Book; use App\Models\Book;
Route::get('/', function () { Route::get('/', function () {
@@ -72,3 +73,11 @@ Route::get('/home', [App\Http\Controllers\HomeController::class, 'index'])->name
// Sitemap // Sitemap
Route::get('/sitemap.xml', [SitemapController::class, 'index'])->name('sitemap'); Route::get('/sitemap.xml', [SitemapController::class, 'index'])->name('sitemap');
// Admin - Gutendex Management (requires authentication and admin privileges)
Route::middleware(['auth', 'admin'])->prefix('admin')->name('admin.')->group(function () {
Route::get('/gutendex', [GutendexController::class, 'dashboard'])->name('gutendex.dashboard');
Route::get('/gutendex/status', [GutendexController::class, 'status'])->name('gutendex.status');
Route::post('/gutendex/import', [GutendexController::class, 'import'])->name('gutendex.import');
Route::post('/gutendex/retry/{book}', [GutendexController::class, 'retry'])->name('gutendex.retry');
});