gutendex integration, phase 6
This commit is contained in:
Doar Mihai
@@ -0,0 +1,267 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Jobs\GenerateSummary;
|
||||
use App\Models\Book;
|
||||
use App\Models\Summary;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class GenerateSummaryCommand extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'summary:generate
|
||||
{book_id? : The book ID to generate summary for}
|
||||
{--all : Generate summaries for all books without summaries}
|
||||
{--pending : Retry pending summaries}
|
||||
{--failed : Retry failed summaries}
|
||||
{--limit=50 : Maximum number of summaries to generate (for batch operations)}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Generate or regenerate book summaries using Ollama';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(): int
|
||||
{
|
||||
$bookId = $this->argument('book_id');
|
||||
$all = $this->option('all');
|
||||
$pending = $this->option('pending');
|
||||
$failed = $this->option('failed');
|
||||
$limit = (int) $this->option('limit');
|
||||
|
||||
// Handle single book
|
||||
if ($bookId) {
|
||||
return $this->generateForBook((int) $bookId);
|
||||
}
|
||||
|
||||
// Handle batch operations
|
||||
if ($all) {
|
||||
return $this->generateForBooksWithoutSummaries($limit);
|
||||
}
|
||||
|
||||
if ($pending) {
|
||||
return $this->generateForPendingSummaries($limit);
|
||||
}
|
||||
|
||||
if ($failed) {
|
||||
return $this->generateForFailedSummaries($limit);
|
||||
}
|
||||
|
||||
// No option provided
|
||||
$this->error('Please specify a book ID or use one of the options: --all, --pending, --failed');
|
||||
$this->newLine();
|
||||
$this->comment('Examples:');
|
||||
$this->comment(' php artisan summary:generate 123');
|
||||
$this->comment(' php artisan summary:generate --all');
|
||||
$this->comment(' php artisan summary:generate --failed --limit=10');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate summary for a specific book.
|
||||
*/
|
||||
protected function generateForBook(int $bookId): int
|
||||
{
|
||||
$book = Book::find($bookId);
|
||||
|
||||
if (!$book) {
|
||||
$this->error("Book with ID {$bookId} not found.");
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$this->info("Generating summary for: {$book->title}");
|
||||
$this->newLine();
|
||||
|
||||
if (!$book->full_text_url) {
|
||||
$this->error('Book does not have a full text URL. Cannot generate summary.');
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
// Check existing summary
|
||||
if ($book->summary) {
|
||||
$this->warn('Book already has a summary:');
|
||||
$this->table(
|
||||
['Field', 'Value'],
|
||||
[
|
||||
['Status', $book->summary->processing_status],
|
||||
['Processed At', $book->summary->processed_at?->format('Y-m-d H:i:s') ?? 'N/A'],
|
||||
['Model Used', $book->summary->model_used ?? 'N/A'],
|
||||
['Has Error', $book->summary->processing_error ? 'Yes' : 'No'],
|
||||
]
|
||||
);
|
||||
$this->newLine();
|
||||
|
||||
if (!$this->confirm('Do you want to regenerate the summary?', false)) {
|
||||
$this->info('Operation cancelled.');
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
$this->info('Dispatching summary generation job...');
|
||||
GenerateSummary::dispatch($bookId);
|
||||
|
||||
$this->newLine();
|
||||
$this->info('✓ Summary generation job dispatched!');
|
||||
$this->comment('Run the queue worker: php artisan queue:work');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate summaries for books without summaries.
|
||||
*/
|
||||
protected function generateForBooksWithoutSummaries(int $limit): int
|
||||
{
|
||||
$this->info('Finding books without summaries...');
|
||||
|
||||
$books = Book::doesntHave('summary')
|
||||
->whereNotNull('full_text_url')
|
||||
->where('source', 'gutendex')
|
||||
->limit($limit)
|
||||
->get();
|
||||
|
||||
if ($books->isEmpty()) {
|
||||
$this->info('No books found without summaries.');
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$this->info("Found {$books->count()} books without summaries.");
|
||||
$this->newLine();
|
||||
|
||||
if (!$this->confirm("Dispatch {$books->count()} summary generation jobs?", true)) {
|
||||
$this->info('Operation cancelled.');
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$progressBar = $this->output->createProgressBar($books->count());
|
||||
$progressBar->start();
|
||||
|
||||
foreach ($books as $index => $book) {
|
||||
GenerateSummary::dispatch($book->id)
|
||||
->delay(now()->addSeconds($index * 10)); // Stagger by 10 seconds
|
||||
$progressBar->advance();
|
||||
}
|
||||
|
||||
$progressBar->finish();
|
||||
$this->newLine(2);
|
||||
|
||||
$this->info("✓ Dispatched {$books->count()} summary generation jobs!");
|
||||
$this->comment('Jobs will be processed with 10-second intervals.');
|
||||
$this->comment('Run the queue worker: php artisan queue:work');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate summaries for pending summaries.
|
||||
*/
|
||||
protected function generateForPendingSummaries(int $limit): int
|
||||
{
|
||||
$this->info('Finding pending summaries...');
|
||||
|
||||
$summaries = Summary::pending()
|
||||
->with('book')
|
||||
->limit($limit)
|
||||
->get();
|
||||
|
||||
if ($summaries->isEmpty()) {
|
||||
$this->info('No pending summaries found.');
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$this->info("Found {$summaries->count()} pending summaries.");
|
||||
$this->newLine();
|
||||
|
||||
if (!$this->confirm("Retry {$summaries->count()} pending summaries?", true)) {
|
||||
$this->info('Operation cancelled.');
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$progressBar = $this->output->createProgressBar($summaries->count());
|
||||
$progressBar->start();
|
||||
|
||||
foreach ($summaries as $index => $summary) {
|
||||
if ($summary->book) {
|
||||
GenerateSummary::dispatch($summary->book->id)
|
||||
->delay(now()->addSeconds($index * 10));
|
||||
}
|
||||
$progressBar->advance();
|
||||
}
|
||||
|
||||
$progressBar->finish();
|
||||
$this->newLine(2);
|
||||
|
||||
$this->info("✓ Dispatched {$summaries->count()} retry jobs!");
|
||||
$this->comment('Run the queue worker: php artisan queue:work');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate summaries for failed summaries.
|
||||
*/
|
||||
protected function generateForFailedSummaries(int $limit): int
|
||||
{
|
||||
$this->info('Finding failed summaries...');
|
||||
|
||||
$summaries = Summary::failed()
|
||||
->with('book')
|
||||
->limit($limit)
|
||||
->get();
|
||||
|
||||
if ($summaries->isEmpty()) {
|
||||
$this->info('No failed summaries found.');
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$this->warn("Found {$summaries->count()} failed summaries.");
|
||||
$this->newLine();
|
||||
|
||||
// Show error details
|
||||
$this->table(
|
||||
['Book ID', 'Title', 'Error'],
|
||||
$summaries->map(fn($s) => [
|
||||
$s->book_id,
|
||||
$s->book?->title ?? 'Unknown',
|
||||
substr($s->processing_error ?? 'Unknown error', 0, 60) . '...',
|
||||
])->toArray()
|
||||
);
|
||||
$this->newLine();
|
||||
|
||||
if (!$this->confirm("Retry {$summaries->count()} failed summaries?", true)) {
|
||||
$this->info('Operation cancelled.');
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$progressBar = $this->output->createProgressBar($summaries->count());
|
||||
$progressBar->start();
|
||||
|
||||
foreach ($summaries as $index => $summary) {
|
||||
if ($summary->book) {
|
||||
GenerateSummary::dispatch($summary->book->id)
|
||||
->delay(now()->addSeconds($index * 15)); // Longer delay for failures
|
||||
}
|
||||
$progressBar->advance();
|
||||
}
|
||||
|
||||
$progressBar->finish();
|
||||
$this->newLine(2);
|
||||
|
||||
$this->info("✓ Dispatched {$summaries->count()} retry jobs!");
|
||||
$this->comment('Jobs will be processed with 15-second intervals.');
|
||||
$this->comment('Run the queue worker: php artisan queue:work');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Jobs\BatchImportFromGutendex;
|
||||
use App\Jobs\ImportBookFromGutendex;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class ImportFromGutendex extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'gutendex:import
|
||||
{--language=en : Language code to filter books}
|
||||
{--limit=100 : Maximum number of books to import}
|
||||
{--page=1 : Starting page number}
|
||||
{--queue : Run the import via queue jobs}
|
||||
{--no-summary : Skip summary generation}
|
||||
{--copyright=false : Copyright filter (false for public domain)}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Import books from Gutendex API with optional filters';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(): int
|
||||
{
|
||||
$language = $this->option('language');
|
||||
$limit = (int) $this->option('limit');
|
||||
$page = (int) $this->option('page');
|
||||
$useQueue = $this->option('queue');
|
||||
$generateSummary = !$this->option('no-summary');
|
||||
$copyright = $this->option('copyright');
|
||||
|
||||
$this->info('Gutendex Import');
|
||||
$this->info('==============');
|
||||
$this->newLine();
|
||||
|
||||
// Build filters
|
||||
$filters = [
|
||||
'languages' => $language,
|
||||
'copyright' => $copyright,
|
||||
];
|
||||
|
||||
$this->table(
|
||||
['Setting', 'Value'],
|
||||
[
|
||||
['Language', $language],
|
||||
['Limit', $limit],
|
||||
['Starting Page', $page],
|
||||
['Mode', $useQueue ? 'Queued' : 'Immediate'],
|
||||
['Generate Summaries', $generateSummary ? 'Yes' : 'No'],
|
||||
['Copyright Filter', $copyright],
|
||||
]
|
||||
);
|
||||
|
||||
if (!$this->confirm('Do you want to proceed with this import?', true)) {
|
||||
$this->warn('Import cancelled.');
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
|
||||
if ($useQueue) {
|
||||
return $this->handleQueuedImport($filters, $page, $limit, $generateSummary);
|
||||
} else {
|
||||
return $this->handleImmediateImport($filters, $page, $limit, $generateSummary);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle queued import.
|
||||
*/
|
||||
protected function handleQueuedImport(array $filters, int $page, int $limit, bool $generateSummary): int
|
||||
{
|
||||
$this->info('Dispatching batch import job...');
|
||||
|
||||
// Calculate end page based on limit (assuming ~32 books per page)
|
||||
$booksPerPage = 32;
|
||||
$endPage = $page + (int) ceil($limit / $booksPerPage) - 1;
|
||||
|
||||
BatchImportFromGutendex::dispatch(
|
||||
$filters,
|
||||
$page,
|
||||
$endPage,
|
||||
$limit,
|
||||
$generateSummary
|
||||
);
|
||||
|
||||
$this->info('✓ Batch import job dispatched successfully!');
|
||||
$this->newLine();
|
||||
$this->comment('The import will be processed in the background.');
|
||||
$this->comment('Run the queue worker: php artisan queue:work');
|
||||
$this->comment('Monitor progress: php artisan summary:status');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle immediate import (synchronous).
|
||||
*/
|
||||
protected function handleImmediateImport(array $filters, int $page, int $limit, bool $generateSummary): int
|
||||
{
|
||||
$this->warn('⚠ Running immediate import. This may take a while...');
|
||||
$this->warn('⚠ For large imports, consider using --queue option.');
|
||||
$this->newLine();
|
||||
|
||||
$gutendexClient = app(\App\Services\GutendexClient::class);
|
||||
|
||||
$totalImported = 0;
|
||||
$totalSkipped = 0;
|
||||
$currentPage = $page;
|
||||
|
||||
$progressBar = $this->output->createProgressBar($limit);
|
||||
$progressBar->start();
|
||||
|
||||
while ($totalImported < $limit) {
|
||||
$response = $gutendexClient->searchBooks(null, $currentPage, $filters);
|
||||
|
||||
if (empty($response['results'])) {
|
||||
$this->newLine();
|
||||
$this->warn('No more results available.');
|
||||
break;
|
||||
}
|
||||
|
||||
foreach ($response['results'] as $bookData) {
|
||||
if ($totalImported >= $limit) {
|
||||
break 2;
|
||||
}
|
||||
|
||||
$gutenbergId = $bookData['id'] ?? null;
|
||||
if (!$gutenbergId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if already exists
|
||||
if (\App\Models\Book::where('gutenberg_id', $gutenbergId)->exists()) {
|
||||
$totalSkipped++;
|
||||
$progressBar->advance();
|
||||
continue;
|
||||
}
|
||||
|
||||
// Dispatch job
|
||||
ImportBookFromGutendex::dispatch($gutenbergId, $generateSummary);
|
||||
$totalImported++;
|
||||
$progressBar->advance();
|
||||
}
|
||||
|
||||
if (empty($response['next'])) {
|
||||
break;
|
||||
}
|
||||
|
||||
$currentPage++;
|
||||
}
|
||||
|
||||
$progressBar->finish();
|
||||
$this->newLine(2);
|
||||
|
||||
$this->info('✓ Import jobs dispatched successfully!');
|
||||
$this->newLine();
|
||||
$this->table(
|
||||
['Metric', 'Count'],
|
||||
[
|
||||
['Jobs Dispatched', $totalImported],
|
||||
['Books Skipped (existing)', $totalSkipped],
|
||||
['Pages Processed', $currentPage - $page + 1],
|
||||
]
|
||||
);
|
||||
|
||||
$this->newLine();
|
||||
$this->comment('Jobs have been dispatched to the queue.');
|
||||
$this->comment('Run the queue worker: php artisan queue:work');
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\Book;
|
||||
use App\Models\Summary;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
class SummaryStatus extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'summary:status
|
||||
{--detailed : Show detailed breakdown}
|
||||
{--recent=10 : Number of recent activities to show}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Display statistics and status of summary generation';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(): int
|
||||
{
|
||||
$this->info('Summary Generation Status');
|
||||
$this->info('========================');
|
||||
$this->newLine();
|
||||
|
||||
$this->displayOverallStatistics();
|
||||
$this->newLine();
|
||||
|
||||
if ($this->option('detailed')) {
|
||||
$this->displayDetailedBreakdown();
|
||||
$this->newLine();
|
||||
}
|
||||
|
||||
$this->displayRecentActivity();
|
||||
$this->newLine();
|
||||
|
||||
$this->displayQueueStatus();
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Display overall statistics.
|
||||
*/
|
||||
protected function displayOverallStatistics(): void
|
||||
{
|
||||
$totalBooks = Book::count();
|
||||
$gutendexBooks = Book::gutendex()->count();
|
||||
$booksWithText = Book::whereNotNull('full_text_url')->count();
|
||||
|
||||
$totalSummaries = Summary::count();
|
||||
$completedSummaries = Summary::completed()->count();
|
||||
$pendingSummaries = Summary::pending()->count();
|
||||
$processingSummaries = Summary::processing()->count();
|
||||
$failedSummaries = Summary::failed()->count();
|
||||
|
||||
$booksWithoutSummaries = Book::gutendex()
|
||||
->whereNotNull('full_text_url')
|
||||
->doesntHave('summary')
|
||||
->count();
|
||||
|
||||
$completionRate = $totalSummaries > 0
|
||||
? round(($completedSummaries / $totalSummaries) * 100, 2)
|
||||
: 0;
|
||||
|
||||
$this->table(
|
||||
['Metric', 'Count', 'Percentage'],
|
||||
[
|
||||
['Total Books', number_format($totalBooks), '-'],
|
||||
['Gutendex Books', number_format($gutendexBooks), round(($gutendexBooks / max($totalBooks, 1)) * 100, 1) . '%'],
|
||||
['Books with Full Text', number_format($booksWithText), '-'],
|
||||
['Books without Summaries', number_format($booksWithoutSummaries), '-'],
|
||||
['---', '---', '---'],
|
||||
['Total Summaries', number_format($totalSummaries), '-'],
|
||||
['✓ Completed', number_format($completedSummaries), $completionRate . '%'],
|
||||
['⏳ Pending', number_format($pendingSummaries), round(($pendingSummaries / max($totalSummaries, 1)) * 100, 1) . '%'],
|
||||
['⚙ Processing', number_format($processingSummaries), round(($processingSummaries / max($totalSummaries, 1)) * 100, 1) . '%'],
|
||||
['✗ Failed', number_format($failedSummaries), round(($failedSummaries / max($totalSummaries, 1)) * 100, 1) . '%'],
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Display detailed breakdown by model and category.
|
||||
*/
|
||||
protected function displayDetailedBreakdown(): void
|
||||
{
|
||||
$this->info('Detailed Breakdown');
|
||||
$this->info('------------------');
|
||||
$this->newLine();
|
||||
|
||||
// By model
|
||||
$byModel = Summary::completed()
|
||||
->select('model_used', DB::raw('COUNT(*) as count'))
|
||||
->groupBy('model_used')
|
||||
->get();
|
||||
|
||||
if ($byModel->isNotEmpty()) {
|
||||
$this->info('Summaries by Model:');
|
||||
$this->table(
|
||||
['Model', 'Count'],
|
||||
$byModel->map(fn($item) => [
|
||||
$item->model_used ?? 'Unknown',
|
||||
number_format($item->count),
|
||||
])->toArray()
|
||||
);
|
||||
$this->newLine();
|
||||
}
|
||||
|
||||
// By language
|
||||
$byLanguage = Book::gutendex()
|
||||
->select('language', DB::raw('COUNT(*) as count'))
|
||||
->whereNotNull('language')
|
||||
->groupBy('language')
|
||||
->orderByDesc('count')
|
||||
->limit(10)
|
||||
->get();
|
||||
|
||||
if ($byLanguage->isNotEmpty()) {
|
||||
$this->info('Books by Language (Top 10):');
|
||||
$this->table(
|
||||
['Language', 'Count'],
|
||||
$byLanguage->map(fn($item) => [
|
||||
$item->language,
|
||||
number_format($item->count),
|
||||
])->toArray()
|
||||
);
|
||||
$this->newLine();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Display recent activity.
|
||||
*/
|
||||
protected function displayRecentActivity(): void
|
||||
{
|
||||
$recentCount = (int) $this->option('recent');
|
||||
|
||||
$this->info("Recent Activity (Last {$recentCount})");
|
||||
$this->info('------------------');
|
||||
$this->newLine();
|
||||
|
||||
// Recent completed summaries
|
||||
$recentCompleted = Summary::completed()
|
||||
->with('book')
|
||||
->orderByDesc('processed_at')
|
||||
->limit($recentCount)
|
||||
->get();
|
||||
|
||||
if ($recentCompleted->isNotEmpty()) {
|
||||
$this->info('✓ Recently Completed:');
|
||||
$this->table(
|
||||
['Book ID', 'Title', 'Processed At', 'Model'],
|
||||
$recentCompleted->map(fn($s) => [
|
||||
$s->book_id,
|
||||
substr($s->book?->title ?? 'Unknown', 0, 40),
|
||||
$s->processed_at?->diffForHumans() ?? 'N/A',
|
||||
$s->model_used ?? 'N/A',
|
||||
])->toArray()
|
||||
);
|
||||
$this->newLine();
|
||||
}
|
||||
|
||||
// Recent failures
|
||||
$recentFailed = Summary::failed()
|
||||
->with('book')
|
||||
->orderByDesc('updated_at')
|
||||
->limit($recentCount)
|
||||
->get();
|
||||
|
||||
if ($recentFailed->isNotEmpty()) {
|
||||
$this->warn('✗ Recent Failures:');
|
||||
$this->table(
|
||||
['Book ID', 'Title', 'Error'],
|
||||
$recentFailed->map(fn($s) => [
|
||||
$s->book_id,
|
||||
substr($s->book?->title ?? 'Unknown', 0, 30),
|
||||
substr($s->processing_error ?? 'Unknown error', 0, 60),
|
||||
])->toArray()
|
||||
);
|
||||
$this->newLine();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Display queue status.
|
||||
*/
|
||||
protected function displayQueueStatus(): void
|
||||
{
|
||||
$this->info('Queue Status');
|
||||
$this->info('------------');
|
||||
$this->newLine();
|
||||
|
||||
// Count pending jobs (if using database queue)
|
||||
try {
|
||||
$pendingJobs = DB::table('jobs')->count();
|
||||
$failedJobs = DB::table('failed_jobs')->count();
|
||||
|
||||
$this->table(
|
||||
['Queue', 'Count'],
|
||||
[
|
||||
['Pending Jobs', number_format($pendingJobs)],
|
||||
['Failed Jobs', number_format($failedJobs)],
|
||||
]
|
||||
);
|
||||
|
||||
if ($pendingJobs > 0) {
|
||||
$this->comment('Run the queue worker: php artisan queue:work');
|
||||
}
|
||||
|
||||
if ($failedJobs > 0) {
|
||||
$this->warn("You have {$failedJobs} failed jobs. Review with: php artisan queue:failed");
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
$this->comment('Queue status unavailable (database driver not configured).');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Jobs\ImportBookFromGutendex;
|
||||
use App\Models\Book;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class SyncBookFromGutendex extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'gutendex:sync-book
|
||||
{gutenberg_id : The Gutenberg book ID to import}
|
||||
{--force : Force re-import even if book exists}
|
||||
{--no-summary : Skip summary generation}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Import or update a specific book from Gutendex by its ID';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(): int
|
||||
{
|
||||
$gutenbergId = (int) $this->argument('gutenberg_id');
|
||||
$force = $this->option('force');
|
||||
$generateSummary = !$this->option('no-summary');
|
||||
|
||||
if ($gutenbergId <= 0) {
|
||||
$this->error('Invalid Gutenberg ID. Must be a positive integer.');
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$this->info("Syncing book from Gutendex (ID: {$gutenbergId})");
|
||||
$this->newLine();
|
||||
|
||||
// Check if book already exists
|
||||
$existingBook = Book::where('gutenberg_id', $gutenbergId)->first();
|
||||
|
||||
if ($existingBook && !$force) {
|
||||
$this->warn('Book already exists in the database!');
|
||||
$this->newLine();
|
||||
$this->table(
|
||||
['Field', 'Value'],
|
||||
[
|
||||
['Database ID', $existingBook->id],
|
||||
['Title', $existingBook->title],
|
||||
['Author', $existingBook->author],
|
||||
['Slug', $existingBook->slug],
|
||||
['Category', $existingBook->category->name ?? 'N/A'],
|
||||
['Has Summary', $existingBook->summary ? 'Yes' : 'No'],
|
||||
['Summary Status', $existingBook->summary->processing_status ?? 'N/A'],
|
||||
]
|
||||
);
|
||||
$this->newLine();
|
||||
|
||||
if (!$this->confirm('Use --force to re-import. Continue anyway?', false)) {
|
||||
$this->info('Operation cancelled.');
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
// Delete existing book to re-import
|
||||
$this->warn('Deleting existing book...');
|
||||
$existingBook->delete();
|
||||
$this->info('✓ Existing book deleted.');
|
||||
$this->newLine();
|
||||
}
|
||||
|
||||
// Fetch book info from Gutendex first to show details
|
||||
$this->info('Fetching book details from Gutendex...');
|
||||
$gutendexClient = app(\App\Services\GutendexClient::class);
|
||||
$gutendexBook = $gutendexClient->getBook($gutenbergId);
|
||||
|
||||
if (!$gutendexBook) {
|
||||
$this->error("Failed to fetch book {$gutenbergId} from Gutendex API.");
|
||||
$this->comment('The book may not exist or the API may be unavailable.');
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
// Display book info
|
||||
$this->info('✓ Book found on Gutendex');
|
||||
$this->newLine();
|
||||
$this->table(
|
||||
['Field', 'Value'],
|
||||
[
|
||||
['Gutenberg ID', $gutendexBook->id],
|
||||
['Title', $gutendexBook->title],
|
||||
['Author(s)', $gutendexBook->getAuthorNamesString()],
|
||||
['Language', $gutendexBook->getPrimaryLanguage()],
|
||||
['Public Domain', $gutendexBook->isPublicDomain() ? 'Yes' : 'No'],
|
||||
['Plain Text Available', $gutendexBook->getPlainTextUrl() ? 'Yes' : 'No'],
|
||||
['Download Count', number_format($gutendexBook->downloadCount)],
|
||||
]
|
||||
);
|
||||
$this->newLine();
|
||||
|
||||
// Validate book
|
||||
if (!$gutendexBook->isPublicDomain()) {
|
||||
$this->error('This book is not in the public domain and cannot be imported.');
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
if (!$gutendexBook->getPlainTextUrl()) {
|
||||
$this->error('This book does not have a plain text version available.');
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
// Dispatch import job
|
||||
$this->info('Dispatching import job...');
|
||||
ImportBookFromGutendex::dispatch($gutenbergId, $generateSummary);
|
||||
|
||||
$this->newLine();
|
||||
$this->info('✓ Import job dispatched successfully!');
|
||||
$this->newLine();
|
||||
$this->comment('The book will be imported in the background.');
|
||||
$this->comment('Run the queue worker: php artisan queue:work');
|
||||
|
||||
if ($generateSummary) {
|
||||
$this->comment('Summary will be generated automatically after import.');
|
||||
} else {
|
||||
$this->comment('Summary generation was skipped (--no-summary flag).');
|
||||
}
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Services\OllamaService;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
class TestOllama extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'ollama:test
|
||||
{--model= : Specific model to test (overrides config)}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Test connection to Ollama API and run a sample summary generation';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(OllamaService $ollamaService): int
|
||||
{
|
||||
$this->info('Ollama API Test');
|
||||
$this->info('===============');
|
||||
$this->newLine();
|
||||
|
||||
// Override model if specified
|
||||
$modelOption = $this->option('model');
|
||||
if ($modelOption) {
|
||||
$ollamaService->setModel($modelOption);
|
||||
$this->comment("Using model: {$modelOption}");
|
||||
$this->newLine();
|
||||
}
|
||||
|
||||
// Display configuration
|
||||
$this->displayConfiguration();
|
||||
$this->newLine();
|
||||
|
||||
// Test connection
|
||||
if (!$this->testConnection($ollamaService)) {
|
||||
return self::FAILURE;
|
||||
}
|
||||
$this->newLine();
|
||||
|
||||
// List available models
|
||||
$this->listAvailableModels($ollamaService);
|
||||
$this->newLine();
|
||||
|
||||
// Run test generation
|
||||
if ($this->confirm('Do you want to run a test summary generation?', true)) {
|
||||
$this->newLine();
|
||||
$this->testSummaryGeneration($ollamaService);
|
||||
}
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Display Ollama configuration.
|
||||
*/
|
||||
protected function displayConfiguration(): void
|
||||
{
|
||||
$this->info('Configuration:');
|
||||
$this->table(
|
||||
['Setting', 'Value'],
|
||||
[
|
||||
['API URL', config('ollama.api_url')],
|
||||
['Model', config('ollama.model')],
|
||||
['Timeout', config('ollama.timeout') . ' seconds'],
|
||||
['Temperature', config('ollama.temperature')],
|
||||
['Max Tokens', config('ollama.max_tokens')],
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test connection to Ollama API.
|
||||
*/
|
||||
protected function testConnection(OllamaService $ollamaService): bool
|
||||
{
|
||||
$this->info('Testing connection to Ollama API...');
|
||||
|
||||
if ($ollamaService->checkConnection()) {
|
||||
$this->info('✓ Connection successful!');
|
||||
return true;
|
||||
} else {
|
||||
$this->error('✗ Connection failed!');
|
||||
$this->newLine();
|
||||
$this->warn('Possible issues:');
|
||||
$this->comment(' 1. Ollama is not running (start with: ollama serve)');
|
||||
$this->comment(' 2. Wrong API URL in config/ollama.php or .env');
|
||||
$this->comment(' 3. Network/firewall issues');
|
||||
$this->newLine();
|
||||
$this->comment('Current API URL: ' . config('ollama.api_url'));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List available models.
|
||||
*/
|
||||
protected function listAvailableModels(OllamaService $ollamaService): void
|
||||
{
|
||||
$this->info('Available Models:');
|
||||
|
||||
$models = $ollamaService->listModels();
|
||||
|
||||
if (empty($models)) {
|
||||
$this->warn('No models found or unable to fetch models.');
|
||||
$this->newLine();
|
||||
$this->comment('Pull a model with: ollama pull llama3');
|
||||
return;
|
||||
}
|
||||
|
||||
$currentModel = $ollamaService->getModel();
|
||||
$rows = [];
|
||||
|
||||
foreach ($models as $model) {
|
||||
$isCurrent = $model === $currentModel ? '✓' : '';
|
||||
$rows[] = [$model, $isCurrent];
|
||||
}
|
||||
|
||||
$this->table(['Model Name', 'Current'], $rows);
|
||||
|
||||
if (!in_array($currentModel, $models)) {
|
||||
$this->warn("⚠ Configured model '{$currentModel}' is not available!");
|
||||
$this->comment("Pull it with: ollama pull {$currentModel}");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test summary generation.
|
||||
*/
|
||||
protected function testSummaryGeneration(OllamaService $ollamaService): void
|
||||
{
|
||||
$this->info('Running test summary generation...');
|
||||
$this->newLine();
|
||||
|
||||
$testResult = $ollamaService->testGeneration();
|
||||
|
||||
if ($testResult['success']) {
|
||||
$this->info('✓ Test generation successful!');
|
||||
$this->newLine();
|
||||
|
||||
$this->table(
|
||||
['Metric', 'Value'],
|
||||
[
|
||||
['Duration', $testResult['duration'] . ' seconds'],
|
||||
['Model Used', $testResult['model']],
|
||||
['Word Count', $testResult['word_count']],
|
||||
]
|
||||
);
|
||||
|
||||
$this->newLine();
|
||||
$this->info('Generated Summary:');
|
||||
$this->line('------------------');
|
||||
$this->line($testResult['summary']);
|
||||
$this->line('------------------');
|
||||
$this->newLine();
|
||||
|
||||
// Quality assessment
|
||||
$wordCount = $testResult['word_count'];
|
||||
$minWords = config('ollama.quality_checks.min_word_count', 100);
|
||||
$maxWords = config('ollama.quality_checks.max_word_count', 1000);
|
||||
|
||||
if ($wordCount < $minWords) {
|
||||
$this->warn("⚠ Summary is shorter than recommended minimum ({$minWords} words)");
|
||||
} elseif ($wordCount > $maxWords) {
|
||||
$this->warn("⚠ Summary is longer than recommended maximum ({$maxWords} words)");
|
||||
} else {
|
||||
$this->info('✓ Summary length is within recommended range');
|
||||
}
|
||||
|
||||
$avgTimePerWord = $testResult['duration'] / max($wordCount, 1);
|
||||
$this->comment("Average: " . round($avgTimePerWord, 2) . " seconds per word");
|
||||
} else {
|
||||
$this->error('✗ Test generation failed!');
|
||||
$this->newLine();
|
||||
$this->warn('This could indicate:');
|
||||
$this->comment(' 1. Model is not properly loaded');
|
||||
$this->comment(' 2. Insufficient system resources');
|
||||
$this->comment(' 3. Timeout issues (increase OLLAMA_TIMEOUT in .env)');
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
<?php
|
||||
|
||||
namespace App\Http\Controllers;
|
||||
|
||||
use App\Jobs\BatchImportFromGutendex;
|
||||
use App\Jobs\GenerateSummary;
|
||||
use App\Models\Book;
|
||||
use App\Models\Summary;
|
||||
use Illuminate\Http\Request;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
class GutendexController extends Controller
|
||||
{
|
||||
/**
|
||||
* Display the Gutendex admin dashboard.
|
||||
*/
|
||||
public function dashboard()
|
||||
{
|
||||
$stats = $this->getStatistics();
|
||||
$recentImports = $this->getRecentImports();
|
||||
$recentFailures = $this->getRecentFailures();
|
||||
|
||||
return view('admin.gutendex.dashboard', compact('stats', 'recentImports', 'recentFailures'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Display processing status.
|
||||
*/
|
||||
public function status()
|
||||
{
|
||||
$stats = $this->getStatistics();
|
||||
$processing = Summary::processing()->with('book')->get();
|
||||
$pending = Summary::pending()->with('book')->limit(20)->get();
|
||||
$failed = Summary::failed()->with('book')->limit(20)->get();
|
||||
|
||||
$queueStats = $this->getQueueStatistics();
|
||||
|
||||
return view('admin.gutendex.status', compact('stats', 'processing', 'pending', 'failed', 'queueStats'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger a new import.
|
||||
*/
|
||||
public function import(Request $request)
|
||||
{
|
||||
$request->validate([
|
||||
'language' => 'required|string|max:10',
|
||||
'limit' => 'required|integer|min:1|max:500',
|
||||
'page' => 'required|integer|min:1',
|
||||
'generate_summaries' => 'boolean',
|
||||
]);
|
||||
|
||||
$language = $request->input('language', 'en');
|
||||
$limit = (int) $request->input('limit', 50);
|
||||
$page = (int) $request->input('page', 1);
|
||||
$generateSummaries = $request->boolean('generate_summaries', true);
|
||||
|
||||
$filters = [
|
||||
'languages' => $language,
|
||||
'copyright' => 'false',
|
||||
];
|
||||
|
||||
// Calculate end page
|
||||
$booksPerPage = 32;
|
||||
$endPage = $page + (int) ceil($limit / $booksPerPage) - 1;
|
||||
|
||||
// Dispatch batch import job
|
||||
BatchImportFromGutendex::dispatch(
|
||||
$filters,
|
||||
$page,
|
||||
$endPage,
|
||||
$limit,
|
||||
$generateSummaries
|
||||
);
|
||||
|
||||
return response()->json([
|
||||
'success' => true,
|
||||
'message' => "Import job dispatched for up to {$limit} books",
|
||||
'details' => [
|
||||
'language' => $language,
|
||||
'limit' => $limit,
|
||||
'start_page' => $page,
|
||||
'end_page' => $endPage,
|
||||
'generate_summaries' => $generateSummaries,
|
||||
],
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry a failed book summary.
|
||||
*/
|
||||
public function retry(Book $book)
|
||||
{
|
||||
if (!$book->summary) {
|
||||
return response()->json([
|
||||
'success' => false,
|
||||
'message' => 'Book does not have a summary record',
|
||||
], 404);
|
||||
}
|
||||
|
||||
if ($book->summary->processing_status === 'completed') {
|
||||
return response()->json([
|
||||
'success' => false,
|
||||
'message' => 'Summary is already completed',
|
||||
], 400);
|
||||
}
|
||||
|
||||
// Dispatch job to regenerate summary
|
||||
GenerateSummary::dispatch($book->id);
|
||||
|
||||
return response()->json([
|
||||
'success' => true,
|
||||
'message' => "Retry job dispatched for: {$book->title}",
|
||||
'book_id' => $book->id,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get overall statistics.
|
||||
*/
|
||||
protected function getStatistics(): array
|
||||
{
|
||||
$totalBooks = Book::count();
|
||||
$gutendexBooks = Book::gutendex()->count();
|
||||
$booksWithText = Book::whereNotNull('full_text_url')->count();
|
||||
$booksWithoutSummaries = Book::gutendex()
|
||||
->whereNotNull('full_text_url')
|
||||
->doesntHave('summary')
|
||||
->count();
|
||||
|
||||
$totalSummaries = Summary::count();
|
||||
$completedSummaries = Summary::completed()->count();
|
||||
$pendingSummaries = Summary::pending()->count();
|
||||
$processingSummaries = Summary::processing()->count();
|
||||
$failedSummaries = Summary::failed()->count();
|
||||
|
||||
$completionRate = $totalSummaries > 0
|
||||
? round(($completedSummaries / $totalSummaries) * 100, 2)
|
||||
: 0;
|
||||
|
||||
return [
|
||||
'total_books' => $totalBooks,
|
||||
'gutendex_books' => $gutendexBooks,
|
||||
'books_with_text' => $booksWithText,
|
||||
'books_without_summaries' => $booksWithoutSummaries,
|
||||
'total_summaries' => $totalSummaries,
|
||||
'completed_summaries' => $completedSummaries,
|
||||
'pending_summaries' => $pendingSummaries,
|
||||
'processing_summaries' => $processingSummaries,
|
||||
'failed_summaries' => $failedSummaries,
|
||||
'completion_rate' => $completionRate,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recent imports.
|
||||
*/
|
||||
protected function getRecentImports(int $limit = 10): array
|
||||
{
|
||||
return Book::gutendex()
|
||||
->with('category')
|
||||
->orderByDesc('created_at')
|
||||
->limit($limit)
|
||||
->get()
|
||||
->toArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recent failures.
|
||||
*/
|
||||
protected function getRecentFailures(int $limit = 10): array
|
||||
{
|
||||
return Summary::failed()
|
||||
->with('book')
|
||||
->orderByDesc('updated_at')
|
||||
->limit($limit)
|
||||
->get()
|
||||
->toArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get queue statistics.
|
||||
*/
|
||||
protected function getQueueStatistics(): array
|
||||
{
|
||||
try {
|
||||
$pendingJobs = DB::table('jobs')->count();
|
||||
$failedJobs = DB::table('failed_jobs')->count();
|
||||
|
||||
return [
|
||||
'pending_jobs' => $pendingJobs,
|
||||
'failed_jobs' => $failedJobs,
|
||||
];
|
||||
} catch (\Exception $e) {
|
||||
return [
|
||||
'pending_jobs' => 0,
|
||||
'failed_jobs' => 0,
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
<?php
|
||||
|
||||
namespace App\Http\Middleware;
|
||||
|
||||
use Closure;
|
||||
use Illuminate\Http\Request;
|
||||
use Symfony\Component\HttpFoundation\Response;
|
||||
|
||||
class IsAdmin
|
||||
{
|
||||
/**
|
||||
* Handle an incoming request.
|
||||
*
|
||||
* @param \Closure(\Illuminate\Http\Request): (\Symfony\Component\HttpFoundation\Response) $next
|
||||
*/
|
||||
public function handle(Request $request, Closure $next): Response
|
||||
{
|
||||
// Check if user is authenticated
|
||||
if (!auth()->check()) {
|
||||
return redirect()->route('login')->with('error', 'You must be logged in to access this page.');
|
||||
}
|
||||
|
||||
// Check if user is admin
|
||||
if (!auth()->user()->is_admin) {
|
||||
abort(403, 'Unauthorized access. Admin privileges required.');
|
||||
}
|
||||
|
||||
return $next($request);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user