opentextsummaries/app/Jobs/BatchImportFromGutendex.php

<?php

namespace App\Jobs;

use App\Models\Book;
use App\Services\GutendexClient;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;

class BatchImportFromGutendex implements ShouldQueue
{
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

    public array $filters;
    public int $startPage;
    public int $endPage;
    public int $limit;
    public bool $generateSummaries;

    /**
     * The number of times the job may be attempted.
     *
     * @var int
     */
    public $tries = 3;

    /**
     * The number of seconds to wait before retrying the job.
     *
     * @var int
     */
    public $backoff = 120;

    /**
     * The number of seconds the job can run before timing out.
     *
     * @var int
     */
    public $timeout = 3600; // 1 hour for batch operations

    /**
     * Create a new job instance.
     */
    public function __construct(
        array $filters = [],
        int $startPage = 1,
        int $endPage = 1,
        int $limit = 50,
        bool $generateSummaries = true
    ) {
        $this->filters = $filters;
        $this->startPage = $startPage;
        $this->endPage = $endPage;
        $this->limit = $limit;
        $this->generateSummaries = $generateSummaries;
        $this->onQueue('gutendex-import');
    }

    /**
     * Execute the job.
     */
    public function handle(GutendexClient $gutendexClient): void
    {
        Log::info('BatchImportFromGutendex: Starting batch import', [
            'filters' => $this->filters,
            'start_page' => $this->startPage,
            'end_page' => $this->endPage,
            'limit' => $this->limit,
        ]);

        $totalImported = 0;
        $totalSkipped = 0;
        $totalErrors = 0;

        try {
            for ($page = $this->startPage; $page <= $this->endPage; $page++) {
                Log::info('BatchImportFromGutendex: Processing page', [
                    'page' => $page,
                    'of' => $this->endPage,
                ]);

                // Fetch books from Gutendex
                $response = $gutendexClient->searchBooks(null, $page, $this->filters);

                if (empty($response['results'])) {
                    Log::warning('BatchImportFromGutendex: No results on page', [
                        'page' => $page,
                    ]);
                    break;
                }

                $booksOnPage = count($response['results']);
                $imported = 0;
                $skipped = 0;

                foreach ($response['results'] as $bookData) {
                    // Check if we've reached the limit
                    if ($totalImported >= $this->limit) {
                        Log::info('BatchImportFromGutendex: Limit reached', [
                            'limit' => $this->limit,
                            'imported' => $totalImported,
                        ]);
                        break 2; // Break out of both loops
                    }

                    $gutenbergId = $bookData['id'] ?? null;

                    if (!$gutenbergId) {
                        Log::warning('BatchImportFromGutendex: Book missing ID', [
                            'book_data' => $bookData,
                        ]);
                        $totalErrors++;
                        continue;
                    }

                    // Check if book already exists
                    if (Book::where('gutenberg_id', $gutenbergId)->exists()) {
                        Log::debug('BatchImportFromGutendex: Book already exists', [
                            'gutenberg_id' => $gutenbergId,
                        ]);
                        $skipped++;
                        $totalSkipped++;
                        continue;
                    }

                    // Check if book has plain text available
                    $formats = $bookData['formats'] ?? [];
                    $hasPlainText = false;
                    foreach ($formats as $format => $url) {
                        if (str_contains($format, 'text/plain')) {
                            $hasPlainText = true;
                            break;
                        }
                    }

                    if (!$hasPlainText) {
                        Log::debug('BatchImportFromGutendex: No plain text available', [
                            'gutenberg_id' => $gutenbergId,
                        ]);
                        $skipped++;
                        $totalSkipped++;
                        continue;
                    }

                    // Dispatch individual import job
                    ImportBookFromGutendex::dispatch($gutenbergId, $this->generateSummaries)
                        ->delay(now()->addSeconds($totalImported * 2)); // Stagger dispatching

                    $imported++;
                    $totalImported++;

                    // Rate limiting: add delay after dispatching
                    $rateLimit = config('gutendex.rate_limit_delay', 1000);
                    if ($rateLimit > 0) {
                        usleep($rateLimit * 1000);
                    }
                }

                Log::info('BatchImportFromGutendex: Page processed', [
                    'page' => $page,
                    'books_on_page' => $booksOnPage,
                    'imported' => $imported,
                    'skipped' => $skipped,
                ]);

                // Stop if we've reached the limit
                if ($totalImported >= $this->limit) {
                    break;
                }

                // Stop if there are no more results
                if (empty($response['next'])) {
                    Log::info('BatchImportFromGutendex: No more pages available');
                    break;
                }
            }

            Log::info('BatchImportFromGutendex: Batch import completed', [
                'total_imported' => $totalImported,
                'total_skipped' => $totalSkipped,
                'total_errors' => $totalErrors,
                'pages_processed' => $page - $this->startPage + 1,
            ]);
        } catch (\Exception $e) {
            Log::error('BatchImportFromGutendex: Batch import failed', [
                'error' => $e->getMessage(),
                'trace' => $e->getTraceAsString(),
                'imported_before_error' => $totalImported,
            ]);
            throw $e;
        }
    }

    /**
     * Handle a job failure.
     */
    public function failed(\Throwable $exception): void
    {
        Log::error('BatchImportFromGutendex: Job failed permanently', [
            'filters' => $this->filters,
            'start_page' => $this->startPage,
            'end_page' => $this->endPage,
            'error' => $exception->getMessage(),
        ]);
    }
}