backfill command and minor fixes

2026-05-31 16:14:40 +03:00
parent b99ad721da
commit ea3e226398
5 changed files with 221 additions and 90 deletions
@@ -0,0 +1,103 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Console\Commands\Concerns\UpsertsGames;
+use App\Services\IgdbService;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Cache;
+
+class BackfillGamesCommand extends Command
+{
+    use UpsertsGames;
+
+    protected $signature = 'games:backfill
+                            {--start=     : Override the starting offset (ignores saved progress)}
+                            {--total=50000 : Stop after importing this many games total}
+                            {--batch=500  : Games per IGDB request (max 500)}
+                            {--sleep=2    : Seconds to sleep between batches}
+                            {--reset      : Clear saved progress and start from 0}
+                            {--min-ratings=5 : Minimum rating_count to include}';
+
+    protected $description = 'Bulk-import games from IGDB, resuming from last saved offset if interrupted';
+
+    private const PROGRESS_KEY = 'games_backfill_offset';
+
+    public function handle(IgdbService $igdb): int
+    {
+        $batch      = min((int) $this->option('batch'), 500);
+        $total      = (int) $this->option('total');
+        $sleep      = (int) $this->option('sleep');
+        $minRatings = (int) $this->option('min-ratings');
+
+        if ($this->option('reset')) {
+            Cache::forget(self::PROGRESS_KEY);
+            $this->info('Progress reset.');
+        }
+
+        $startOption = $this->option('start');
+        $offset = $startOption !== null
+            ? (int) $startOption
+            : (int) Cache::get(self::PROGRESS_KEY, 0);
+
+        $this->info("Starting backfill — offset: {$offset}, target: {$total} games, batch: {$batch}, sleep: {$sleep}s");
+        $this->newLine();
+
+        $imported  = 0;
+        $skipped   = 0;
+        $batchNum  = 0;
+
+        while ($imported < $total) {
+            $batchNum++;
+            $limit = min($batch, $total - $imported);
+
+            $this->line("Batch #{$batchNum} — offset {$offset}, fetching {$limit}…");
+
+            try {
+                $results = $igdb->getPopularGamesForBackfill($limit, $offset, $minRatings);
+            } catch (\Exception $e) {
+                $this->error('IGDB request failed: ' . $e->getMessage());
+                $this->warn("Progress saved at offset {$offset}. Re-run to resume.");
+                Cache::put(self::PROGRESS_KEY, $offset);
+                return self::FAILURE;
+            }
+
+            if (empty($results)) {
+                $this->info('IGDB returned no more results. Backfill complete.');
+                Cache::forget(self::PROGRESS_KEY);
+                break;
+            }
+
+            $batchImported = 0;
+            foreach ($results as $data) {
+                try {
+                    $this->upsertGame($data, $igdb);
+                    $batchImported++;
+                } catch (\Exception $e) {
+                    $skipped++;
+                    $this->warn("  Skipped igdb_id={$data['id']}: " . $e->getMessage());
+                }
+            }
+
+            $imported += $batchImported;
+            $offset   += count($results);
+
+            Cache::put(self::PROGRESS_KEY, $offset);
+
+            $this->line("  → {$batchImported} upserted | total so far: {$imported} | next offset: {$offset}");
+
+            if ($imported < $total && count($results) === $limit) {
+                sleep($sleep);
+            }
+        }
+
+        $this->newLine();
+        $this->info("Backfill finished. Imported/updated: {$imported}, skipped: {$skipped}.");
+
+        if ($imported >= $total) {
+            $this->comment("Reached --total={$total}. Run again (or with a higher --total) to continue from offset {$offset}.");
+        }
+
+        return self::SUCCESS;
+    }
+}