From ea3e226398239f697b5b4e0bdba6ab2e4b388b39 Mon Sep 17 00:00:00 2001 From: Doar Mihai Date: Sun, 31 May 2026 16:14:40 +0300 Subject: [PATCH] backfill command and minor fixes --- app/Console/Commands/BackfillGamesCommand.php | 103 ++++++++++++++++++ .../Commands/Concerns/UpsertsGames.php | 93 ++++++++++++++++ .../Commands/GenerateSitemapCommand.php | 4 +- app/Console/Commands/ImportGamesCommand.php | 90 +-------------- app/Services/IgdbService.php | 21 ++++ 5 files changed, 221 insertions(+), 90 deletions(-) create mode 100644 app/Console/Commands/BackfillGamesCommand.php create mode 100644 app/Console/Commands/Concerns/UpsertsGames.php diff --git a/app/Console/Commands/BackfillGamesCommand.php b/app/Console/Commands/BackfillGamesCommand.php new file mode 100644 index 0000000..4389368 --- /dev/null +++ b/app/Console/Commands/BackfillGamesCommand.php @@ -0,0 +1,103 @@ +option('batch'), 500); + $total = (int) $this->option('total'); + $sleep = (int) $this->option('sleep'); + $minRatings = (int) $this->option('min-ratings'); + + if ($this->option('reset')) { + Cache::forget(self::PROGRESS_KEY); + $this->info('Progress reset.'); + } + + $startOption = $this->option('start'); + $offset = $startOption !== null + ? (int) $startOption + : (int) Cache::get(self::PROGRESS_KEY, 0); + + $this->info("Starting backfill — offset: {$offset}, target: {$total} games, batch: {$batch}, sleep: {$sleep}s"); + $this->newLine(); + + $imported = 0; + $skipped = 0; + $batchNum = 0; + + while ($imported < $total) { + $batchNum++; + $limit = min($batch, $total - $imported); + + $this->line("Batch #{$batchNum} — offset {$offset}, fetching {$limit}…"); + + try { + $results = $igdb->getPopularGamesForBackfill($limit, $offset, $minRatings); + } catch (\Exception $e) { + $this->error('IGDB request failed: ' . $e->getMessage()); + $this->warn("Progress saved at offset {$offset}. Re-run to resume."); + Cache::put(self::PROGRESS_KEY, $offset); + return self::FAILURE; + } + + if (empty($results)) { + $this->info('IGDB returned no more results. Backfill complete.'); + Cache::forget(self::PROGRESS_KEY); + break; + } + + $batchImported = 0; + foreach ($results as $data) { + try { + $this->upsertGame($data, $igdb); + $batchImported++; + } catch (\Exception $e) { + $skipped++; + $this->warn(" Skipped igdb_id={$data['id']}: " . $e->getMessage()); + } + } + + $imported += $batchImported; + $offset += count($results); + + Cache::put(self::PROGRESS_KEY, $offset); + + $this->line(" → {$batchImported} upserted | total so far: {$imported} | next offset: {$offset}"); + + if ($imported < $total && count($results) === $limit) { + sleep($sleep); + } + } + + $this->newLine(); + $this->info("Backfill finished. Imported/updated: {$imported}, skipped: {$skipped}."); + + if ($imported >= $total) { + $this->comment("Reached --total={$total}. Run again (or with a higher --total) to continue from offset {$offset}."); + } + + return self::SUCCESS; + } +} \ No newline at end of file diff --git a/app/Console/Commands/Concerns/UpsertsGames.php b/app/Console/Commands/Concerns/UpsertsGames.php new file mode 100644 index 0000000..7890bcf --- /dev/null +++ b/app/Console/Commands/Concerns/UpsertsGames.php @@ -0,0 +1,93 @@ + $igdbId], + [ + 'title' => $title, + 'summary' => $data['summary'] ?? null, + 'cover_url' => $coverUrl, + 'cover_thumb_url' => $coverThumbUrl, + 'release_date' => $releaseDate, + 'release_year' => $releaseYear, + 'status' => 'published', + ] + ); + + if (!empty($data['genres'])) { + $genreIds = []; + foreach ($data['genres'] as $g) { + $genre = Genre::firstOrCreate( + ['igdb_id' => $g['id']], + ['name' => $g['name'], 'slug' => \Str::slug($g['name'])] + ); + $genreIds[] = $genre->id; + } + $game->genres()->sync($genreIds); + } + + if (!empty($data['platforms'])) { + $platformIds = []; + foreach ($data['platforms'] as $p) { + $platform = Platform::firstOrCreate( + ['igdb_id' => $p['id']], + ['name' => $p['name'], 'slug' => \Str::slug($p['name'])] + ); + $platformIds[] = $platform->id; + } + $game->platforms()->sync($platformIds); + } + + if (!empty($data['involved_companies'])) { + $companyPivot = []; + foreach ($data['involved_companies'] as $ic) { + if (empty($ic['company']['name'])) continue; + $company = Company::firstOrCreate( + ['igdb_id' => $ic['company']['id']], + ['name' => $ic['company']['name'], 'slug' => \Str::slug($ic['company']['name'])] + ); + $role = $ic['developer'] ? 'developer' : ($ic['publisher'] ? 'publisher' : 'other'); + $companyPivot[$company->id] = ['role' => $role]; + } + $game->companies()->sync($companyPivot); + } + + if (!empty($data['screenshots'])) { + GameScreenshot::where('game_id', $game->id)->delete(); + foreach ($data['screenshots'] as $i => $shot) { + GameScreenshot::create([ + 'game_id' => $game->id, + 'url' => $igdb::screenshotUrl($shot['image_id'], 'screenshot_big'), + 'thumb_url' => $igdb::screenshotUrl($shot['image_id'], 'screenshot_med'), + 'order' => $i, + ]); + } + } + } +} \ No newline at end of file diff --git a/app/Console/Commands/GenerateSitemapCommand.php b/app/Console/Commands/GenerateSitemapCommand.php index 8fdaf0b..ce24b50 100644 --- a/app/Console/Commands/GenerateSitemapCommand.php +++ b/app/Console/Commands/GenerateSitemapCommand.php @@ -33,11 +33,11 @@ class GenerateSitemapCommand extends Command }); User::where('is_private', false) - ->select('username', 'updated_at') + ->select('slug', 'updated_at') ->chunk(500, function ($users) use ($sitemap) { foreach ($users as $user) { $sitemap->add( - Url::create("/users/{$user->username}") + Url::create("/users/{$user->slug}") ->setLastModificationDate($user->updated_at) ->setPriority(0.5) ->setChangeFrequency(Url::CHANGE_FREQUENCY_WEEKLY) diff --git a/app/Console/Commands/ImportGamesCommand.php b/app/Console/Commands/ImportGamesCommand.php index 2ec30a1..7664f11 100644 --- a/app/Console/Commands/ImportGamesCommand.php +++ b/app/Console/Commands/ImportGamesCommand.php @@ -2,16 +2,13 @@ namespace App\Console\Commands; -use App\Models\Company; -use App\Models\Game; -use App\Models\GameScreenshot; -use App\Models\Genre; -use App\Models\Platform; +use App\Console\Commands\Concerns\UpsertsGames; use App\Services\IgdbService; use Illuminate\Console\Command; class ImportGamesCommand extends Command { + use UpsertsGames; protected $signature = 'games:import {--limit=100} {--offset=0} {--popular} {--new} {--year=} {--platform=}'; protected $description = 'Import games from IGDB into the local database'; @@ -74,87 +71,4 @@ class ImportGamesCommand extends Command return self::SUCCESS; } - private function upsertGame(array $data, IgdbService $igdb): void - { - $igdbId = $data['id']; - $title = $data['name'] ?? 'Unknown'; - $releaseDate = isset($data['first_release_date']) - ? date('Y-m-d', $data['first_release_date']) - : null; - $releaseYear = $releaseDate ? substr($releaseDate, 0, 4) : null; - - $coverUrl = null; - $coverThumbUrl = null; - if (!empty($data['cover']['image_id'])) { - $coverUrl = $igdb::coverUrl($data['cover']['image_id'], 'cover_big'); - $coverThumbUrl = $igdb::coverUrl($data['cover']['image_id'], 'cover_small'); - } - - $game = Game::updateOrCreate( - ['igdb_id' => $igdbId], - [ - 'title' => $title, - 'summary' => $data['summary'] ?? null, - 'cover_url' => $coverUrl, - 'cover_thumb_url'=> $coverThumbUrl, - 'release_date' => $releaseDate, - 'release_year' => $releaseYear, - 'status' => 'published', - ] - ); - - // Genres - if (!empty($data['genres'])) { - $genreIds = []; - foreach ($data['genres'] as $g) { - $genre = Genre::firstOrCreate( - ['igdb_id' => $g['id']], - ['name' => $g['name'], 'slug' => \Str::slug($g['name'])] - ); - $genreIds[] = $genre->id; - } - $game->genres()->sync($genreIds); - } - - // Platforms - if (!empty($data['platforms'])) { - $platformIds = []; - foreach ($data['platforms'] as $p) { - $platform = Platform::firstOrCreate( - ['igdb_id' => $p['id']], - ['name' => $p['name'], 'slug' => \Str::slug($p['name'])] - ); - $platformIds[] = $platform->id; - } - $game->platforms()->sync($platformIds); - } - - // Companies (involved_companies) - if (!empty($data['involved_companies'])) { - $companyPivot = []; - foreach ($data['involved_companies'] as $ic) { - if (empty($ic['company']['name'])) continue; - $company = Company::firstOrCreate( - ['igdb_id' => $ic['company']['id']], - ['name' => $ic['company']['name'], 'slug' => \Str::slug($ic['company']['name'])] - ); - $role = $ic['developer'] ? 'developer' : ($ic['publisher'] ? 'publisher' : 'other'); - $companyPivot[$company->id] = ['role' => $role]; - } - $game->companies()->sync($companyPivot); - } - - // Screenshots - if (!empty($data['screenshots'])) { - GameScreenshot::where('game_id', $game->id)->delete(); - foreach ($data['screenshots'] as $i => $shot) { - GameScreenshot::create([ - 'game_id' => $game->id, - 'url' => $igdb::screenshotUrl($shot['image_id'], 'screenshot_big'), - 'thumb_url' => $igdb::screenshotUrl($shot['image_id'], 'screenshot_med'), - 'order' => $i, - ]); - } - } - } } diff --git a/app/Services/IgdbService.php b/app/Services/IgdbService.php index 5b534d1..027cfd3 100644 --- a/app/Services/IgdbService.php +++ b/app/Services/IgdbService.php @@ -116,6 +116,27 @@ class IgdbService "); } + /** + * Get games for backfill — broader filter than getPopularGames. + */ + public function getPopularGamesForBackfill(int $limit = 500, int $offset = 0, int $minRatings = 5): array + { + return $this->query('games', " + fields id,name,slug,summary,cover.image_id,first_release_date, + genres.id,genres.name,genres.slug, + platforms.id,platforms.name,platforms.slug,platforms.abbreviation, + involved_companies.company.id,involved_companies.company.name, + involved_companies.company.slug,involved_companies.developer, + involved_companies.publisher, + screenshots.image_id, + rating,rating_count; + where rating != null & rating_count > {$minRatings} & cover != null; + sort rating_count desc; + limit {$limit}; + offset {$offset}; + "); + } + /** * Get recently released games. */