progress

app/Console/Commands/FetchIgdbDumps.php

@@ -0,0 +1,122 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Services\IgdbAuthService;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Storage;
+use Illuminate\Support\Facades\Log;
+use ZipArchive;
+
+class FetchIgdbDumps extends Command
+{
+    protected $signature = 'igdb:fetch-dumps
+                            {--type=games : Type of dump to fetch (games, genres, platforms)}
+                            {--limit=1000 : Number of items to fetch per request}
+                            {--offset=0 : Starting offset for pagination}
+                            {--output=storage/dumps : Output directory for the dumps}';
+
+    protected $description = 'Fetch data dumps from IGDB API';
+
+    private const ENDPOINTS = [
+        'games' => 'https://api.igdb.com/v4/games',
+        'genres' => 'https://api.igdb.com/v4/genres',
+        'platforms' => 'https://api.igdb.com/v4/platforms',
+    ];
+
+    private const FIELDS = [
+        'games' => 'id,name,summary,cover.url,slug,genres,platforms,first_release_date,rating,rating_count,category,status,url,version_title,artworks.url,artworks.width,artworks.height,artworks.image_id,screenshots.url,screenshots.width,screenshots.height,screenshots.image_id,videos.name,videos.video_id,videos.url,videos.checksum,bundles.id,collections.id,dlcs.id,expanded_games.id,franchises.id,ports.id,remakes.id,similar_games.id',
+        'genres' => 'id,name,slug',
+        'platforms' => 'id,name,slug,abbreviation',
+    ];
+
+    public function handle(IgdbAuthService $authService): int
+    {
+        $type = $this->option('type');
+        $limit = (int) $this->option('limit');
+        $offset = (int) $this->option('offset');
+        $outputDir = $this->option('output');
+
+        if (!isset(self::ENDPOINTS[$type])) {
+            $this->error("Invalid type: {$type}");
+            return Command::FAILURE;
+        }
+
+        if (!is_dir($outputDir)) {
+            if (!mkdir($outputDir, 0755, true)) {
+                $this->error("Failed to create output directory: {$outputDir}");
+                return Command::FAILURE;
+            }
+        }
+
+        $this->info("Fetching {$type} dumps from IGDB API");
+        $this->info("Output directory: {$outputDir}");
+
+        try {
+            $this->fetchDumps($type, $limit, $offset, $outputDir, $authService);
+        } catch (\Exception $e) {
+            $this->error("Error fetching dumps: {$e->getMessage()}");
+            Log::error('Error fetching IGDB dumps', [
+                'error' => $e->getMessage(),
+                'trace' => $e->getTraceAsString()
+            ]);
+            return Command::FAILURE;
+        }
+
+        $this->info('Dump fetching completed successfully');
+        return Command::SUCCESS;
+    }
+
+    private function fetchDumps(string $type, int $limit, int $offset, string $outputDir, IgdbAuthService $authService): void
+    {
+        $endpoint = self::ENDPOINTS[$type];
+        $fields = self::FIELDS[$type];
+        $totalFetched = 0;
+        $fileCount = 1;
+
+        do {
+            $this->info("Fetching batch {$fileCount} (offset: {$offset})");
+
+            $accessToken = $authService->getAccessToken();
+
+            $response = Http::withHeaders([
+                'Client-ID' => config('services.igdb.credentials.client_id'),
+                'Authorization' => 'Bearer ' . $accessToken,
+            ])->post($endpoint, [
+                'fields' => $fields,
+                'limit' => $limit,
+                'offset' => $offset,
+                'where' => 'rating_count > 0',
+            ]);
+
+            if (!$response->successful()) {
+                throw new \Exception("API request failed: " . $response->body());
+            }
+
+            $data = $response->json();
+            $count = count($data);
+
+            if ($count === 0) {
+                break;
+            }
+
+            $filename = "{$type}_{$fileCount}.json";
+            $filepath = "{$outputDir}/{$filename}";
+
+            if (!file_put_contents($filepath, json_encode($data, JSON_PRETTY_PRINT))) {
+                throw new \Exception("Failed to write to file: {$filepath}");
+            }
+
+            $this->info("Saved {$count} items to {$filename}");
+            $totalFetched += $count;
+            $offset += $limit;
+            $fileCount++;
+
+            // Sleep to avoid rate limiting
+            sleep(1);
+        } while ($count === $limit);
+
+        $this->info("Total items fetched: {$totalFetched}");
+    }
+}

app/Console/Commands/ProcessIgdbDumps.php

@@ -0,0 +1,202 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Services\IgdbService;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Storage;
+use Illuminate\Support\Facades\Log;
+use ZipArchive;
+
+class ProcessIgdbDumps extends Command
+{
+    protected $signature = 'igdb:process-dumps
+                            {--path= : Path to the directory containing IGDB dumps}
+                            {--type=games : Type of dump to process (games, genres, platforms)}';
+
+    protected $description = 'Process IGDB data dumps and import them into the database';
+
+    public function handle(IgdbService $igdbService): int
+    {
+        $path = $this->option('path');
+        $type = $this->option('type');
+
+        if (!$path) {
+//            $this->error('Please provide a path to the IGDB dumps directory');
+//            return Command::FAILURE;
+            $path = storage_path('/dumps');
+        }
+
+        if (!is_dir($path)) {
+            $this->error('The provided path is not a directory');
+            return Command::FAILURE;
+        }
+
+        $this->info("Processing {$type} dumps from {$path}");
+
+        try {
+            switch ($type) {
+                case 'games':
+                    $this->processGamesDump($path, $igdbService);
+                    break;
+                case 'genres':
+                    $this->processGenresDump($path, $igdbService);
+                    break;
+                case 'platforms':
+                    $this->processPlatformsDump($path, $igdbService);
+                    break;
+                default:
+                    $this->error("Unknown dump type: {$type}");
+                    return Command::FAILURE;
+            }
+        } catch (\Exception $e) {
+            $this->error("Error processing dumps: {$e->getMessage()}");
+            Log::error('Error processing IGDB dumps', [
+                'error' => $e->getMessage(),
+                'trace' => $e->getTraceAsString()
+            ]);
+            return Command::FAILURE;
+        }
+
+        $this->info('Dump processing completed successfully');
+        return Command::SUCCESS;
+    }
+
+    private function processGamesDump(string $path, IgdbService $igdbService): void
+    {
+        $files = glob($path . '/games_*.json');
+        $totalFiles = count($files);
+        $processedFiles = 0;
+
+        $this->info("Found {$totalFiles} game dump files");
+
+        foreach ($files as $file) {
+            $this->info("Processing file: " . basename($file));
+
+            $content = file_get_contents($file);
+            $games = json_decode($content, true);
+
+            if (json_last_error() !== JSON_ERROR_NONE) {
+                $this->error("Error decoding JSON in file: " . basename($file));
+                continue;
+            }
+
+            $totalGames = count($games);
+            $this->info("Found {$totalGames} games in file");
+
+            $progressBar = $this->output->createProgressBar($totalGames);
+            $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%');
+
+            foreach ($games as $gameData) {
+                try {
+                    $igdbService->processGameData($gameData);
+                    $progressBar->advance();
+                } catch (\Exception $e) {
+                    Log::error('Failed to process game', [
+                        'game_id' => $gameData['id'] ?? 'unknown',
+                        'error' => $e->getMessage()
+                    ]);
+                    $this->warn("Failed to process game: {$e->getMessage()}");
+                }
+            }
+
+            $progressBar->finish();
+            $this->newLine();
+            $processedFiles++;
+        }
+
+        $this->info("Processed {$processedFiles} out of {$totalFiles} files");
+    }
+
+    private function processGenresDump(string $path, IgdbService $igdbService): void
+    {
+        $files = glob($path . '/genres_*.json');
+        $totalFiles = count($files);
+        $processedFiles = 0;
+
+        $this->info("Found {$totalFiles} genre dump files");
+
+        foreach ($files as $file) {
+            $this->info("Processing file: " . basename($file));
+
+            $content = file_get_contents($file);
+            $genres = json_decode($content, true);
+
+            if (json_last_error() !== JSON_ERROR_NONE) {
+                $this->error("Error decoding JSON in file: " . basename($file));
+                continue;
+            }
+
+            $totalGenres = count($genres);
+            $this->info("Found {$totalGenres} genres in file");
+
+            $progressBar = $this->output->createProgressBar($totalGenres);
+            $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%');
+
+            foreach ($genres as $genreData) {
+                try {
+                    $igdbService->processGenreData($genreData);
+                    $progressBar->advance();
+                } catch (\Exception $e) {
+                    Log::error('Failed to process genre', [
+                        'genre_id' => $genreData['id'] ?? 'unknown',
+                        'error' => $e->getMessage()
+                    ]);
+                    $this->warn("Failed to process genre: {$e->getMessage()}");
+                }
+            }
+
+            $progressBar->finish();
+            $this->newLine();
+            $processedFiles++;
+        }
+
+        $this->info("Processed {$processedFiles} out of {$totalFiles} files");
+    }
+
+    private function processPlatformsDump(string $path, IgdbService $igdbService): void
+    {
+        $files = glob($path . '/platforms_*.json');
+        $totalFiles = count($files);
+        $processedFiles = 0;
+
+        $this->info("Found {$totalFiles} platform dump files");
+
+        foreach ($files as $file) {
+            $this->info("Processing file: " . basename($file));
+
+            $content = file_get_contents($file);
+            $platforms = json_decode($content, true);
+
+            if (json_last_error() !== JSON_ERROR_NONE) {
+                $this->error("Error decoding JSON in file: " . basename($file));
+                continue;
+            }
+
+            $totalPlatforms = count($platforms);
+            $this->info("Found {$totalPlatforms} platforms in file");
+
+            $progressBar = $this->output->createProgressBar($totalPlatforms);
+            $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%');
+
+            foreach ($platforms as $platformData) {
+                try {
+                    $igdbService->processPlatformData($platformData);
+                    $progressBar->advance();
+                } catch (\Exception $e) {
+                    Log::error('Failed to process platform', [
+                        'platform_id' => $platformData['id'] ?? 'unknown',
+                        'error' => $e->getMessage()
+                    ]);
+                    $this->warn("Failed to process platform: {$e->getMessage()}");
+                }
+            }
+
+            $progressBar->finish();
+            $this->newLine();
+            $processedFiles++;
+        }
+
+        $this->info("Processed {$processedFiles} out of {$totalFiles} files");
+    }
+}

app/Services/IgdbAuthService.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Log;
+
+class IgdbAuthService
+{
+    private const TOKEN_CACHE_KEY = 'igdb_access_token';
+    private const TOKEN_EXPIRY = 3600; // 1 hour in seconds
+
+    public function getAccessToken(): string
+    {
+        return Cache::remember(self::TOKEN_CACHE_KEY, self::TOKEN_EXPIRY, function () {
+            return $this->fetchNewAccessToken();
+        });
+    }
+
+    private function fetchNewAccessToken(): string
+    {
+        try {
+            $response = Http::post('https://id.twitch.tv/oauth2/token', [
+                'client_id' => config('services.igdb.client_id'),
+                'client_secret' => config('services.igdb.client_secret'),
+                'grant_type' => 'client_credentials',
+            ]);
+
+            if (!$response->successful()) {
+                throw new \Exception("Failed to get access token: " . $response->body());
+            }
+
+            $data = $response->json();
+            return $data['access_token'];
+        } catch (\Exception $e) {
+            Log::error('Failed to fetch IGDB access token', [
+                'error' => $e->getMessage()
+            ]);
+            throw $e;
+        }
+    }
+} 

app/Services/IgdbService.php

@@ -223,20 +223,23 @@ class IgdbService
         // Process platforms
         $platforms = [];
         if (isset($gameData->platforms) && is_array($gameData->platforms)) {
-            foreach ($gameData->platforms as $platformId) {
+            $platformData = IgdbPlatform::select(['id', 'name', 'slug', 'abbreviation'])
-                if ($platformDetails = $this->getPlatformDetails($platformId)) {
+                ->whereIn('id', $gameData->platforms)
-                    $platform = Platform::updateOrCreate(
+                ->get();
-                        ['igdb_id' => $platformDetails['id']],
+
-                        [
+            foreach ($platformData as $platformDetails) {
-                            'name' => $platformDetails['name'],
+                $platform = Platform::updateOrCreate(
-                            'slug' => $platformDetails['slug'],
+                    ['igdb_id' => $platformDetails->id],
-                            'abbreviation' => $platformDetails['abbreviation'],
+                    [
-                        ]
+                        'name' => $platformDetails->name,
-                    );
+                        'slug' => $platformDetails->slug,
-                    $platforms[] = $platform->id;
+                        'abbreviation' => $platformDetails->abbreviation,
-                }
+                    ]
+                );
+                $platforms[] = $platform->id;
             }
         }
+
         // Create or update the game
         $game = Game::updateOrCreate(
             ['igdb_id' => $gameData->id],
@@ -252,55 +255,101 @@ class IgdbService
         );
 
         // Process artworks
-        if (isset($gameData->artworks) && is_array($gameData->artworks)) {
+        try {
-            foreach ($gameData->artworks as $artwork) {
+            $artworks = \MarcReichel\IGDBLaravel\Models\Artwork::select([
+                'id',
+                'url',
+                'width', 
+                'height',
+                'image_id'
+            ])
+            ->where('game', '=', $gameData->id)
+            ->get();
+
+            foreach ($artworks as $artworkData) {
                 Artwork::updateOrCreate(
-                    ['igdb_id' => $artwork->id],
+                    ['igdb_id' => $artworkData->id],
                     [
                         'game_id' => $game->id,
-                        'url' => $artwork->url,
+                        'url' => $artworkData->url,
-                        'width' => $artwork->width ?? null,
+                        'width' => $artworkData->width ?? null,
-                        'height' => $artwork->height ?? null,
+                        'height' => $artworkData->height ?? null,
-                        'image_id' => $artwork->image_id ?? null,
+                        'image_id' => $artworkData->image_id ?? null,
                     ]
                 );
             }
+        } catch (\Exception $e) {
+            Log::error('Failed to get artworks for game', [
+                'game_id' => $game->id,
+                'error' => $e->getMessage()
+            ]);
         }
+
         // Process platforms relationship
         if (!empty($platforms)) {
             $game->platforms()->sync($platforms);
         }
 
         // Process screenshots
-        if (isset($gameData->screenshots) && is_array($gameData->screenshots)) {
+        try {
-            foreach ($gameData->screenshots as $screenshot) {
+            $screenshots = \MarcReichel\IGDBLaravel\Models\Screenshot::select([
+                'id',
+                'url',
+                'width',
+                'height',
+                'image_id'
+            ])
+            ->where('game', '=', $gameData->id)
+            ->get();
+
+            foreach ($screenshots as $screenshotData) {
                 Screenshot::updateOrCreate(
-                    ['igdb_id' => $screenshot->id],
+                    ['igdb_id' => $screenshotData->id],
                     [
                         'game_id' => $game->id,
-                        'url' => $screenshot->url,
+                        'url' => $screenshotData->url,
-                        'width' => $screenshot->width ?? null,
+                        'width' => $screenshotData->width ?? null,
-                        'height' => $screenshot->height ?? null,
+                        'height' => $screenshotData->height ?? null,
-                        'image_id' => $screenshot->image_id ?? null,
+                        'image_id' => $screenshotData->image_id ?? null,
                     ]
                 );
             }
+        } catch (\Exception $e) {
+            Log::error('Failed to get screenshots for game', [
+                'game_id' => $game->id,
+                'error' => $e->getMessage()
+            ]);
         }
 
         // Process videos
-        if (isset($gameData->videos) && is_array($gameData->videos)) {
+        try {
-            foreach ($gameData->videos as $video) {
+            $videos = \MarcReichel\IGDBLaravel\Models\GameVideo::select([
+                'id',
+                'name', 
+                'video_id',
+                'url',
+                'checksum'
+            ])
+            ->where('game', '=', $gameData->id)
+            ->get();
+
+            foreach ($videos as $videoData) {
                 Video::updateOrCreate(
-                    ['igdb_id' => $video->id],
+                    ['igdb_id' => $videoData->id],
                     [
                         'game_id' => $game->id,
-                        'name' => $video->name,
+                        'name' => $videoData->name,
-                        'video_id' => $video->video_id,
+                        'video_id' => $videoData->video_id,
-                        'url' => $video->url,
+                        'url' => $videoData->url,
-                        'checksum' => $video->checksum ?? null,
+                        'checksum' => $videoData->checksum ?? null,
                     ]
                 );
             }
+        } catch (\Exception $e) {
+            Log::error('Failed to get videos for game', [
+                'game_id' => $game->id,
+                'error' => $e->getMessage()
+            ]);
         }
 
         // Process relationships
@@ -316,24 +365,18 @@ class IgdbService
             throw new \InvalidArgumentException('Game data must be an instance of IgdbGame');
         }
 
+        // Fetch related games data using IGDB API
         $relationships = [
-            'bundles' => 'bundles',
+            'bundles' => IgdbGame::select(['id'])->where('id', 'in', $gameData->bundles ?? [])->get()->pluck('id')->toArray(),
-            'collections' => 'collections',
+            'collections' => IgdbGame::select(['id'])->where('id', 'in', $gameData->collections ?? [])->get()->pluck('id')->toArray(),
-            'dlcs' => 'dlcs',
+            'dlcs' => IgdbGame::select(['id'])->where('id', 'in', $gameData->dlcs ?? [])->get()->pluck('id')->toArray(),
-            'expanded_games' => 'expansions',
+            'expanded_games' => IgdbGame::select(['id'])->where('id', 'in', $gameData->expanded_games ?? [])->get()->pluck('id')->toArray(),
-            'franchises' => 'franchises',
+            'franchises' => IgdbGame::select(['id'])->where('id', 'in', $gameData->franchises ?? [])->get()->pluck('id')->toArray(),
-            'ports' => 'ports',
+            'ports' => IgdbGame::select(['id'])->where('id', 'in', $gameData->ports ?? [])->get()->pluck('id')->toArray(),
-            'remakes' => 'remakes',
+            'remakes' => IgdbGame::select(['id'])->where('id', 'in', $gameData->remakes ?? [])->get()->pluck('id')->toArray(),
-            'similar_games' => 'similarGames',
+            'similar_games' => IgdbGame::select(['id'])->where('id', 'in', $gameData->similar_games ?? [])->get()->pluck('id')->toArray(),
         ];
 
-        $collections = Collection::where('games', $gameData->identifier)->first();
-
-        dd(
-            $gameData,
-            $collections
-        );
-
         foreach ($relationships as $igdbKey => $relationship) {
             $relatedGames = [];
             if (isset($gameData->$igdbKey) && !empty($gameData->$igdbKey)) {
@@ -343,4 +386,43 @@ class IgdbService
             $game->$relationship()->sync($relatedGames);
         }
     }
+
+    public function processGenreData(array $genreData): void
+    {
+        try {
+            Genre::updateOrCreate(
+                ['igdb_id' => $genreData['id']],
+                [
+                    'name' => $genreData['name'],
+                    'slug' => $genreData['slug'],
+                ]
+            );
+        } catch (\Exception $e) {
+            Log::error('Failed to process genre data', [
+                'genre_id' => $genreData['id'],
+                'error' => $e->getMessage()
+            ]);
+            throw $e;
+        }
+    }
+
+    public function processPlatformData(array $platformData): void
+    {
+        try {
+            Platform::updateOrCreate(
+                ['igdb_id' => $platformData['id']],
+                [
+                    'name' => $platformData['name'],
+                    'slug' => $platformData['slug'],
+                    'abbreviation' => $platformData['abbreviation'] ?? null,
+                ]
+            );
+        } catch (\Exception $e) {
+            Log::error('Failed to process platform data', [
+                'platform_id' => $platformData['id'],
+                'error' => $e->getMessage()
+            ]);
+            throw $e;
+        }
+    }
 }