diff --git a/app/Console/Commands/FetchIgdbDumps.php b/app/Console/Commands/FetchIgdbDumps.php new file mode 100644 index 0000000..bdc58d0 --- /dev/null +++ b/app/Console/Commands/FetchIgdbDumps.php @@ -0,0 +1,122 @@ + 'https://api.igdb.com/v4/games', + 'genres' => 'https://api.igdb.com/v4/genres', + 'platforms' => 'https://api.igdb.com/v4/platforms', + ]; + + private const FIELDS = [ + 'games' => 'id,name,summary,cover.url,slug,genres,platforms,first_release_date,rating,rating_count,category,status,url,version_title,artworks.url,artworks.width,artworks.height,artworks.image_id,screenshots.url,screenshots.width,screenshots.height,screenshots.image_id,videos.name,videos.video_id,videos.url,videos.checksum,bundles.id,collections.id,dlcs.id,expanded_games.id,franchises.id,ports.id,remakes.id,similar_games.id', + 'genres' => 'id,name,slug', + 'platforms' => 'id,name,slug,abbreviation', + ]; + + public function handle(IgdbAuthService $authService): int + { + $type = $this->option('type'); + $limit = (int) $this->option('limit'); + $offset = (int) $this->option('offset'); + $outputDir = $this->option('output'); + + if (!isset(self::ENDPOINTS[$type])) { + $this->error("Invalid type: {$type}"); + return Command::FAILURE; + } + + if (!is_dir($outputDir)) { + if (!mkdir($outputDir, 0755, true)) { + $this->error("Failed to create output directory: {$outputDir}"); + return Command::FAILURE; + } + } + + $this->info("Fetching {$type} dumps from IGDB API"); + $this->info("Output directory: {$outputDir}"); + + try { + $this->fetchDumps($type, $limit, $offset, $outputDir, $authService); + } catch (\Exception $e) { + $this->error("Error fetching dumps: {$e->getMessage()}"); + Log::error('Error fetching IGDB dumps', [ + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString() + ]); + return Command::FAILURE; + } + + $this->info('Dump fetching completed successfully'); + return Command::SUCCESS; + } + + private function fetchDumps(string $type, int $limit, int $offset, string $outputDir, IgdbAuthService $authService): void + { + $endpoint = self::ENDPOINTS[$type]; + $fields = self::FIELDS[$type]; + $totalFetched = 0; + $fileCount = 1; + + do { + $this->info("Fetching batch {$fileCount} (offset: {$offset})"); + + $accessToken = $authService->getAccessToken(); + + $response = Http::withHeaders([ + 'Client-ID' => config('services.igdb.credentials.client_id'), + 'Authorization' => 'Bearer ' . $accessToken, + ])->post($endpoint, [ + 'fields' => $fields, + 'limit' => $limit, + 'offset' => $offset, + 'where' => 'rating_count > 0', + ]); + + if (!$response->successful()) { + throw new \Exception("API request failed: " . $response->body()); + } + + $data = $response->json(); + $count = count($data); + + if ($count === 0) { + break; + } + + $filename = "{$type}_{$fileCount}.json"; + $filepath = "{$outputDir}/{$filename}"; + + if (!file_put_contents($filepath, json_encode($data, JSON_PRETTY_PRINT))) { + throw new \Exception("Failed to write to file: {$filepath}"); + } + + $this->info("Saved {$count} items to {$filename}"); + $totalFetched += $count; + $offset += $limit; + $fileCount++; + + // Sleep to avoid rate limiting + sleep(1); + } while ($count === $limit); + + $this->info("Total items fetched: {$totalFetched}"); + } +} diff --git a/app/Console/Commands/ProcessIgdbDumps.php b/app/Console/Commands/ProcessIgdbDumps.php new file mode 100644 index 0000000..22abd9d --- /dev/null +++ b/app/Console/Commands/ProcessIgdbDumps.php @@ -0,0 +1,202 @@ +option('path'); + $type = $this->option('type'); + + if (!$path) { +// $this->error('Please provide a path to the IGDB dumps directory'); +// return Command::FAILURE; + $path = storage_path('/dumps'); + } + + if (!is_dir($path)) { + $this->error('The provided path is not a directory'); + return Command::FAILURE; + } + + $this->info("Processing {$type} dumps from {$path}"); + + try { + switch ($type) { + case 'games': + $this->processGamesDump($path, $igdbService); + break; + case 'genres': + $this->processGenresDump($path, $igdbService); + break; + case 'platforms': + $this->processPlatformsDump($path, $igdbService); + break; + default: + $this->error("Unknown dump type: {$type}"); + return Command::FAILURE; + } + } catch (\Exception $e) { + $this->error("Error processing dumps: {$e->getMessage()}"); + Log::error('Error processing IGDB dumps', [ + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString() + ]); + return Command::FAILURE; + } + + $this->info('Dump processing completed successfully'); + return Command::SUCCESS; + } + + private function processGamesDump(string $path, IgdbService $igdbService): void + { + $files = glob($path . '/games_*.json'); + $totalFiles = count($files); + $processedFiles = 0; + + $this->info("Found {$totalFiles} game dump files"); + + foreach ($files as $file) { + $this->info("Processing file: " . basename($file)); + + $content = file_get_contents($file); + $games = json_decode($content, true); + + if (json_last_error() !== JSON_ERROR_NONE) { + $this->error("Error decoding JSON in file: " . basename($file)); + continue; + } + + $totalGames = count($games); + $this->info("Found {$totalGames} games in file"); + + $progressBar = $this->output->createProgressBar($totalGames); + $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%'); + + foreach ($games as $gameData) { + try { + $igdbService->processGameData($gameData); + $progressBar->advance(); + } catch (\Exception $e) { + Log::error('Failed to process game', [ + 'game_id' => $gameData['id'] ?? 'unknown', + 'error' => $e->getMessage() + ]); + $this->warn("Failed to process game: {$e->getMessage()}"); + } + } + + $progressBar->finish(); + $this->newLine(); + $processedFiles++; + } + + $this->info("Processed {$processedFiles} out of {$totalFiles} files"); + } + + private function processGenresDump(string $path, IgdbService $igdbService): void + { + $files = glob($path . '/genres_*.json'); + $totalFiles = count($files); + $processedFiles = 0; + + $this->info("Found {$totalFiles} genre dump files"); + + foreach ($files as $file) { + $this->info("Processing file: " . basename($file)); + + $content = file_get_contents($file); + $genres = json_decode($content, true); + + if (json_last_error() !== JSON_ERROR_NONE) { + $this->error("Error decoding JSON in file: " . basename($file)); + continue; + } + + $totalGenres = count($genres); + $this->info("Found {$totalGenres} genres in file"); + + $progressBar = $this->output->createProgressBar($totalGenres); + $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%'); + + foreach ($genres as $genreData) { + try { + $igdbService->processGenreData($genreData); + $progressBar->advance(); + } catch (\Exception $e) { + Log::error('Failed to process genre', [ + 'genre_id' => $genreData['id'] ?? 'unknown', + 'error' => $e->getMessage() + ]); + $this->warn("Failed to process genre: {$e->getMessage()}"); + } + } + + $progressBar->finish(); + $this->newLine(); + $processedFiles++; + } + + $this->info("Processed {$processedFiles} out of {$totalFiles} files"); + } + + private function processPlatformsDump(string $path, IgdbService $igdbService): void + { + $files = glob($path . '/platforms_*.json'); + $totalFiles = count($files); + $processedFiles = 0; + + $this->info("Found {$totalFiles} platform dump files"); + + foreach ($files as $file) { + $this->info("Processing file: " . basename($file)); + + $content = file_get_contents($file); + $platforms = json_decode($content, true); + + if (json_last_error() !== JSON_ERROR_NONE) { + $this->error("Error decoding JSON in file: " . basename($file)); + continue; + } + + $totalPlatforms = count($platforms); + $this->info("Found {$totalPlatforms} platforms in file"); + + $progressBar = $this->output->createProgressBar($totalPlatforms); + $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%'); + + foreach ($platforms as $platformData) { + try { + $igdbService->processPlatformData($platformData); + $progressBar->advance(); + } catch (\Exception $e) { + Log::error('Failed to process platform', [ + 'platform_id' => $platformData['id'] ?? 'unknown', + 'error' => $e->getMessage() + ]); + $this->warn("Failed to process platform: {$e->getMessage()}"); + } + } + + $progressBar->finish(); + $this->newLine(); + $processedFiles++; + } + + $this->info("Processed {$processedFiles} out of {$totalFiles} files"); + } +} diff --git a/app/Services/IgdbAuthService.php b/app/Services/IgdbAuthService.php new file mode 100644 index 0000000..c2d9bec --- /dev/null +++ b/app/Services/IgdbAuthService.php @@ -0,0 +1,43 @@ +fetchNewAccessToken(); + }); + } + + private function fetchNewAccessToken(): string + { + try { + $response = Http::post('https://id.twitch.tv/oauth2/token', [ + 'client_id' => config('services.igdb.client_id'), + 'client_secret' => config('services.igdb.client_secret'), + 'grant_type' => 'client_credentials', + ]); + + if (!$response->successful()) { + throw new \Exception("Failed to get access token: " . $response->body()); + } + + $data = $response->json(); + return $data['access_token']; + } catch (\Exception $e) { + Log::error('Failed to fetch IGDB access token', [ + 'error' => $e->getMessage() + ]); + throw $e; + } + } +} \ No newline at end of file diff --git a/app/Services/IgdbService.php b/app/Services/IgdbService.php index 160a8d2..f7d747b 100644 --- a/app/Services/IgdbService.php +++ b/app/Services/IgdbService.php @@ -223,20 +223,23 @@ class IgdbService // Process platforms $platforms = []; if (isset($gameData->platforms) && is_array($gameData->platforms)) { - foreach ($gameData->platforms as $platformId) { - if ($platformDetails = $this->getPlatformDetails($platformId)) { - $platform = Platform::updateOrCreate( - ['igdb_id' => $platformDetails['id']], - [ - 'name' => $platformDetails['name'], - 'slug' => $platformDetails['slug'], - 'abbreviation' => $platformDetails['abbreviation'], - ] - ); - $platforms[] = $platform->id; - } + $platformData = IgdbPlatform::select(['id', 'name', 'slug', 'abbreviation']) + ->whereIn('id', $gameData->platforms) + ->get(); + + foreach ($platformData as $platformDetails) { + $platform = Platform::updateOrCreate( + ['igdb_id' => $platformDetails->id], + [ + 'name' => $platformDetails->name, + 'slug' => $platformDetails->slug, + 'abbreviation' => $platformDetails->abbreviation, + ] + ); + $platforms[] = $platform->id; } } + // Create or update the game $game = Game::updateOrCreate( ['igdb_id' => $gameData->id], @@ -252,55 +255,101 @@ class IgdbService ); // Process artworks - if (isset($gameData->artworks) && is_array($gameData->artworks)) { - foreach ($gameData->artworks as $artwork) { + try { + $artworks = \MarcReichel\IGDBLaravel\Models\Artwork::select([ + 'id', + 'url', + 'width', + 'height', + 'image_id' + ]) + ->where('game', '=', $gameData->id) + ->get(); + + foreach ($artworks as $artworkData) { Artwork::updateOrCreate( - ['igdb_id' => $artwork->id], + ['igdb_id' => $artworkData->id], [ 'game_id' => $game->id, - 'url' => $artwork->url, - 'width' => $artwork->width ?? null, - 'height' => $artwork->height ?? null, - 'image_id' => $artwork->image_id ?? null, + 'url' => $artworkData->url, + 'width' => $artworkData->width ?? null, + 'height' => $artworkData->height ?? null, + 'image_id' => $artworkData->image_id ?? null, ] ); } + } catch (\Exception $e) { + Log::error('Failed to get artworks for game', [ + 'game_id' => $game->id, + 'error' => $e->getMessage() + ]); } + // Process platforms relationship if (!empty($platforms)) { $game->platforms()->sync($platforms); } // Process screenshots - if (isset($gameData->screenshots) && is_array($gameData->screenshots)) { - foreach ($gameData->screenshots as $screenshot) { + try { + $screenshots = \MarcReichel\IGDBLaravel\Models\Screenshot::select([ + 'id', + 'url', + 'width', + 'height', + 'image_id' + ]) + ->where('game', '=', $gameData->id) + ->get(); + + foreach ($screenshots as $screenshotData) { Screenshot::updateOrCreate( - ['igdb_id' => $screenshot->id], + ['igdb_id' => $screenshotData->id], [ 'game_id' => $game->id, - 'url' => $screenshot->url, - 'width' => $screenshot->width ?? null, - 'height' => $screenshot->height ?? null, - 'image_id' => $screenshot->image_id ?? null, + 'url' => $screenshotData->url, + 'width' => $screenshotData->width ?? null, + 'height' => $screenshotData->height ?? null, + 'image_id' => $screenshotData->image_id ?? null, ] ); } + } catch (\Exception $e) { + Log::error('Failed to get screenshots for game', [ + 'game_id' => $game->id, + 'error' => $e->getMessage() + ]); } // Process videos - if (isset($gameData->videos) && is_array($gameData->videos)) { - foreach ($gameData->videos as $video) { + try { + $videos = \MarcReichel\IGDBLaravel\Models\GameVideo::select([ + 'id', + 'name', + 'video_id', + 'url', + 'checksum' + ]) + ->where('game', '=', $gameData->id) + ->get(); + + foreach ($videos as $videoData) { Video::updateOrCreate( - ['igdb_id' => $video->id], + ['igdb_id' => $videoData->id], [ 'game_id' => $game->id, - 'name' => $video->name, - 'video_id' => $video->video_id, - 'url' => $video->url, - 'checksum' => $video->checksum ?? null, + 'name' => $videoData->name, + 'video_id' => $videoData->video_id, + 'url' => $videoData->url, + 'checksum' => $videoData->checksum ?? null, ] ); } + } catch (\Exception $e) { + Log::error('Failed to get videos for game', [ + 'game_id' => $game->id, + 'error' => $e->getMessage() + ]); } // Process relationships @@ -316,24 +365,18 @@ class IgdbService throw new \InvalidArgumentException('Game data must be an instance of IgdbGame'); } + // Fetch related games data using IGDB API $relationships = [ - 'bundles' => 'bundles', - 'collections' => 'collections', - 'dlcs' => 'dlcs', - 'expanded_games' => 'expansions', - 'franchises' => 'franchises', - 'ports' => 'ports', - 'remakes' => 'remakes', - 'similar_games' => 'similarGames', + 'bundles' => IgdbGame::select(['id'])->where('id', 'in', $gameData->bundles ?? [])->get()->pluck('id')->toArray(), + 'collections' => IgdbGame::select(['id'])->where('id', 'in', $gameData->collections ?? [])->get()->pluck('id')->toArray(), + 'dlcs' => IgdbGame::select(['id'])->where('id', 'in', $gameData->dlcs ?? [])->get()->pluck('id')->toArray(), + 'expanded_games' => IgdbGame::select(['id'])->where('id', 'in', $gameData->expanded_games ?? [])->get()->pluck('id')->toArray(), + 'franchises' => IgdbGame::select(['id'])->where('id', 'in', $gameData->franchises ?? [])->get()->pluck('id')->toArray(), + 'ports' => IgdbGame::select(['id'])->where('id', 'in', $gameData->ports ?? [])->get()->pluck('id')->toArray(), + 'remakes' => IgdbGame::select(['id'])->where('id', 'in', $gameData->remakes ?? [])->get()->pluck('id')->toArray(), + 'similar_games' => IgdbGame::select(['id'])->where('id', 'in', $gameData->similar_games ?? [])->get()->pluck('id')->toArray(), ]; - $collections = Collection::where('games', $gameData->identifier)->first(); - - dd( - $gameData, - $collections - ); - foreach ($relationships as $igdbKey => $relationship) { $relatedGames = []; if (isset($gameData->$igdbKey) && !empty($gameData->$igdbKey)) { @@ -343,4 +386,43 @@ class IgdbService $game->$relationship()->sync($relatedGames); } } + + public function processGenreData(array $genreData): void + { + try { + Genre::updateOrCreate( + ['igdb_id' => $genreData['id']], + [ + 'name' => $genreData['name'], + 'slug' => $genreData['slug'], + ] + ); + } catch (\Exception $e) { + Log::error('Failed to process genre data', [ + 'genre_id' => $genreData['id'], + 'error' => $e->getMessage() + ]); + throw $e; + } + } + + public function processPlatformData(array $platformData): void + { + try { + Platform::updateOrCreate( + ['igdb_id' => $platformData['id']], + [ + 'name' => $platformData['name'], + 'slug' => $platformData['slug'], + 'abbreviation' => $platformData['abbreviation'] ?? null, + ] + ); + } catch (\Exception $e) { + Log::error('Failed to process platform data', [ + 'platform_id' => $platformData['id'], + 'error' => $e->getMessage() + ]); + throw $e; + } + } }