progress
This commit is contained in:
@@ -0,0 +1,122 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Services\IgdbAuthService;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use ZipArchive;
|
||||
|
||||
class FetchIgdbDumps extends Command
|
||||
{
|
||||
protected $signature = 'igdb:fetch-dumps
|
||||
{--type=games : Type of dump to fetch (games, genres, platforms)}
|
||||
{--limit=1000 : Number of items to fetch per request}
|
||||
{--offset=0 : Starting offset for pagination}
|
||||
{--output=storage/dumps : Output directory for the dumps}';
|
||||
|
||||
protected $description = 'Fetch data dumps from IGDB API';
|
||||
|
||||
private const ENDPOINTS = [
|
||||
'games' => 'https://api.igdb.com/v4/games',
|
||||
'genres' => 'https://api.igdb.com/v4/genres',
|
||||
'platforms' => 'https://api.igdb.com/v4/platforms',
|
||||
];
|
||||
|
||||
private const FIELDS = [
|
||||
'games' => 'id,name,summary,cover.url,slug,genres,platforms,first_release_date,rating,rating_count,category,status,url,version_title,artworks.url,artworks.width,artworks.height,artworks.image_id,screenshots.url,screenshots.width,screenshots.height,screenshots.image_id,videos.name,videos.video_id,videos.url,videos.checksum,bundles.id,collections.id,dlcs.id,expanded_games.id,franchises.id,ports.id,remakes.id,similar_games.id',
|
||||
'genres' => 'id,name,slug',
|
||||
'platforms' => 'id,name,slug,abbreviation',
|
||||
];
|
||||
|
||||
public function handle(IgdbAuthService $authService): int
|
||||
{
|
||||
$type = $this->option('type');
|
||||
$limit = (int) $this->option('limit');
|
||||
$offset = (int) $this->option('offset');
|
||||
$outputDir = $this->option('output');
|
||||
|
||||
if (!isset(self::ENDPOINTS[$type])) {
|
||||
$this->error("Invalid type: {$type}");
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
if (!is_dir($outputDir)) {
|
||||
if (!mkdir($outputDir, 0755, true)) {
|
||||
$this->error("Failed to create output directory: {$outputDir}");
|
||||
return Command::FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
$this->info("Fetching {$type} dumps from IGDB API");
|
||||
$this->info("Output directory: {$outputDir}");
|
||||
|
||||
try {
|
||||
$this->fetchDumps($type, $limit, $offset, $outputDir, $authService);
|
||||
} catch (\Exception $e) {
|
||||
$this->error("Error fetching dumps: {$e->getMessage()}");
|
||||
Log::error('Error fetching IGDB dumps', [
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString()
|
||||
]);
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$this->info('Dump fetching completed successfully');
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
private function fetchDumps(string $type, int $limit, int $offset, string $outputDir, IgdbAuthService $authService): void
|
||||
{
|
||||
$endpoint = self::ENDPOINTS[$type];
|
||||
$fields = self::FIELDS[$type];
|
||||
$totalFetched = 0;
|
||||
$fileCount = 1;
|
||||
|
||||
do {
|
||||
$this->info("Fetching batch {$fileCount} (offset: {$offset})");
|
||||
|
||||
$accessToken = $authService->getAccessToken();
|
||||
|
||||
$response = Http::withHeaders([
|
||||
'Client-ID' => config('services.igdb.credentials.client_id'),
|
||||
'Authorization' => 'Bearer ' . $accessToken,
|
||||
])->post($endpoint, [
|
||||
'fields' => $fields,
|
||||
'limit' => $limit,
|
||||
'offset' => $offset,
|
||||
'where' => 'rating_count > 0',
|
||||
]);
|
||||
|
||||
if (!$response->successful()) {
|
||||
throw new \Exception("API request failed: " . $response->body());
|
||||
}
|
||||
|
||||
$data = $response->json();
|
||||
$count = count($data);
|
||||
|
||||
if ($count === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
$filename = "{$type}_{$fileCount}.json";
|
||||
$filepath = "{$outputDir}/{$filename}";
|
||||
|
||||
if (!file_put_contents($filepath, json_encode($data, JSON_PRETTY_PRINT))) {
|
||||
throw new \Exception("Failed to write to file: {$filepath}");
|
||||
}
|
||||
|
||||
$this->info("Saved {$count} items to {$filename}");
|
||||
$totalFetched += $count;
|
||||
$offset += $limit;
|
||||
$fileCount++;
|
||||
|
||||
// Sleep to avoid rate limiting
|
||||
sleep(1);
|
||||
} while ($count === $limit);
|
||||
|
||||
$this->info("Total items fetched: {$totalFetched}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Services\IgdbService;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use ZipArchive;
|
||||
|
||||
class ProcessIgdbDumps extends Command
|
||||
{
|
||||
protected $signature = 'igdb:process-dumps
|
||||
{--path= : Path to the directory containing IGDB dumps}
|
||||
{--type=games : Type of dump to process (games, genres, platforms)}';
|
||||
|
||||
protected $description = 'Process IGDB data dumps and import them into the database';
|
||||
|
||||
public function handle(IgdbService $igdbService): int
|
||||
{
|
||||
$path = $this->option('path');
|
||||
$type = $this->option('type');
|
||||
|
||||
if (!$path) {
|
||||
// $this->error('Please provide a path to the IGDB dumps directory');
|
||||
// return Command::FAILURE;
|
||||
$path = storage_path('/dumps');
|
||||
}
|
||||
|
||||
if (!is_dir($path)) {
|
||||
$this->error('The provided path is not a directory');
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$this->info("Processing {$type} dumps from {$path}");
|
||||
|
||||
try {
|
||||
switch ($type) {
|
||||
case 'games':
|
||||
$this->processGamesDump($path, $igdbService);
|
||||
break;
|
||||
case 'genres':
|
||||
$this->processGenresDump($path, $igdbService);
|
||||
break;
|
||||
case 'platforms':
|
||||
$this->processPlatformsDump($path, $igdbService);
|
||||
break;
|
||||
default:
|
||||
$this->error("Unknown dump type: {$type}");
|
||||
return Command::FAILURE;
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
$this->error("Error processing dumps: {$e->getMessage()}");
|
||||
Log::error('Error processing IGDB dumps', [
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString()
|
||||
]);
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$this->info('Dump processing completed successfully');
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
private function processGamesDump(string $path, IgdbService $igdbService): void
|
||||
{
|
||||
$files = glob($path . '/games_*.json');
|
||||
$totalFiles = count($files);
|
||||
$processedFiles = 0;
|
||||
|
||||
$this->info("Found {$totalFiles} game dump files");
|
||||
|
||||
foreach ($files as $file) {
|
||||
$this->info("Processing file: " . basename($file));
|
||||
|
||||
$content = file_get_contents($file);
|
||||
$games = json_decode($content, true);
|
||||
|
||||
if (json_last_error() !== JSON_ERROR_NONE) {
|
||||
$this->error("Error decoding JSON in file: " . basename($file));
|
||||
continue;
|
||||
}
|
||||
|
||||
$totalGames = count($games);
|
||||
$this->info("Found {$totalGames} games in file");
|
||||
|
||||
$progressBar = $this->output->createProgressBar($totalGames);
|
||||
$progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%');
|
||||
|
||||
foreach ($games as $gameData) {
|
||||
try {
|
||||
$igdbService->processGameData($gameData);
|
||||
$progressBar->advance();
|
||||
} catch (\Exception $e) {
|
||||
Log::error('Failed to process game', [
|
||||
'game_id' => $gameData['id'] ?? 'unknown',
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
$this->warn("Failed to process game: {$e->getMessage()}");
|
||||
}
|
||||
}
|
||||
|
||||
$progressBar->finish();
|
||||
$this->newLine();
|
||||
$processedFiles++;
|
||||
}
|
||||
|
||||
$this->info("Processed {$processedFiles} out of {$totalFiles} files");
|
||||
}
|
||||
|
||||
private function processGenresDump(string $path, IgdbService $igdbService): void
|
||||
{
|
||||
$files = glob($path . '/genres_*.json');
|
||||
$totalFiles = count($files);
|
||||
$processedFiles = 0;
|
||||
|
||||
$this->info("Found {$totalFiles} genre dump files");
|
||||
|
||||
foreach ($files as $file) {
|
||||
$this->info("Processing file: " . basename($file));
|
||||
|
||||
$content = file_get_contents($file);
|
||||
$genres = json_decode($content, true);
|
||||
|
||||
if (json_last_error() !== JSON_ERROR_NONE) {
|
||||
$this->error("Error decoding JSON in file: " . basename($file));
|
||||
continue;
|
||||
}
|
||||
|
||||
$totalGenres = count($genres);
|
||||
$this->info("Found {$totalGenres} genres in file");
|
||||
|
||||
$progressBar = $this->output->createProgressBar($totalGenres);
|
||||
$progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%');
|
||||
|
||||
foreach ($genres as $genreData) {
|
||||
try {
|
||||
$igdbService->processGenreData($genreData);
|
||||
$progressBar->advance();
|
||||
} catch (\Exception $e) {
|
||||
Log::error('Failed to process genre', [
|
||||
'genre_id' => $genreData['id'] ?? 'unknown',
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
$this->warn("Failed to process genre: {$e->getMessage()}");
|
||||
}
|
||||
}
|
||||
|
||||
$progressBar->finish();
|
||||
$this->newLine();
|
||||
$processedFiles++;
|
||||
}
|
||||
|
||||
$this->info("Processed {$processedFiles} out of {$totalFiles} files");
|
||||
}
|
||||
|
||||
private function processPlatformsDump(string $path, IgdbService $igdbService): void
|
||||
{
|
||||
$files = glob($path . '/platforms_*.json');
|
||||
$totalFiles = count($files);
|
||||
$processedFiles = 0;
|
||||
|
||||
$this->info("Found {$totalFiles} platform dump files");
|
||||
|
||||
foreach ($files as $file) {
|
||||
$this->info("Processing file: " . basename($file));
|
||||
|
||||
$content = file_get_contents($file);
|
||||
$platforms = json_decode($content, true);
|
||||
|
||||
if (json_last_error() !== JSON_ERROR_NONE) {
|
||||
$this->error("Error decoding JSON in file: " . basename($file));
|
||||
continue;
|
||||
}
|
||||
|
||||
$totalPlatforms = count($platforms);
|
||||
$this->info("Found {$totalPlatforms} platforms in file");
|
||||
|
||||
$progressBar = $this->output->createProgressBar($totalPlatforms);
|
||||
$progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%');
|
||||
|
||||
foreach ($platforms as $platformData) {
|
||||
try {
|
||||
$igdbService->processPlatformData($platformData);
|
||||
$progressBar->advance();
|
||||
} catch (\Exception $e) {
|
||||
Log::error('Failed to process platform', [
|
||||
'platform_id' => $platformData['id'] ?? 'unknown',
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
$this->warn("Failed to process platform: {$e->getMessage()}");
|
||||
}
|
||||
}
|
||||
|
||||
$progressBar->finish();
|
||||
$this->newLine();
|
||||
$processedFiles++;
|
||||
}
|
||||
|
||||
$this->info("Processed {$processedFiles} out of {$totalFiles} files");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user