Add bulk CDX shard ingestion command as an API-independent fallback
crawlindex:bulk-fetch walks Common Crawl's CDX shard files directly on data.commoncrawl.org (via cc-index.paths.gz), instead of the index.commoncrawl.org query API — usable for bulk backfill/refresh or as a fallback when that API is down. Shares the group/save pipeline with crawlindex:fetch via a new SavesCommonCrawlCaptures trait. Also fixes a real bug this surfaced under load: site_pages.urlkey was a varchar(255), which is too narrow for real-world captured URLs (long query strings on spam/malformed pages overflow it). Widened to text and switched the uniqueness constraint to a sha1 hash column, since MySQL can't fully index a TEXT column.
This commit is contained in:
m105
@@ -0,0 +1,117 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Console\Commands\Concerns\SavesCommonCrawlCaptures;
|
||||
use App\Services\CommonCrawl\CommonCrawlIndexService;
|
||||
use Illuminate\Console\Attributes\Description;
|
||||
use Illuminate\Console\Attributes\Signature;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
#[Signature('crawlindex:bulk-fetch
|
||||
{--collection= : Common Crawl collection id, e.g. CC-MAIN-2026-25. Defaults to the latest collection (resolved via the query API — pass this explicitly to avoid touching the API at all)}
|
||||
{--tld= : Only ingest captures whose host ends in this TLD (e.g. es). Filtered locally after download, never via the query API}
|
||||
{--shard-start=0 : Index (0-based) of the first CDX shard file to process}
|
||||
{--shard-count=1 : Number of consecutive shards to process, starting at --shard-start}
|
||||
{--all-shards : Process every remaining shard, starting at --shard-start (overrides --shard-count)}
|
||||
{--with-metadata : Also fetch each page from the WARC archive and extract title/description/keywords/content (slower, one extra request per saved page — expensive at bulk scale, consider pairing with --tld)}
|
||||
')]
|
||||
#[Description('Bulk-populate/refresh sites and pages by walking Common Crawl\'s CDX shard files directly from data.commoncrawl.org, bypassing the index.commoncrawl.org query API')]
|
||||
class BulkFetchCommonCrawlIndex extends Command
|
||||
{
|
||||
use SavesCommonCrawlCaptures;
|
||||
|
||||
/** Raw shard lines are grouped and saved in batches this large, to bound memory use per shard. */
|
||||
private const CHUNK_SIZE = 5000;
|
||||
|
||||
public function __construct(private readonly CommonCrawlIndexService $service)
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(): int
|
||||
{
|
||||
$tld = $this->option('tld') ? strtolower(ltrim((string) $this->option('tld'), '.')) : null;
|
||||
$shardStart = max(0, (int) $this->option('shard-start'));
|
||||
$allShards = (bool) $this->option('all-shards');
|
||||
$shardCount = $allShards ? null : max(1, (int) $this->option('shard-count'));
|
||||
$withMetadata = (bool) $this->option('with-metadata');
|
||||
|
||||
$collection = $this->option('collection') ?: $this->service->resolveLatestCollection();
|
||||
|
||||
if (! $collection) {
|
||||
$this->error('Could not resolve a Common Crawl collection id. Pass one explicitly with --collection.');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$this->info("Listing CDX shards for collection {$collection}...");
|
||||
$shardPaths = $this->service->listCdxShardPaths($collection);
|
||||
|
||||
if ($shardPaths === null) {
|
||||
$this->error('Could not download or parse cc-index.paths.gz for this collection.');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$totalShards = count($shardPaths);
|
||||
|
||||
if ($totalShards === 0) {
|
||||
$this->error('cc-index.paths.gz for this collection listed no CDX shards.');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
if ($shardStart >= $totalShards) {
|
||||
$this->error("--shard-start ({$shardStart}) is beyond the last shard index ({$totalShards} shards total, 0-".($totalShards - 1).').');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$shardEnd = $allShards ? $totalShards - 1 : min($totalShards - 1, $shardStart + $shardCount - 1);
|
||||
|
||||
$this->info("Collection has {$totalShards} CDX shard(s). Processing shard(s) {$shardStart}-{$shardEnd}".($tld ? " filtered to .{$tld}" : '').'.');
|
||||
|
||||
$totals = ['sitesCreated' => 0, 'sitesUpdated' => 0, 'pagesCreated' => 0, 'pagesUpdated' => 0];
|
||||
$contentCache = [];
|
||||
|
||||
for ($shardIndex = $shardStart; $shardIndex <= $shardEnd; $shardIndex++) {
|
||||
$path = $shardPaths[$shardIndex];
|
||||
$this->info("Fetching shard {$shardIndex}/{$shardEnd}: {$path}");
|
||||
|
||||
$lines = $this->service->fetchShardLines($path);
|
||||
|
||||
if ($lines === null) {
|
||||
$this->warn(" Failed to download or decompress shard {$shardIndex}, skipping.");
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($tld) {
|
||||
$prefix = $tld.',';
|
||||
$lines = array_values(array_filter($lines, fn (string $line) => str_starts_with($line, $prefix)));
|
||||
}
|
||||
|
||||
$this->line(' '.count($lines).' matching raw record(s) in this shard.');
|
||||
|
||||
foreach (array_chunk($lines, self::CHUNK_SIZE) as $chunk) {
|
||||
$jsonLines = array_values(array_filter(array_map(
|
||||
fn (string $line) => $this->service->parseShardLine($line),
|
||||
$chunk
|
||||
)));
|
||||
|
||||
if (! empty($jsonLines)) {
|
||||
$this->saveCaptures($jsonLines, $collection, $withMetadata, $contentCache, $totals);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
$this->info("Done. Processed shard(s) {$shardStart}-{$shardEnd}. Sites: {$totals['sitesCreated']} new, {$totals['sitesUpdated']} updated. Pages: {$totals['pagesCreated']} new, {$totals['pagesUpdated']} updated.");
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands\Concerns;
|
||||
|
||||
/**
|
||||
* Shared by console commands that feed CDX-API-shaped JSON lines (one capture per line)
|
||||
* through CommonCrawlIndexService's group/save pipeline, with progress-bar output.
|
||||
* Requires the using class to be a Command with a $service property.
|
||||
*/
|
||||
trait SavesCommonCrawlCaptures
|
||||
{
|
||||
private function saveCaptures(array $lines, string $collection, bool $withContent, array &$contentCache, array &$totals): void
|
||||
{
|
||||
$grouped = $this->service->groupIntoSites($lines, $collection);
|
||||
$siteGroups = $grouped['sites'];
|
||||
$totalPages = $siteGroups->sum(fn (array $group) => $group['pages']->count());
|
||||
|
||||
$this->line(" {$grouped['captureCount']} capture(s) covering {$siteGroups->count()} site(s) and {$totalPages} unique page(s).");
|
||||
|
||||
if ($withContent) {
|
||||
$this->comment(" Fetching page content from the WARC archive for up to {$totalPages} page(s), this will be slower.");
|
||||
}
|
||||
|
||||
$this->withProgressBar($siteGroups, function (array $group) use ($collection, $withContent, &$contentCache, &$totals) {
|
||||
$result = $this->service->saveSiteGroup($group, $collection, $withContent, $contentCache);
|
||||
|
||||
$result['wasNewSite'] ? $totals['sitesCreated']++ : $totals['sitesUpdated']++;
|
||||
$totals['pagesCreated'] += $result['pagesCreated'];
|
||||
$totals['pagesUpdated'] += $result['pagesUpdated'];
|
||||
});
|
||||
|
||||
$this->newLine();
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Console\Commands\Concerns\SavesCommonCrawlCaptures;
|
||||
use App\Services\CommonCrawl\CommonCrawlIndexService;
|
||||
use Illuminate\Console\Attributes\Description;
|
||||
use Illuminate\Console\Attributes\Signature;
|
||||
@@ -22,6 +23,8 @@ use Illuminate\Console\Command;
|
||||
#[Description('Fetch Common Crawl URL index records and save them as sites with their pages')]
|
||||
class FetchCommonCrawlIndex extends Command
|
||||
{
|
||||
use SavesCommonCrawlCaptures;
|
||||
|
||||
private const VALID_MATCH_TYPES = ['exact', 'prefix', 'host', 'domain'];
|
||||
|
||||
public function __construct(private readonly CommonCrawlIndexService $service)
|
||||
@@ -135,27 +138,4 @@ class FetchCommonCrawlIndex extends Command
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
private function saveCaptures(array $lines, string $collection, bool $withContent, array &$contentCache, array &$totals): void
|
||||
{
|
||||
$grouped = $this->service->groupIntoSites($lines, $collection);
|
||||
$siteGroups = $grouped['sites'];
|
||||
$totalPages = $siteGroups->sum(fn (array $group) => $group['pages']->count());
|
||||
|
||||
$this->line(" {$grouped['captureCount']} capture(s) covering {$siteGroups->count()} site(s) and {$totalPages} unique page(s).");
|
||||
|
||||
if ($withContent) {
|
||||
$this->comment(" --with-metadata is set: fetching page content from the WARC archive for up to {$totalPages} page(s), this will be slower.");
|
||||
}
|
||||
|
||||
$this->withProgressBar($siteGroups, function (array $group) use ($collection, $withContent, &$contentCache, &$totals) {
|
||||
$result = $this->service->saveSiteGroup($group, $collection, $withContent, $contentCache);
|
||||
|
||||
$result['wasNewSite'] ? $totals['sitesCreated']++ : $totals['sitesUpdated']++;
|
||||
$totals['pagesCreated'] += $result['pagesCreated'];
|
||||
$totals['pagesUpdated'] += $result['pagesUpdated'];
|
||||
});
|
||||
|
||||
$this->newLine();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user