root/huginn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add bulk CDX shard ingestion command as an API-independent fallback

Browse Source 
crawlindex:bulk-fetch walks Common Crawl's CDX shard files directly on
data.commoncrawl.org (via cc-index.paths.gz), instead of the
index.commoncrawl.org query API — usable for bulk backfill/refresh or
as a fallback when that API is down. Shares the group/save pipeline
with crawlindex:fetch via a new SavesCommonCrawlCaptures trait.

Also fixes a real bug this surfaced under load: site_pages.urlkey was
a varchar(255), which is too narrow for real-world captured URLs
(long query strings on spam/malformed pages overflow it). Widened to
text and switched the uniqueness constraint to a sha1 hash column,
since MySQL can't fully index a TEXT column.
This commit is contained in:
m105
2026-07-20 00:26:27 +03:00
parent 1c9fc5d423
commit 4dc9b84eb8
6 changed files with 324 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/FetchCommonCrawlIndex.php
+3 -23
View File
@@ -2,6 +2,7 @@
namespace App\Console\Commands;
use App\Console\Commands\Concerns\SavesCommonCrawlCaptures;
use App\Services\CommonCrawl\CommonCrawlIndexService;
use Illuminate\Console\Attributes\Description;
use Illuminate\Console\Attributes\Signature;
@@ -22,6 +23,8 @@ use Illuminate\Console\Command;
#[Description('Fetch Common Crawl URL index records and save them as sites with their pages')]
class FetchCommonCrawlIndex extends Command
{
use SavesCommonCrawlCaptures;
private const VALID_MATCH_TYPES = ['exact', 'prefix', 'host', 'domain'];
public function __construct(private readonly CommonCrawlIndexService $service)
@@ -135,27 +138,4 @@ class FetchCommonCrawlIndex extends Command
return self::SUCCESS;
}
private function saveCaptures(array $lines, string $collection, bool $withContent, array &$contentCache, array &$totals): void
{
$grouped = $this->service->groupIntoSites($lines, $collection);
$siteGroups = $grouped['sites'];
$totalPages = $siteGroups->sum(fn (array $group) => $group['pages']->count());
$this->line(" {$grouped['captureCount']} capture(s) covering {$siteGroups->count()} site(s) and {$totalPages} unique page(s).");
if ($withContent) {
$this->comment(" --with-metadata is set: fetching page content from the WARC archive for up to {$totalPages} page(s), this will be slower.");
}
$this->withProgressBar($siteGroups, function (array $group) use ($collection, $withContent, &$contentCache, &$totals) {
$result = $this->service->saveSiteGroup($group, $collection, $withContent, $contentCache);
$result['wasNewSite'] ? $totals['sitesCreated']++ : $totals['sitesUpdated']++;
$totals['pagesCreated'] += $result['pagesCreated'];
$totals['pagesUpdated'] += $result['pagesUpdated'];
});
$this->newLine();
}
}