Add bulk CDX shard ingestion command as an API-independent fallback

crawlindex:bulk-fetch walks Common Crawl's CDX shard files directly on
data.commoncrawl.org (via cc-index.paths.gz), instead of the
index.commoncrawl.org query API — usable for bulk backfill/refresh or
as a fallback when that API is down. Shares the group/save pipeline
with crawlindex:fetch via a new SavesCommonCrawlCaptures trait.

Also fixes a real bug this surfaced under load: site_pages.urlkey was
a varchar(255), which is too narrow for real-world captured URLs
(long query strings on spam/malformed pages overflow it). Widened to
text and switched the uniqueness constraint to a sha1 hash column,
since MySQL can't fully index a TEXT column.

app/Console/Commands/Concerns/SavesCommonCrawlCaptures.php

@@ -0,0 +1,34 @@
+<?php
+
+namespace App\Console\Commands\Concerns;
+
+/**
+ * Shared by console commands that feed CDX-API-shaped JSON lines (one capture per line)
+ * through CommonCrawlIndexService's group/save pipeline, with progress-bar output.
+ * Requires the using class to be a Command with a $service property.
+ */
+trait SavesCommonCrawlCaptures
+{
+    private function saveCaptures(array $lines, string $collection, bool $withContent, array &$contentCache, array &$totals): void
+    {
+        $grouped = $this->service->groupIntoSites($lines, $collection);
+        $siteGroups = $grouped['sites'];
+        $totalPages = $siteGroups->sum(fn (array $group) => $group['pages']->count());
+
+        $this->line("  {$grouped['captureCount']} capture(s) covering {$siteGroups->count()} site(s) and {$totalPages} unique page(s).");
+
+        if ($withContent) {
+            $this->comment("  Fetching page content from the WARC archive for up to {$totalPages} page(s), this will be slower.");
+        }
+
+        $this->withProgressBar($siteGroups, function (array $group) use ($collection, $withContent, &$contentCache, &$totals) {
+            $result = $this->service->saveSiteGroup($group, $collection, $withContent, $contentCache);
+
+            $result['wasNewSite'] ? $totals['sitesCreated']++ : $totals['sitesUpdated']++;
+            $totals['pagesCreated'] += $result['pagesCreated'];
+            $totals['pagesUpdated'] += $result['pagesUpdated'];
+        });
+
+        $this->newLine();
+    }
+}