Add bulk CDX shard ingestion command as an API-independent fallback

crawlindex:bulk-fetch walks Common Crawl's CDX shard files directly on
data.commoncrawl.org (via cc-index.paths.gz), instead of the
index.commoncrawl.org query API — usable for bulk backfill/refresh or
as a fallback when that API is down. Shares the group/save pipeline
with crawlindex:fetch via a new SavesCommonCrawlCaptures trait.

Also fixes a real bug this surfaced under load: site_pages.urlkey was
a varchar(255), which is too narrow for real-world captured URLs
(long query strings on spam/malformed pages overflow it). Widened to
text and switched the uniqueness constraint to a sha1 hash column,
since MySQL can't fully index a TEXT column.

database/migrations/2026_07_20_001940_widen_urlkey_on_site_pages_table.php

@@ -0,0 +1,84 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * urlkey is unbounded in practice (long, spammy query strings show up in the wild),
+     * so it can no longer be the indexed/unique column directly — MySQL can't put a
+     * full-length index on TEXT. Widen it to text and key uniqueness off a sha1 hash
+     * of it instead, backfilled for existing rows before the new index is created.
+     */
+    public function up(): void
+    {
+        // A plain index on site_id temporarily takes over from the composite unique
+        // below as what satisfies MySQL's "FK columns must be indexed" requirement, so
+        // that unique index is free to drop. It's dropped again at the end, once the
+        // new (site_id, urlkey_hash) unique index — which also starts with site_id —
+        // is in place to cover the FK itself.
+        Schema::table('site_pages', function (Blueprint $table) {
+            $table->index('site_id', 'site_pages_site_id_temp_index');
+        });
+
+        Schema::table('site_pages', function (Blueprint $table) {
+            $table->dropUnique(['site_id', 'urlkey']);
+        });
+
+        // urlkey can't be widened to text while any index still covers its full length
+        // (MySQL requires a key length for TEXT/BLOB columns in an index), so this can
+        // only happen once the drop above has gone through.
+        Schema::table('site_pages', function (Blueprint $table) {
+            $table->text('urlkey')->change();
+            $table->char('urlkey_hash', 40)->nullable()->after('urlkey');
+        });
+
+        // Hashed in PHP, chunked, rather than a SQL SHA1()/MD5() call — not every MySQL
+        // build ships those (this one doesn't), and this guarantees the backfilled hash
+        // is byte-for-byte what CommonCrawlIndexService computes for new rows.
+        DB::table('site_pages')->orderBy('id')->chunkById(500, function ($rows) {
+            foreach ($rows as $row) {
+                DB::table('site_pages')->where('id', $row->id)->update(['urlkey_hash' => sha1($row->urlkey)]);
+            }
+        });
+
+        Schema::table('site_pages', function (Blueprint $table) {
+            $table->char('urlkey_hash', 40)->nullable(false)->change();
+            $table->unique(['site_id', 'urlkey_hash']);
+        });
+
+        Schema::table('site_pages', function (Blueprint $table) {
+            $table->dropIndex('site_pages_site_id_temp_index');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('site_pages', function (Blueprint $table) {
+            $table->index('site_id', 'site_pages_site_id_temp_index');
+        });
+
+        Schema::table('site_pages', function (Blueprint $table) {
+            $table->dropUnique(['site_id', 'urlkey_hash']);
+        });
+
+        Schema::table('site_pages', function (Blueprint $table) {
+            $table->string('urlkey', 255)->change();
+            $table->dropColumn('urlkey_hash');
+        });
+
+        Schema::table('site_pages', function (Blueprint $table) {
+            $table->unique(['site_id', 'urlkey']);
+        });
+
+        Schema::table('site_pages', function (Blueprint $table) {
+            $table->dropIndex('site_pages_site_id_temp_index');
+        });
+    }
+};