huginn/app/Console/Commands/FetchCommonCrawlIndex.php

<?php

namespace App\Console\Commands;

use App\Services\CommonCrawl\CommonCrawlIndexService;
use Illuminate\Console\Attributes\Description;
use Illuminate\Console\Attributes\Signature;
use Illuminate\Console\Command;

#[Signature('crawlindex:fetch
    {url? : The URL, host, or domain pattern to search for (e.g. example.com/*). Omit this and use --tld to browse a whole TLD}
    {--tld= : Restrict the query to a top-level domain (e.g. com, net) instead of passing a url. Requires --match-type=domain}
    {--collection= : Common Crawl collection id, e.g. CC-MAIN-2026-05. Defaults to the latest collection}
    {--match-type=domain : exact|prefix|host|domain}
    {--limit=1000 : Maximum number of index records to fetch per page}
    {--page=0 : CDX index page to start from (the index is split into fixed pages; there is no "skip N records" option, see --pages/--all-pages)}
    {--pages=1 : Number of consecutive CDX index pages to fetch, starting at --page}
    {--all-pages : Keep fetching consecutive pages, starting at --page, until the index is exhausted (overrides --pages)}
    {--page-size=5 : Number of index blocks per CDX page, passed through as pageSize}
    {--with-metadata : Also fetch each page from the WARC archive and extract title/description/keywords/content (slower, one extra request per saved page)}
')]
#[Description('Fetch Common Crawl URL index records and save them as sites with their pages')]
class FetchCommonCrawlIndex extends Command
{
    private const VALID_MATCH_TYPES = ['exact', 'prefix', 'host', 'domain'];

    public function __construct(private readonly CommonCrawlIndexService $service)
    {
        parent::__construct();
    }

    /**
     * Execute the console command.
     */
    public function handle(): int
    {
        $url = $this->argument('url');
        $tld = $this->option('tld');
        $matchType = $this->option('match-type');
        $limit = (int) $this->option('limit');
        $page = (int) $this->option('page');
        $pageSize = (int) $this->option('page-size');
        $allPages = (bool) $this->option('all-pages');
        $pagesToFetch = $allPages ? null : max(1, (int) $this->option('pages'));
        $withMetadata = (bool) $this->option('with-metadata');

        if (! in_array($matchType, self::VALID_MATCH_TYPES, true)) {
            $this->error('--match-type must be one of: '.implode(', ', self::VALID_MATCH_TYPES));

            return self::FAILURE;
        }

        if (! $url && ! $tld) {
            $this->error('Provide a url argument or a --tld option (e.g. --tld=com).');

            return self::FAILURE;
        }

        if ($url && $tld) {
            $this->error('Pass either a url argument or --tld, not both.');

            return self::FAILURE;
        }

        if ($tld) {
            if ($matchType !== 'domain') {
                $this->error('--tld can only be used with --match-type=domain (the default).');

                return self::FAILURE;
            }

            $url = ltrim($tld, '.');
        }

        $collection = $this->option('collection') ?: $this->service->resolveLatestCollection();

        if (! $collection) {
            $this->error('Could not resolve a Common Crawl collection id. Pass one explicitly with --collection.');

            return self::FAILURE;
        }

        $totals = ['sitesCreated' => 0, 'sitesUpdated' => 0, 'pagesCreated' => 0, 'pagesUpdated' => 0];
        $contentCache = [];
        $pagesFetched = 0;
        $currentPage = $page;

        while ($allPages || $pagesFetched < $pagesToFetch) {
            $this->info("Fetching CDX page {$currentPage} for [{$url}] (collection={$collection}, matchType={$matchType}, limit={$limit}, pageSize={$pageSize})...");

            $result = $this->service->fetchIndexPage($collection, $url, $matchType, $limit, $currentPage, $pageSize);

            if ($result['status'] === 'end') {
                if ($pagesFetched === 0) {
                    $this->error("Common Crawl rejected page {$currentPage}.");

                    return self::FAILURE;
                }

                $this->info('Reached the end of the index, no more pages.');
                break;
            }

            if ($result['status'] === 'not_found') {
                if ($pagesFetched === 0) {
                    $this->warn('No captures found for this URL/collection.');
                }

                break;
            }

            if ($result['status'] === 'error') {
                $this->error("Common Crawl index request failed with status {$result['httpStatus']}: {$result['body']}");

                return self::FAILURE;
            }

            $pagesFetched++;

            if (! empty($result['lines'])) {
                $this->saveCaptures($result['lines'], $collection, $withMetadata, $contentCache, $totals);
            } else {
                $this->comment('(empty page)');
            }

            $currentPage++;

            if ($allPages || $pagesFetched < $pagesToFetch) {
                usleep(300_000); // be polite to the shared index server between page requests
            }
        }

        $this->newLine();
        $this->info("Done. Fetched {$pagesFetched} CDX page(s). Sites: {$totals['sitesCreated']} new, {$totals['sitesUpdated']} updated. Pages: {$totals['pagesCreated']} new, {$totals['pagesUpdated']} updated.");

        return self::SUCCESS;
    }

    private function saveCaptures(array $lines, string $collection, bool $withContent, array &$contentCache, array &$totals): void
    {
        $grouped = $this->service->groupIntoSites($lines, $collection);
        $siteGroups = $grouped['sites'];
        $totalPages = $siteGroups->sum(fn (array $group) => $group['pages']->count());

        $this->line("  {$grouped['captureCount']} capture(s) covering {$siteGroups->count()} site(s) and {$totalPages} unique page(s).");

        if ($withContent) {
            $this->comment("  --with-metadata is set: fetching page content from the WARC archive for up to {$totalPages} page(s), this will be slower.");
        }

        $this->withProgressBar($siteGroups, function (array $group) use ($collection, $withContent, &$contentCache, &$totals) {
            $result = $this->service->saveSiteGroup($group, $collection, $withContent, $contentCache);

            $result['wasNewSite'] ? $totals['sitesCreated']++ : $totals['sitesUpdated']++;
            $totals['pagesCreated'] += $result['pagesCreated'];
            $totals['pagesUpdated'] += $result['pagesUpdated'];
        });

        $this->newLine();
    }
}