Initial commit: Laravel scaffold + Common Crawl site indexing

Adds Site/SitePage models and the crawlindex:fetch command, which pulls CDX index records from Common Crawl and optionally enriches them with page title/description/keywords/body text extracted from the WARC archive. CDX/WARC access and the site/page upsert logic live in CommonCrawlIndexService and WarcRecordFetcher.
2026-07-20 00:04:45 +03:00
commit 1c9fc5d423
64 changed files with 11338 additions and 0 deletions
@@ -0,0 +1,130 @@
+<?php
+
+namespace App\Support\CommonCrawl;
+
+use DOMDocument;
+use DOMXPath;
+use Illuminate\Support\Facades\Http;
+
+/**
+ * Downloads a single record's bytes out of a Common Crawl WARC file via an HTTP
+ * Range request (using the offset/length recorded from the CDX index) and extracts
+ * page metadata from it. Only ever touches data.commoncrawl.org, never the CDX
+ * index server, and never pulls more than one record's worth of bytes at a time.
+ */
+class WarcRecordFetcher
+{
+    public const DEFAULT_USER_AGENT = 'huginn-crawlindex/1.0 (+https://github.com/)';
+
+    private const WARC_BASE_URL = 'https://data.commoncrawl.org/';
+
+    /** Cap on stored page text, in characters — keeps a single page's row bounded. */
+    private const MAX_CONTENT_LENGTH = 20000;
+
+    public function __construct(private readonly string $userAgent = self::DEFAULT_USER_AGENT) {}
+
+    public function fetchMetadata(array $attributes): array
+    {
+        if (($attributes['mime'] ?? null) !== 'text/html'
+            || ($attributes['status'] ?? null) !== 200
+            || empty($attributes['filename'])
+            || $attributes['offset'] === null
+            || $attributes['length'] === null
+        ) {
+            return [];
+        }
+
+        $start = $attributes['offset'];
+        $end = $start + $attributes['length'] - 1;
+
+        $response = Http::withUserAgent($this->userAgent)
+            ->timeout(30)
+            ->retry(2, 1000)
+            ->withHeaders(['Range' => "bytes={$start}-{$end}"])
+            ->get(self::WARC_BASE_URL.$attributes['filename']);
+
+        if (! $response->successful()) {
+            return [];
+        }
+
+        $html = $this->extractHtmlFromWarcSegment($response->body());
+
+        return $html === null ? [] : $this->extractHtmlMeta($html);
+    }
+
+    /**
+     * A WARC segment (gzip-compressed) contains WARC headers, then the raw HTTP
+     * response (its own headers followed by the HTML body). Strip both header blocks.
+     */
+    private function extractHtmlFromWarcSegment(string $gzipped): ?string
+    {
+        $decompressed = @gzdecode($gzipped);
+
+        if ($decompressed === false) {
+            return null;
+        }
+
+        [$warcHeaders, $rest] = array_pad(preg_split("/\r?\n\r?\n/", $decompressed, 2), 2, null);
+
+        if ($rest === null) {
+            return null;
+        }
+
+        [$httpHeaders, $html] = array_pad(preg_split("/\r?\n\r?\n/", $rest, 2), 2, null);
+
+        return $html;
+    }
+
+    private function extractHtmlMeta(string $html): array
+    {
+        $meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null];
+
+        $document = new DOMDocument;
+
+        libxml_use_internal_errors(true);
+        $document->loadHTML('<?xml encoding="UTF-8">'.$html, LIBXML_NOERROR | LIBXML_NOWARNING);
+        libxml_clear_errors();
+
+        $xpath = new DOMXPath($document);
+
+        $titleNode = $xpath->query('//title')->item(0);
+
+        if ($titleNode) {
+            $meta['title'] = trim(preg_replace('/\s+/', ' ', $titleNode->textContent)) ?: null;
+        }
+
+        foreach (['description', 'keywords'] as $name) {
+            $node = $xpath->query("//meta[translate(@name, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')='{$name}']")->item(0);
+
+            if ($node && $node->getAttribute('content') !== '') {
+                $meta[$name] = trim(preg_replace('/\s+/', ' ', $node->getAttribute('content')));
+            }
+        }
+
+        $meta['content'] = $this->extractVisibleText($xpath);
+
+        return $meta;
+    }
+
+    /**
+     * Pull the page's visible body text (for later keyword/product search — e.g. "does
+     * this site sell honey?"), stripped of script/style/noscript content and collapsed
+     * whitespace, capped at MAX_CONTENT_LENGTH so a single page can't blow up storage.
+     */
+    private function extractVisibleText(DOMXPath $xpath): ?string
+    {
+        foreach ($xpath->query('//script | //style | //noscript') as $node) {
+            $node->parentNode?->removeChild($node);
+        }
+
+        $body = $xpath->query('//body')->item(0);
+
+        if (! $body) {
+            return null;
+        }
+
+        $text = trim(preg_replace('/\s+/', ' ', $body->textContent));
+
+        return $text === '' ? null : mb_substr($text, 0, self::MAX_CONTENT_LENGTH);
+    }
+}