huginn/app/Support/CommonCrawl/WarcRecordFetcher.php

<?php

namespace App\Support\CommonCrawl;

use DOMDocument;
use DOMXPath;
use Illuminate\Support\Facades\Http;

/**
 * Downloads a single record's bytes out of a Common Crawl WARC file via an HTTP
 * Range request (using the offset/length recorded from the CDX index) and extracts
 * page metadata from it. Only ever touches data.commoncrawl.org, never the CDX
 * index server, and never pulls more than one record's worth of bytes at a time.
 */
class WarcRecordFetcher
{
    public const DEFAULT_USER_AGENT = 'huginn-crawlindex/1.0 (+https://github.com/)';

    private const WARC_BASE_URL = 'https://data.commoncrawl.org/';

    /** Cap on stored page text, in characters — keeps a single page's row bounded. */
    private const MAX_CONTENT_LENGTH = 20000;

    public function __construct(private readonly string $userAgent = self::DEFAULT_USER_AGENT) {}

    public function fetchMetadata(array $attributes): array
    {
        if (($attributes['mime'] ?? null) !== 'text/html'
            || ($attributes['status'] ?? null) !== 200
            || empty($attributes['filename'])
            || $attributes['offset'] === null
            || $attributes['length'] === null
        ) {
            return [];
        }

        $start = $attributes['offset'];
        $end = $start + $attributes['length'] - 1;

        $response = Http::withUserAgent($this->userAgent)
            ->timeout(30)
            ->retry(2, 1000)
            ->withHeaders(['Range' => "bytes={$start}-{$end}"])
            ->get(self::WARC_BASE_URL.$attributes['filename']);

        if (! $response->successful()) {
            return [];
        }

        $segment = $this->extractHtmlFromWarcSegment($response->body());

        return $segment === null
            ? []
            : $this->extractHtmlMeta($segment['html'], $attributes['url'], $segment['headers']);
    }

    /**
     * A WARC segment (gzip-compressed) contains WARC headers, then the raw HTTP
     * response (its own headers followed by the HTML body). Strip the WARC header
     * block; the HTTP header block is kept (for Server/X-Powered-By) alongside the HTML.
     */
    private function extractHtmlFromWarcSegment(string $gzipped): ?array
    {
        $decompressed = @gzdecode($gzipped);

        if ($decompressed === false) {
            return null;
        }

        [$warcHeaders, $rest] = array_pad(preg_split("/\r?\n\r?\n/", $decompressed, 2), 2, null);

        if ($rest === null) {
            return null;
        }

        [$httpHeaders, $html] = array_pad(preg_split("/\r?\n\r?\n/", $rest, 2), 2, null);

        return ['headers' => $httpHeaders ?? '', 'html' => $html];
    }

    /**
     * Parses a raw HTTP header block (status line + "Name: value" lines) into a
     * lowercase-keyed map. First occurrence of a header name wins.
     */
    private function parseHttpHeaders(string $raw): array
    {
        $headers = [];

        foreach (preg_split("/\r?\n/", $raw) as $index => $line) {
            if ($index === 0 || ! str_contains($line, ':')) {
                continue; // line 0 is the "HTTP/1.1 200 OK" status line, not a header
            }

            [$name, $value] = explode(':', $line, 2);
            $key = strtolower(trim($name));

            $headers[$key] ??= trim($value);
        }

        return $headers;
    }

    private function extractHtmlMeta(string $html, string $pageUrl, string $rawHttpHeaders): array
    {
        $headers = $this->parseHttpHeaders($rawHttpHeaders);

        $meta = [
            'title' => null,
            'description' => null,
            'keywords' => null,
            'generator' => null,
            'server' => isset($headers['server']) ? mb_substr($headers['server'], 0, 255) : null,
            'powered_by' => isset($headers['x-powered-by']) ? mb_substr($headers['x-powered-by'], 0, 255) : null,
            'content' => null,
            'links' => [],
        ];

        $document = new DOMDocument;

        libxml_use_internal_errors(true);
        $document->loadHTML('<?xml encoding="UTF-8">'.$html, LIBXML_NOERROR | LIBXML_NOWARNING);
        libxml_clear_errors();

        $xpath = new DOMXPath($document);

        $titleNode = $xpath->query('//title')->item(0);

        if ($titleNode) {
            $meta['title'] = trim(preg_replace('/\s+/', ' ', $titleNode->textContent)) ?: null;
        }

        foreach (['description', 'keywords', 'generator'] as $name) {
            $node = $xpath->query("//meta[translate(@name, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')='{$name}']")->item(0);

            if ($node && $node->getAttribute('content') !== '') {
                $value = trim(preg_replace('/\s+/', ' ', $node->getAttribute('content')));
                // generator is stored in a varchar column; description/keywords are text.
                $meta[$name] = $name === 'generator' ? mb_substr($value, 0, 255) : $value;
            }
        }

        $meta['content'] = $this->extractVisibleText($xpath);
        $meta['links'] = $this->extractOutboundLinks($xpath, $pageUrl);

        return $meta;
    }

    /**
     * Every <a href> that resolves to an absolute http(s) URL on a *different* host than
     * the page itself — same-site navigation links aren't backlinks. Relative/root-
     * relative/protocol-relative hrefs are resolved against the page's own URL.
     */
    private function extractOutboundLinks(DOMXPath $xpath, string $pageUrl): array
    {
        $pageHost = strtolower((string) parse_url($pageUrl, PHP_URL_HOST));
        $links = [];

        foreach ($xpath->query('//a[@href]') as $node) {
            $resolved = $this->resolveUrl($pageUrl, $node->getAttribute('href'));

            if ($resolved === null) {
                continue;
            }

            $targetHost = strtolower((string) parse_url($resolved, PHP_URL_HOST));

            if ($targetHost === '' || $targetHost === $pageHost) {
                continue;
            }

            $links[] = [
                'url' => $resolved,
                'domain' => $targetHost,
                'anchor_text' => mb_substr(trim(preg_replace('/\s+/', ' ', $node->textContent)) ?: '', 0, 255) ?: null,
                'rel' => $node->getAttribute('rel') ?: null,
            ];
        }

        return $links;
    }

    /**
     * Resolve a (possibly relative) href against the page's own URL into an absolute
     * http(s) URL. Handles absolute, protocol-relative ("//host/path"), root-relative
     * ("/path"), query-only ("?x=1"), and plain relative ("path") hrefs; not a full
     * RFC 3986 implementation, but covers what real-world HTML actually uses.
     */
    private function resolveUrl(string $base, string $href): ?string
    {
        $href = trim($href);

        if ($href === '' || str_starts_with($href, '#')) {
            return null;
        }

        if (preg_match('/^[a-z][a-z0-9+.\-]*:/i', $href) === 1) {
            return str_starts_with(strtolower($href), 'http') ? $href : null;
        }

        $baseParts = parse_url($base);

        if (! $baseParts || empty($baseParts['scheme']) || empty($baseParts['host'])) {
            return null;
        }

        $origin = $baseParts['scheme'].'://'.$baseParts['host'].(isset($baseParts['port']) ? ':'.$baseParts['port'] : '');

        if (str_starts_with($href, '//')) {
            return $baseParts['scheme'].':'.$href;
        }

        if (str_starts_with($href, '/')) {
            return $origin.$href;
        }

        if (str_starts_with($href, '?')) {
            return $origin.($baseParts['path'] ?? '/').$href;
        }

        $basePath = $baseParts['path'] ?? '/';
        $baseDir = substr($basePath, 0, strrpos($basePath, '/') + 1) ?: '/';

        return $origin.$this->normalizePath($baseDir.$href);
    }

    private function normalizePath(string $path): string
    {
        $stack = [];

        foreach (explode('/', $path) as $segment) {
            if ($segment === '.' || $segment === '') {
                continue;
            }

            if ($segment === '..') {
                array_pop($stack);

                continue;
            }

            $stack[] = $segment;
        }

        return '/'.implode('/', $stack).(str_ends_with($path, '/') ? '/' : '');
    }

    /**
     * Pull the page's visible body text (for later keyword/product search — e.g. "does
     * this site sell honey?"), stripped of script/style/noscript content and collapsed
     * whitespace, capped at MAX_CONTENT_LENGTH so a single page can't blow up storage.
     */
    private function extractVisibleText(DOMXPath $xpath): ?string
    {
        foreach ($xpath->query('//script | //style | //noscript') as $node) {
            $node->parentNode?->removeChild($node);
        }

        $body = $xpath->query('//body')->item(0);

        if (! $body) {
            return null;
        }

        $text = trim(preg_replace('/\s+/', ' ', $body->textContent));

        return $text === '' ? null : mb_substr($text, 0, self::MAX_CONTENT_LENGTH);
    }
}