<?php

namespace App\Support\CommonCrawl;

use DOMDocument;
use DOMXPath;
use Illuminate\Support\Facades\Http;

/**
 * Downloads a single record's bytes out of a Common Crawl WARC file via an HTTP
 * Range request (using the offset/length recorded from the CDX index) and extracts
 * page metadata from it. Only ever touches data.commoncrawl.org, never the CDX
 * index server, and never pulls more than one record's worth of bytes at a time.
 */
class WarcRecordFetcher
{
    public const DEFAULT_USER_AGENT = 'huginn-crawlindex/1.0 (+https://github.com/)';

    private const WARC_BASE_URL = 'https://data.commoncrawl.org/';

    /** Cap on stored page text, in characters — keeps a single page's row bounded. */
    private const MAX_CONTENT_LENGTH = 20000;

    public function __construct(private readonly string $userAgent = self::DEFAULT_USER_AGENT) {}

    public function fetchMetadata(array $attributes): array
    {
        if (($attributes['mime'] ?? null) !== 'text/html'
            || ($attributes['status'] ?? null) !== 200
            || empty($attributes['filename'])
            || $attributes['offset'] === null
            || $attributes['length'] === null
        ) {
            return [];
        }

        $start = $attributes['offset'];
        $end = $start + $attributes['length'] - 1;

        $response = Http::withUserAgent($this->userAgent)
            ->timeout(30)
            ->retry(2, 1000)
            ->withHeaders(['Range' => "bytes={$start}-{$end}"])
            ->get(self::WARC_BASE_URL.$attributes['filename']);

        if (! $response->successful()) {
            return [];
        }

        $html = $this->extractHtmlFromWarcSegment($response->body());

        return $html === null ? [] : $this->extractHtmlMeta($html);
    }

    /**
     * A WARC segment (gzip-compressed) contains WARC headers, then the raw HTTP
     * response (its own headers followed by the HTML body). Strip both header blocks.
     */
    private function extractHtmlFromWarcSegment(string $gzipped): ?string
    {
        $decompressed = @gzdecode($gzipped);

        if ($decompressed === false) {
            return null;
        }

        [$warcHeaders, $rest] = array_pad(preg_split("/\r?\n\r?\n/", $decompressed, 2), 2, null);

        if ($rest === null) {
            return null;
        }

        [$httpHeaders, $html] = array_pad(preg_split("/\r?\n\r?\n/", $rest, 2), 2, null);

        return $html;
    }

    private function extractHtmlMeta(string $html): array
    {
        $meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null];

        $document = new DOMDocument;

        libxml_use_internal_errors(true);
        $document->loadHTML('<?xml encoding="UTF-8">'.$html, LIBXML_NOERROR | LIBXML_NOWARNING);
        libxml_clear_errors();

        $xpath = new DOMXPath($document);

        $titleNode = $xpath->query('//title')->item(0);

        if ($titleNode) {
            $meta['title'] = trim(preg_replace('/\s+/', ' ', $titleNode->textContent)) ?: null;
        }

        foreach (['description', 'keywords'] as $name) {
            $node = $xpath->query("//meta[translate(@name, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')='{$name}']")->item(0);

            if ($node && $node->getAttribute('content') !== '') {
                $meta[$name] = trim(preg_replace('/\s+/', ' ', $node->getAttribute('content')));
            }
        }

        $meta['content'] = $this->extractVisibleText($xpath);

        return $meta;
    }

    /**
     * Pull the page's visible body text (for later keyword/product search — e.g. "does
     * this site sell honey?"), stripped of script/style/noscript content and collapsed
     * whitespace, capped at MAX_CONTENT_LENGTH so a single page can't blow up storage.
     */
    private function extractVisibleText(DOMXPath $xpath): ?string
    {
        foreach ($xpath->query('//script | //style | //noscript') as $node) {
            $node->parentNode?->removeChild($node);
        }

        $body = $xpath->query('//body')->item(0);

        if (! $body) {
            return null;
        }

        $text = trim(preg_replace('/\s+/', ' ', $body->textContent));

        return $text === '' ? null : mb_substr($text, 0, self::MAX_CONTENT_LENGTH);
    }
}