userAgent) ->timeout(30) ->retry(2, 1000) ->withHeaders(['Range' => "bytes={$start}-{$end}"]) ->get(self::WARC_BASE_URL.$attributes['filename']); if (! $response->successful()) { return []; } $html = $this->extractHtmlFromWarcSegment($response->body()); return $html === null ? [] : $this->extractHtmlMeta($html, $attributes['url']); } /** * A WARC segment (gzip-compressed) contains WARC headers, then the raw HTTP * response (its own headers followed by the HTML body). Strip both header blocks. */ private function extractHtmlFromWarcSegment(string $gzipped): ?string { $decompressed = @gzdecode($gzipped); if ($decompressed === false) { return null; } [$warcHeaders, $rest] = array_pad(preg_split("/\r?\n\r?\n/", $decompressed, 2), 2, null); if ($rest === null) { return null; } [$httpHeaders, $html] = array_pad(preg_split("/\r?\n\r?\n/", $rest, 2), 2, null); return $html; } private function extractHtmlMeta(string $html, string $pageUrl): array { $meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null, 'links' => []]; $document = new DOMDocument; libxml_use_internal_errors(true); $document->loadHTML(''.$html, LIBXML_NOERROR | LIBXML_NOWARNING); libxml_clear_errors(); $xpath = new DOMXPath($document); $titleNode = $xpath->query('//title')->item(0); if ($titleNode) { $meta['title'] = trim(preg_replace('/\s+/', ' ', $titleNode->textContent)) ?: null; } foreach (['description', 'keywords'] as $name) { $node = $xpath->query("//meta[translate(@name, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')='{$name}']")->item(0); if ($node && $node->getAttribute('content') !== '') { $meta[$name] = trim(preg_replace('/\s+/', ' ', $node->getAttribute('content'))); } } $meta['content'] = $this->extractVisibleText($xpath); $meta['links'] = $this->extractOutboundLinks($xpath, $pageUrl); return $meta; } /** * Every that resolves to an absolute http(s) URL on a *different* host than * the page itself — same-site navigation links aren't backlinks. Relative/root- * relative/protocol-relative hrefs are resolved against the page's own URL. */ private function extractOutboundLinks(DOMXPath $xpath, string $pageUrl): array { $pageHost = strtolower((string) parse_url($pageUrl, PHP_URL_HOST)); $links = []; foreach ($xpath->query('//a[@href]') as $node) { $resolved = $this->resolveUrl($pageUrl, $node->getAttribute('href')); if ($resolved === null) { continue; } $targetHost = strtolower((string) parse_url($resolved, PHP_URL_HOST)); if ($targetHost === '' || $targetHost === $pageHost) { continue; } $links[] = [ 'url' => $resolved, 'domain' => $targetHost, 'anchor_text' => mb_substr(trim(preg_replace('/\s+/', ' ', $node->textContent)) ?: '', 0, 255) ?: null, 'rel' => $node->getAttribute('rel') ?: null, ]; } return $links; } /** * Resolve a (possibly relative) href against the page's own URL into an absolute * http(s) URL. Handles absolute, protocol-relative ("//host/path"), root-relative * ("/path"), query-only ("?x=1"), and plain relative ("path") hrefs; not a full * RFC 3986 implementation, but covers what real-world HTML actually uses. */ private function resolveUrl(string $base, string $href): ?string { $href = trim($href); if ($href === '' || str_starts_with($href, '#')) { return null; } if (preg_match('/^[a-z][a-z0-9+.\-]*:/i', $href) === 1) { return str_starts_with(strtolower($href), 'http') ? $href : null; } $baseParts = parse_url($base); if (! $baseParts || empty($baseParts['scheme']) || empty($baseParts['host'])) { return null; } $origin = $baseParts['scheme'].'://'.$baseParts['host'].(isset($baseParts['port']) ? ':'.$baseParts['port'] : ''); if (str_starts_with($href, '//')) { return $baseParts['scheme'].':'.$href; } if (str_starts_with($href, '/')) { return $origin.$href; } if (str_starts_with($href, '?')) { return $origin.($baseParts['path'] ?? '/').$href; } $basePath = $baseParts['path'] ?? '/'; $baseDir = substr($basePath, 0, strrpos($basePath, '/') + 1) ?: '/'; return $origin.$this->normalizePath($baseDir.$href); } private function normalizePath(string $path): string { $stack = []; foreach (explode('/', $path) as $segment) { if ($segment === '.' || $segment === '') { continue; } if ($segment === '..') { array_pop($stack); continue; } $stack[] = $segment; } return '/'.implode('/', $stack).(str_ends_with($path, '/') ? '/' : ''); } /** * Pull the page's visible body text (for later keyword/product search — e.g. "does * this site sell honey?"), stripped of script/style/noscript content and collapsed * whitespace, capped at MAX_CONTENT_LENGTH so a single page can't blow up storage. */ private function extractVisibleText(DOMXPath $xpath): ?string { foreach ($xpath->query('//script | //style | //noscript') as $node) { $node->parentNode?->removeChild($node); } $body = $xpath->query('//body')->item(0); if (! $body) { return null; } $text = trim(preg_replace('/\s+/', ' ', $body->textContent)); return $text === '' ? null : mb_substr($text, 0, self::MAX_CONTENT_LENGTH); } }