Extract and store outbound links (backlinks) during content fetch

Adds a backlinks table: one row per <a href> on a captured page that points at a different domain (same-site nav links are skipped). Piggybacks on the existing --with-metadata WARC fetch/DOM-parse in WarcRecordFetcher rather than a separate command/job, since extracting links needs the same HTML already being downloaded for title/description/content — no extra requests. WarcRecordFetcher gains href resolution (absolute, protocol-relative, root-relative, query-only, and plain relative hrefs) against the page's own URL. CommonCrawlIndexService::saveBacklinks() replaces a page's link set wholesale on each (re)fetch, so reruns stay idempotent. Site gets outboundLinks()/inboundLinks() relations, the latter keyed by domain string since a link's target site may not be indexed yet. Verified against real .es data: 781 backlinks extracted across 205 distinct target domains (social platforms, Trustpilot, genuine inter-site links with correct anchor text), and a rerun of the same fetch left the count unchanged.
2026-07-22 21:29:11 +03:00
parent cefb846a19
commit 3d0b9a6523
6 changed files with 236 additions and 4 deletions
@@ -49,7 +49,7 @@ class WarcRecordFetcher

        $html = $this->extractHtmlFromWarcSegment($response->body());

-        return $html === null ? [] : $this->extractHtmlMeta($html);
+        return $html === null ? [] : $this->extractHtmlMeta($html, $attributes['url']);
    }

    /**
@@ -75,9 +75,9 @@ class WarcRecordFetcher
        return $html;
    }

-    private function extractHtmlMeta(string $html): array
+    private function extractHtmlMeta(string $html, string $pageUrl): array
    {
-        $meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null];
+        $meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null, 'links' => []];

        $document = new DOMDocument;

@@ -102,10 +102,110 @@ class WarcRecordFetcher
        }

        $meta['content'] = $this->extractVisibleText($xpath);
+        $meta['links'] = $this->extractOutboundLinks($xpath, $pageUrl);

        return $meta;
    }

+    /**
+     * Every <a href> that resolves to an absolute http(s) URL on a *different* host than
+     * the page itself — same-site navigation links aren't backlinks. Relative/root-
+     * relative/protocol-relative hrefs are resolved against the page's own URL.
+     */
+    private function extractOutboundLinks(DOMXPath $xpath, string $pageUrl): array
+    {
+        $pageHost = strtolower((string) parse_url($pageUrl, PHP_URL_HOST));
+        $links = [];
+
+        foreach ($xpath->query('//a[@href]') as $node) {
+            $resolved = $this->resolveUrl($pageUrl, $node->getAttribute('href'));
+
+            if ($resolved === null) {
+                continue;
+            }
+
+            $targetHost = strtolower((string) parse_url($resolved, PHP_URL_HOST));
+
+            if ($targetHost === '' || $targetHost === $pageHost) {
+                continue;
+            }
+
+            $links[] = [
+                'url' => $resolved,
+                'domain' => $targetHost,
+                'anchor_text' => mb_substr(trim(preg_replace('/\s+/', ' ', $node->textContent)) ?: '', 0, 255) ?: null,
+                'rel' => $node->getAttribute('rel') ?: null,
+            ];
+        }
+
+        return $links;
+    }
+
+    /**
+     * Resolve a (possibly relative) href against the page's own URL into an absolute
+     * http(s) URL. Handles absolute, protocol-relative ("//host/path"), root-relative
+     * ("/path"), query-only ("?x=1"), and plain relative ("path") hrefs; not a full
+     * RFC 3986 implementation, but covers what real-world HTML actually uses.
+     */
+    private function resolveUrl(string $base, string $href): ?string
+    {
+        $href = trim($href);
+
+        if ($href === '' || str_starts_with($href, '#')) {
+            return null;
+        }
+
+        if (preg_match('/^[a-z][a-z0-9+.\-]*:/i', $href) === 1) {
+            return str_starts_with(strtolower($href), 'http') ? $href : null;
+        }
+
+        $baseParts = parse_url($base);
+
+        if (! $baseParts || empty($baseParts['scheme']) || empty($baseParts['host'])) {
+            return null;
+        }
+
+        $origin = $baseParts['scheme'].'://'.$baseParts['host'].(isset($baseParts['port']) ? ':'.$baseParts['port'] : '');
+
+        if (str_starts_with($href, '//')) {
+            return $baseParts['scheme'].':'.$href;
+        }
+
+        if (str_starts_with($href, '/')) {
+            return $origin.$href;
+        }
+
+        if (str_starts_with($href, '?')) {
+            return $origin.($baseParts['path'] ?? '/').$href;
+        }
+
+        $basePath = $baseParts['path'] ?? '/';
+        $baseDir = substr($basePath, 0, strrpos($basePath, '/') + 1) ?: '/';
+
+        return $origin.$this->normalizePath($baseDir.$href);
+    }
+
+    private function normalizePath(string $path): string
+    {
+        $stack = [];
+
+        foreach (explode('/', $path) as $segment) {
+            if ($segment === '.' || $segment === '') {
+                continue;
+            }
+
+            if ($segment === '..') {
+                array_pop($stack);
+
+                continue;
+            }
+
+            $stack[] = $segment;
+        }
+
+        return '/'.implode('/', $stack).(str_ends_with($path, '/') ? '/' : '');
+    }
+
    /**
     * Pull the page's visible body text (for later keyword/product search — e.g. "does
     * this site sell honey?"), stripped of script/style/noscript content and collapsed