Extract and store outbound links (backlinks) during content fetch

Adds a backlinks table: one row per <a href> on a captured page that
points at a different domain (same-site nav links are skipped).
Piggybacks on the existing --with-metadata WARC fetch/DOM-parse in
WarcRecordFetcher rather than a separate command/job, since extracting
links needs the same HTML already being downloaded for
title/description/content — no extra requests.

WarcRecordFetcher gains href resolution (absolute, protocol-relative,
root-relative, query-only, and plain relative hrefs) against the
page's own URL. CommonCrawlIndexService::saveBacklinks() replaces a
page's link set wholesale on each (re)fetch, so reruns stay
idempotent. Site gets outboundLinks()/inboundLinks() relations, the
latter keyed by domain string since a link's target site may not be
indexed yet.

Verified against real .es data: 781 backlinks extracted across 205
distinct target domains (social platforms, Trustpilot, genuine
inter-site links with correct anchor text), and a rerun of the same
fetch left the count unchanged.

app/Models/Backlink.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
+
+class Backlink extends Model
+{
+    protected $fillable = [
+        'site_id',
+        'site_page_id',
+        'target_url',
+        'target_url_hash',
+        'target_domain',
+        'anchor_text',
+        'rel',
+    ];
+
+    public function site(): BelongsTo
+    {
+        return $this->belongsTo(Site::class);
+    }
+
+    public function sitePage(): BelongsTo
+    {
+        return $this->belongsTo(SitePage::class);
+    }
+}

app/Models/Site.php

@@ -20,4 +20,16 @@ class Site extends Model
     {
         return $this->hasMany(SitePage::class);
     }
+
+    /** Links this site's pages make out to other domains. */
+    public function outboundLinks(): HasMany
+    {
+        return $this->hasMany(Backlink::class);
+    }
+
+    /** Backlinks pointing at this site, i.e. rows recorded against its domain as the target. */
+    public function inboundLinks(): HasMany
+    {
+        return $this->hasMany(Backlink::class, 'target_domain', 'domain');
+    }
 }

app/Models/SitePage.php

@@ -4,6 +4,7 @@ namespace App\Models;
 
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\Relations\BelongsTo;
+use Illuminate\Database\Eloquent\Relations\HasMany;
 
 class SitePage extends Model
 {
@@ -42,4 +43,9 @@ class SitePage extends Model
     {
         return $this->belongsTo(Site::class);
     }
+
+    public function links(): HasMany
+    {
+        return $this->hasMany(Backlink::class);
+    }
 }

app/Services/CommonCrawl/CommonCrawlIndexService.php

@@ -2,6 +2,7 @@
 
 namespace App\Services\CommonCrawl;
 
+use App\Models\Backlink;
 use App\Models\Site;
 use App\Models\SitePage;
 use App\Support\CommonCrawl\WarcRecordFetcher;
@@ -138,8 +139,13 @@ class CommonCrawlIndexService
             unset($attributes['collection'], $attributes['domain']);
             $attributes['urlkey_hash'] = sha1($page['urlkey']);
 
+            $links = [];
+
             if ($withContent) {
-                $attributes = array_merge($attributes, $this->fetchContent($page, $contentCache));
+                $content = $this->fetchContent($page, $contentCache);
+                $links = $content['links'] ?? [];
+                unset($content['links']);
+                $attributes = array_merge($attributes, $content);
             }
 
             $result = SitePage::updateOrCreate(
@@ -148,6 +154,10 @@ class CommonCrawlIndexService
             );
 
             $result->wasRecentlyCreated ? $pagesCreated++ : $pagesUpdated++;
+
+            if ($withContent) {
+                $this->saveBacklinks($result, $site, $links);
+            }
         }
 
         return [
@@ -158,6 +168,40 @@ class CommonCrawlIndexService
         ];
     }
 
+    /**
+     * Replace a page's recorded outbound links wholesale — simpler and more correct
+     * than diffing against what was there before, since a page's actual link set can
+     * change between crawls.
+     */
+    private function saveBacklinks(SitePage $sitePage, Site $site, array $links): void
+    {
+        $sitePage->links()->delete();
+
+        if (empty($links)) {
+            return;
+        }
+
+        $now = now();
+
+        $rows = collect($links)
+            ->unique('url')
+            ->map(fn (array $link) => [
+                'site_id' => $site->id,
+                'site_page_id' => $sitePage->id,
+                'target_url' => $link['url'],
+                'target_url_hash' => sha1($link['url']),
+                'target_domain' => $link['domain'],
+                'anchor_text' => $link['anchor_text'],
+                'rel' => $link['rel'],
+                'created_at' => $now,
+                'updated_at' => $now,
+            ])
+            ->values()
+            ->all();
+
+        Backlink::insert($rows);
+    }
+
     /**
      * List every CDX shard file for a collection, straight from the bulk data storage
      * (data.commoncrawl.org) rather than the index.commoncrawl.org query API — this is

app/Support/CommonCrawl/WarcRecordFetcher.php

@@ -49,7 +49,7 @@ class WarcRecordFetcher
 
         $html = $this->extractHtmlFromWarcSegment($response->body());
 
-        return $html === null ? [] : $this->extractHtmlMeta($html);
+        return $html === null ? [] : $this->extractHtmlMeta($html, $attributes['url']);
     }
 
     /**
@@ -75,9 +75,9 @@ class WarcRecordFetcher
         return $html;
     }
 
-    private function extractHtmlMeta(string $html): array
+    private function extractHtmlMeta(string $html, string $pageUrl): array
     {
-        $meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null];
+        $meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null, 'links' => []];
 
         $document = new DOMDocument;
 
@@ -102,10 +102,110 @@ class WarcRecordFetcher
         }
 
         $meta['content'] = $this->extractVisibleText($xpath);
+        $meta['links'] = $this->extractOutboundLinks($xpath, $pageUrl);
 
         return $meta;
     }
 
+    /**
+     * Every <a href> that resolves to an absolute http(s) URL on a *different* host than
+     * the page itself — same-site navigation links aren't backlinks. Relative/root-
+     * relative/protocol-relative hrefs are resolved against the page's own URL.
+     */
+    private function extractOutboundLinks(DOMXPath $xpath, string $pageUrl): array
+    {
+        $pageHost = strtolower((string) parse_url($pageUrl, PHP_URL_HOST));
+        $links = [];
+
+        foreach ($xpath->query('//a[@href]') as $node) {
+            $resolved = $this->resolveUrl($pageUrl, $node->getAttribute('href'));
+
+            if ($resolved === null) {
+                continue;
+            }
+
+            $targetHost = strtolower((string) parse_url($resolved, PHP_URL_HOST));
+
+            if ($targetHost === '' || $targetHost === $pageHost) {
+                continue;
+            }
+
+            $links[] = [
+                'url' => $resolved,
+                'domain' => $targetHost,
+                'anchor_text' => mb_substr(trim(preg_replace('/\s+/', ' ', $node->textContent)) ?: '', 0, 255) ?: null,
+                'rel' => $node->getAttribute('rel') ?: null,
+            ];
+        }
+
+        return $links;
+    }
+
+    /**
+     * Resolve a (possibly relative) href against the page's own URL into an absolute
+     * http(s) URL. Handles absolute, protocol-relative ("//host/path"), root-relative
+     * ("/path"), query-only ("?x=1"), and plain relative ("path") hrefs; not a full
+     * RFC 3986 implementation, but covers what real-world HTML actually uses.
+     */
+    private function resolveUrl(string $base, string $href): ?string
+    {
+        $href = trim($href);
+
+        if ($href === '' || str_starts_with($href, '#')) {
+            return null;
+        }
+
+        if (preg_match('/^[a-z][a-z0-9+.\-]*:/i', $href) === 1) {
+            return str_starts_with(strtolower($href), 'http') ? $href : null;
+        }
+
+        $baseParts = parse_url($base);
+
+        if (! $baseParts || empty($baseParts['scheme']) || empty($baseParts['host'])) {
+            return null;
+        }
+
+        $origin = $baseParts['scheme'].'://'.$baseParts['host'].(isset($baseParts['port']) ? ':'.$baseParts['port'] : '');
+
+        if (str_starts_with($href, '//')) {
+            return $baseParts['scheme'].':'.$href;
+        }
+
+        if (str_starts_with($href, '/')) {
+            return $origin.$href;
+        }
+
+        if (str_starts_with($href, '?')) {
+            return $origin.($baseParts['path'] ?? '/').$href;
+        }
+
+        $basePath = $baseParts['path'] ?? '/';
+        $baseDir = substr($basePath, 0, strrpos($basePath, '/') + 1) ?: '/';
+
+        return $origin.$this->normalizePath($baseDir.$href);
+    }
+
+    private function normalizePath(string $path): string
+    {
+        $stack = [];
+
+        foreach (explode('/', $path) as $segment) {
+            if ($segment === '.' || $segment === '') {
+                continue;
+            }
+
+            if ($segment === '..') {
+                array_pop($stack);
+
+                continue;
+            }
+
+            $stack[] = $segment;
+        }
+
+        return '/'.implode('/', $stack).(str_ends_with($path, '/') ? '/' : '');
+    }
+
     /**
      * Pull the page's visible body text (for later keyword/product search — e.g. "does
      * this site sell honey?"), stripped of script/style/noscript content and collapsed

database/migrations/2026_07_22_212610_create_backlinks_table.php

@@ -0,0 +1,41 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * One row per outbound link found on a captured page, pointing at a different
+     * domain — the edges of a web graph, queryable from either end: "what does this
+     * site link out to" (site_id) or "who links to this domain" (target_domain).
+     */
+    public function up(): void
+    {
+        Schema::create('backlinks', function (Blueprint $table) {
+            $table->id();
+            $table->foreignId('site_id')->constrained()->cascadeOnDelete();
+            $table->foreignId('site_page_id')->constrained()->cascadeOnDelete();
+            $table->text('target_url');
+            // target_url is unbounded like site_pages.urlkey was — hash it for the
+            // unique index rather than repeat that earlier mistake.
+            $table->char('target_url_hash', 40);
+            $table->string('target_domain')->nullable();
+            $table->string('anchor_text')->nullable();
+            $table->string('rel')->nullable();
+            $table->timestamps();
+
+            $table->unique(['site_page_id', 'target_url_hash']);
+            $table->index('target_domain');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::dropIfExists('backlinks');
+    }
+};