Extract and store outbound links (backlinks) during content fetch

Adds a backlinks table: one row per <a href> on a captured page that
points at a different domain (same-site nav links are skipped).
Piggybacks on the existing --with-metadata WARC fetch/DOM-parse in
WarcRecordFetcher rather than a separate command/job, since extracting
links needs the same HTML already being downloaded for
title/description/content — no extra requests.

WarcRecordFetcher gains href resolution (absolute, protocol-relative,
root-relative, query-only, and plain relative hrefs) against the
page's own URL. CommonCrawlIndexService::saveBacklinks() replaces a
page's link set wholesale on each (re)fetch, so reruns stay
idempotent. Site gets outboundLinks()/inboundLinks() relations, the
latter keyed by domain string since a link's target site may not be
indexed yet.

Verified against real .es data: 781 backlinks extracted across 205
distinct target domains (social platforms, Trustpilot, genuine
inter-site links with correct anchor text), and a rerun of the same
fetch left the count unchanged.
This commit is contained in:
m105
2026-07-22 21:29:11 +03:00
parent cefb846a19
commit 3d0b9a6523
6 changed files with 236 additions and 4 deletions
@@ -2,6 +2,7 @@
namespace App\Services\CommonCrawl;
use App\Models\Backlink;
use App\Models\Site;
use App\Models\SitePage;
use App\Support\CommonCrawl\WarcRecordFetcher;
@@ -138,8 +139,13 @@ class CommonCrawlIndexService
unset($attributes['collection'], $attributes['domain']);
$attributes['urlkey_hash'] = sha1($page['urlkey']);
$links = [];
if ($withContent) {
$attributes = array_merge($attributes, $this->fetchContent($page, $contentCache));
$content = $this->fetchContent($page, $contentCache);
$links = $content['links'] ?? [];
unset($content['links']);
$attributes = array_merge($attributes, $content);
}
$result = SitePage::updateOrCreate(
@@ -148,6 +154,10 @@ class CommonCrawlIndexService
);
$result->wasRecentlyCreated ? $pagesCreated++ : $pagesUpdated++;
if ($withContent) {
$this->saveBacklinks($result, $site, $links);
}
}
return [
@@ -158,6 +168,40 @@ class CommonCrawlIndexService
];
}
/**
* Replace a page's recorded outbound links wholesale — simpler and more correct
* than diffing against what was there before, since a page's actual link set can
* change between crawls.
*/
private function saveBacklinks(SitePage $sitePage, Site $site, array $links): void
{
$sitePage->links()->delete();
if (empty($links)) {
return;
}
$now = now();
$rows = collect($links)
->unique('url')
->map(fn (array $link) => [
'site_id' => $site->id,
'site_page_id' => $sitePage->id,
'target_url' => $link['url'],
'target_url_hash' => sha1($link['url']),
'target_domain' => $link['domain'],
'anchor_text' => $link['anchor_text'],
'rel' => $link['rel'],
'created_at' => $now,
'updated_at' => $now,
])
->values()
->all();
Backlink::insert($rows);
}
/**
* List every CDX shard file for a collection, straight from the bulk data storage
* (data.commoncrawl.org) rather than the index.commoncrawl.org query API — this is