Extract and store outbound links (backlinks) during content fetch
Adds a backlinks table: one row per <a href> on a captured page that points at a different domain (same-site nav links are skipped). Piggybacks on the existing --with-metadata WARC fetch/DOM-parse in WarcRecordFetcher rather than a separate command/job, since extracting links needs the same HTML already being downloaded for title/description/content — no extra requests. WarcRecordFetcher gains href resolution (absolute, protocol-relative, root-relative, query-only, and plain relative hrefs) against the page's own URL. CommonCrawlIndexService::saveBacklinks() replaces a page's link set wholesale on each (re)fetch, so reruns stay idempotent. Site gets outboundLinks()/inboundLinks() relations, the latter keyed by domain string since a link's target site may not be indexed yet. Verified against real .es data: 781 backlinks extracted across 205 distinct target domains (social platforms, Trustpilot, genuine inter-site links with correct anchor text), and a rerun of the same fetch left the count unchanged.
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
|
||||
namespace App\Services\CommonCrawl;
|
||||
|
||||
use App\Models\Backlink;
|
||||
use App\Models\Site;
|
||||
use App\Models\SitePage;
|
||||
use App\Support\CommonCrawl\WarcRecordFetcher;
|
||||
@@ -138,8 +139,13 @@ class CommonCrawlIndexService
|
||||
unset($attributes['collection'], $attributes['domain']);
|
||||
$attributes['urlkey_hash'] = sha1($page['urlkey']);
|
||||
|
||||
$links = [];
|
||||
|
||||
if ($withContent) {
|
||||
$attributes = array_merge($attributes, $this->fetchContent($page, $contentCache));
|
||||
$content = $this->fetchContent($page, $contentCache);
|
||||
$links = $content['links'] ?? [];
|
||||
unset($content['links']);
|
||||
$attributes = array_merge($attributes, $content);
|
||||
}
|
||||
|
||||
$result = SitePage::updateOrCreate(
|
||||
@@ -148,6 +154,10 @@ class CommonCrawlIndexService
|
||||
);
|
||||
|
||||
$result->wasRecentlyCreated ? $pagesCreated++ : $pagesUpdated++;
|
||||
|
||||
if ($withContent) {
|
||||
$this->saveBacklinks($result, $site, $links);
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
@@ -158,6 +168,40 @@ class CommonCrawlIndexService
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace a page's recorded outbound links wholesale — simpler and more correct
|
||||
* than diffing against what was there before, since a page's actual link set can
|
||||
* change between crawls.
|
||||
*/
|
||||
private function saveBacklinks(SitePage $sitePage, Site $site, array $links): void
|
||||
{
|
||||
$sitePage->links()->delete();
|
||||
|
||||
if (empty($links)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$now = now();
|
||||
|
||||
$rows = collect($links)
|
||||
->unique('url')
|
||||
->map(fn (array $link) => [
|
||||
'site_id' => $site->id,
|
||||
'site_page_id' => $sitePage->id,
|
||||
'target_url' => $link['url'],
|
||||
'target_url_hash' => sha1($link['url']),
|
||||
'target_domain' => $link['domain'],
|
||||
'anchor_text' => $link['anchor_text'],
|
||||
'rel' => $link['rel'],
|
||||
'created_at' => $now,
|
||||
'updated_at' => $now,
|
||||
])
|
||||
->values()
|
||||
->all();
|
||||
|
||||
Backlink::insert($rows);
|
||||
}
|
||||
|
||||
/**
|
||||
* List every CDX shard file for a collection, straight from the bulk data storage
|
||||
* (data.commoncrawl.org) rather than the index.commoncrawl.org query API — this is
|
||||
|
||||
Reference in New Issue
Block a user