Extract and store outbound links (backlinks) during content fetch
Adds a backlinks table: one row per <a href> on a captured page that points at a different domain (same-site nav links are skipped). Piggybacks on the existing --with-metadata WARC fetch/DOM-parse in WarcRecordFetcher rather than a separate command/job, since extracting links needs the same HTML already being downloaded for title/description/content — no extra requests. WarcRecordFetcher gains href resolution (absolute, protocol-relative, root-relative, query-only, and plain relative hrefs) against the page's own URL. CommonCrawlIndexService::saveBacklinks() replaces a page's link set wholesale on each (re)fetch, so reruns stay idempotent. Site gets outboundLinks()/inboundLinks() relations, the latter keyed by domain string since a link's target site may not be indexed yet. Verified against real .es data: 781 backlinks extracted across 205 distinct target domains (social platforms, Trustpilot, genuine inter-site links with correct anchor text), and a rerun of the same fetch left the count unchanged.
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Database\Eloquent\Relations\BelongsTo;
|
||||
|
||||
class Backlink extends Model
|
||||
{
|
||||
protected $fillable = [
|
||||
'site_id',
|
||||
'site_page_id',
|
||||
'target_url',
|
||||
'target_url_hash',
|
||||
'target_domain',
|
||||
'anchor_text',
|
||||
'rel',
|
||||
];
|
||||
|
||||
public function site(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(Site::class);
|
||||
}
|
||||
|
||||
public function sitePage(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(SitePage::class);
|
||||
}
|
||||
}
|
||||
@@ -20,4 +20,16 @@ class Site extends Model
|
||||
{
|
||||
return $this->hasMany(SitePage::class);
|
||||
}
|
||||
|
||||
/** Links this site's pages make out to other domains. */
|
||||
public function outboundLinks(): HasMany
|
||||
{
|
||||
return $this->hasMany(Backlink::class);
|
||||
}
|
||||
|
||||
/** Backlinks pointing at this site, i.e. rows recorded against its domain as the target. */
|
||||
public function inboundLinks(): HasMany
|
||||
{
|
||||
return $this->hasMany(Backlink::class, 'target_domain', 'domain');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Database\Eloquent\Relations\BelongsTo;
|
||||
use Illuminate\Database\Eloquent\Relations\HasMany;
|
||||
|
||||
class SitePage extends Model
|
||||
{
|
||||
@@ -42,4 +43,9 @@ class SitePage extends Model
|
||||
{
|
||||
return $this->belongsTo(Site::class);
|
||||
}
|
||||
|
||||
public function links(): HasMany
|
||||
{
|
||||
return $this->hasMany(Backlink::class);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user