m105
3d0b9a6523
Adds a backlinks table: one row per <a href> on a captured page that points at a different domain (same-site nav links are skipped). Piggybacks on the existing --with-metadata WARC fetch/DOM-parse in WarcRecordFetcher rather than a separate command/job, since extracting links needs the same HTML already being downloaded for title/description/content — no extra requests. WarcRecordFetcher gains href resolution (absolute, protocol-relative, root-relative, query-only, and plain relative hrefs) against the page's own URL. CommonCrawlIndexService::saveBacklinks() replaces a page's link set wholesale on each (re)fetch, so reruns stay idempotent. Site gets outboundLinks()/inboundLinks() relations, the latter keyed by domain string since a link's target site may not be indexed yet. Verified against real .es data: 781 backlinks extracted across 205 distinct target domains (social platforms, Trustpilot, genuine inter-site links with correct anchor text), and a rerun of the same fetch left the count unchanged.
52 lines
998 B
PHP
52 lines
998 B
PHP
<?php
|
|
|
|
namespace App\Models;
|
|
|
|
use Illuminate\Database\Eloquent\Model;
|
|
use Illuminate\Database\Eloquent\Relations\BelongsTo;
|
|
use Illuminate\Database\Eloquent\Relations\HasMany;
|
|
|
|
class SitePage extends Model
|
|
{
|
|
protected $fillable = [
|
|
'site_id',
|
|
'urlkey',
|
|
'urlkey_hash',
|
|
'url',
|
|
'timestamp',
|
|
'mime',
|
|
'mime_detected',
|
|
'status',
|
|
'digest',
|
|
'length',
|
|
'offset',
|
|
'filename',
|
|
'languages',
|
|
'charset',
|
|
'redirect',
|
|
'title',
|
|
'description',
|
|
'keywords',
|
|
'content',
|
|
'raw',
|
|
'record_hash',
|
|
];
|
|
|
|
protected $casts = [
|
|
'raw' => 'array',
|
|
'status' => 'integer',
|
|
'length' => 'integer',
|
|
'offset' => 'integer',
|
|
];
|
|
|
|
public function site(): BelongsTo
|
|
{
|
|
return $this->belongsTo(Site::class);
|
|
}
|
|
|
|
public function links(): HasMany
|
|
{
|
|
return $this->hasMany(Backlink::class);
|
|
}
|
|
}
|