m105
3d0b9a6523
Adds a backlinks table: one row per <a href> on a captured page that points at a different domain (same-site nav links are skipped). Piggybacks on the existing --with-metadata WARC fetch/DOM-parse in WarcRecordFetcher rather than a separate command/job, since extracting links needs the same HTML already being downloaded for title/description/content — no extra requests. WarcRecordFetcher gains href resolution (absolute, protocol-relative, root-relative, query-only, and plain relative hrefs) against the page's own URL. CommonCrawlIndexService::saveBacklinks() replaces a page's link set wholesale on each (re)fetch, so reruns stay idempotent. Site gets outboundLinks()/inboundLinks() relations, the latter keyed by domain string since a link's target site may not be indexed yet. Verified against real .es data: 781 backlinks extracted across 205 distinct target domains (social platforms, Trustpilot, genuine inter-site links with correct anchor text), and a rerun of the same fetch left the count unchanged.
42 lines
1.4 KiB
PHP
42 lines
1.4 KiB
PHP
<?php
|
|
|
|
use Illuminate\Database\Migrations\Migration;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
use Illuminate\Support\Facades\Schema;
|
|
|
|
return new class extends Migration
|
|
{
|
|
/**
|
|
* One row per outbound link found on a captured page, pointing at a different
|
|
* domain — the edges of a web graph, queryable from either end: "what does this
|
|
* site link out to" (site_id) or "who links to this domain" (target_domain).
|
|
*/
|
|
public function up(): void
|
|
{
|
|
Schema::create('backlinks', function (Blueprint $table) {
|
|
$table->id();
|
|
$table->foreignId('site_id')->constrained()->cascadeOnDelete();
|
|
$table->foreignId('site_page_id')->constrained()->cascadeOnDelete();
|
|
$table->text('target_url');
|
|
// target_url is unbounded like site_pages.urlkey was — hash it for the
|
|
// unique index rather than repeat that earlier mistake.
|
|
$table->char('target_url_hash', 40);
|
|
$table->string('target_domain')->nullable();
|
|
$table->string('anchor_text')->nullable();
|
|
$table->string('rel')->nullable();
|
|
$table->timestamps();
|
|
|
|
$table->unique(['site_page_id', 'target_url_hash']);
|
|
$table->index('target_domain');
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Reverse the migrations.
|
|
*/
|
|
public function down(): void
|
|
{
|
|
Schema::dropIfExists('backlinks');
|
|
}
|
|
};
|