diff --git a/app/Models/Backlink.php b/app/Models/Backlink.php new file mode 100644 index 0000000..cc5c995 --- /dev/null +++ b/app/Models/Backlink.php @@ -0,0 +1,29 @@ +belongsTo(Site::class); + } + + public function sitePage(): BelongsTo + { + return $this->belongsTo(SitePage::class); + } +} diff --git a/app/Models/Site.php b/app/Models/Site.php index 283d690..44d4832 100644 --- a/app/Models/Site.php +++ b/app/Models/Site.php @@ -20,4 +20,16 @@ class Site extends Model { return $this->hasMany(SitePage::class); } + + /** Links this site's pages make out to other domains. */ + public function outboundLinks(): HasMany + { + return $this->hasMany(Backlink::class); + } + + /** Backlinks pointing at this site, i.e. rows recorded against its domain as the target. */ + public function inboundLinks(): HasMany + { + return $this->hasMany(Backlink::class, 'target_domain', 'domain'); + } } diff --git a/app/Models/SitePage.php b/app/Models/SitePage.php index 3af342b..d365d6b 100644 --- a/app/Models/SitePage.php +++ b/app/Models/SitePage.php @@ -4,6 +4,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; +use Illuminate\Database\Eloquent\Relations\HasMany; class SitePage extends Model { @@ -42,4 +43,9 @@ class SitePage extends Model { return $this->belongsTo(Site::class); } + + public function links(): HasMany + { + return $this->hasMany(Backlink::class); + } } diff --git a/app/Services/CommonCrawl/CommonCrawlIndexService.php b/app/Services/CommonCrawl/CommonCrawlIndexService.php index ad47ef2..4894938 100644 --- a/app/Services/CommonCrawl/CommonCrawlIndexService.php +++ b/app/Services/CommonCrawl/CommonCrawlIndexService.php @@ -2,6 +2,7 @@ namespace App\Services\CommonCrawl; +use App\Models\Backlink; use App\Models\Site; use App\Models\SitePage; use App\Support\CommonCrawl\WarcRecordFetcher; @@ -138,8 +139,13 @@ class CommonCrawlIndexService unset($attributes['collection'], $attributes['domain']); $attributes['urlkey_hash'] = sha1($page['urlkey']); + $links = []; + if ($withContent) { - $attributes = array_merge($attributes, $this->fetchContent($page, $contentCache)); + $content = $this->fetchContent($page, $contentCache); + $links = $content['links'] ?? []; + unset($content['links']); + $attributes = array_merge($attributes, $content); } $result = SitePage::updateOrCreate( @@ -148,6 +154,10 @@ class CommonCrawlIndexService ); $result->wasRecentlyCreated ? $pagesCreated++ : $pagesUpdated++; + + if ($withContent) { + $this->saveBacklinks($result, $site, $links); + } } return [ @@ -158,6 +168,40 @@ class CommonCrawlIndexService ]; } + /** + * Replace a page's recorded outbound links wholesale — simpler and more correct + * than diffing against what was there before, since a page's actual link set can + * change between crawls. + */ + private function saveBacklinks(SitePage $sitePage, Site $site, array $links): void + { + $sitePage->links()->delete(); + + if (empty($links)) { + return; + } + + $now = now(); + + $rows = collect($links) + ->unique('url') + ->map(fn (array $link) => [ + 'site_id' => $site->id, + 'site_page_id' => $sitePage->id, + 'target_url' => $link['url'], + 'target_url_hash' => sha1($link['url']), + 'target_domain' => $link['domain'], + 'anchor_text' => $link['anchor_text'], + 'rel' => $link['rel'], + 'created_at' => $now, + 'updated_at' => $now, + ]) + ->values() + ->all(); + + Backlink::insert($rows); + } + /** * List every CDX shard file for a collection, straight from the bulk data storage * (data.commoncrawl.org) rather than the index.commoncrawl.org query API — this is diff --git a/app/Support/CommonCrawl/WarcRecordFetcher.php b/app/Support/CommonCrawl/WarcRecordFetcher.php index 83b0755..435acbb 100644 --- a/app/Support/CommonCrawl/WarcRecordFetcher.php +++ b/app/Support/CommonCrawl/WarcRecordFetcher.php @@ -49,7 +49,7 @@ class WarcRecordFetcher $html = $this->extractHtmlFromWarcSegment($response->body()); - return $html === null ? [] : $this->extractHtmlMeta($html); + return $html === null ? [] : $this->extractHtmlMeta($html, $attributes['url']); } /** @@ -75,9 +75,9 @@ class WarcRecordFetcher return $html; } - private function extractHtmlMeta(string $html): array + private function extractHtmlMeta(string $html, string $pageUrl): array { - $meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null]; + $meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null, 'links' => []]; $document = new DOMDocument; @@ -102,10 +102,110 @@ class WarcRecordFetcher } $meta['content'] = $this->extractVisibleText($xpath); + $meta['links'] = $this->extractOutboundLinks($xpath, $pageUrl); return $meta; } + /** + * Every that resolves to an absolute http(s) URL on a *different* host than + * the page itself — same-site navigation links aren't backlinks. Relative/root- + * relative/protocol-relative hrefs are resolved against the page's own URL. + */ + private function extractOutboundLinks(DOMXPath $xpath, string $pageUrl): array + { + $pageHost = strtolower((string) parse_url($pageUrl, PHP_URL_HOST)); + $links = []; + + foreach ($xpath->query('//a[@href]') as $node) { + $resolved = $this->resolveUrl($pageUrl, $node->getAttribute('href')); + + if ($resolved === null) { + continue; + } + + $targetHost = strtolower((string) parse_url($resolved, PHP_URL_HOST)); + + if ($targetHost === '' || $targetHost === $pageHost) { + continue; + } + + $links[] = [ + 'url' => $resolved, + 'domain' => $targetHost, + 'anchor_text' => mb_substr(trim(preg_replace('/\s+/', ' ', $node->textContent)) ?: '', 0, 255) ?: null, + 'rel' => $node->getAttribute('rel') ?: null, + ]; + } + + return $links; + } + + /** + * Resolve a (possibly relative) href against the page's own URL into an absolute + * http(s) URL. Handles absolute, protocol-relative ("//host/path"), root-relative + * ("/path"), query-only ("?x=1"), and plain relative ("path") hrefs; not a full + * RFC 3986 implementation, but covers what real-world HTML actually uses. + */ + private function resolveUrl(string $base, string $href): ?string + { + $href = trim($href); + + if ($href === '' || str_starts_with($href, '#')) { + return null; + } + + if (preg_match('/^[a-z][a-z0-9+.\-]*:/i', $href) === 1) { + return str_starts_with(strtolower($href), 'http') ? $href : null; + } + + $baseParts = parse_url($base); + + if (! $baseParts || empty($baseParts['scheme']) || empty($baseParts['host'])) { + return null; + } + + $origin = $baseParts['scheme'].'://'.$baseParts['host'].(isset($baseParts['port']) ? ':'.$baseParts['port'] : ''); + + if (str_starts_with($href, '//')) { + return $baseParts['scheme'].':'.$href; + } + + if (str_starts_with($href, '/')) { + return $origin.$href; + } + + if (str_starts_with($href, '?')) { + return $origin.($baseParts['path'] ?? '/').$href; + } + + $basePath = $baseParts['path'] ?? '/'; + $baseDir = substr($basePath, 0, strrpos($basePath, '/') + 1) ?: '/'; + + return $origin.$this->normalizePath($baseDir.$href); + } + + private function normalizePath(string $path): string + { + $stack = []; + + foreach (explode('/', $path) as $segment) { + if ($segment === '.' || $segment === '') { + continue; + } + + if ($segment === '..') { + array_pop($stack); + + continue; + } + + $stack[] = $segment; + } + + return '/'.implode('/', $stack).(str_ends_with($path, '/') ? '/' : ''); + } + /** * Pull the page's visible body text (for later keyword/product search — e.g. "does * this site sell honey?"), stripped of script/style/noscript content and collapsed diff --git a/database/migrations/2026_07_22_212610_create_backlinks_table.php b/database/migrations/2026_07_22_212610_create_backlinks_table.php new file mode 100644 index 0000000..47819a1 --- /dev/null +++ b/database/migrations/2026_07_22_212610_create_backlinks_table.php @@ -0,0 +1,41 @@ +id(); + $table->foreignId('site_id')->constrained()->cascadeOnDelete(); + $table->foreignId('site_page_id')->constrained()->cascadeOnDelete(); + $table->text('target_url'); + // target_url is unbounded like site_pages.urlkey was — hash it for the + // unique index rather than repeat that earlier mistake. + $table->char('target_url_hash', 40); + $table->string('target_domain')->nullable(); + $table->string('anchor_text')->nullable(); + $table->string('rel')->nullable(); + $table->timestamps(); + + $table->unique(['site_page_id', 'target_url_hash']); + $table->index('target_domain'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('backlinks'); + } +};