Track and display subdomains under their root domain

Adds sites.root_domain (indexed string, backfilled for existing rows)
computed via a naive "last two labels" heuristic (App\Support\RootDomain)
— good enough for example.com/somesite.es, but wrong for two-part
public suffixes like .co.uk. Doing that correctly needs the Mozilla
Public Suffix List; flagged as a known simplification rather than
built now.

Deliberately not a hard FK: like Backlink.target_domain, a subdomain
can get indexed before its root domain does (or vice versa), so
Site::subdomains()/rootSite() match on the domain string rather than
requiring the relationship to exist at write time.

Site detail page: a root domain shows a paginated "Subdomains"
section (same guest-preview cap as Pages/Backlinks); a subdomain
shows "Subdomain of {root}" linking back, when the root is itself
indexed.

Verified against real data: 21k.ro correctly lists its two known
subdomains (buzau.21k.ro, timisoara.21k.ro), each of which correctly
links back to 21k.ro. Added SubdomainsTest (root lists subdomains,
subdomain links to root, empty state) — full suite (31 tests) passes.
This commit is contained in:
m105
2026-07-25 15:44:12 +03:00
parent 6435bccb20
commit 72b87ab579
8 changed files with 196 additions and 0 deletions
+8
View File
@@ -21,12 +21,20 @@ class SiteController extends Controller
$referringDomainsCount = $site->inboundLinks()->distinct()->count('site_id');
$outboundLinksCount = $site->outboundLinks()->count();
$subdomains = GuestPreview::paginate(
$site->subdomains()->withCount('pages')->orderBy('domain'),
50,
'subdomains_page'
);
return view('sites.show', [
'site' => $site,
'pages' => $pages,
'backlinks' => $backlinks,
'referringDomainsCount' => $referringDomainsCount,
'outboundLinksCount' => $outboundLinksCount,
'subdomains' => $subdomains,
'rootSite' => $site->rootSite(),
]);
}
}
+17
View File
@@ -10,6 +10,7 @@ class Site extends Model
protected $fillable = [
'collection',
'domain',
'root_domain',
'url',
'title',
'description',
@@ -35,4 +36,20 @@ class Site extends Model
{
return $this->hasMany(Backlink::class, 'target_domain', 'domain');
}
/** Other indexed sites whose root domain is this site's domain, e.g. blog.example.com under example.com. */
public function subdomains(): HasMany
{
return $this->hasMany(Site::class, 'root_domain', 'domain')->where('id', '!=', $this->id);
}
/** The root-domain Site this one is a subdomain of, if it's indexed and if this site even is a subdomain. */
public function rootSite(): ?Site
{
if ($this->domain === $this->root_domain) {
return null;
}
return static::where('domain', $this->root_domain)->first();
}
}
@@ -6,6 +6,7 @@ use App\Models\Backlink;
use App\Models\Site;
use App\Models\SitePage;
use App\Support\CommonCrawl\WarcRecordFetcher;
use App\Support\RootDomain;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Http;
@@ -116,6 +117,7 @@ class CommonCrawlIndexService
$homepage = $pages->sort(fn ($a, $b) => $this->compareCaptures($a, $b))->first();
$site->url = $homepage['url'];
$site->root_domain = RootDomain::of($group['domain']);
if ($withContent) {
$content = $this->fetchContent($homepage, $contentCache);
+24
View File
@@ -0,0 +1,24 @@
<?php
namespace App\Support;
/**
* Naive "last two labels" root-domain heuristic — good enough for the common case
* (example.com, somesite.es) but wrong for real-world two-part public suffixes like
* .co.uk or .com.au, where it would treat "co.uk" as the root instead of
* "example.co.uk". Doing that correctly means integrating the Mozilla Public Suffix
* List; this is a deliberate simplification until that's worth the added complexity.
*/
class RootDomain
{
public static function of(string $domain): string
{
$labels = explode('.', strtolower(trim($domain, '.')));
if (count($labels) <= 2) {
return implode('.', $labels);
}
return implode('.', array_slice($labels, -2));
}
}