From 72b87ab5799ee188392f59da0e467434cd1315df Mon Sep 17 00:00:00 2001
From: m105 <m105@corrupt.webhop.me>
Date: Sat, 25 Jul 2026 15:44:12 +0300
Subject: [PATCH] Track and display subdomains under their root domain
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds sites.root_domain (indexed string, backfilled for existing rows)
computed via a naive "last two labels" heuristic (App\Support\RootDomain)
— good enough for example.com/somesite.es, but wrong for two-part
public suffixes like .co.uk. Doing that correctly needs the Mozilla
Public Suffix List; flagged as a known simplification rather than
built now.

Deliberately not a hard FK: like Backlink.target_domain, a subdomain
can get indexed before its root domain does (or vice versa), so
Site::subdomains()/rootSite() match on the domain string rather than
requiring the relationship to exist at write time.

Site detail page: a root domain shows a paginated "Subdomains"
section (same guest-preview cap as Pages/Backlinks); a subdomain
shows "Subdomain of {root}" linking back, when the root is itself
indexed.

Verified against real data: 21k.ro correctly lists its two known
subdomains (buzau.21k.ro, timisoara.21k.ro), each of which correctly
links back to 21k.ro. Added SubdomainsTest (root lists subdomains,
subdomain links to root, empty state) — full suite (31 tests) passes.
---
 app/Http/Controllers/SiteController.php       |  8 +++
 app/Models/Site.php                           | 17 ++++++
 .../CommonCrawl/CommonCrawlIndexService.php   |  2 +
 app/Support/RootDomain.php                    | 24 +++++++++
 ..._154034_add_root_domain_to_sites_table.php | 45 ++++++++++++++++
 resources/views/sites/show.blade.php          | 46 ++++++++++++++++
 tests/Feature/GuestPreviewTest.php            |  1 +
 tests/Feature/SubdomainsTest.php              | 53 +++++++++++++++++++
 8 files changed, 196 insertions(+)
 create mode 100644 app/Support/RootDomain.php
 create mode 100644 database/migrations/2026_07_25_154034_add_root_domain_to_sites_table.php
 create mode 100644 tests/Feature/SubdomainsTest.php

diff --git a/app/Http/Controllers/SiteController.php b/app/Http/Controllers/SiteController.php
index 843db21..2b08d7b 100644
--- a/app/Http/Controllers/SiteController.php
+++ b/app/Http/Controllers/SiteController.php
@@ -21,12 +21,20 @@ class SiteController extends Controller
         $referringDomainsCount = $site->inboundLinks()->distinct()->count('site_id');
         $outboundLinksCount = $site->outboundLinks()->count();
 
+        $subdomains = GuestPreview::paginate(
+            $site->subdomains()->withCount('pages')->orderBy('domain'),
+            50,
+            'subdomains_page'
+        );
+
         return view('sites.show', [
             'site' => $site,
             'pages' => $pages,
             'backlinks' => $backlinks,
             'referringDomainsCount' => $referringDomainsCount,
             'outboundLinksCount' => $outboundLinksCount,
+            'subdomains' => $subdomains,
+            'rootSite' => $site->rootSite(),
         ]);
     }
 }
diff --git a/app/Models/Site.php b/app/Models/Site.php
index 58b5825..406a9e0 100644
--- a/app/Models/Site.php
+++ b/app/Models/Site.php
@@ -10,6 +10,7 @@ class Site extends Model
     protected $fillable = [
         'collection',
         'domain',
+        'root_domain',
         'url',
         'title',
         'description',
@@ -35,4 +36,20 @@ class Site extends Model
     {
         return $this->hasMany(Backlink::class, 'target_domain', 'domain');
     }
+
+    /** Other indexed sites whose root domain is this site's domain, e.g. blog.example.com under example.com. */
+    public function subdomains(): HasMany
+    {
+        return $this->hasMany(Site::class, 'root_domain', 'domain')->where('id', '!=', $this->id);
+    }
+
+    /** The root-domain Site this one is a subdomain of, if it's indexed and if this site even is a subdomain. */
+    public function rootSite(): ?Site
+    {
+        if ($this->domain === $this->root_domain) {
+            return null;
+        }
+
+        return static::where('domain', $this->root_domain)->first();
+    }
 }
diff --git a/app/Services/CommonCrawl/CommonCrawlIndexService.php b/app/Services/CommonCrawl/CommonCrawlIndexService.php
index 43e8cfe..9d5c3f1 100644
--- a/app/Services/CommonCrawl/CommonCrawlIndexService.php
+++ b/app/Services/CommonCrawl/CommonCrawlIndexService.php
@@ -6,6 +6,7 @@ use App\Models\Backlink;
 use App\Models\Site;
 use App\Models\SitePage;
 use App\Support\CommonCrawl\WarcRecordFetcher;
+use App\Support\RootDomain;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Http;
 
@@ -116,6 +117,7 @@ class CommonCrawlIndexService
 
         $homepage = $pages->sort(fn ($a, $b) => $this->compareCaptures($a, $b))->first();
         $site->url = $homepage['url'];
+        $site->root_domain = RootDomain::of($group['domain']);
 
         if ($withContent) {
             $content = $this->fetchContent($homepage, $contentCache);
diff --git a/app/Support/RootDomain.php b/app/Support/RootDomain.php
new file mode 100644
index 0000000..86db33d
--- /dev/null
+++ b/app/Support/RootDomain.php
@@ -0,0 +1,24 @@
+<?php
+
+namespace App\Support;
+
+/**
+ * Naive "last two labels" root-domain heuristic — good enough for the common case
+ * (example.com, somesite.es) but wrong for real-world two-part public suffixes like
+ * .co.uk or .com.au, where it would treat "co.uk" as the root instead of
+ * "example.co.uk". Doing that correctly means integrating the Mozilla Public Suffix
+ * List; this is a deliberate simplification until that's worth the added complexity.
+ */
+class RootDomain
+{
+    public static function of(string $domain): string
+    {
+        $labels = explode('.', strtolower(trim($domain, '.')));
+
+        if (count($labels) <= 2) {
+            return implode('.', $labels);
+        }
+
+        return implode('.', array_slice($labels, -2));
+    }
+}
diff --git a/database/migrations/2026_07_25_154034_add_root_domain_to_sites_table.php b/database/migrations/2026_07_25_154034_add_root_domain_to_sites_table.php
new file mode 100644
index 0000000..cfe9992
--- /dev/null
+++ b/database/migrations/2026_07_25_154034_add_root_domain_to_sites_table.php
@@ -0,0 +1,45 @@
+<?php
+
+use App\Support\RootDomain;
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * root_domain lets a site's subdomains be found without a hard FK (which would
+     * need backfilling later if the root domain gets crawled after its subdomains do)
+     * — same reasoning as Backlink.target_domain being a plain indexed string rather
+     * than a foreign key to a site that might not exist yet.
+     */
+    public function up(): void
+    {
+        Schema::table('sites', function (Blueprint $table) {
+            $table->string('root_domain')->nullable()->after('domain');
+        });
+
+        DB::table('sites')->orderBy('id')->select('id', 'domain')->chunkById(500, function ($rows) {
+            foreach ($rows as $row) {
+                DB::table('sites')->where('id', $row->id)->update(['root_domain' => RootDomain::of($row->domain)]);
+            }
+        });
+
+        Schema::table('sites', function (Blueprint $table) {
+            $table->string('root_domain')->nullable(false)->change();
+            $table->index('root_domain');
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('sites', function (Blueprint $table) {
+            $table->dropIndex(['root_domain']);
+            $table->dropColumn('root_domain');
+        });
+    }
+};
diff --git a/resources/views/sites/show.blade.php b/resources/views/sites/show.blade.php
index b543de2..9e89eab 100644
--- a/resources/views/sites/show.blade.php
+++ b/resources/views/sites/show.blade.php
@@ -4,6 +4,13 @@
 
         <h1 class="mt-2 text-2xl font-semibold">{{ $site->domain }}</h1>
 
+        @if ($rootSite)
+            <p class="mt-1 text-sm text-neutral-500">
+                Subdomain of
+                <a href="{{ route('sites.show', $rootSite) }}" class="text-blue-600 hover:underline dark:text-blue-400">{{ $rootSite->domain }}</a>
+            </p>
+        @endif
+
         @if ($site->title)
             <p class="mt-1 text-neutral-700 dark:text-neutral-300">{{ $site->title }}</p>
         @endif
@@ -137,5 +144,44 @@
                 <x-guest-preview-notice :paginator="$backlinks" />
             @endauth
         </div>
+
+        <h2 class="mt-10 text-lg font-medium">Subdomains ({{ $subdomains->total() }})</h2>
+
+        <div class="mt-4 overflow-hidden rounded-lg border border-neutral-200 dark:border-neutral-800">
+            <table class="min-w-full divide-y divide-neutral-200 dark:divide-neutral-800">
+                <thead class="bg-neutral-50 dark:bg-neutral-900">
+                    <tr>
+                        <th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-neutral-500">Subdomain</th>
+                        <th class="px-4 py-2 text-left text-xs font-medium uppercase tracking-wide text-neutral-500">Title</th>
+                        <th class="px-4 py-2 text-right text-xs font-medium uppercase tracking-wide text-neutral-500">Pages</th>
+                    </tr>
+                </thead>
+                <tbody class="divide-y divide-neutral-200 dark:divide-neutral-800">
+                    @forelse ($subdomains as $subdomain)
+                        <tr>
+                            <td class="px-4 py-2 text-sm">
+                                <a href="{{ route('sites.show', $subdomain) }}" class="font-medium text-blue-600 hover:underline dark:text-blue-400">
+                                    {{ $subdomain->domain }}
+                                </a>
+                            </td>
+                            <td class="px-4 py-2 text-sm text-neutral-600 dark:text-neutral-400">{{ $subdomain->title ?? '—' }}</td>
+                            <td class="px-4 py-2 text-right text-sm text-neutral-600 dark:text-neutral-400">{{ $subdomain->pages_count }}</td>
+                        </tr>
+                    @empty
+                        <tr>
+                            <td colspan="3" class="px-4 py-6 text-center text-sm text-neutral-500">No subdomains found.</td>
+                        </tr>
+                    @endforelse
+                </tbody>
+            </table>
+        </div>
+
+        <div class="mt-4">
+            @auth
+                {{ $subdomains->links() }}
+            @else
+                <x-guest-preview-notice :paginator="$subdomains" />
+            @endauth
+        </div>
     </div>
 </x-layouts.app>
diff --git a/tests/Feature/GuestPreviewTest.php b/tests/Feature/GuestPreviewTest.php
index 26dc99b..771a3dc 100644
--- a/tests/Feature/GuestPreviewTest.php
+++ b/tests/Feature/GuestPreviewTest.php
@@ -17,6 +17,7 @@ class GuestPreviewTest extends TestCase
             Site::create([
                 'collection' => 'CC-MAIN-TEST',
                 'domain' => "example{$i}.com",
+                'root_domain' => "example{$i}.com",
                 'url' => "https://example{$i}.com/",
             ]);
         }
diff --git a/tests/Feature/SubdomainsTest.php b/tests/Feature/SubdomainsTest.php
new file mode 100644
index 0000000..7c82719
--- /dev/null
+++ b/tests/Feature/SubdomainsTest.php
@@ -0,0 +1,53 @@
+<?php
+
+namespace Tests\Feature;
+
+use App\Models\Site;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Tests\TestCase;
+
+class SubdomainsTest extends TestCase
+{
+    use RefreshDatabase;
+
+    public function test_root_domain_page_lists_its_subdomains(): void
+    {
+        $root = Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'example.com', 'root_domain' => 'example.com']);
+        Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'blog.example.com', 'root_domain' => 'example.com']);
+        Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'shop.example.com', 'root_domain' => 'example.com']);
+        Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'unrelated.net', 'root_domain' => 'unrelated.net']);
+
+        $response = $this->actingAs(User::factory()->create())->get(route('sites.show', $root));
+
+        $response->assertOk();
+        $response->assertSeeText('Subdomains (2)');
+        $response->assertSeeText('blog.example.com');
+        $response->assertSeeText('shop.example.com');
+        $response->assertDontSeeText('unrelated.net');
+    }
+
+    public function test_subdomain_page_links_back_to_its_root(): void
+    {
+        $root = Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'example.com', 'root_domain' => 'example.com']);
+        $sub = Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'blog.example.com', 'root_domain' => 'example.com']);
+
+        $response = $this->actingAs(User::factory()->create())->get(route('sites.show', $sub));
+
+        $response->assertOk();
+        $response->assertSeeText('Subdomain of');
+        $response->assertSeeText($root->domain);
+    }
+
+    public function test_root_domain_with_no_subdomains_shows_empty_state(): void
+    {
+        $root = Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'lonely.com', 'root_domain' => 'lonely.com']);
+
+        $response = $this->actingAs(User::factory()->create())->get(route('sites.show', $root));
+
+        $response->assertOk();
+        $response->assertSeeText('Subdomains (0)');
+        $response->assertSeeText('No subdomains found.');
+        $response->assertDontSeeText('Subdomain of');
+    }
+}