diff --git a/app/Http/Controllers/SiteController.php b/app/Http/Controllers/SiteController.php index 843db21..2b08d7b 100644 --- a/app/Http/Controllers/SiteController.php +++ b/app/Http/Controllers/SiteController.php @@ -21,12 +21,20 @@ class SiteController extends Controller $referringDomainsCount = $site->inboundLinks()->distinct()->count('site_id'); $outboundLinksCount = $site->outboundLinks()->count(); + $subdomains = GuestPreview::paginate( + $site->subdomains()->withCount('pages')->orderBy('domain'), + 50, + 'subdomains_page' + ); + return view('sites.show', [ 'site' => $site, 'pages' => $pages, 'backlinks' => $backlinks, 'referringDomainsCount' => $referringDomainsCount, 'outboundLinksCount' => $outboundLinksCount, + 'subdomains' => $subdomains, + 'rootSite' => $site->rootSite(), ]); } } diff --git a/app/Models/Site.php b/app/Models/Site.php index 58b5825..406a9e0 100644 --- a/app/Models/Site.php +++ b/app/Models/Site.php @@ -10,6 +10,7 @@ class Site extends Model protected $fillable = [ 'collection', 'domain', + 'root_domain', 'url', 'title', 'description', @@ -35,4 +36,20 @@ class Site extends Model { return $this->hasMany(Backlink::class, 'target_domain', 'domain'); } + + /** Other indexed sites whose root domain is this site's domain, e.g. blog.example.com under example.com. */ + public function subdomains(): HasMany + { + return $this->hasMany(Site::class, 'root_domain', 'domain')->where('id', '!=', $this->id); + } + + /** The root-domain Site this one is a subdomain of, if it's indexed and if this site even is a subdomain. */ + public function rootSite(): ?Site + { + if ($this->domain === $this->root_domain) { + return null; + } + + return static::where('domain', $this->root_domain)->first(); + } } diff --git a/app/Services/CommonCrawl/CommonCrawlIndexService.php b/app/Services/CommonCrawl/CommonCrawlIndexService.php index 43e8cfe..9d5c3f1 100644 --- a/app/Services/CommonCrawl/CommonCrawlIndexService.php +++ b/app/Services/CommonCrawl/CommonCrawlIndexService.php @@ -6,6 +6,7 @@ use App\Models\Backlink; use App\Models\Site; use App\Models\SitePage; use App\Support\CommonCrawl\WarcRecordFetcher; +use App\Support\RootDomain; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Http; @@ -116,6 +117,7 @@ class CommonCrawlIndexService $homepage = $pages->sort(fn ($a, $b) => $this->compareCaptures($a, $b))->first(); $site->url = $homepage['url']; + $site->root_domain = RootDomain::of($group['domain']); if ($withContent) { $content = $this->fetchContent($homepage, $contentCache); diff --git a/app/Support/RootDomain.php b/app/Support/RootDomain.php new file mode 100644 index 0000000..86db33d --- /dev/null +++ b/app/Support/RootDomain.php @@ -0,0 +1,24 @@ +string('root_domain')->nullable()->after('domain'); + }); + + DB::table('sites')->orderBy('id')->select('id', 'domain')->chunkById(500, function ($rows) { + foreach ($rows as $row) { + DB::table('sites')->where('id', $row->id)->update(['root_domain' => RootDomain::of($row->domain)]); + } + }); + + Schema::table('sites', function (Blueprint $table) { + $table->string('root_domain')->nullable(false)->change(); + $table->index('root_domain'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('sites', function (Blueprint $table) { + $table->dropIndex(['root_domain']); + $table->dropColumn('root_domain'); + }); + } +}; diff --git a/resources/views/sites/show.blade.php b/resources/views/sites/show.blade.php index b543de2..9e89eab 100644 --- a/resources/views/sites/show.blade.php +++ b/resources/views/sites/show.blade.php @@ -4,6 +4,13 @@

{{ $site->domain }}

+ @if ($rootSite) +

+ Subdomain of + {{ $rootSite->domain }} +

+ @endif + @if ($site->title)

{{ $site->title }}

@endif @@ -137,5 +144,44 @@ @endauth + +

Subdomains ({{ $subdomains->total() }})

+ +
+ + + + + + + + + + @forelse ($subdomains as $subdomain) + + + + + + @empty + + + + @endforelse + +
SubdomainTitlePages
+ + {{ $subdomain->domain }} + + {{ $subdomain->title ?? '—' }}{{ $subdomain->pages_count }}
No subdomains found.
+
+ +
+ @auth + {{ $subdomains->links() }} + @else + + @endauth +
diff --git a/tests/Feature/GuestPreviewTest.php b/tests/Feature/GuestPreviewTest.php index 26dc99b..771a3dc 100644 --- a/tests/Feature/GuestPreviewTest.php +++ b/tests/Feature/GuestPreviewTest.php @@ -17,6 +17,7 @@ class GuestPreviewTest extends TestCase Site::create([ 'collection' => 'CC-MAIN-TEST', 'domain' => "example{$i}.com", + 'root_domain' => "example{$i}.com", 'url' => "https://example{$i}.com/", ]); } diff --git a/tests/Feature/SubdomainsTest.php b/tests/Feature/SubdomainsTest.php new file mode 100644 index 0000000..7c82719 --- /dev/null +++ b/tests/Feature/SubdomainsTest.php @@ -0,0 +1,53 @@ + 'CC-MAIN-TEST', 'domain' => 'example.com', 'root_domain' => 'example.com']); + Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'blog.example.com', 'root_domain' => 'example.com']); + Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'shop.example.com', 'root_domain' => 'example.com']); + Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'unrelated.net', 'root_domain' => 'unrelated.net']); + + $response = $this->actingAs(User::factory()->create())->get(route('sites.show', $root)); + + $response->assertOk(); + $response->assertSeeText('Subdomains (2)'); + $response->assertSeeText('blog.example.com'); + $response->assertSeeText('shop.example.com'); + $response->assertDontSeeText('unrelated.net'); + } + + public function test_subdomain_page_links_back_to_its_root(): void + { + $root = Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'example.com', 'root_domain' => 'example.com']); + $sub = Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'blog.example.com', 'root_domain' => 'example.com']); + + $response = $this->actingAs(User::factory()->create())->get(route('sites.show', $sub)); + + $response->assertOk(); + $response->assertSeeText('Subdomain of'); + $response->assertSeeText($root->domain); + } + + public function test_root_domain_with_no_subdomains_shows_empty_state(): void + { + $root = Site::create(['collection' => 'CC-MAIN-TEST', 'domain' => 'lonely.com', 'root_domain' => 'lonely.com']); + + $response = $this->actingAs(User::factory()->create())->get(route('sites.show', $root)); + + $response->assertOk(); + $response->assertSeeText('Subdomains (0)'); + $response->assertSeeText('No subdomains found.'); + $response->assertDontSeeText('Subdomain of'); + } +}