From c7299fe92814238bc94a55ed9980e164ad4c654d Mon Sep 17 00:00:00 2001 From: m105 Date: Wed, 22 Jul 2026 22:14:40 +0300 Subject: [PATCH] Add tech-stack fingerprinting (server, powered-by, generator) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WarcRecordFetcher was splitting out and discarding the HTTP response headers to get at the HTML body — now it keeps and parses them, so Server/X-Powered-By headers get captured alongside the tag already sitting in the parsed DOM. No extra requests: same WARC segment already fetched for --with-metadata. Site copies these from its homepage capture, same as title/description/keywords. New badges on the site detail page show the detected platform (e.g. "Wix.com Website Builder") and server signature at a glance. Verified against real .es data: correctly detected Wix ("00padel.es"), nginx/Apache/Cloudflare server headers, and PHP X-Powered-By headers across the batch. --- app/Models/Site.php | 3 + app/Models/SitePage.php | 3 + .../CommonCrawl/CommonCrawlIndexService.php | 3 + app/Support/CommonCrawl/WarcRecordFetcher.php | 56 ++++++++++++++++--- ...26_07_22_221213_add_tech_stack_columns.php | 42 ++++++++++++++ resources/views/sites/show.blade.php | 14 +++++ 6 files changed, 112 insertions(+), 9 deletions(-) create mode 100644 database/migrations/2026_07_22_221213_add_tech_stack_columns.php diff --git a/app/Models/Site.php b/app/Models/Site.php index 44d4832..58b5825 100644 --- a/app/Models/Site.php +++ b/app/Models/Site.php @@ -14,6 +14,9 @@ class Site extends Model 'title', 'description', 'keywords', + 'server', + 'powered_by', + 'generator', ]; public function pages(): HasMany diff --git a/app/Models/SitePage.php b/app/Models/SitePage.php index d365d6b..a081e4a 100644 --- a/app/Models/SitePage.php +++ b/app/Models/SitePage.php @@ -27,6 +27,9 @@ class SitePage extends Model 'title', 'description', 'keywords', + 'server', + 'powered_by', + 'generator', 'content', 'raw', 'record_hash', diff --git a/app/Services/CommonCrawl/CommonCrawlIndexService.php b/app/Services/CommonCrawl/CommonCrawlIndexService.php index 4894938..43e8cfe 100644 --- a/app/Services/CommonCrawl/CommonCrawlIndexService.php +++ b/app/Services/CommonCrawl/CommonCrawlIndexService.php @@ -125,6 +125,9 @@ class CommonCrawlIndexService 'title' => $content['title'], 'description' => $content['description'], 'keywords' => $content['keywords'], + 'server' => $content['server'], + 'powered_by' => $content['powered_by'], + 'generator' => $content['generator'], ]); } } diff --git a/app/Support/CommonCrawl/WarcRecordFetcher.php b/app/Support/CommonCrawl/WarcRecordFetcher.php index 435acbb..bcfcc61 100644 --- a/app/Support/CommonCrawl/WarcRecordFetcher.php +++ b/app/Support/CommonCrawl/WarcRecordFetcher.php @@ -47,16 +47,19 @@ class WarcRecordFetcher return []; } - $html = $this->extractHtmlFromWarcSegment($response->body()); + $segment = $this->extractHtmlFromWarcSegment($response->body()); - return $html === null ? [] : $this->extractHtmlMeta($html, $attributes['url']); + return $segment === null + ? [] + : $this->extractHtmlMeta($segment['html'], $attributes['url'], $segment['headers']); } /** * A WARC segment (gzip-compressed) contains WARC headers, then the raw HTTP - * response (its own headers followed by the HTML body). Strip both header blocks. + * response (its own headers followed by the HTML body). Strip the WARC header + * block; the HTTP header block is kept (for Server/X-Powered-By) alongside the HTML. */ - private function extractHtmlFromWarcSegment(string $gzipped): ?string + private function extractHtmlFromWarcSegment(string $gzipped): ?array { $decompressed = @gzdecode($gzipped); @@ -72,12 +75,45 @@ class WarcRecordFetcher [$httpHeaders, $html] = array_pad(preg_split("/\r?\n\r?\n/", $rest, 2), 2, null); - return $html; + return ['headers' => $httpHeaders ?? '', 'html' => $html]; } - private function extractHtmlMeta(string $html, string $pageUrl): array + /** + * Parses a raw HTTP header block (status line + "Name: value" lines) into a + * lowercase-keyed map. First occurrence of a header name wins. + */ + private function parseHttpHeaders(string $raw): array { - $meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null, 'links' => []]; + $headers = []; + + foreach (preg_split("/\r?\n/", $raw) as $index => $line) { + if ($index === 0 || ! str_contains($line, ':')) { + continue; // line 0 is the "HTTP/1.1 200 OK" status line, not a header + } + + [$name, $value] = explode(':', $line, 2); + $key = strtolower(trim($name)); + + $headers[$key] ??= trim($value); + } + + return $headers; + } + + private function extractHtmlMeta(string $html, string $pageUrl, string $rawHttpHeaders): array + { + $headers = $this->parseHttpHeaders($rawHttpHeaders); + + $meta = [ + 'title' => null, + 'description' => null, + 'keywords' => null, + 'generator' => null, + 'server' => isset($headers['server']) ? mb_substr($headers['server'], 0, 255) : null, + 'powered_by' => isset($headers['x-powered-by']) ? mb_substr($headers['x-powered-by'], 0, 255) : null, + 'content' => null, + 'links' => [], + ]; $document = new DOMDocument; @@ -93,11 +129,13 @@ class WarcRecordFetcher $meta['title'] = trim(preg_replace('/\s+/', ' ', $titleNode->textContent)) ?: null; } - foreach (['description', 'keywords'] as $name) { + foreach (['description', 'keywords', 'generator'] as $name) { $node = $xpath->query("//meta[translate(@name, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')='{$name}']")->item(0); if ($node && $node->getAttribute('content') !== '') { - $meta[$name] = trim(preg_replace('/\s+/', ' ', $node->getAttribute('content'))); + $value = trim(preg_replace('/\s+/', ' ', $node->getAttribute('content'))); + // generator is stored in a varchar column; description/keywords are text. + $meta[$name] = $name === 'generator' ? mb_substr($value, 0, 255) : $value; } } diff --git a/database/migrations/2026_07_22_221213_add_tech_stack_columns.php b/database/migrations/2026_07_22_221213_add_tech_stack_columns.php new file mode 100644 index 0000000..c774172 --- /dev/null +++ b/database/migrations/2026_07_22_221213_add_tech_stack_columns.php @@ -0,0 +1,42 @@ +) — cheap + * tech-stack signals captured for free during the existing --with-metadata WARC fetch. + */ + public function up(): void + { + Schema::table('site_pages', function (Blueprint $table) { + $table->string('server')->nullable()->after('keywords'); + $table->string('powered_by')->nullable()->after('server'); + $table->string('generator')->nullable()->after('powered_by'); + }); + + Schema::table('sites', function (Blueprint $table) { + $table->string('server')->nullable()->after('keywords'); + $table->string('powered_by')->nullable()->after('server'); + $table->string('generator')->nullable()->after('powered_by'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('site_pages', function (Blueprint $table) { + $table->dropColumn(['server', 'powered_by', 'generator']); + }); + + Schema::table('sites', function (Blueprint $table) { + $table->dropColumn(['server', 'powered_by', 'generator']); + }); + } +}; diff --git a/resources/views/sites/show.blade.php b/resources/views/sites/show.blade.php index 1260bd7..29bb858 100644 --- a/resources/views/sites/show.blade.php +++ b/resources/views/sites/show.blade.php @@ -18,6 +18,20 @@

@endif + @if ($site->server || $site->powered_by || $site->generator) +
+ @if ($site->generator) + {{ $site->generator }} + @endif + @if ($site->server) + Server: {{ $site->server }} + @endif + @if ($site->powered_by) + Powered by: {{ $site->powered_by }} + @endif +
+ @endif +

Pages ({{ $pages->total() }})