root/huginn
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add tech-stack fingerprinting (server, powered-by, generator)

Browse Source 
WarcRecordFetcher was splitting out and discarding the HTTP response
headers to get at the HTML body — now it keeps and parses them, so
Server/X-Powered-By headers get captured alongside the <meta
name="generator"> tag already sitting in the parsed DOM. No extra
requests: same WARC segment already fetched for --with-metadata.

Site copies these from its homepage capture, same as
title/description/keywords. New badges on the site detail page show
the detected platform (e.g. "Wix.com Website Builder") and server
signature at a glance.

Verified against real .es data: correctly detected Wix
("00padel.es"), nginx/Apache/Cloudflare server headers, and PHP
X-Powered-By headers across the batch.
This commit is contained in:
m105
2026-07-22 22:14:40 +03:00
parent 6e6be7d7c6
commit c7299fe928
6 changed files with 112 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Support/CommonCrawl/WarcRecordFetcher.php
+47 -9
View File
@@ -47,16 +47,19 @@ class WarcRecordFetcher
return [];
}
$html = $this->extractHtmlFromWarcSegment($response->body());
$segment = $this->extractHtmlFromWarcSegment($response->body());
return $html === null ? [] : $this->extractHtmlMeta($html, $attributes['url']);
return $segment === null
? []
: $this->extractHtmlMeta($segment['html'], $attributes['url'], $segment['headers']);
}
/**
* A WARC segment (gzip-compressed) contains WARC headers, then the raw HTTP
* response (its own headers followed by the HTML body). Strip both header blocks.
* response (its own headers followed by the HTML body). Strip the WARC header
* block; the HTTP header block is kept (for Server/X-Powered-By) alongside the HTML.
*/
private function extractHtmlFromWarcSegment(string $gzipped): ?string
private function extractHtmlFromWarcSegment(string $gzipped): ?array
{
$decompressed = @gzdecode($gzipped);
@@ -72,12 +75,45 @@ class WarcRecordFetcher
[$httpHeaders, $html] = array_pad(preg_split("/\r?\n\r?\n/", $rest, 2), 2, null);
return $html;
return ['headers' => $httpHeaders ?? '', 'html' => $html];
}
private function extractHtmlMeta(string $html, string $pageUrl): array
/**
* Parses a raw HTTP header block (status line + "Name: value" lines) into a
* lowercase-keyed map. First occurrence of a header name wins.
*/
private function parseHttpHeaders(string $raw): array
{
$meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null, 'links' => []];
$headers = [];
foreach (preg_split("/\r?\n/", $raw) as $index => $line) {
if ($index === 0 || ! str_contains($line, ':')) {
continue; // line 0 is the "HTTP/1.1 200 OK" status line, not a header
}
[$name, $value] = explode(':', $line, 2);
$key = strtolower(trim($name));
$headers[$key] ??= trim($value);
}
return $headers;
}
private function extractHtmlMeta(string $html, string $pageUrl, string $rawHttpHeaders): array
{
$headers = $this->parseHttpHeaders($rawHttpHeaders);
$meta = [
'title' => null,
'description' => null,
'keywords' => null,
'generator' => null,
'server' => isset($headers['server']) ? mb_substr($headers['server'], 0, 255) : null,
'powered_by' => isset($headers['x-powered-by']) ? mb_substr($headers['x-powered-by'], 0, 255) : null,
'content' => null,
'links' => [],
];
$document = new DOMDocument;
@@ -93,11 +129,13 @@ class WarcRecordFetcher
$meta['title'] = trim(preg_replace('/\s+/', ' ', $titleNode->textContent)) ?: null;
}
foreach (['description', 'keywords'] as $name) {
foreach (['description', 'keywords', 'generator'] as $name) {
$node = $xpath->query("//meta[translate(@name, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')='{$name}']")->item(0);
if ($node && $node->getAttribute('content') !== '') {
$meta[$name] = trim(preg_replace('/\s+/', ' ', $node->getAttribute('content')));
$value = trim(preg_replace('/\s+/', ' ', $node->getAttribute('content')));
// generator is stored in a varchar column; description/keywords are text.
$meta[$name] = $name === 'generator' ? mb_substr($value, 0, 255) : $value;
}
}