Add tech-stack fingerprinting (server, powered-by, generator)
WarcRecordFetcher was splitting out and discarding the HTTP response
headers to get at the HTML body — now it keeps and parses them, so
Server/X-Powered-By headers get captured alongside the <meta
name="generator"> tag already sitting in the parsed DOM. No extra
requests: same WARC segment already fetched for --with-metadata.
Site copies these from its homepage capture, same as
title/description/keywords. New badges on the site detail page show
the detected platform (e.g. "Wix.com Website Builder") and server
signature at a glance.
Verified against real .es data: correctly detected Wix
("00padel.es"), nginx/Apache/Cloudflare server headers, and PHP
X-Powered-By headers across the batch.
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* Server/X-Powered-By (from HTTP response headers, already downloaded alongside the
|
||||
* HTML but previously discarded) and generator (from <meta name="generator">) — cheap
|
||||
* tech-stack signals captured for free during the existing --with-metadata WARC fetch.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
Schema::table('site_pages', function (Blueprint $table) {
|
||||
$table->string('server')->nullable()->after('keywords');
|
||||
$table->string('powered_by')->nullable()->after('server');
|
||||
$table->string('generator')->nullable()->after('powered_by');
|
||||
});
|
||||
|
||||
Schema::table('sites', function (Blueprint $table) {
|
||||
$table->string('server')->nullable()->after('keywords');
|
||||
$table->string('powered_by')->nullable()->after('server');
|
||||
$table->string('generator')->nullable()->after('powered_by');
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the migrations.
|
||||
*/
|
||||
public function down(): void
|
||||
{
|
||||
Schema::table('site_pages', function (Blueprint $table) {
|
||||
$table->dropColumn(['server', 'powered_by', 'generator']);
|
||||
});
|
||||
|
||||
Schema::table('sites', function (Blueprint $table) {
|
||||
$table->dropColumn(['server', 'powered_by', 'generator']);
|
||||
});
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user