Initial commit: Laravel scaffold + Common Crawl site indexing

Adds Site/SitePage models and the crawlindex:fetch command, which pulls
CDX index records from Common Crawl and optionally enriches them with
page title/description/keywords/body text extracted from the WARC
archive. CDX/WARC access and the site/page upsert logic live in
CommonCrawlIndexService and WarcRecordFetcher.
This commit is contained in:
m105
2026-07-20 00:04:45 +03:00
commit 1c9fc5d423
64 changed files with 11338 additions and 0 deletions
@@ -0,0 +1,35 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::create('sites', function (Blueprint $table) {
$table->id();
$table->string('collection');
$table->string('domain');
$table->text('url')->nullable();
$table->string('title')->nullable();
$table->text('description')->nullable();
$table->text('keywords')->nullable();
$table->timestamps();
$table->unique(['collection', 'domain']);
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::dropIfExists('sites');
}
};