Initial commit: Laravel scaffold + Common Crawl site indexing
Adds Site/SitePage models and the crawlindex:fetch command, which pulls CDX index records from Common Crawl and optionally enriches them with page title/description/keywords/body text extracted from the WARC archive. CDX/WARC access and the site/page upsert logic live in CommonCrawlIndexService and WarcRecordFetcher.
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* Run the migrations.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
Schema::create('sites', function (Blueprint $table) {
|
||||
$table->id();
|
||||
$table->string('collection');
|
||||
$table->string('domain');
|
||||
$table->text('url')->nullable();
|
||||
$table->string('title')->nullable();
|
||||
$table->text('description')->nullable();
|
||||
$table->text('keywords')->nullable();
|
||||
$table->timestamps();
|
||||
|
||||
$table->unique(['collection', 'domain']);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the migrations.
|
||||
*/
|
||||
public function down(): void
|
||||
{
|
||||
Schema::dropIfExists('sites');
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user