m105
1c9fc5d423
Adds Site/SitePage models and the crawlindex:fetch command, which pulls CDX index records from Common Crawl and optionally enriches them with page title/description/keywords/body text extracted from the WARC archive. CDX/WARC access and the site/page upsert logic live in CommonCrawlIndexService and WarcRecordFetcher.
36 lines
859 B
PHP
36 lines
859 B
PHP
<?php
|
|
|
|
use Illuminate\Database\Migrations\Migration;
|
|
use Illuminate\Database\Schema\Blueprint;
|
|
use Illuminate\Support\Facades\Schema;
|
|
|
|
return new class extends Migration
|
|
{
|
|
/**
|
|
* Run the migrations.
|
|
*/
|
|
public function up(): void
|
|
{
|
|
Schema::create('sites', function (Blueprint $table) {
|
|
$table->id();
|
|
$table->string('collection');
|
|
$table->string('domain');
|
|
$table->text('url')->nullable();
|
|
$table->string('title')->nullable();
|
|
$table->text('description')->nullable();
|
|
$table->text('keywords')->nullable();
|
|
$table->timestamps();
|
|
|
|
$table->unique(['collection', 'domain']);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Reverse the migrations.
|
|
*/
|
|
public function down(): void
|
|
{
|
|
Schema::dropIfExists('sites');
|
|
}
|
|
};
|