Initial commit: Laravel scaffold + Common Crawl site indexing

Adds Site/SitePage models and the crawlindex:fetch command, which pulls
CDX index records from Common Crawl and optionally enriches them with
page title/description/keywords/body text extracted from the WARC
archive. CDX/WARC access and the site/page upsert logic live in
CommonCrawlIndexService and WarcRecordFetcher.
This commit is contained in:
m105
2026-07-20 00:04:45 +03:00
commit 1c9fc5d423
64 changed files with 11338 additions and 0 deletions
+44
View File
@@ -0,0 +1,44 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
class SitePage extends Model
{
protected $fillable = [
'site_id',
'urlkey',
'url',
'timestamp',
'mime',
'mime_detected',
'status',
'digest',
'length',
'offset',
'filename',
'languages',
'charset',
'redirect',
'title',
'description',
'keywords',
'content',
'raw',
'record_hash',
];
protected $casts = [
'raw' => 'array',
'status' => 'integer',
'length' => 'integer',
'offset' => 'integer',
];
public function site(): BelongsTo
{
return $this->belongsTo(Site::class);
}
}