m105
1c9fc5d423
Adds Site/SitePage models and the crawlindex:fetch command, which pulls CDX index records from Common Crawl and optionally enriches them with page title/description/keywords/body text extracted from the WARC archive. CDX/WARC access and the site/page upsert logic live in CommonCrawlIndexService and WarcRecordFetcher.
24 lines
400 B
PHP
24 lines
400 B
PHP
<?php
|
|
|
|
namespace App\Models;
|
|
|
|
use Illuminate\Database\Eloquent\Model;
|
|
use Illuminate\Database\Eloquent\Relations\HasMany;
|
|
|
|
class Site extends Model
|
|
{
|
|
protected $fillable = [
|
|
'collection',
|
|
'domain',
|
|
'url',
|
|
'title',
|
|
'description',
|
|
'keywords',
|
|
];
|
|
|
|
public function pages(): HasMany
|
|
{
|
|
return $this->hasMany(SitePage::class);
|
|
}
|
|
}
|