Initial commit: Laravel scaffold + Common Crawl site indexing
Adds Site/SitePage models and the crawlindex:fetch command, which pulls CDX index records from Common Crawl and optionally enriches them with page title/description/keywords/body text extracted from the WARC archive. CDX/WARC access and the site/page upsert logic live in CommonCrawlIndexService and WarcRecordFetcher.
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Database\Eloquent\Relations\HasMany;
|
||||
|
||||
class Site extends Model
|
||||
{
|
||||
protected $fillable = [
|
||||
'collection',
|
||||
'domain',
|
||||
'url',
|
||||
'title',
|
||||
'description',
|
||||
'keywords',
|
||||
];
|
||||
|
||||
public function pages(): HasMany
|
||||
{
|
||||
return $this->hasMany(SitePage::class);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Database\Eloquent\Relations\BelongsTo;
|
||||
|
||||
class SitePage extends Model
|
||||
{
|
||||
protected $fillable = [
|
||||
'site_id',
|
||||
'urlkey',
|
||||
'url',
|
||||
'timestamp',
|
||||
'mime',
|
||||
'mime_detected',
|
||||
'status',
|
||||
'digest',
|
||||
'length',
|
||||
'offset',
|
||||
'filename',
|
||||
'languages',
|
||||
'charset',
|
||||
'redirect',
|
||||
'title',
|
||||
'description',
|
||||
'keywords',
|
||||
'content',
|
||||
'raw',
|
||||
'record_hash',
|
||||
];
|
||||
|
||||
protected $casts = [
|
||||
'raw' => 'array',
|
||||
'status' => 'integer',
|
||||
'length' => 'integer',
|
||||
'offset' => 'integer',
|
||||
];
|
||||
|
||||
public function site(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(Site::class);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
// use Illuminate\Contracts\Auth\MustVerifyEmail;
|
||||
use Database\Factories\UserFactory;
|
||||
use Illuminate\Database\Eloquent\Attributes\Fillable;
|
||||
use Illuminate\Database\Eloquent\Attributes\Hidden;
|
||||
use Illuminate\Database\Eloquent\Factories\HasFactory;
|
||||
use Illuminate\Foundation\Auth\User as Authenticatable;
|
||||
use Illuminate\Notifications\Notifiable;
|
||||
|
||||
#[Fillable(['name', 'email', 'password'])]
|
||||
#[Hidden(['password', 'remember_token'])]
|
||||
class User extends Authenticatable
|
||||
{
|
||||
/** @use HasFactory<UserFactory> */
|
||||
use HasFactory, Notifiable;
|
||||
|
||||
/**
|
||||
* Get the attributes that should be cast.
|
||||
*
|
||||
* @return array<string, string>
|
||||
*/
|
||||
protected function casts(): array
|
||||
{
|
||||
return [
|
||||
'email_verified_at' => 'datetime',
|
||||
'password' => 'hashed',
|
||||
];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user