m105
4dc9b84eb8
crawlindex:bulk-fetch walks Common Crawl's CDX shard files directly on data.commoncrawl.org (via cc-index.paths.gz), instead of the index.commoncrawl.org query API — usable for bulk backfill/refresh or as a fallback when that API is down. Shares the group/save pipeline with crawlindex:fetch via a new SavesCommonCrawlCaptures trait. Also fixes a real bug this surfaced under load: site_pages.urlkey was a varchar(255), which is too narrow for real-world captured URLs (long query strings on spam/malformed pages overflow it). Widened to text and switched the uniqueness constraint to a sha1 hash column, since MySQL can't fully index a TEXT column.
46 lines
848 B
PHP
46 lines
848 B
PHP
<?php
|
|
|
|
namespace App\Models;
|
|
|
|
use Illuminate\Database\Eloquent\Model;
|
|
use Illuminate\Database\Eloquent\Relations\BelongsTo;
|
|
|
|
class SitePage extends Model
|
|
{
|
|
protected $fillable = [
|
|
'site_id',
|
|
'urlkey',
|
|
'urlkey_hash',
|
|
'url',
|
|
'timestamp',
|
|
'mime',
|
|
'mime_detected',
|
|
'status',
|
|
'digest',
|
|
'length',
|
|
'offset',
|
|
'filename',
|
|
'languages',
|
|
'charset',
|
|
'redirect',
|
|
'title',
|
|
'description',
|
|
'keywords',
|
|
'content',
|
|
'raw',
|
|
'record_hash',
|
|
];
|
|
|
|
protected $casts = [
|
|
'raw' => 'array',
|
|
'status' => 'integer',
|
|
'length' => 'integer',
|
|
'offset' => 'integer',
|
|
];
|
|
|
|
public function site(): BelongsTo
|
|
{
|
|
return $this->belongsTo(Site::class);
|
|
}
|
|
}
|