Initial commit: Laravel scaffold + Common Crawl site indexing
Adds Site/SitePage models and the crawlindex:fetch command, which pulls CDX index records from Common Crawl and optionally enriches them with page title/description/keywords/body text extracted from the WARC archive. CDX/WARC access and the site/page upsert logic live in CommonCrawlIndexService and WarcRecordFetcher.
This commit is contained in:
m105
@@ -0,0 +1,161 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Services\CommonCrawl\CommonCrawlIndexService;
|
||||
use Illuminate\Console\Attributes\Description;
|
||||
use Illuminate\Console\Attributes\Signature;
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
#[Signature('crawlindex:fetch
|
||||
{url? : The URL, host, or domain pattern to search for (e.g. example.com/*). Omit this and use --tld to browse a whole TLD}
|
||||
{--tld= : Restrict the query to a top-level domain (e.g. com, net) instead of passing a url. Requires --match-type=domain}
|
||||
{--collection= : Common Crawl collection id, e.g. CC-MAIN-2026-05. Defaults to the latest collection}
|
||||
{--match-type=domain : exact|prefix|host|domain}
|
||||
{--limit=1000 : Maximum number of index records to fetch per page}
|
||||
{--page=0 : CDX index page to start from (the index is split into fixed pages; there is no "skip N records" option, see --pages/--all-pages)}
|
||||
{--pages=1 : Number of consecutive CDX index pages to fetch, starting at --page}
|
||||
{--all-pages : Keep fetching consecutive pages, starting at --page, until the index is exhausted (overrides --pages)}
|
||||
{--page-size=5 : Number of index blocks per CDX page, passed through as pageSize}
|
||||
{--with-metadata : Also fetch each page from the WARC archive and extract title/description/keywords/content (slower, one extra request per saved page)}
|
||||
')]
|
||||
#[Description('Fetch Common Crawl URL index records and save them as sites with their pages')]
|
||||
class FetchCommonCrawlIndex extends Command
|
||||
{
|
||||
private const VALID_MATCH_TYPES = ['exact', 'prefix', 'host', 'domain'];
|
||||
|
||||
public function __construct(private readonly CommonCrawlIndexService $service)
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle(): int
|
||||
{
|
||||
$url = $this->argument('url');
|
||||
$tld = $this->option('tld');
|
||||
$matchType = $this->option('match-type');
|
||||
$limit = (int) $this->option('limit');
|
||||
$page = (int) $this->option('page');
|
||||
$pageSize = (int) $this->option('page-size');
|
||||
$allPages = (bool) $this->option('all-pages');
|
||||
$pagesToFetch = $allPages ? null : max(1, (int) $this->option('pages'));
|
||||
$withMetadata = (bool) $this->option('with-metadata');
|
||||
|
||||
if (! in_array($matchType, self::VALID_MATCH_TYPES, true)) {
|
||||
$this->error('--match-type must be one of: '.implode(', ', self::VALID_MATCH_TYPES));
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
if (! $url && ! $tld) {
|
||||
$this->error('Provide a url argument or a --tld option (e.g. --tld=com).');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
if ($url && $tld) {
|
||||
$this->error('Pass either a url argument or --tld, not both.');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
if ($tld) {
|
||||
if ($matchType !== 'domain') {
|
||||
$this->error('--tld can only be used with --match-type=domain (the default).');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$url = ltrim($tld, '.');
|
||||
}
|
||||
|
||||
$collection = $this->option('collection') ?: $this->service->resolveLatestCollection();
|
||||
|
||||
if (! $collection) {
|
||||
$this->error('Could not resolve a Common Crawl collection id. Pass one explicitly with --collection.');
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$totals = ['sitesCreated' => 0, 'sitesUpdated' => 0, 'pagesCreated' => 0, 'pagesUpdated' => 0];
|
||||
$contentCache = [];
|
||||
$pagesFetched = 0;
|
||||
$currentPage = $page;
|
||||
|
||||
while ($allPages || $pagesFetched < $pagesToFetch) {
|
||||
$this->info("Fetching CDX page {$currentPage} for [{$url}] (collection={$collection}, matchType={$matchType}, limit={$limit}, pageSize={$pageSize})...");
|
||||
|
||||
$result = $this->service->fetchIndexPage($collection, $url, $matchType, $limit, $currentPage, $pageSize);
|
||||
|
||||
if ($result['status'] === 'end') {
|
||||
if ($pagesFetched === 0) {
|
||||
$this->error("Common Crawl rejected page {$currentPage}.");
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$this->info('Reached the end of the index, no more pages.');
|
||||
break;
|
||||
}
|
||||
|
||||
if ($result['status'] === 'not_found') {
|
||||
if ($pagesFetched === 0) {
|
||||
$this->warn('No captures found for this URL/collection.');
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
if ($result['status'] === 'error') {
|
||||
$this->error("Common Crawl index request failed with status {$result['httpStatus']}: {$result['body']}");
|
||||
|
||||
return self::FAILURE;
|
||||
}
|
||||
|
||||
$pagesFetched++;
|
||||
|
||||
if (! empty($result['lines'])) {
|
||||
$this->saveCaptures($result['lines'], $collection, $withMetadata, $contentCache, $totals);
|
||||
} else {
|
||||
$this->comment('(empty page)');
|
||||
}
|
||||
|
||||
$currentPage++;
|
||||
|
||||
if ($allPages || $pagesFetched < $pagesToFetch) {
|
||||
usleep(300_000); // be polite to the shared index server between page requests
|
||||
}
|
||||
}
|
||||
|
||||
$this->newLine();
|
||||
$this->info("Done. Fetched {$pagesFetched} CDX page(s). Sites: {$totals['sitesCreated']} new, {$totals['sitesUpdated']} updated. Pages: {$totals['pagesCreated']} new, {$totals['pagesUpdated']} updated.");
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
private function saveCaptures(array $lines, string $collection, bool $withContent, array &$contentCache, array &$totals): void
|
||||
{
|
||||
$grouped = $this->service->groupIntoSites($lines, $collection);
|
||||
$siteGroups = $grouped['sites'];
|
||||
$totalPages = $siteGroups->sum(fn (array $group) => $group['pages']->count());
|
||||
|
||||
$this->line(" {$grouped['captureCount']} capture(s) covering {$siteGroups->count()} site(s) and {$totalPages} unique page(s).");
|
||||
|
||||
if ($withContent) {
|
||||
$this->comment(" --with-metadata is set: fetching page content from the WARC archive for up to {$totalPages} page(s), this will be slower.");
|
||||
}
|
||||
|
||||
$this->withProgressBar($siteGroups, function (array $group) use ($collection, $withContent, &$contentCache, &$totals) {
|
||||
$result = $this->service->saveSiteGroup($group, $collection, $withContent, $contentCache);
|
||||
|
||||
$result['wasNewSite'] ? $totals['sitesCreated']++ : $totals['sitesUpdated']++;
|
||||
$totals['pagesCreated'] += $result['pagesCreated'];
|
||||
$totals['pagesUpdated'] += $result['pagesUpdated'];
|
||||
});
|
||||
|
||||
$this->newLine();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
<?php
|
||||
|
||||
namespace App\Http\Controllers;
|
||||
|
||||
abstract class Controller
|
||||
{
|
||||
//
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Database\Eloquent\Relations\HasMany;
|
||||
|
||||
class Site extends Model
|
||||
{
|
||||
protected $fillable = [
|
||||
'collection',
|
||||
'domain',
|
||||
'url',
|
||||
'title',
|
||||
'description',
|
||||
'keywords',
|
||||
];
|
||||
|
||||
public function pages(): HasMany
|
||||
{
|
||||
return $this->hasMany(SitePage::class);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
use Illuminate\Database\Eloquent\Relations\BelongsTo;
|
||||
|
||||
class SitePage extends Model
|
||||
{
|
||||
protected $fillable = [
|
||||
'site_id',
|
||||
'urlkey',
|
||||
'url',
|
||||
'timestamp',
|
||||
'mime',
|
||||
'mime_detected',
|
||||
'status',
|
||||
'digest',
|
||||
'length',
|
||||
'offset',
|
||||
'filename',
|
||||
'languages',
|
||||
'charset',
|
||||
'redirect',
|
||||
'title',
|
||||
'description',
|
||||
'keywords',
|
||||
'content',
|
||||
'raw',
|
||||
'record_hash',
|
||||
];
|
||||
|
||||
protected $casts = [
|
||||
'raw' => 'array',
|
||||
'status' => 'integer',
|
||||
'length' => 'integer',
|
||||
'offset' => 'integer',
|
||||
];
|
||||
|
||||
public function site(): BelongsTo
|
||||
{
|
||||
return $this->belongsTo(Site::class);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
// use Illuminate\Contracts\Auth\MustVerifyEmail;
|
||||
use Database\Factories\UserFactory;
|
||||
use Illuminate\Database\Eloquent\Attributes\Fillable;
|
||||
use Illuminate\Database\Eloquent\Attributes\Hidden;
|
||||
use Illuminate\Database\Eloquent\Factories\HasFactory;
|
||||
use Illuminate\Foundation\Auth\User as Authenticatable;
|
||||
use Illuminate\Notifications\Notifiable;
|
||||
|
||||
#[Fillable(['name', 'email', 'password'])]
|
||||
#[Hidden(['password', 'remember_token'])]
|
||||
class User extends Authenticatable
|
||||
{
|
||||
/** @use HasFactory<UserFactory> */
|
||||
use HasFactory, Notifiable;
|
||||
|
||||
/**
|
||||
* Get the attributes that should be cast.
|
||||
*
|
||||
* @return array<string, string>
|
||||
*/
|
||||
protected function casts(): array
|
||||
{
|
||||
return [
|
||||
'email_verified_at' => 'datetime',
|
||||
'password' => 'hashed',
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
<?php
|
||||
|
||||
namespace App\Providers;
|
||||
|
||||
use Illuminate\Support\ServiceProvider;
|
||||
|
||||
class AppServiceProvider extends ServiceProvider
|
||||
{
|
||||
/**
|
||||
* Register any application services.
|
||||
*/
|
||||
public function register(): void
|
||||
{
|
||||
//
|
||||
}
|
||||
|
||||
/**
|
||||
* Bootstrap any application services.
|
||||
*/
|
||||
public function boot(): void
|
||||
{
|
||||
//
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\CommonCrawl;
|
||||
|
||||
use App\Models\Site;
|
||||
use App\Models\SitePage;
|
||||
use App\Support\CommonCrawl\WarcRecordFetcher;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
|
||||
/**
|
||||
* Talks to the Common Crawl CDX index, groups raw capture records into sites/pages, and
|
||||
* upserts them. Fetching a capture's actual page content (title/description/keywords/
|
||||
* body text) from its WARC file is delegated to WarcRecordFetcher.
|
||||
*/
|
||||
class CommonCrawlIndexService
|
||||
{
|
||||
private const COLLINFO_URL = 'https://index.commoncrawl.org/collinfo.json';
|
||||
|
||||
public function __construct(private readonly WarcRecordFetcher $warcFetcher) {}
|
||||
|
||||
public function resolveLatestCollection(): ?string
|
||||
{
|
||||
$response = Http::withUserAgent(WarcRecordFetcher::DEFAULT_USER_AGENT)
|
||||
->timeout(30)
|
||||
->retry(3, 2000)
|
||||
->get(self::COLLINFO_URL);
|
||||
|
||||
if (! $response->successful()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return $response->json('0.id');
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch one CDX index page. Returns an array with a 'status' of:
|
||||
* - 'ok' — 'lines' holds the raw JSON-per-line capture records
|
||||
* - 'end' — matchType=domain/prefix ranges are exhausted, stop paging
|
||||
* - 'not_found' — no captures at all for this query
|
||||
* - 'error' — 'httpStatus'/'body' describe what went wrong
|
||||
*/
|
||||
public function fetchIndexPage(string $collection, string $url, string $matchType, int $limit, int $page, int $pageSize): array
|
||||
{
|
||||
$response = Http::withUserAgent(WarcRecordFetcher::DEFAULT_USER_AGENT)
|
||||
->timeout(60)
|
||||
->retry(3, 2000)
|
||||
->get("https://index.commoncrawl.org/{$collection}-index", [
|
||||
'url' => $url,
|
||||
'output' => 'json',
|
||||
'matchType' => $matchType,
|
||||
'limit' => $limit,
|
||||
'page' => $page,
|
||||
'pageSize' => $pageSize,
|
||||
]);
|
||||
|
||||
if ($response->status() === 400) {
|
||||
return ['status' => 'end'];
|
||||
}
|
||||
|
||||
if ($response->status() === 404) {
|
||||
return ['status' => 'not_found'];
|
||||
}
|
||||
|
||||
if (! $response->successful()) {
|
||||
return ['status' => 'error', 'httpStatus' => $response->status(), 'body' => $response->body()];
|
||||
}
|
||||
|
||||
return ['status' => 'ok', 'lines' => array_values(array_filter(explode("\n", trim($response->body()))))];
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one CDX page's worth of lines and group into sites (by domain) and, within
|
||||
* each site, into pages (by SURT urlkey) — different forms of the same page
|
||||
* (http/https, www/non-www, different capture timestamps) collapse to one page row.
|
||||
*
|
||||
* Returns ['captureCount' => int, 'sites' => Collection<{domain, pages}>].
|
||||
*/
|
||||
public function groupIntoSites(array $lines, string $collection): array
|
||||
{
|
||||
$captures = collect($lines)
|
||||
->map(fn (string $line) => json_decode($line, true))
|
||||
->filter(fn ($record) => is_array($record) && ! empty($record['urlkey']))
|
||||
->map(fn (array $record) => $this->mapCapture($record, $collection));
|
||||
|
||||
$sites = $captures
|
||||
->groupBy('domain')
|
||||
->map(fn (Collection $domainCaptures) => $domainCaptures
|
||||
->groupBy('urlkey')
|
||||
->map(fn (Collection $pageCaptures) => $pageCaptures->sort(fn ($a, $b) => $this->compareCaptures($a, $b))->first())
|
||||
->values())
|
||||
->map(fn (Collection $pages, string $domain) => ['domain' => $domain, 'pages' => $pages])
|
||||
->values();
|
||||
|
||||
return ['captureCount' => $captures->count(), 'sites' => $sites];
|
||||
}
|
||||
|
||||
/**
|
||||
* Upsert one site (and its pages) from a group produced by groupIntoSites(). When
|
||||
* $withContent is set, fetches each distinct capture's HTML from its WARC file to
|
||||
* fill in title/description/keywords/content — one extra request per capture,
|
||||
* memoized in $contentCache across the whole run.
|
||||
*
|
||||
* Returns ['site' => Site, 'wasNewSite' => bool, 'pagesCreated' => int, 'pagesUpdated' => int].
|
||||
*/
|
||||
public function saveSiteGroup(array $group, string $collection, bool $withContent, array &$contentCache): array
|
||||
{
|
||||
/** @var Collection $pages */
|
||||
$pages = $group['pages'];
|
||||
|
||||
$site = Site::firstOrNew(['collection' => $collection, 'domain' => $group['domain']]);
|
||||
$wasNewSite = ! $site->exists;
|
||||
|
||||
$homepage = $pages->sort(fn ($a, $b) => $this->compareCaptures($a, $b))->first();
|
||||
$site->url = $homepage['url'];
|
||||
|
||||
if ($withContent) {
|
||||
$content = $this->fetchContent($homepage, $contentCache);
|
||||
|
||||
if (! empty($content)) {
|
||||
$site->fill([
|
||||
'title' => $content['title'],
|
||||
'description' => $content['description'],
|
||||
'keywords' => $content['keywords'],
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
$site->save();
|
||||
|
||||
$pagesCreated = 0;
|
||||
$pagesUpdated = 0;
|
||||
|
||||
foreach ($pages as $page) {
|
||||
$attributes = $page;
|
||||
unset($attributes['collection'], $attributes['domain']);
|
||||
|
||||
if ($withContent) {
|
||||
$attributes = array_merge($attributes, $this->fetchContent($page, $contentCache));
|
||||
}
|
||||
|
||||
$result = SitePage::updateOrCreate(
|
||||
['site_id' => $site->id, 'urlkey' => $page['urlkey']],
|
||||
$attributes,
|
||||
);
|
||||
|
||||
$result->wasRecentlyCreated ? $pagesCreated++ : $pagesUpdated++;
|
||||
}
|
||||
|
||||
return [
|
||||
'site' => $site,
|
||||
'wasNewSite' => $wasNewSite,
|
||||
'pagesCreated' => $pagesCreated,
|
||||
'pagesUpdated' => $pagesUpdated,
|
||||
];
|
||||
}
|
||||
|
||||
private function fetchContent(array $page, array &$contentCache): array
|
||||
{
|
||||
$contentCache[$page['record_hash']] ??= $this->warcFetcher->fetchMetadata($page);
|
||||
|
||||
return $contentCache[$page['record_hash']];
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort captures best-first: a live HTML homepage beats a live HTML subpage, which
|
||||
* beats any other 200, which beats everything else; ties broken by the most recent
|
||||
* capture timestamp. Used both to pick a site's homepage and to pick which capture
|
||||
* of a given page (across timestamps) to keep.
|
||||
*/
|
||||
private function compareCaptures(array $a, array $b): int
|
||||
{
|
||||
$scoreDiff = $this->captureScore($b) <=> $this->captureScore($a);
|
||||
|
||||
return $scoreDiff !== 0 ? $scoreDiff : strcmp($b['timestamp'], $a['timestamp']);
|
||||
}
|
||||
|
||||
private function captureScore(array $attributes): int
|
||||
{
|
||||
$isHtml200 = ($attributes['status'] ?? null) === 200 && ($attributes['mime'] ?? null) === 'text/html';
|
||||
|
||||
return match (true) {
|
||||
$isHtml200 && in_array($this->pathFromUrlkey($attributes['urlkey']), ['/', ''], true) => 3,
|
||||
$isHtml200 => 2,
|
||||
($attributes['status'] ?? null) === 200 => 1,
|
||||
default => 0,
|
||||
};
|
||||
}
|
||||
|
||||
private function mapCapture(array $record, string $collection): array
|
||||
{
|
||||
$filename = $record['filename'] ?? null;
|
||||
$offset = isset($record['offset']) ? (int) $record['offset'] : null;
|
||||
$status = isset($record['status']) && is_numeric($record['status']) ? (int) $record['status'] : null;
|
||||
|
||||
return [
|
||||
'collection' => $collection,
|
||||
'urlkey' => $record['urlkey'],
|
||||
'domain' => $this->domainFromUrlkey($record['urlkey']),
|
||||
'timestamp' => $record['timestamp'] ?? '',
|
||||
'url' => $record['url'] ?? '',
|
||||
'mime' => $record['mime'] ?? null,
|
||||
'mime_detected' => $record['mime-detected'] ?? null,
|
||||
'status' => $status,
|
||||
'digest' => $record['digest'] ?? null,
|
||||
'length' => isset($record['length']) ? (int) $record['length'] : null,
|
||||
'offset' => $offset,
|
||||
'filename' => $filename,
|
||||
'languages' => $record['languages'] ?? null,
|
||||
'charset' => $record['charset'] ?? null,
|
||||
'redirect' => $record['redirect'] ?? null,
|
||||
'raw' => $record,
|
||||
'record_hash' => sha1($collection.'|'.$filename.'|'.$offset),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The urlkey is a SURT key, e.g. "com,example)/path" — reversed, comma-separated
|
||||
* host labels (with a leading "www." already stripped by Common Crawl's
|
||||
* canonicalization) followed by the path after ")". Reversing the host labels back
|
||||
* gives the plain domain, which is what pages are grouped into sites by.
|
||||
*/
|
||||
private function domainFromUrlkey(string $urlkey): string
|
||||
{
|
||||
$hostPart = strstr($urlkey, ')', true);
|
||||
$hostPart = $hostPart === false ? $urlkey : $hostPart;
|
||||
|
||||
$labels = array_filter(explode(',', $hostPart), fn (string $label) => $label !== '');
|
||||
|
||||
return implode('.', array_reverse($labels)) ?: $urlkey;
|
||||
}
|
||||
|
||||
private function pathFromUrlkey(string $urlkey): string
|
||||
{
|
||||
$position = strpos($urlkey, ')');
|
||||
|
||||
return $position === false ? '' : substr($urlkey, $position + 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
<?php
|
||||
|
||||
namespace App\Support\CommonCrawl;
|
||||
|
||||
use DOMDocument;
|
||||
use DOMXPath;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
|
||||
/**
|
||||
* Downloads a single record's bytes out of a Common Crawl WARC file via an HTTP
|
||||
* Range request (using the offset/length recorded from the CDX index) and extracts
|
||||
* page metadata from it. Only ever touches data.commoncrawl.org, never the CDX
|
||||
* index server, and never pulls more than one record's worth of bytes at a time.
|
||||
*/
|
||||
class WarcRecordFetcher
|
||||
{
|
||||
public const DEFAULT_USER_AGENT = 'huginn-crawlindex/1.0 (+https://github.com/)';
|
||||
|
||||
private const WARC_BASE_URL = 'https://data.commoncrawl.org/';
|
||||
|
||||
/** Cap on stored page text, in characters — keeps a single page's row bounded. */
|
||||
private const MAX_CONTENT_LENGTH = 20000;
|
||||
|
||||
public function __construct(private readonly string $userAgent = self::DEFAULT_USER_AGENT) {}
|
||||
|
||||
public function fetchMetadata(array $attributes): array
|
||||
{
|
||||
if (($attributes['mime'] ?? null) !== 'text/html'
|
||||
|| ($attributes['status'] ?? null) !== 200
|
||||
|| empty($attributes['filename'])
|
||||
|| $attributes['offset'] === null
|
||||
|| $attributes['length'] === null
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$start = $attributes['offset'];
|
||||
$end = $start + $attributes['length'] - 1;
|
||||
|
||||
$response = Http::withUserAgent($this->userAgent)
|
||||
->timeout(30)
|
||||
->retry(2, 1000)
|
||||
->withHeaders(['Range' => "bytes={$start}-{$end}"])
|
||||
->get(self::WARC_BASE_URL.$attributes['filename']);
|
||||
|
||||
if (! $response->successful()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
$html = $this->extractHtmlFromWarcSegment($response->body());
|
||||
|
||||
return $html === null ? [] : $this->extractHtmlMeta($html);
|
||||
}
|
||||
|
||||
/**
|
||||
* A WARC segment (gzip-compressed) contains WARC headers, then the raw HTTP
|
||||
* response (its own headers followed by the HTML body). Strip both header blocks.
|
||||
*/
|
||||
private function extractHtmlFromWarcSegment(string $gzipped): ?string
|
||||
{
|
||||
$decompressed = @gzdecode($gzipped);
|
||||
|
||||
if ($decompressed === false) {
|
||||
return null;
|
||||
}
|
||||
|
||||
[$warcHeaders, $rest] = array_pad(preg_split("/\r?\n\r?\n/", $decompressed, 2), 2, null);
|
||||
|
||||
if ($rest === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
[$httpHeaders, $html] = array_pad(preg_split("/\r?\n\r?\n/", $rest, 2), 2, null);
|
||||
|
||||
return $html;
|
||||
}
|
||||
|
||||
private function extractHtmlMeta(string $html): array
|
||||
{
|
||||
$meta = ['title' => null, 'description' => null, 'keywords' => null, 'content' => null];
|
||||
|
||||
$document = new DOMDocument;
|
||||
|
||||
libxml_use_internal_errors(true);
|
||||
$document->loadHTML('<?xml encoding="UTF-8">'.$html, LIBXML_NOERROR | LIBXML_NOWARNING);
|
||||
libxml_clear_errors();
|
||||
|
||||
$xpath = new DOMXPath($document);
|
||||
|
||||
$titleNode = $xpath->query('//title')->item(0);
|
||||
|
||||
if ($titleNode) {
|
||||
$meta['title'] = trim(preg_replace('/\s+/', ' ', $titleNode->textContent)) ?: null;
|
||||
}
|
||||
|
||||
foreach (['description', 'keywords'] as $name) {
|
||||
$node = $xpath->query("//meta[translate(@name, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')='{$name}']")->item(0);
|
||||
|
||||
if ($node && $node->getAttribute('content') !== '') {
|
||||
$meta[$name] = trim(preg_replace('/\s+/', ' ', $node->getAttribute('content')));
|
||||
}
|
||||
}
|
||||
|
||||
$meta['content'] = $this->extractVisibleText($xpath);
|
||||
|
||||
return $meta;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the page's visible body text (for later keyword/product search — e.g. "does
|
||||
* this site sell honey?"), stripped of script/style/noscript content and collapsed
|
||||
* whitespace, capped at MAX_CONTENT_LENGTH so a single page can't blow up storage.
|
||||
*/
|
||||
private function extractVisibleText(DOMXPath $xpath): ?string
|
||||
{
|
||||
foreach ($xpath->query('//script | //style | //noscript') as $node) {
|
||||
$node->parentNode?->removeChild($node);
|
||||
}
|
||||
|
||||
$body = $xpath->query('//body')->item(0);
|
||||
|
||||
if (! $body) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$text = trim(preg_replace('/\s+/', ' ', $body->textContent));
|
||||
|
||||
return $text === '' ? null : mb_substr($text, 0, self::MAX_CONTENT_LENGTH);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user