Initial commit: Laravel scaffold + Common Crawl site indexing
Adds Site/SitePage models and the crawlindex:fetch command, which pulls CDX index records from Common Crawl and optionally enriches them with page title/description/keywords/body text extracted from the WARC archive. CDX/WARC access and the site/page upsert logic live in CommonCrawlIndexService and WarcRecordFetcher.
This commit is contained in:
m105
@@ -0,0 +1,239 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services\CommonCrawl;
|
||||
|
||||
use App\Models\Site;
|
||||
use App\Models\SitePage;
|
||||
use App\Support\CommonCrawl\WarcRecordFetcher;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
|
||||
/**
|
||||
* Talks to the Common Crawl CDX index, groups raw capture records into sites/pages, and
|
||||
* upserts them. Fetching a capture's actual page content (title/description/keywords/
|
||||
* body text) from its WARC file is delegated to WarcRecordFetcher.
|
||||
*/
|
||||
class CommonCrawlIndexService
|
||||
{
|
||||
private const COLLINFO_URL = 'https://index.commoncrawl.org/collinfo.json';
|
||||
|
||||
public function __construct(private readonly WarcRecordFetcher $warcFetcher) {}
|
||||
|
||||
public function resolveLatestCollection(): ?string
|
||||
{
|
||||
$response = Http::withUserAgent(WarcRecordFetcher::DEFAULT_USER_AGENT)
|
||||
->timeout(30)
|
||||
->retry(3, 2000)
|
||||
->get(self::COLLINFO_URL);
|
||||
|
||||
if (! $response->successful()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return $response->json('0.id');
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch one CDX index page. Returns an array with a 'status' of:
|
||||
* - 'ok' — 'lines' holds the raw JSON-per-line capture records
|
||||
* - 'end' — matchType=domain/prefix ranges are exhausted, stop paging
|
||||
* - 'not_found' — no captures at all for this query
|
||||
* - 'error' — 'httpStatus'/'body' describe what went wrong
|
||||
*/
|
||||
public function fetchIndexPage(string $collection, string $url, string $matchType, int $limit, int $page, int $pageSize): array
|
||||
{
|
||||
$response = Http::withUserAgent(WarcRecordFetcher::DEFAULT_USER_AGENT)
|
||||
->timeout(60)
|
||||
->retry(3, 2000)
|
||||
->get("https://index.commoncrawl.org/{$collection}-index", [
|
||||
'url' => $url,
|
||||
'output' => 'json',
|
||||
'matchType' => $matchType,
|
||||
'limit' => $limit,
|
||||
'page' => $page,
|
||||
'pageSize' => $pageSize,
|
||||
]);
|
||||
|
||||
if ($response->status() === 400) {
|
||||
return ['status' => 'end'];
|
||||
}
|
||||
|
||||
if ($response->status() === 404) {
|
||||
return ['status' => 'not_found'];
|
||||
}
|
||||
|
||||
if (! $response->successful()) {
|
||||
return ['status' => 'error', 'httpStatus' => $response->status(), 'body' => $response->body()];
|
||||
}
|
||||
|
||||
return ['status' => 'ok', 'lines' => array_values(array_filter(explode("\n", trim($response->body()))))];
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one CDX page's worth of lines and group into sites (by domain) and, within
|
||||
* each site, into pages (by SURT urlkey) — different forms of the same page
|
||||
* (http/https, www/non-www, different capture timestamps) collapse to one page row.
|
||||
*
|
||||
* Returns ['captureCount' => int, 'sites' => Collection<{domain, pages}>].
|
||||
*/
|
||||
public function groupIntoSites(array $lines, string $collection): array
|
||||
{
|
||||
$captures = collect($lines)
|
||||
->map(fn (string $line) => json_decode($line, true))
|
||||
->filter(fn ($record) => is_array($record) && ! empty($record['urlkey']))
|
||||
->map(fn (array $record) => $this->mapCapture($record, $collection));
|
||||
|
||||
$sites = $captures
|
||||
->groupBy('domain')
|
||||
->map(fn (Collection $domainCaptures) => $domainCaptures
|
||||
->groupBy('urlkey')
|
||||
->map(fn (Collection $pageCaptures) => $pageCaptures->sort(fn ($a, $b) => $this->compareCaptures($a, $b))->first())
|
||||
->values())
|
||||
->map(fn (Collection $pages, string $domain) => ['domain' => $domain, 'pages' => $pages])
|
||||
->values();
|
||||
|
||||
return ['captureCount' => $captures->count(), 'sites' => $sites];
|
||||
}
|
||||
|
||||
/**
|
||||
* Upsert one site (and its pages) from a group produced by groupIntoSites(). When
|
||||
* $withContent is set, fetches each distinct capture's HTML from its WARC file to
|
||||
* fill in title/description/keywords/content — one extra request per capture,
|
||||
* memoized in $contentCache across the whole run.
|
||||
*
|
||||
* Returns ['site' => Site, 'wasNewSite' => bool, 'pagesCreated' => int, 'pagesUpdated' => int].
|
||||
*/
|
||||
public function saveSiteGroup(array $group, string $collection, bool $withContent, array &$contentCache): array
|
||||
{
|
||||
/** @var Collection $pages */
|
||||
$pages = $group['pages'];
|
||||
|
||||
$site = Site::firstOrNew(['collection' => $collection, 'domain' => $group['domain']]);
|
||||
$wasNewSite = ! $site->exists;
|
||||
|
||||
$homepage = $pages->sort(fn ($a, $b) => $this->compareCaptures($a, $b))->first();
|
||||
$site->url = $homepage['url'];
|
||||
|
||||
if ($withContent) {
|
||||
$content = $this->fetchContent($homepage, $contentCache);
|
||||
|
||||
if (! empty($content)) {
|
||||
$site->fill([
|
||||
'title' => $content['title'],
|
||||
'description' => $content['description'],
|
||||
'keywords' => $content['keywords'],
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
$site->save();
|
||||
|
||||
$pagesCreated = 0;
|
||||
$pagesUpdated = 0;
|
||||
|
||||
foreach ($pages as $page) {
|
||||
$attributes = $page;
|
||||
unset($attributes['collection'], $attributes['domain']);
|
||||
|
||||
if ($withContent) {
|
||||
$attributes = array_merge($attributes, $this->fetchContent($page, $contentCache));
|
||||
}
|
||||
|
||||
$result = SitePage::updateOrCreate(
|
||||
['site_id' => $site->id, 'urlkey' => $page['urlkey']],
|
||||
$attributes,
|
||||
);
|
||||
|
||||
$result->wasRecentlyCreated ? $pagesCreated++ : $pagesUpdated++;
|
||||
}
|
||||
|
||||
return [
|
||||
'site' => $site,
|
||||
'wasNewSite' => $wasNewSite,
|
||||
'pagesCreated' => $pagesCreated,
|
||||
'pagesUpdated' => $pagesUpdated,
|
||||
];
|
||||
}
|
||||
|
||||
private function fetchContent(array $page, array &$contentCache): array
|
||||
{
|
||||
$contentCache[$page['record_hash']] ??= $this->warcFetcher->fetchMetadata($page);
|
||||
|
||||
return $contentCache[$page['record_hash']];
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort captures best-first: a live HTML homepage beats a live HTML subpage, which
|
||||
* beats any other 200, which beats everything else; ties broken by the most recent
|
||||
* capture timestamp. Used both to pick a site's homepage and to pick which capture
|
||||
* of a given page (across timestamps) to keep.
|
||||
*/
|
||||
private function compareCaptures(array $a, array $b): int
|
||||
{
|
||||
$scoreDiff = $this->captureScore($b) <=> $this->captureScore($a);
|
||||
|
||||
return $scoreDiff !== 0 ? $scoreDiff : strcmp($b['timestamp'], $a['timestamp']);
|
||||
}
|
||||
|
||||
private function captureScore(array $attributes): int
|
||||
{
|
||||
$isHtml200 = ($attributes['status'] ?? null) === 200 && ($attributes['mime'] ?? null) === 'text/html';
|
||||
|
||||
return match (true) {
|
||||
$isHtml200 && in_array($this->pathFromUrlkey($attributes['urlkey']), ['/', ''], true) => 3,
|
||||
$isHtml200 => 2,
|
||||
($attributes['status'] ?? null) === 200 => 1,
|
||||
default => 0,
|
||||
};
|
||||
}
|
||||
|
||||
private function mapCapture(array $record, string $collection): array
|
||||
{
|
||||
$filename = $record['filename'] ?? null;
|
||||
$offset = isset($record['offset']) ? (int) $record['offset'] : null;
|
||||
$status = isset($record['status']) && is_numeric($record['status']) ? (int) $record['status'] : null;
|
||||
|
||||
return [
|
||||
'collection' => $collection,
|
||||
'urlkey' => $record['urlkey'],
|
||||
'domain' => $this->domainFromUrlkey($record['urlkey']),
|
||||
'timestamp' => $record['timestamp'] ?? '',
|
||||
'url' => $record['url'] ?? '',
|
||||
'mime' => $record['mime'] ?? null,
|
||||
'mime_detected' => $record['mime-detected'] ?? null,
|
||||
'status' => $status,
|
||||
'digest' => $record['digest'] ?? null,
|
||||
'length' => isset($record['length']) ? (int) $record['length'] : null,
|
||||
'offset' => $offset,
|
||||
'filename' => $filename,
|
||||
'languages' => $record['languages'] ?? null,
|
||||
'charset' => $record['charset'] ?? null,
|
||||
'redirect' => $record['redirect'] ?? null,
|
||||
'raw' => $record,
|
||||
'record_hash' => sha1($collection.'|'.$filename.'|'.$offset),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* The urlkey is a SURT key, e.g. "com,example)/path" — reversed, comma-separated
|
||||
* host labels (with a leading "www." already stripped by Common Crawl's
|
||||
* canonicalization) followed by the path after ")". Reversing the host labels back
|
||||
* gives the plain domain, which is what pages are grouped into sites by.
|
||||
*/
|
||||
private function domainFromUrlkey(string $urlkey): string
|
||||
{
|
||||
$hostPart = strstr($urlkey, ')', true);
|
||||
$hostPart = $hostPart === false ? $urlkey : $hostPart;
|
||||
|
||||
$labels = array_filter(explode(',', $hostPart), fn (string $label) => $label !== '');
|
||||
|
||||
return implode('.', array_reverse($labels)) ?: $urlkey;
|
||||
}
|
||||
|
||||
private function pathFromUrlkey(string $urlkey): string
|
||||
{
|
||||
$position = strpos($urlkey, ')');
|
||||
|
||||
return $position === false ? '' : substr($urlkey, $position + 1);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user