35 lines
1.4 KiB
PHP
35 lines
1.4 KiB
PHP
|
<?php
|
||
|
|
|
||
|
|
namespace App\Console\Commands\Concerns;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Shared by console commands that feed CDX-API-shaped JSON lines (one capture per line)
|
||
|
|
* through CommonCrawlIndexService's group/save pipeline, with progress-bar output.
|
||
|
|
* Requires the using class to be a Command with a $service property.
|
||
|
|
*/
|
||
|
|
trait SavesCommonCrawlCaptures
|
||
|
|
{
|
||
|
|
private function saveCaptures(array $lines, string $collection, bool $withContent, array &$contentCache, array &$totals): void
|
||
|
|
{
|
||
|
|
$grouped = $this->service->groupIntoSites($lines, $collection);
|
||
|
|
$siteGroups = $grouped['sites'];
|
||
|
|
$totalPages = $siteGroups->sum(fn (array $group) => $group['pages']->count());
|
||
|
|
|
||
|
|
$this->line(" {$grouped['captureCount']} capture(s) covering {$siteGroups->count()} site(s) and {$totalPages} unique page(s).");
|
||
|
|
|
||
|
|
if ($withContent) {
|
||
|
|
$this->comment(" Fetching page content from the WARC archive for up to {$totalPages} page(s), this will be slower.");
|
||
|
|
}
|
||
|
|
|
||
|
|
$this->withProgressBar($siteGroups, function (array $group) use ($collection, $withContent, &$contentCache, &$totals) {
|
||
|
|
$result = $this->service->saveSiteGroup($group, $collection, $withContent, $contentCache);
|
||
|
|
|
||
|
|
$result['wasNewSite'] ? $totals['sitesCreated']++ : $totals['sitesUpdated']++;
|
||
|
|
$totals['pagesCreated'] += $result['pagesCreated'];
|
||
|
|
$totals['pagesUpdated'] += $result['pagesUpdated'];
|
||
|
|
});
|
||
|
|
|
||
|
|
$this->newLine();
|
||
|
|
}
|
||
|
|
}
|