Initial commit: Laravel scaffold + Common Crawl site indexing

Adds Site/SitePage models and the crawlindex:fetch command, which pulls
CDX index records from Common Crawl and optionally enriches them with
page title/description/keywords/body text extracted from the WARC
archive. CDX/WARC access and the site/page upsert logic live in
CommonCrawlIndexService and WarcRecordFetcher.
This commit is contained in:
m105
2026-07-20 00:04:45 +03:00
commit 1c9fc5d423
64 changed files with 11338 additions and 0 deletions
+24
View File
@@ -0,0 +1,24 @@
import { defineConfig } from 'vite';
import laravel from 'laravel-vite-plugin';
import { bunny } from 'laravel-vite-plugin/fonts';
import tailwindcss from '@tailwindcss/vite';
export default defineConfig({
plugins: [
laravel({
input: ['resources/css/app.css', 'resources/js/app.js'],
refresh: true,
fonts: [
bunny('Instrument Sans', {
weights: [400, 500, 600],
}),
],
}),
tailwindcss(),
],
server: {
watch: {
ignored: ['**/storage/framework/views/**'],
},
},
});