lots of progress

2026-01-17 13:46:20 +02:00
parent ab5d3cae98
commit 030a105992
77 changed files with 8507 additions and 20 deletions
@@ -0,0 +1,156 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\BrokenLink;
+use App\Models\Site;
+use App\Notifications\BrokenLinkAlert;
+use GuzzleHttp\Exception\RequestException;
+use Illuminate\Bus\Batchable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Support\Facades\Log;
+use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\UriInterface;
+use Spatie\Crawler\Crawler;
+use Spatie\Crawler\CrawlObservers\CrawlObserver;
+
+class CheckBrokenLinks implements ShouldQueue
+{
+    use Queueable, Batchable;
+
+    public $timeout = 300; // 5 minutes timeout for crawling
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(
+        public Site $site
+    ) {
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        try {
+            Log::info("Starting broken link check for site {$this->site->id} ({$this->site->url})");
+
+            // Parse the base URL to get the domain
+            $parsedUrl = parse_url($this->site->url);
+
+            if (!isset($parsedUrl['host'])) {
+                Log::warning("Invalid URL for broken link check on site {$this->site->id}: {$this->site->url}");
+                return;
+            }
+
+            $baseDomain = $parsedUrl['host'];
+
+            // Create a crawler observer to handle broken links
+            $observer = new class($this->site, $baseDomain) extends CrawlObserver {
+                private Site $site;
+                private string $baseDomain;
+
+                public function __construct(Site $site, string $baseDomain)
+                {
+                    $this->site = $site;
+                    $this->baseDomain = $baseDomain;
+                }
+
+                public function crawled(
+                    UriInterface $url,
+                    ResponseInterface $response,
+                    ?UriInterface $foundOnUrl = null,
+                    ?string $linkText = null
+                ): void {
+                    $statusCode = $response->getStatusCode();
+
+                    // Check if the link is broken (4xx or 5xx)
+                    if ($statusCode >= 400) {
+                        // Only log internal links (same domain)
+                        $linkHost = $url->getHost();
+
+                        if ($linkHost === $this->baseDomain) {
+                            BrokenLink::updateOrCreate(
+                                [
+                                    'site_id' => $this->site->id,
+                                    'url' => (string) $url,
+                                ],
+                                [
+                                    'status_code' => $statusCode,
+                                    'last_checked_at' => now(),
+                                ]
+                            );
+
+                            Log::info("Broken link detected on site {$this->site->id}: {$url} (HTTP {$statusCode})");
+                        }
+                    }
+                }
+
+                public function crawlFailed(
+                    UriInterface $url,
+                    RequestException $requestException,
+                    ?UriInterface $foundOnUrl = null,
+                    ?string $linkText = null
+                ): void {
+                    // Only log internal links (same domain)
+                    $linkHost = $url->getHost();
+
+                    if ($linkHost === $this->baseDomain) {
+                        $statusCode = $requestException->getResponse()
+                            ? $requestException->getResponse()->getStatusCode()
+                            : null;
+
+                        BrokenLink::updateOrCreate(
+                            [
+                                'site_id' => $this->site->id,
+                                'url' => (string) $url,
+                            ],
+                            [
+                                'status_code' => $statusCode,
+                                'last_checked_at' => now(),
+                            ]
+                        );
+
+                        Log::info("Crawl failed for site {$this->site->id}: {$url}");
+                    }
+                }
+            };
+
+            // Configure and start the crawler
+            Crawler::create()
+                ->setConcurrency(3)
+                ->setMaximumDepth(3)
+                ->setMaximumCrawlCount(50)
+                ->ignoreRobots()
+                ->acceptNofollowLinks()
+                ->setCrawlObserver($observer)
+                ->startCrawling($this->site->url);
+
+            Log::info("Completed broken link check for site {$this->site->id}");
+
+            // Get all broken links for this site from current scan
+            $brokenLinks = BrokenLink::where('site_id', $this->site->id)
+                ->where('last_checked_at', '>=', now()->subMinutes(10))
+                ->get()
+                ->map(fn($link) => [
+                    'url' => $link->url,
+                    'status_code' => $link->status_code,
+                ])
+                ->toArray();
+
+            // Send notification if broken links were found
+            if (!empty($brokenLinks)) {
+                $this->site->user->notify(new BrokenLinkAlert(
+                    $this->site,
+                    $brokenLinks
+                ));
+
+                Log::info("Broken link alert sent for site {$this->site->id}: " . count($brokenLinks) . " broken links found");
+            }
+        } catch (\Exception $e) {
+            Log::error("Broken link check failed for site {$this->site->id}: {$e->getMessage()}");
+        }
+    }
+}