ismysiteup.online/app/Jobs/CheckBrokenLinks.php

<?php

namespace App\Jobs;

use App\Models\BrokenLink;
use App\Models\Site;
use App\Notifications\BrokenLinkAlert;
use GuzzleHttp\Exception\RequestException;
use Illuminate\Bus\Batchable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Queue\Queueable;
use Illuminate\Support\Facades\Log;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\UriInterface;
use Spatie\Crawler\Crawler;
use Spatie\Crawler\CrawlObservers\CrawlObserver;

class CheckBrokenLinks implements ShouldQueue
{
    use Queueable, Batchable;

    public $timeout = 300; // 5 minutes timeout for crawling

    /**
     * Create a new job instance.
     */
    public function __construct(
        public Site $site
    ) {
    }

    /**
     * Execute the job.
     */
    public function handle(): void
    {
        try {
            Log::info("Starting broken link check for site {$this->site->id} ({$this->site->url})");

            // Parse the base URL to get the domain
            $parsedUrl = parse_url($this->site->url);

            if (!isset($parsedUrl['host'])) {
                Log::warning("Invalid URL for broken link check on site {$this->site->id}: {$this->site->url}");
                return;
            }

            $baseDomain = $parsedUrl['host'];

            // Create a crawler observer to handle broken links
            $observer = new class($this->site, $baseDomain) extends CrawlObserver {
                private Site $site;
                private string $baseDomain;

                public function __construct(Site $site, string $baseDomain)
                {
                    $this->site = $site;
                    $this->baseDomain = $baseDomain;
                }

                public function crawled(
                    UriInterface $url,
                    ResponseInterface $response,
                    ?UriInterface $foundOnUrl = null,
                    ?string $linkText = null
                ): void {
                    $statusCode = $response->getStatusCode();

                    // Check if the link is broken (4xx or 5xx)
                    if ($statusCode >= 400) {
                        // Only log internal links (same domain)
                        $linkHost = $url->getHost();

                        if ($linkHost === $this->baseDomain) {
                            BrokenLink::updateOrCreate(
                                [
                                    'site_id' => $this->site->id,
                                    'url' => (string) $url,
                                ],
                                [
                                    'status_code' => $statusCode,
                                    'last_checked_at' => now(),
                                ]
                            );

                            Log::info("Broken link detected on site {$this->site->id}: {$url} (HTTP {$statusCode})");
                        }
                    }
                }

                public function crawlFailed(
                    UriInterface $url,
                    RequestException $requestException,
                    ?UriInterface $foundOnUrl = null,
                    ?string $linkText = null
                ): void {
                    // Only log internal links (same domain)
                    $linkHost = $url->getHost();

                    if ($linkHost === $this->baseDomain) {
                        $statusCode = $requestException->getResponse()
                            ? $requestException->getResponse()->getStatusCode()
                            : null;

                        BrokenLink::updateOrCreate(
                            [
                                'site_id' => $this->site->id,
                                'url' => (string) $url,
                            ],
                            [
                                'status_code' => $statusCode,
                                'last_checked_at' => now(),
                            ]
                        );

                        Log::info("Crawl failed for site {$this->site->id}: {$url}");
                    }
                }
            };

            // Configure and start the crawler
            Crawler::create()
                ->setConcurrency(3)
                ->setMaximumDepth(3)
                ->setMaximumCrawlCount(50)
                ->ignoreRobots()
                ->acceptNofollowLinks()
                ->setCrawlObserver($observer)
                ->startCrawling($this->site->url);

            Log::info("Completed broken link check for site {$this->site->id}");

            // Get all broken links for this site from current scan
            $brokenLinks = BrokenLink::where('site_id', $this->site->id)
                ->where('last_checked_at', '>=', now()->subMinutes(10))
                ->get()
                ->map(fn($link) => [
                    'url' => $link->url,
                    'status_code' => $link->status_code,
                ])
                ->toArray();

            // Send notification if broken links were found
            if (!empty($brokenLinks)) {
                $this->site->user->notify(new BrokenLinkAlert(
                    $this->site,
                    $brokenLinks
                ));

                Log::info("Broken link alert sent for site {$this->site->id}: " . count($brokenLinks) . " broken links found");
            }
        } catch (\Exception $e) {
            Log::error("Broken link check failed for site {$this->site->id}: {$e->getMessage()}");
        }
    }
}