some progress

app/Console/Commands/FetchAdzunaDescriptionCommand.php

@@ -0,0 +1,108 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Job;
+use App\Models\JobSource;
+use Illuminate\Console\Attributes\Description;
+use Illuminate\Console\Attributes\Signature;
+use Illuminate\Console\Command;
+
+#[Signature('adzuna:enrich {url?}')]
+#[Description('Fetch adzuna job description from puppeteer.')]
+class FetchAdzunaDescriptionCommand extends Command
+{
+    public function handle(): int
+    {
+        $adzunaUrl = $this->argument('url');
+
+        if ($adzunaUrl) {
+            $this->fetchSingleJobDescription($adzunaUrl);
+        } else {
+            $this->fetchLatest();
+        }
+    }
+
+    private function fetchSingleJobDescription(string $url)
+    {
+        $job = Job::where('source_url', '$url')->first();
+
+        if (!$job) {
+            return;
+        }
+
+        $description = $this->executeScraper($job->source_url);
+
+        if (strlen($description) > 0 && strlen($description) > strlen($job->description)) {
+            $job->description = $description;
+            $job->save();
+        }
+
+        sleep(
+            rand(1, 7)
+        );
+    }
+
+    private function fetchLatest()
+    {
+        $jobSource = JobSource::where('name', 'Adzuna')->first();
+
+        $jobs = Job::where('job_source_id', $jobSource->id)->latest()->limit(50)->get();
+
+        foreach ($jobs as $job) {
+            $parsedUrl = parse_url($job->source_url);
+            $path = $parsedUrl['path'];
+            $explodedPath = explode('/', $path);
+            $id = array_pop($explodedPath);
+
+            echo "Processing $id (slug ".$job->slug.") \n";
+
+            $description = $this->executeScraper($id);
+            $description = $this->processDescription($description);
+
+            if (
+                strlen($description) > 0
+               // && strlen($description) > strlen($job->description)
+            ) {
+                $job->description = $description;
+
+                $job->save();
+                echo "Saved description for $id\n";
+            }
+
+            sleep(
+                rand(1, 7)
+            );
+        }
+    }
+
+    private function processDescription(string $description)
+    {
+        $description = trim($description);
+        $description = str_replace('Sorry, this job is not available in your region', '', $description);
+        return str_replace('Search for similar jobs in your region', '', $description);
+    }
+
+    private function executeScraper(string $id)
+    {
+        $scraperUrl = 'https://jobifyscr.c0rrup7.net/adzuna/'.$id;
+
+        $guzzleClient = new \GuzzleHttp\Client();
+        $response = $guzzleClient->request(
+            'GET',
+            $scraperUrl,
+            [
+                'headers' => [
+                    'xapikeyban' => 'ed7a13919091820bd9e4f61023ec2ec8'
+//                    'xapikeyban' => 'asd'
+                ]
+            ]
+        );
+
+        $content = $response->getBody()->getContents();
+
+        $decodedContent = json_decode($content, true);
+
+        return $decodedContent['description'] ?? null;
+    }
+}

app/Services/JobApi/Transformers/RemotiveTransformer.php

@@ -47,10 +47,10 @@ class RemotiveTransformer implements JobTransformer
     private function sanitizeDescription(string $description): string
     {
         // Remove HTML tags and clean up whitespace
-        $clean = strip_tags($description);
-        $clean = html_entity_decode($clean, ENT_QUOTES | ENT_HTML5, 'UTF-8');
-        $clean = preg_replace('/\s+/', ' ', $clean);
-        return trim($clean);
+//        $clean = strip_tags($description);
+//        $clean = html_entity_decode($clean, ENT_QUOTES | ENT_HTML5, 'UTF-8');
+//        $clean = preg_replace('/\s+/', ' ', $clean);
+        return trim($description);
     }
 
     private function parseLocation(array $data): ?string

app/Services/JobApi/Transformers/TheMuseTransformer.php

@@ -46,10 +46,10 @@ class TheMuseTransformer implements JobTransformer
 
     private function sanitizeDescription(string $description): string
     {
-        $clean = strip_tags($description);
-        $clean = html_entity_decode($clean, ENT_QUOTES | ENT_HTML5, 'UTF-8');
-        $clean = preg_replace('/\s+/', ' ', $clean);
-        return trim($clean);
+//        $clean = strip_tags($description);
+//        $clean = html_entity_decode($clean, ENT_QUOTES | ENT_HTML5, 'UTF-8');
+//        $clean = preg_replace('/\s+/', ' ', $clean);
+        return trim($description);
     }
 
     private function parseLocation(array $data): ?string