some progress

2026-04-18 19:42:44 +03:00
parent 626d109ffb
commit 1ab6fcc00c
11 changed files with 2551 additions and 10 deletions
@@ -0,0 +1,161 @@
+// Description:
+// Puppeteer scraper that fetches Adzuna job details using a headless browser
+
+// This should be called something like this: `node adzuna_scraper.js https://www.adzuna.com/details/5694360484`
+// The script uses puppeteer-core with chromium binary to scrape the page
+// I included the html of a page for example in the sidecars/page_example.html file
+
+import puppeteer from 'puppeteer-core';
+import chromium from '@sparticuz/chromium';
+
+// Get URL from command line arguments
+const url = process.argv[2];
+
+if (!url) {
+    console.error('Error: Please provide a URL as an argument');
+    console.error('Usage: node adzuna_scraper.js <url>');
+    process.exit(1);
+}
+
+async function scrapeAdzunaJob(url) {
+    let browser;
+
+    try {
+        console.log(`Launching browser...`);
+
+        // Launch browser with chromium binary
+        browser = await puppeteer.launch({
+            executablePath: await chromium.executablePath(),
+            headless: chromium.headless,
+            args: [
+                ...chromium.args,
+                '--no-sandbox',
+                '--disable-setuid-sandbox',
+                '--disable-dev-shm-usage',
+                '--disable-accelerated-2d-canvas',
+                '--disable-gpu',
+                '--window-size=1920x1080'
+            ]
+        });
+
+        const page = await browser.newPage();
+
+        // Set user agent to avoid being blocked
+        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
+
+        // Navigate to the URL
+        console.log(`Navigating to ${url}...`);
+        await page.goto(url, {
+            waitUntil: 'networkidle2',
+            timeout: 30000
+        });
+
+        // Wait a bit for dynamic content to load
+        await new Promise(resolve => setTimeout(resolve, 3000));
+
+        // Extract the job description content
+        console.log('Extracting job description...');
+
+        let jobDescription = null;
+
+        // First, check if the page loaded correctly by getting the HTML content
+        const bodyHTML = await page.content();
+
+        // If page is very small, it might be an error page
+        if (bodyHTML.length < 1000) {
+            console.log('Warning: Page content is very small, might be blocked');
+            console.log('First 500 chars:', bodyHTML.substring(0, 500));
+        }
+
+        // Try multiple selectors for the job description
+        const descriptionSelectors = [
+            'section.adp-body',
+            '.ui-foreign-click-description section',
+            'div[class*="description"] section',
+            'section[class*="body"]'
+        ];
+
+        for (const selector of descriptionSelectors) {
+            const element = await page.$(selector);
+            if (element) {
+                jobDescription = await page.evaluate(el => {
+                    return {
+                        html: el.innerHTML,
+                        text: el.textContent.trim()
+                    };
+                }, element);
+                console.log(`Found job description with selector: ${selector}`);
+                break;
+            }
+        }
+
+        // If not found with puppeteer, try regex extraction as fallback
+        if (!jobDescription) {
+            console.log('Trying fallback regex extraction...');
+            const patterns = [
+                /<section\s+class="adp-body[^"]*"[^>]*>([\s\S]*?)<\/section>/i,
+                /<section\s+class="[^"]*adp-body[^"]*"[^>]*>([\s\S]*?)<\/section>/i
+            ];
+
+            for (const pattern of patterns) {
+                const match = bodyHTML.match(pattern);
+                if (match && match[1]) {
+                    const htmlContent = match[1].trim();
+                    const textContent = htmlContent
+                        .replace(/<br\s*\/?>/gi, '\n')
+                        .replace(/<\/?(p|div|li|ul|ol|h\d)[^>]*>/gi, '\n')
+                        .replace(/<[^>]+>/g, '')
+                        .replace(/&amp;/g, '&')
+                        .replace(/&lt;/g, '<')
+                        .replace(/&gt;/g, '>')
+                        .replace(/&quot;/g, '"')
+                        .replace(/&#39;/g, "'")
+                        .replace(/\n\s*\n\s*\n/g, '\n\n')
+                        .trim();
+
+                    jobDescription = {
+                        html: htmlContent,
+                        text: textContent
+                    };
+                    console.log('Found job description with regex fallback');
+                    break;
+                }
+            }
+        }
+
+        if (!jobDescription) {
+            throw new Error('Could not find job description on the page');
+        }
+
+        // Output the result as JSON
+        const result = {
+            url: url,
+            jobDescription: jobDescription,
+            timestamp: new Date().toISOString()
+        };
+
+        console.log('\n=== RESULT ===');
+        console.log(JSON.stringify(result, null, 2));
+
+        return result;
+
+    } catch (error) {
+        console.error('Error scraping job:', error.message);
+        throw error;
+    } finally {
+        if (browser) {
+            await browser.close();
+        }
+    }
+}
+
+// Run the scraper
+scrapeAdzunaJob(url)
+    .then(() => {
+        console.log('\nScraping completed successfully');
+        process.exit(0);
+    })
+    .catch((error) => {
+        console.error('\nScraping failed:', error);
+        process.exit(1);
+    });