jobifygarden/sidecars/scraper/adzuna_scraper.js

// Description:
// Puppeteer scraper that fetches Adzuna job details using a headless browser

// This should be called something like this: `node adzuna_scraper.js https://www.adzuna.com/details/5694360484`
// The script uses puppeteer-core with chromium binary to scrape the page
// I included the html of a page for example in the sidecars/page_example.html file

import puppeteer from 'puppeteer-core';
import chromium from '@sparticuz/chromium';

// Get URL from command line arguments
const url = process.argv[2];

if (!url) {
    console.error('Error: Please provide a URL as an argument');
    console.error('Usage: node adzuna_scraper.js <url>');
    process.exit(1);
}

async function scrapeAdzunaJob(url) {
    let browser;

    try {
        console.log(`Launching browser...`);

        // Launch browser with chromium binary
        browser = await puppeteer.launch({
            executablePath: await chromium.executablePath(),
            headless: chromium.headless,
            args: [
                ...chromium.args,
                '--no-sandbox',
                '--disable-setuid-sandbox',
                '--disable-dev-shm-usage',
                '--disable-accelerated-2d-canvas',
                '--disable-gpu',
                '--window-size=1920x1080'
            ]
        });

        const page = await browser.newPage();

        // Set user agent to avoid being blocked
        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');

        // Navigate to the URL
        console.log(`Navigating to ${url}...`);
        await page.goto(url, {
            waitUntil: 'networkidle2',
            timeout: 30000
        });

        // Wait a bit for dynamic content to load
        await new Promise(resolve => setTimeout(resolve, 3000));

        // Extract the job description content
        console.log('Extracting job description...');

        let jobDescription = null;

        // First, check if the page loaded correctly by getting the HTML content
        const bodyHTML = await page.content();

        // If page is very small, it might be an error page
        if (bodyHTML.length < 1000) {
            console.log('Warning: Page content is very small, might be blocked');
            console.log('First 500 chars:', bodyHTML.substring(0, 500));
        }

        // Try multiple selectors for the job description
        const descriptionSelectors = [
            'section.adp-body',
            '.ui-foreign-click-description section',
            'div[class*="description"] section',
            'section[class*="body"]'
        ];

        for (const selector of descriptionSelectors) {
            const element = await page.$(selector);
            if (element) {
                jobDescription = await page.evaluate(el => {
                    return {
                        html: el.innerHTML,
                        text: el.textContent.trim()
                    };
                }, element);
                console.log(`Found job description with selector: ${selector}`);
                break;
            }
        }

        // If not found with puppeteer, try regex extraction as fallback
        if (!jobDescription) {
            console.log('Trying fallback regex extraction...');
            const patterns = [
                /<section\s+class="adp-body[^"]*"[^>]*>([\s\S]*?)<\/section>/i,
                /<section\s+class="[^"]*adp-body[^"]*"[^>]*>([\s\S]*?)<\/section>/i
            ];

            for (const pattern of patterns) {
                const match = bodyHTML.match(pattern);
                if (match && match[1]) {
                    const htmlContent = match[1].trim();
                    const textContent = htmlContent
                        .replace(/<br\s*\/?>/gi, '\n')
                        .replace(/<\/?(p|div|li|ul|ol|h\d)[^>]*>/gi, '\n')
                        .replace(/<[^>]+>/g, '')
                        .replace(/&amp;/g, '&')
                        .replace(/&lt;/g, '<')
                        .replace(/&gt;/g, '>')
                        .replace(/&quot;/g, '"')
                        .replace(/&#39;/g, "'")
                        .replace(/\n\s*\n\s*\n/g, '\n\n')
                        .trim();

                    jobDescription = {
                        html: htmlContent,
                        text: textContent
                    };
                    console.log('Found job description with regex fallback');
                    break;
                }
            }
        }

        if (!jobDescription) {
            throw new Error('Could not find job description on the page');
        }

        // Output the result as JSON
        const result = {
            url: url,
            jobDescription: jobDescription,
            timestamp: new Date().toISOString()
        };

        console.log('\n=== RESULT ===');
        console.log(JSON.stringify(result, null, 2));

        return result;

    } catch (error) {
        console.error('Error scraping job:', error.message);
        throw error;
    } finally {
        if (browser) {
            await browser.close();
        }
    }
}

// Run the scraper
scrapeAdzunaJob(url)
    .then(() => {
        console.log('\nScraping completed successfully');
        process.exit(0);
    })
    .catch((error) => {
        console.error('\nScraping failed:', error);
        process.exit(1);
    });