webproxyscan/app/Console/Commands/FetchProxyLists.php

<?php

namespace App\Console\Commands;

use Illuminate\Console\Command;
use App\Models\ProxiesList;
use App\Models\Proxy;

class FetchProxyLists extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'proxies:fetch-lists {--timeout=30}';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Fetch proxy lists that haven\'t been scraped in the last 4 hours';

    protected $timeout;

    public function __construct()
    {
        parent::__construct();
    }

    /**
     * Execute the console command.
     */
    public function handle()
    {
        $this->timeout = $this->option('timeout');
        $this->info('Starting proxy list fetching...');

        // Get lists that haven't been scraped in the last 4 hours
        $lists = ProxiesList::where(function($query) {
            $query->whereNull('last_scraped_at')
                  ->orWhere('last_scraped_at', '<', now()->subHours(12));
        })->get();

        if ($lists->isEmpty()) {
            $this->info('No proxy lists need to be fetched.');
            return 0;
        }

        $this->info('Found ' . $lists->count() . ' lists to process');
        $bar = $this->output->createProgressBar($lists->count());
        $bar->start();

        $results = [
            'success' => 0,
            'failed' => 0,
            'proxies_added' => 0
        ];

        foreach ($lists as $list) {
            try {
                $this->processList($list, $results);
                $list->update(['last_scraped_at' => now()]);
                $results['success']++;
            } catch (\Exception $e) {
                $this->error("\nError processing {$list->url}: " . $e->getMessage());
                $results['failed']++;
            }

            $bar->advance();
        }

        $bar->finish();
        $this->newLine();
        $this->info('Proxy list fetching completed!');
        $this->info("Successfully processed: {$results['success']}");
        $this->info("Failed to process: {$results['failed']}");
        $this->info("New proxies added: {$results['proxies_added']}");

        return 0;
    }

    protected function processList(ProxiesList $list, array &$results)
    {
        $client = new \GuzzleHttp\Client([
            'timeout' => $this->timeout,
            'verify' => false,
            'http_errors' => false
        ]);

        $response = $client->get($list->url);
        
        if ($response->getStatusCode() !== 200) {
            throw new \Exception("Failed to fetch list: HTTP {$response->getStatusCode()}");
        }

        $content = $response->getBody()->getContents();
        
        switch ($list->scrape_type) {
            case 'new_line':
                $this->processNewLineList($content, $list->proxy_type, $results);
                break;
            case 'scrape_website':
                $this->processWebsiteList($content, $list->proxy_type, $results);
                break;
            case 'json':
                $this->processJsonList($content, $list->proxy_type, $results);
                break;
            default:
                throw new \Exception("Unknown scrape type: {$list->scrape_type}");
        }
    }

    protected function processNewLineList(string $content, string $proxyType, array &$results)
    {
        $lines = array_filter(explode("\n", $content));
        
        foreach ($lines as $line) {
            $line = trim($line);
            if (empty($line)) continue;

            // Try to parse IP:PORT format
            if (preg_match('/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)$/', $line, $matches)) {
                $ip = $matches[1];
                $port = (int) $matches[2];
                $proxyUrl = "{$proxyType}://{$ip}:{$port}";

                $this->saveProxy($proxyUrl, $proxyType, $ip, $port, $results);
            }
        }
    }

    protected function processWebsiteList(string $content, string $proxyType, array &$results)
    {
        // Extract IP:PORT patterns from HTML content
        preg_match_all('/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)/', $content, $matches);
        
        if (empty($matches[0])) {
            throw new \Exception('No proxy patterns found in website content');
        }

        foreach ($matches[0] as $match) {
            list($ip, $port) = explode(':', $match);
            $port = (int) $port;
            $proxyUrl = "{$proxyType}://{$ip}:{$port}";

            $this->saveProxy($proxyUrl, $proxyType, $ip, $port, $results);
        }
    }

    protected function processJsonList(string $content, string $proxyType, array &$results)
    {
        $data = json_decode($content, true);
        
        if (json_last_error() !== JSON_ERROR_NONE) {
            throw new \Exception('Invalid JSON content: ' . json_last_error_msg());
        }

        if (!is_array($data)) {
            throw new \Exception('JSON content must be an array');
        }

        foreach ($data as $proxy) {
            if (!isset($proxy['proxy']) || !isset($proxy['protocol']) || !isset($proxy['ip']) || !isset($proxy['port'])) {
                continue;
            }

            $this->saveProxy(
                $proxy['proxy'],
                $proxy['protocol'],
                $proxy['ip'],
                (int) $proxy['port'],
                $results
            );
        }
    }

    protected function saveProxy(string $proxyUrl, string $proxyType, string $ip, int $port, array &$results)
    {
        // Check if proxy already exists
        $exists = Proxy::where('proxy', $proxyUrl)->exists();

        if (!$exists) {
            Proxy::create([
                'proxy' => $proxyUrl,
                'protocol' => $proxyType,
                'ip' => $ip,
                'port' => $port,
                'https' => $proxyType === 'https',
                'anonymity' => 'unknown',
                'score' => 1
            ]);
            $results['proxies_added']++;
        }
    }
}