timeout = $this->option('timeout'); $this->info('Starting proxy list fetching...'); // Get lists that haven't been scraped in the last 4 hours $lists = ProxiesList::where(function($query) { $query->whereNull('last_scraped_at') ->orWhere('last_scraped_at', '<', now()->subHours(12)); })->get(); if ($lists->isEmpty()) { $this->info('No proxy lists need to be fetched.'); return 0; } $this->info('Found ' . $lists->count() . ' lists to process'); $bar = $this->output->createProgressBar($lists->count()); $bar->start(); $results = [ 'success' => 0, 'failed' => 0, 'proxies_added' => 0 ]; foreach ($lists as $list) { try { $this->processList($list, $results); $list->update(['last_scraped_at' => now()]); $results['success']++; } catch (\Exception $e) { $this->error("\nError processing {$list->url}: " . $e->getMessage()); $results['failed']++; } $bar->advance(); } $bar->finish(); $this->newLine(); $this->info('Proxy list fetching completed!'); $this->info("Successfully processed: {$results['success']}"); $this->info("Failed to process: {$results['failed']}"); $this->info("New proxies added: {$results['proxies_added']}"); return 0; } protected function processList(ProxiesList $list, array &$results) { $client = new \GuzzleHttp\Client([ 'timeout' => $this->timeout, 'verify' => false, 'http_errors' => false ]); $response = $client->get($list->url); if ($response->getStatusCode() !== 200) { throw new \Exception("Failed to fetch list: HTTP {$response->getStatusCode()}"); } $content = $response->getBody()->getContents(); switch ($list->scrape_type) { case 'new_line': $this->processNewLineList($content, $list->proxy_type, $results); break; case 'scrape_website': $this->processWebsiteList($content, $list->proxy_type, $results); break; case 'json': $this->processJsonList($content, $list->proxy_type, $results); break; default: throw new \Exception("Unknown scrape type: {$list->scrape_type}"); } } protected function processNewLineList(string $content, string $proxyType, array &$results) { $lines = array_filter(explode("\n", $content)); foreach ($lines as $line) { $line = trim($line); if (empty($line)) continue; // Try to parse IP:PORT format if (preg_match('/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)$/', $line, $matches)) { $ip = $matches[1]; $port = (int) $matches[2]; $proxyUrl = "{$proxyType}://{$ip}:{$port}"; $this->saveProxy($proxyUrl, $proxyType, $ip, $port, $results); } } } protected function processWebsiteList(string $content, string $proxyType, array &$results) { // Extract IP:PORT patterns from HTML content preg_match_all('/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)/', $content, $matches); if (empty($matches[0])) { throw new \Exception('No proxy patterns found in website content'); } foreach ($matches[0] as $match) { list($ip, $port) = explode(':', $match); $port = (int) $port; $proxyUrl = "{$proxyType}://{$ip}:{$port}"; $this->saveProxy($proxyUrl, $proxyType, $ip, $port, $results); } } protected function processJsonList(string $content, string $proxyType, array &$results) { $data = json_decode($content, true); if (json_last_error() !== JSON_ERROR_NONE) { throw new \Exception('Invalid JSON content: ' . json_last_error_msg()); } if (!is_array($data)) { throw new \Exception('JSON content must be an array'); } foreach ($data as $proxy) { if (!isset($proxy['proxy']) || !isset($proxy['protocol']) || !isset($proxy['ip']) || !isset($proxy['port'])) { continue; } $this->saveProxy( $proxy['proxy'], $proxy['protocol'], $proxy['ip'], (int) $proxy['port'], $results ); } } protected function saveProxy(string $proxyUrl, string $proxyType, string $ip, int $port, array &$results) { // Check if proxy already exists $exists = Proxy::where('proxy', $proxyUrl)->exists(); if (!$exists) { Proxy::create([ 'proxy' => $proxyUrl, 'protocol' => $proxyType, 'ip' => $ip, 'port' => $port, 'https' => $proxyType === 'https', 'anonymity' => 'unknown', 'score' => 1 ]); $results['proxies_added']++; } } }