first commit

2025-03-30 19:37:01 +03:00
commit c3686cbf5c
118 changed files with 16821 additions and 0 deletions
@@ -0,0 +1,162 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use App\Models\ProxiesList;
+use App\Models\Proxy;
+
+class FetchProxyLists extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'proxies:fetch-lists {--timeout=30}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Fetch proxy lists that haven\'t been scraped in the last 4 hours';
+
+    protected $timeout;
+
+    public function __construct()
+    {
+        parent::__construct();
+    }
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $this->timeout = $this->option('timeout');
+        $this->info('Starting proxy list fetching...');
+
+        // Get lists that haven't been scraped in the last 4 hours
+        $lists = ProxiesList::where(function($query) {
+            $query->whereNull('last_scraped_at')
+                  ->orWhere('last_scraped_at', '<', now()->subHours(12));
+        })->get();
+
+        if ($lists->isEmpty()) {
+            $this->info('No proxy lists need to be fetched.');
+            return 0;
+        }
+
+        $this->info('Found ' . $lists->count() . ' lists to process');
+        $bar = $this->output->createProgressBar($lists->count());
+        $bar->start();
+
+        $results = [
+            'success' => 0,
+            'failed' => 0,
+            'proxies_added' => 0
+        ];
+
+        foreach ($lists as $list) {
+            try {
+                $this->processList($list, $results);
+                $list->update(['last_scraped_at' => now()]);
+                $results['success']++;
+            } catch (\Exception $e) {
+                $this->error("\nError processing {$list->url}: " . $e->getMessage());
+                $results['failed']++;
+            }
+
+            $bar->advance();
+        }
+
+        $bar->finish();
+        $this->newLine();
+        $this->info('Proxy list fetching completed!');
+        $this->info("Successfully processed: {$results['success']}");
+        $this->info("Failed to process: {$results['failed']}");
+        $this->info("New proxies added: {$results['proxies_added']}");
+
+        return 0;
+    }
+
+    protected function processList(ProxiesList $list, array &$results)
+    {
+        $client = new \GuzzleHttp\Client([
+            'timeout' => $this->timeout,
+            'verify' => false,
+            'http_errors' => false
+        ]);
+
+        $response = $client->get($list->url);
+        
+        if ($response->getStatusCode() !== 200) {
+            throw new \Exception("Failed to fetch list: HTTP {$response->getStatusCode()}");
+        }
+
+        $content = $response->getBody()->getContents();
+        
+        if ($list->scrape_type === 'new_line') {
+            $this->processNewLineList($content, $list->proxy_type, $results);
+        } else {
+            $this->processWebsiteList($content, $list->proxy_type, $results);
+        }
+    }
+
+    protected function processNewLineList(string $content, string $proxyType, array &$results)
+    {
+        $lines = array_filter(explode("\n", $content));
+        
+        foreach ($lines as $line) {
+            $line = trim($line);
+            if (empty($line)) continue;
+
+            // Try to parse IP:PORT format
+            if (preg_match('/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)$/', $line, $matches)) {
+                $ip = $matches[1];
+                $port = (int) $matches[2];
+                $proxyUrl = "{$proxyType}://{$ip}:{$port}";
+
+                $this->saveProxy($proxyUrl, $proxyType, $ip, $port, $results);
+            }
+        }
+    }
+
+    protected function processWebsiteList(string $content, string $proxyType, array &$results)
+    {
+        // Extract IP:PORT patterns from HTML content
+        preg_match_all('/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)/', $content, $matches);
+        
+        if (empty($matches[0])) {
+            throw new \Exception('No proxy patterns found in website content');
+        }
+
+        foreach ($matches[0] as $match) {
+            list($ip, $port) = explode(':', $match);
+            $port = (int) $port;
+            $proxyUrl = "{$proxyType}://{$ip}:{$port}";
+
+            $this->saveProxy($proxyUrl, $proxyType, $ip, $port, $results);
+        }
+    }
+
+    protected function saveProxy(string $proxyUrl, string $proxyType, string $ip, int $port, array &$results)
+    {
+        // Check if proxy already exists
+        $exists = Proxy::where('proxy', $proxyUrl)->exists();
+
+        if (!$exists) {
+            Proxy::create([
+                'proxy' => $proxyUrl,
+                'protocol' => $proxyType,
+                'ip' => $ip,
+                'port' => $port,
+                'https' => $proxyType === 'https',
+                'anonymity' => 'unknown',
+                'score' => 1
+            ]);
+            $results['proxies_added']++;
+        }
+    }
+}