From 7c56f25053d4f3d6ec62aa9a496b92763309719e Mon Sep 17 00:00:00 2001 From: DoarMihai Date: Sun, 30 Mar 2025 20:24:01 +0300 Subject: [PATCH] [#1] added last_tested_at column to proxies and added json to the command to fetch proxies --- app/Console/Commands/FetchProxyLists.php | 43 ++++++++- app/Console/Commands/FetchProxyScrape.php | 92 ------------------- app/Models/Proxy.php | 6 +- ...54_add_last_tested_at_to_proxies_table.php | 28 ++++++ database/seeders/ProxiesListSeeder.php | 7 ++ 5 files changed, 78 insertions(+), 98 deletions(-) delete mode 100644 app/Console/Commands/FetchProxyScrape.php create mode 100644 database/migrations/2025_03_30_172254_add_last_tested_at_to_proxies_table.php diff --git a/app/Console/Commands/FetchProxyLists.php b/app/Console/Commands/FetchProxyLists.php index aaa8ef7..0fa7700 100644 --- a/app/Console/Commands/FetchProxyLists.php +++ b/app/Console/Commands/FetchProxyLists.php @@ -97,10 +97,18 @@ class FetchProxyLists extends Command $content = $response->getBody()->getContents(); - if ($list->scrape_type === 'new_line') { - $this->processNewLineList($content, $list->proxy_type, $results); - } else { - $this->processWebsiteList($content, $list->proxy_type, $results); + switch ($list->scrape_type) { + case 'new_line': + $this->processNewLineList($content, $list->proxy_type, $results); + break; + case 'scrape_website': + $this->processWebsiteList($content, $list->proxy_type, $results); + break; + case 'json': + $this->processJsonList($content, $list->proxy_type, $results); + break; + default: + throw new \Exception("Unknown scrape type: {$list->scrape_type}"); } } @@ -141,6 +149,33 @@ class FetchProxyLists extends Command } } + protected function processJsonList(string $content, string $proxyType, array &$results) + { + $data = json_decode($content, true); + + if (json_last_error() !== JSON_ERROR_NONE) { + throw new \Exception('Invalid JSON content: ' . json_last_error_msg()); + } + + if (!is_array($data)) { + throw new \Exception('JSON content must be an array'); + } + + foreach ($data as $proxy) { + if (!isset($proxy['proxy']) || !isset($proxy['protocol']) || !isset($proxy['ip']) || !isset($proxy['port'])) { + continue; + } + + $this->saveProxy( + $proxy['proxy'], + $proxy['protocol'], + $proxy['ip'], + (int) $proxy['port'], + $results + ); + } + } + protected function saveProxy(string $proxyUrl, string $proxyType, string $ip, int $port, array &$results) { // Check if proxy already exists diff --git a/app/Console/Commands/FetchProxyScrape.php b/app/Console/Commands/FetchProxyScrape.php deleted file mode 100644 index 66de9bc..0000000 --- a/app/Console/Commands/FetchProxyScrape.php +++ /dev/null @@ -1,92 +0,0 @@ -info('Fetching proxies from ProxyScrape API...'); - - try { - $response = Http::get('https://api.proxyscrape.com/v3/free-proxy-list/get?request=displayproxies&protocol=http&proxy_format=ipport&format=text&timeout=20000'); - - if (!$response->successful()) { - throw new \Exception('Failed to fetch proxies: ' . $response->status()); - } - - $proxyList = explode("\n", trim($response->body())); - $this->info('Found ' . count($proxyList) . ' proxies'); - - $bar = $this->output->createProgressBar(count($proxyList)); - $bar->start(); - - $saved = 0; - $skipped = 0; - - foreach ($proxyList as $proxyString) { - $proxyString = trim($proxyString); - if (empty($proxyString)) continue; - - // Parse IP and Port from the proxy string - list($ip, $port) = explode(':', $proxyString); - $proxyUrl = "http://{$ip}:{$port}"; - - // Check if proxy already exists - $exists = Proxy::where('proxy', $proxyUrl)->exists(); - - if (!$exists) { - Proxy::create([ - 'proxy' => $proxyUrl, - 'protocol' => 'http', - 'ip' => $ip, - 'port' => (int) $port, - 'https' => false, // Default to false since these are HTTP proxies - 'anonymity' => 'unknown', // ProxyScrape doesn't provide anonymity info - 'score' => 1, // Default score - 'country' => null, // ProxyScrape doesn't provide geolocation - 'city' => null, - ]); - $saved++; - } else { - $skipped++; - } - - $bar->advance(); - } - - $bar->finish(); - $this->newLine(); - $this->info("Proxies imported successfully!"); - $this->info("New proxies saved: {$saved}"); - $this->info("Existing proxies skipped: {$skipped}"); - - } catch (\Exception $e) { - $this->error('Error: ' . $e->getMessage()); - return 1; - } - - return 0; - } -} diff --git a/app/Models/Proxy.php b/app/Models/Proxy.php index a2c801a..45bddb8 100644 --- a/app/Models/Proxy.php +++ b/app/Models/Proxy.php @@ -18,13 +18,15 @@ class Proxy extends Model 'anonymity', 'score', 'country', - 'city' + 'city', + 'last_tested_at' ]; protected $casts = [ 'https' => 'boolean', 'port' => 'integer', - 'score' => 'integer' + 'score' => 'integer', + 'last_tested_at' => 'datetime' ]; public function getGeolocationAttribute() diff --git a/database/migrations/2025_03_30_172254_add_last_tested_at_to_proxies_table.php b/database/migrations/2025_03_30_172254_add_last_tested_at_to_proxies_table.php new file mode 100644 index 0000000..d8ec293 --- /dev/null +++ b/database/migrations/2025_03_30_172254_add_last_tested_at_to_proxies_table.php @@ -0,0 +1,28 @@ +timestamp('last_tested_at')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('proxies', function (Blueprint $table) { + $table->dropColumn('last_tested_at'); + }); + } +}; diff --git a/database/seeders/ProxiesListSeeder.php b/database/seeders/ProxiesListSeeder.php index be09111..a63468f 100644 --- a/database/seeders/ProxiesListSeeder.php +++ b/database/seeders/ProxiesListSeeder.php @@ -43,6 +43,13 @@ class ProxiesListSeeder extends Seeder 'https://raw.githubusercontent.com/Tsprnay/Proxy-lists/master/proxies/https.txt' ]; + // Add JSON proxy list + ProxiesList::create([ + 'url' => 'https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json', + 'scrape_type' => 'json', + 'proxy_type' => 'http' + ]); + // SOCKS4 proxies with new_line scrape type $socks4Urls = [ 'https://api.proxyscrape.com/v3/free-proxy-list/get?request=displayproxies&protocol=socks4&proxy_format=ipport&format=text&timeout=20000',