first commit
This commit is contained in:
@@ -0,0 +1,85 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use App\Models\Proxy;
|
||||
|
||||
class FetchProxies extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'proxies:fetch';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Fetch proxies from the CDN and save them to the database';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$this->info('Fetching proxies from CDN...');
|
||||
|
||||
try {
|
||||
$response = Http::get('https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json');
|
||||
|
||||
if (!$response->successful()) {
|
||||
throw new \Exception('Failed to fetch proxies: ' . $response->status());
|
||||
}
|
||||
|
||||
$proxies = $response->json();
|
||||
$this->info('Found ' . count($proxies) . ' proxies');
|
||||
|
||||
$bar = $this->output->createProgressBar(count($proxies));
|
||||
$bar->start();
|
||||
|
||||
$saved = 0;
|
||||
$skipped = 0;
|
||||
|
||||
foreach ($proxies as $proxyData) {
|
||||
// Check if proxy already exists
|
||||
$exists = Proxy::where('proxy', $proxyData['proxy'])->exists();
|
||||
|
||||
if (!$exists) {
|
||||
Proxy::create([
|
||||
'proxy' => $proxyData['proxy'],
|
||||
'protocol' => $proxyData['protocol'],
|
||||
'ip' => $proxyData['ip'],
|
||||
'port' => $proxyData['port'],
|
||||
'https' => $proxyData['https'],
|
||||
'anonymity' => $proxyData['anonymity'],
|
||||
'score' => $proxyData['score'],
|
||||
'country' => $proxyData['geolocation']['country'] ?? null,
|
||||
'city' => $proxyData['geolocation']['city'] ?? null,
|
||||
]);
|
||||
$saved++;
|
||||
} else {
|
||||
$skipped++;
|
||||
}
|
||||
|
||||
$bar->advance();
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
$this->newLine();
|
||||
$this->info("Proxies imported successfully!");
|
||||
$this->info("New proxies saved: {$saved}");
|
||||
$this->info("Existing proxies skipped: {$skipped}");
|
||||
|
||||
} catch (\Exception $e) {
|
||||
$this->error('Error: ' . $e->getMessage());
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use App\Models\ProxiesList;
|
||||
use App\Models\Proxy;
|
||||
|
||||
class FetchProxyLists extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'proxies:fetch-lists {--timeout=30}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Fetch proxy lists that haven\'t been scraped in the last 4 hours';
|
||||
|
||||
protected $timeout;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$this->timeout = $this->option('timeout');
|
||||
$this->info('Starting proxy list fetching...');
|
||||
|
||||
// Get lists that haven't been scraped in the last 4 hours
|
||||
$lists = ProxiesList::where(function($query) {
|
||||
$query->whereNull('last_scraped_at')
|
||||
->orWhere('last_scraped_at', '<', now()->subHours(12));
|
||||
})->get();
|
||||
|
||||
if ($lists->isEmpty()) {
|
||||
$this->info('No proxy lists need to be fetched.');
|
||||
return 0;
|
||||
}
|
||||
|
||||
$this->info('Found ' . $lists->count() . ' lists to process');
|
||||
$bar = $this->output->createProgressBar($lists->count());
|
||||
$bar->start();
|
||||
|
||||
$results = [
|
||||
'success' => 0,
|
||||
'failed' => 0,
|
||||
'proxies_added' => 0
|
||||
];
|
||||
|
||||
foreach ($lists as $list) {
|
||||
try {
|
||||
$this->processList($list, $results);
|
||||
$list->update(['last_scraped_at' => now()]);
|
||||
$results['success']++;
|
||||
} catch (\Exception $e) {
|
||||
$this->error("\nError processing {$list->url}: " . $e->getMessage());
|
||||
$results['failed']++;
|
||||
}
|
||||
|
||||
$bar->advance();
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
$this->newLine();
|
||||
$this->info('Proxy list fetching completed!');
|
||||
$this->info("Successfully processed: {$results['success']}");
|
||||
$this->info("Failed to process: {$results['failed']}");
|
||||
$this->info("New proxies added: {$results['proxies_added']}");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
protected function processList(ProxiesList $list, array &$results)
|
||||
{
|
||||
$client = new \GuzzleHttp\Client([
|
||||
'timeout' => $this->timeout,
|
||||
'verify' => false,
|
||||
'http_errors' => false
|
||||
]);
|
||||
|
||||
$response = $client->get($list->url);
|
||||
|
||||
if ($response->getStatusCode() !== 200) {
|
||||
throw new \Exception("Failed to fetch list: HTTP {$response->getStatusCode()}");
|
||||
}
|
||||
|
||||
$content = $response->getBody()->getContents();
|
||||
|
||||
if ($list->scrape_type === 'new_line') {
|
||||
$this->processNewLineList($content, $list->proxy_type, $results);
|
||||
} else {
|
||||
$this->processWebsiteList($content, $list->proxy_type, $results);
|
||||
}
|
||||
}
|
||||
|
||||
protected function processNewLineList(string $content, string $proxyType, array &$results)
|
||||
{
|
||||
$lines = array_filter(explode("\n", $content));
|
||||
|
||||
foreach ($lines as $line) {
|
||||
$line = trim($line);
|
||||
if (empty($line)) continue;
|
||||
|
||||
// Try to parse IP:PORT format
|
||||
if (preg_match('/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)$/', $line, $matches)) {
|
||||
$ip = $matches[1];
|
||||
$port = (int) $matches[2];
|
||||
$proxyUrl = "{$proxyType}://{$ip}:{$port}";
|
||||
|
||||
$this->saveProxy($proxyUrl, $proxyType, $ip, $port, $results);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected function processWebsiteList(string $content, string $proxyType, array &$results)
|
||||
{
|
||||
// Extract IP:PORT patterns from HTML content
|
||||
preg_match_all('/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)/', $content, $matches);
|
||||
|
||||
if (empty($matches[0])) {
|
||||
throw new \Exception('No proxy patterns found in website content');
|
||||
}
|
||||
|
||||
foreach ($matches[0] as $match) {
|
||||
list($ip, $port) = explode(':', $match);
|
||||
$port = (int) $port;
|
||||
$proxyUrl = "{$proxyType}://{$ip}:{$port}";
|
||||
|
||||
$this->saveProxy($proxyUrl, $proxyType, $ip, $port, $results);
|
||||
}
|
||||
}
|
||||
|
||||
protected function saveProxy(string $proxyUrl, string $proxyType, string $ip, int $port, array &$results)
|
||||
{
|
||||
// Check if proxy already exists
|
||||
$exists = Proxy::where('proxy', $proxyUrl)->exists();
|
||||
|
||||
if (!$exists) {
|
||||
Proxy::create([
|
||||
'proxy' => $proxyUrl,
|
||||
'protocol' => $proxyType,
|
||||
'ip' => $ip,
|
||||
'port' => $port,
|
||||
'https' => $proxyType === 'https',
|
||||
'anonymity' => 'unknown',
|
||||
'score' => 1
|
||||
]);
|
||||
$results['proxies_added']++;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use App\Models\Proxy;
|
||||
|
||||
class FetchProxyScrape extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'proxies:fetch-scrape';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Fetch proxies from ProxyScrape API and save them to the database';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$this->info('Fetching proxies from ProxyScrape API...');
|
||||
|
||||
try {
|
||||
$response = Http::get('https://api.proxyscrape.com/v3/free-proxy-list/get?request=displayproxies&protocol=http&proxy_format=ipport&format=text&timeout=20000');
|
||||
|
||||
if (!$response->successful()) {
|
||||
throw new \Exception('Failed to fetch proxies: ' . $response->status());
|
||||
}
|
||||
|
||||
$proxyList = explode("\n", trim($response->body()));
|
||||
$this->info('Found ' . count($proxyList) . ' proxies');
|
||||
|
||||
$bar = $this->output->createProgressBar(count($proxyList));
|
||||
$bar->start();
|
||||
|
||||
$saved = 0;
|
||||
$skipped = 0;
|
||||
|
||||
foreach ($proxyList as $proxyString) {
|
||||
$proxyString = trim($proxyString);
|
||||
if (empty($proxyString)) continue;
|
||||
|
||||
// Parse IP and Port from the proxy string
|
||||
list($ip, $port) = explode(':', $proxyString);
|
||||
$proxyUrl = "http://{$ip}:{$port}";
|
||||
|
||||
// Check if proxy already exists
|
||||
$exists = Proxy::where('proxy', $proxyUrl)->exists();
|
||||
|
||||
if (!$exists) {
|
||||
Proxy::create([
|
||||
'proxy' => $proxyUrl,
|
||||
'protocol' => 'http',
|
||||
'ip' => $ip,
|
||||
'port' => (int) $port,
|
||||
'https' => false, // Default to false since these are HTTP proxies
|
||||
'anonymity' => 'unknown', // ProxyScrape doesn't provide anonymity info
|
||||
'score' => 1, // Default score
|
||||
'country' => null, // ProxyScrape doesn't provide geolocation
|
||||
'city' => null,
|
||||
]);
|
||||
$saved++;
|
||||
} else {
|
||||
$skipped++;
|
||||
}
|
||||
|
||||
$bar->advance();
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
$this->newLine();
|
||||
$this->info("Proxies imported successfully!");
|
||||
$this->info("New proxies saved: {$saved}");
|
||||
$this->info("Existing proxies skipped: {$skipped}");
|
||||
|
||||
} catch (\Exception $e) {
|
||||
$this->error('Error: ' . $e->getMessage());
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use App\Models\Proxy;
|
||||
|
||||
class TestProxies extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'proxies:test {--timeout=10} {--concurrent=10}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Test proxies against the test endpoint';
|
||||
|
||||
protected $testUrl;
|
||||
protected $timeout;
|
||||
protected $concurrent;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
parent::__construct();
|
||||
$this->testUrl = config('app.url') . '/proxy-test';
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$this->timeout = $this->option('timeout');
|
||||
$this->concurrent = $this->option('concurrent');
|
||||
|
||||
$this->info('Starting proxy testing...');
|
||||
$this->info("Test URL: {$this->testUrl}");
|
||||
$this->info("Timeout: {$this->timeout}s");
|
||||
$this->info("Concurrent tests: {$this->concurrent}");
|
||||
|
||||
$proxies = Proxy::all();
|
||||
$this->info('Found ' . $proxies->count() . ' proxies to test');
|
||||
|
||||
$bar = $this->output->createProgressBar($proxies->count());
|
||||
$bar->start();
|
||||
|
||||
$chunks = $proxies->chunk($this->concurrent);
|
||||
$results = [
|
||||
'working' => 0,
|
||||
'failed' => 0,
|
||||
'timeout' => 0
|
||||
];
|
||||
|
||||
foreach ($chunks as $chunk) {
|
||||
$promises = [];
|
||||
foreach ($chunk as $proxy) {
|
||||
$promises[] = $this->testProxy($proxy, $results);
|
||||
}
|
||||
|
||||
// Wait for all promises to complete
|
||||
foreach ($promises as $promise) {
|
||||
$promise->wait();
|
||||
}
|
||||
|
||||
$bar->advance($chunk->count());
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
$this->newLine();
|
||||
$this->info('Proxy testing completed!');
|
||||
$this->info("Working proxies: {$results['working']}");
|
||||
$this->info("Failed proxies: {$results['failed']}");
|
||||
$this->info("Timeout proxies: {$results['timeout']}");
|
||||
}
|
||||
|
||||
protected function testProxy($proxy, &$results)
|
||||
{
|
||||
return new \GuzzleHttp\Promise\Promise(function() use ($proxy, &$results) {
|
||||
try {
|
||||
$client = new \GuzzleHttp\Client([
|
||||
'proxy' => $proxy->proxy,
|
||||
'timeout' => $this->timeout,
|
||||
'verify' => false,
|
||||
'http_errors' => false
|
||||
]);
|
||||
|
||||
$response = $client->get($this->testUrl);
|
||||
$body = json_decode($response->getBody(), true);
|
||||
|
||||
if ($response->getStatusCode() === 200 && isset($body['unique_id'])) {
|
||||
// Proxy is working, increase score
|
||||
$proxy->increment('score');
|
||||
$results['working']++;
|
||||
} else {
|
||||
// Proxy returned invalid response
|
||||
$proxy->decrement('score');
|
||||
$results['failed']++;
|
||||
}
|
||||
} catch (\GuzzleHttp\Exception\ConnectException $e) {
|
||||
// Connection failed
|
||||
$proxy->decrement('score');
|
||||
$results['failed']++;
|
||||
} catch (\GuzzleHttp\Exception\RequestException $e) {
|
||||
if ($e->getCode() === CURLE_OPERATION_TIMEDOUT) {
|
||||
// Request timed out
|
||||
$proxy->decrement('score');
|
||||
$results['timeout']++;
|
||||
} else {
|
||||
// Other request error
|
||||
$proxy->decrement('score');
|
||||
$results['failed']++;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user