root/webproxyscan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[#1] added last_tested_at column to proxies and added json to the command to fetch proxies

Browse Source
This commit is contained in:
root
2025-03-30 20:24:01 +03:00
parent c3686cbf5c
commit 7c56f25053
5 changed files with 78 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/Console/Commands/FetchProxyLists.php
+39 -4
View File
@@ -97,10 +97,18 @@ class FetchProxyLists extends Command
$content = $response->getBody()->getContents();
if ($list->scrape_type === 'new_line') {
$this->processNewLineList($content, $list->proxy_type, $results);
} else {
$this->processWebsiteList($content, $list->proxy_type, $results);
switch ($list->scrape_type) {
case 'new_line':
$this->processNewLineList($content, $list->proxy_type, $results);
break;
case 'scrape_website':
$this->processWebsiteList($content, $list->proxy_type, $results);
break;
case 'json':
$this->processJsonList($content, $list->proxy_type, $results);
break;
default:
throw new \Exception("Unknown scrape type: {$list->scrape_type}");
}
}
@@ -141,6 +149,33 @@ class FetchProxyLists extends Command
}
}
protected function processJsonList(string $content, string $proxyType, array &$results)
{
$data = json_decode($content, true);
if (json_last_error() !== JSON_ERROR_NONE) {
throw new \Exception('Invalid JSON content: ' . json_last_error_msg());
}
if (!is_array($data)) {
throw new \Exception('JSON content must be an array');
}
foreach ($data as $proxy) {
if (!isset($proxy['proxy']) || !isset($proxy['protocol']) || !isset($proxy['ip']) || !isset($proxy['port'])) {
continue;
}
$this->saveProxy(
$proxy['proxy'],
$proxy['protocol'],
$proxy['ip'],
(int) $proxy['port'],
$results
);
}
}
protected function saveProxy(string $proxyUrl, string $proxyType, string $ip, int $port, array &$results)
{
// Check if proxy already exists
app/Console/Commands/FetchProxyScrape.php
-92
View File
@@ -1,92 +0,0 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Http;
use App\Models\Proxy;
class FetchProxyScrape extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'proxies:fetch-scrape';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Fetch proxies from ProxyScrape API and save them to the database';
/**
* Execute the console command.
*/
public function handle()
{
$this->info('Fetching proxies from ProxyScrape API...');
try {
$response = Http::get('https://api.proxyscrape.com/v3/free-proxy-list/get?request=displayproxies&protocol=http&proxy_format=ipport&format=text&timeout=20000');
if (!$response->successful()) {
throw new \Exception('Failed to fetch proxies: ' . $response->status());
}
$proxyList = explode("\n", trim($response->body()));
$this->info('Found ' . count($proxyList) . ' proxies');
$bar = $this->output->createProgressBar(count($proxyList));
$bar->start();
$saved = 0;
$skipped = 0;
foreach ($proxyList as $proxyString) {
$proxyString = trim($proxyString);
if (empty($proxyString)) continue;
// Parse IP and Port from the proxy string
list($ip, $port) = explode(':', $proxyString);
$proxyUrl = "http://{$ip}:{$port}";
// Check if proxy already exists
$exists = Proxy::where('proxy', $proxyUrl)->exists();
if (!$exists) {
Proxy::create([
'proxy' => $proxyUrl,
'protocol' => 'http',
'ip' => $ip,
'port' => (int) $port,
'https' => false, // Default to false since these are HTTP proxies
'anonymity' => 'unknown', // ProxyScrape doesn't provide anonymity info
'score' => 1, // Default score
'country' => null, // ProxyScrape doesn't provide geolocation
'city' => null,
]);
$saved++;
} else {
$skipped++;
}
$bar->advance();
}
$bar->finish();
$this->newLine();
$this->info("Proxies imported successfully!");
$this->info("New proxies saved: {$saved}");
$this->info("Existing proxies skipped: {$skipped}");
} catch (\Exception $e) {
$this->error('Error: ' . $e->getMessage());
return 1;
}
return 0;
}
}
app/Models/Proxy.php
+4 -2
View File
@@ -18,13 +18,15 @@ class Proxy extends Model
'anonymity',
'score',
'country',
'city'
'city',
'last_tested_at'
];
protected $casts = [
'https' => 'boolean',
'port' => 'integer',
'score' => 'integer'
'score' => 'integer',
'last_tested_at' => 'datetime'
];
public function getGeolocationAttribute()
database/migrations/2025_03_30_172254_add_last_tested_at_to_proxies_table.php
+28
View File
@@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('proxies', function (Blueprint $table) {
$table->timestamp('last_tested_at')->nullable();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('proxies', function (Blueprint $table) {
$table->dropColumn('last_tested_at');
});
}
};
database/seeders/ProxiesListSeeder.php
+7
View File
@@ -43,6 +43,13 @@ class ProxiesListSeeder extends Seeder
'https://raw.githubusercontent.com/Tsprnay/Proxy-lists/master/proxies/https.txt'
];
// Add JSON proxy list
ProxiesList::create([
'url' => 'https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json',
'scrape_type' => 'json',
'proxy_type' => 'http'
]);
// SOCKS4 proxies with new_line scrape type
$socks4Urls = [
'https://api.proxyscrape.com/v3/free-proxy-list/get?request=displayproxies&protocol=socks4&proxy_format=ipport&format=text&timeout=20000',