[#1] added last_tested_at column to proxies and added json to the command to fetch proxies

This commit is contained in:
2025-03-30 20:24:01 +03:00
parent c3686cbf5c
commit 7c56f25053
5 changed files with 78 additions and 98 deletions
+39 -4
View File
@@ -97,10 +97,18 @@ class FetchProxyLists extends Command
$content = $response->getBody()->getContents();
if ($list->scrape_type === 'new_line') {
$this->processNewLineList($content, $list->proxy_type, $results);
} else {
$this->processWebsiteList($content, $list->proxy_type, $results);
switch ($list->scrape_type) {
case 'new_line':
$this->processNewLineList($content, $list->proxy_type, $results);
break;
case 'scrape_website':
$this->processWebsiteList($content, $list->proxy_type, $results);
break;
case 'json':
$this->processJsonList($content, $list->proxy_type, $results);
break;
default:
throw new \Exception("Unknown scrape type: {$list->scrape_type}");
}
}
@@ -141,6 +149,33 @@ class FetchProxyLists extends Command
}
}
protected function processJsonList(string $content, string $proxyType, array &$results)
{
$data = json_decode($content, true);
if (json_last_error() !== JSON_ERROR_NONE) {
throw new \Exception('Invalid JSON content: ' . json_last_error_msg());
}
if (!is_array($data)) {
throw new \Exception('JSON content must be an array');
}
foreach ($data as $proxy) {
if (!isset($proxy['proxy']) || !isset($proxy['protocol']) || !isset($proxy['ip']) || !isset($proxy['port'])) {
continue;
}
$this->saveProxy(
$proxy['proxy'],
$proxy['protocol'],
$proxy['ip'],
(int) $proxy['port'],
$results
);
}
}
protected function saveProxy(string $proxyUrl, string $proxyType, string $ip, int $port, array &$results)
{
// Check if proxy already exists