[#1] added last_tested_at column to proxies and added json to the command to fetch proxies
This commit is contained in:
@@ -97,10 +97,18 @@ class FetchProxyLists extends Command
|
||||
|
||||
$content = $response->getBody()->getContents();
|
||||
|
||||
if ($list->scrape_type === 'new_line') {
|
||||
$this->processNewLineList($content, $list->proxy_type, $results);
|
||||
} else {
|
||||
$this->processWebsiteList($content, $list->proxy_type, $results);
|
||||
switch ($list->scrape_type) {
|
||||
case 'new_line':
|
||||
$this->processNewLineList($content, $list->proxy_type, $results);
|
||||
break;
|
||||
case 'scrape_website':
|
||||
$this->processWebsiteList($content, $list->proxy_type, $results);
|
||||
break;
|
||||
case 'json':
|
||||
$this->processJsonList($content, $list->proxy_type, $results);
|
||||
break;
|
||||
default:
|
||||
throw new \Exception("Unknown scrape type: {$list->scrape_type}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -141,6 +149,33 @@ class FetchProxyLists extends Command
|
||||
}
|
||||
}
|
||||
|
||||
protected function processJsonList(string $content, string $proxyType, array &$results)
|
||||
{
|
||||
$data = json_decode($content, true);
|
||||
|
||||
if (json_last_error() !== JSON_ERROR_NONE) {
|
||||
throw new \Exception('Invalid JSON content: ' . json_last_error_msg());
|
||||
}
|
||||
|
||||
if (!is_array($data)) {
|
||||
throw new \Exception('JSON content must be an array');
|
||||
}
|
||||
|
||||
foreach ($data as $proxy) {
|
||||
if (!isset($proxy['proxy']) || !isset($proxy['protocol']) || !isset($proxy['ip']) || !isset($proxy['port'])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->saveProxy(
|
||||
$proxy['proxy'],
|
||||
$proxy['protocol'],
|
||||
$proxy['ip'],
|
||||
(int) $proxy['port'],
|
||||
$results
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
protected function saveProxy(string $proxyUrl, string $proxyType, string $ip, int $port, array &$results)
|
||||
{
|
||||
// Check if proxy already exists
|
||||
|
||||
Reference in New Issue
Block a user