Add bulk-run actions and per-source rate limiting
Run-all buttons (per target and per investigation) queue every applicable, not-already-in-flight source via a new FindingDispatcher, re-attempting prior failures automatically. RunFinding now applies a per-source RateLimited queue middleware so a bulk run can't hammer the free public APIs (ip-api, crt.sh, whois, sherlock, theharvester) it depends on; $tries raised to 30 to survive rate-limiter releases, with a failed() handler so exhausted retries still resolve the Finding instead of leaving it stuck pending. Also fixes CrtShRunner treating crt.sh's intermittent 404/502 (a known flaky free service) as fatal on the first try — added a bounded retry. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
853686f3f3
commit
e4fa57c3d6
@@ -0,0 +1,50 @@
|
||||
<?php
|
||||
|
||||
namespace App\Osint;
|
||||
|
||||
use App\Jobs\RunFinding;
|
||||
use App\Models\Source;
|
||||
use App\Models\Target;
|
||||
use App\Models\User;
|
||||
|
||||
class FindingDispatcher
|
||||
{
|
||||
public function __construct(private readonly RunnerRegistry $registry)
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue a lookup for every enabled, registered source that applies to
|
||||
* the target's type and doesn't already have a pending/running finding
|
||||
* for it. Returns how many were queued.
|
||||
*/
|
||||
public function dispatchAllApplicable(Target $target, User $runBy): int
|
||||
{
|
||||
$applicableSources = Source::where('enabled', true)->get()
|
||||
->filter(fn (Source $source) => in_array($target->type, $source->target_types, true)
|
||||
&& $this->registry->has($source->key));
|
||||
|
||||
$busySourceIds = $target->findings()
|
||||
->whereIn('status', ['pending', 'running'])
|
||||
->pluck('source_id');
|
||||
|
||||
$queued = 0;
|
||||
|
||||
foreach ($applicableSources as $source) {
|
||||
if ($busySourceIds->contains($source->id)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$finding = $target->findings()->create([
|
||||
'source_id' => $source->id,
|
||||
'run_by' => $runBy->id,
|
||||
'status' => 'pending',
|
||||
]);
|
||||
|
||||
RunFinding::dispatch($finding->id);
|
||||
$queued++;
|
||||
}
|
||||
|
||||
return $queued;
|
||||
}
|
||||
}
|
||||
@@ -16,14 +16,20 @@ class CrtShRunner implements Runner
|
||||
throw new RunnerException('crt.sh only supports "domain" targets.');
|
||||
}
|
||||
|
||||
// crt.sh is a free, frequently overloaded public service that
|
||||
// intermittently returns 404/502/503 for queries that succeed
|
||||
// moments later (confirmed by hand: same query, 404 then 200 with
|
||||
// no code changes) — worth a few retries before treating it as a
|
||||
// real failure.
|
||||
$response = Http::timeout((int) config('osint.run_timeout'))
|
||||
->retry(3, 2000, throw: false)
|
||||
->get('https://crt.sh/', [
|
||||
'q' => $target->value,
|
||||
'output' => 'json',
|
||||
]);
|
||||
|
||||
if (! $response->successful()) {
|
||||
throw new RunnerException('crt.sh request failed with status '.$response->status().'.');
|
||||
throw new RunnerException('crt.sh request failed with status '.$response->status().' after retries.');
|
||||
}
|
||||
|
||||
// crt.sh returns an empty body (not valid JSON) when there are no matches.
|
||||
|
||||
Reference in New Issue
Block a user