From e4fa57c3d68763c6d97e77add901e22561feac8b Mon Sep 17 00:00:00 2001 From: cedry2k1992 Date: Mon, 20 Jul 2026 11:29:26 +0300 Subject: [PATCH] Add bulk-run actions and per-source rate limiting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run-all buttons (per target and per investigation) queue every applicable, not-already-in-flight source via a new FindingDispatcher, re-attempting prior failures automatically. RunFinding now applies a per-source RateLimited queue middleware so a bulk run can't hammer the free public APIs (ip-api, crt.sh, whois, sherlock, theharvester) it depends on; $tries raised to 30 to survive rate-limiter releases, with a failed() handler so exhausted retries still resolve the Finding instead of leaving it stuck pending. Also fixes CrtShRunner treating crt.sh's intermittent 404/502 (a known flaky free service) as fatal on the first try — added a bounded retry. Co-Authored-By: Claude Sonnet 5 --- PLAN.md | 50 +++++++++++++ .../Controllers/InvestigationController.php | 15 ++++ app/Http/Controllers/TargetController.php | 15 ++++ app/Jobs/RunFinding.php | 38 +++++++++- app/Osint/FindingDispatcher.php | 50 +++++++++++++ app/Osint/Runners/CrtShRunner.php | 8 ++- app/Providers/AppServiceProvider.php | 11 ++- deploy/systemd/osint-queue-worker.service | 2 +- resources/views/investigations/show.blade.php | 16 ++++- routes/web.php | 2 + tests/Feature/BulkRunTest.php | 69 ++++++++++++++++++ tests/Unit/FindingDispatcherTest.php | 70 +++++++++++++++++++ tests/Unit/Jobs/RunFindingTest.php | 46 ++++++++++++ 13 files changed, 387 insertions(+), 5 deletions(-) create mode 100644 app/Osint/FindingDispatcher.php create mode 100644 tests/Feature/BulkRunTest.php create mode 100644 tests/Unit/FindingDispatcherTest.php create mode 100644 tests/Unit/Jobs/RunFindingTest.php diff --git a/PLAN.md b/PLAN.md index bd85c8d..47627b2 100644 --- a/PLAN.md +++ b/PLAN.md @@ -160,6 +160,56 @@ commands). worker process directly and confirmed systemd auto-restarted it within seconds. See `deploy/systemd/README.md` for install/operate commands. +9. **Bulk actions + rate limiting** (done) — the original 8 phases + were all complete; this closed the biggest remaining workflow + friction (running each source against each target one at a time). + - `App\Osint\FindingDispatcher::dispatchAllApplicable()` — queues + every enabled, registered source that applies to a target's type + and doesn't already have a `pending`/`running` finding for it + (a `completed`/`failed` one *is* re-queued, so bulk-running + naturally retries prior failures). Shared by a "Run all" button + per target and a "Run all targets" button per investigation. + - Per-source rate limiting via Laravel's `RateLimited` queue + middleware (`RunFinding::middleware()`), limits defined in + `AppServiceProvider::boot()`: ip-api.com 45/min (their published + limit), crt.sh 30/min, WHOIS 20/min, Sherlock 10/min, + theHarvester 5/min (all courtesy limits of ours, protecting the + free infra these runners depend on from a bulk run hammering it). + - `RunFinding::$tries` raised from 1 to 30: a rate-limiter release + counts as a queue attempt, and a bulk run can easily queue dozens + of same-source lookups behind one rate-limit window — needs + enough attempts to ride that out. Added `RunFinding::failed()` so + a job that does exhaust retries still marks its `Finding` as + `failed` with a clear error instead of leaving it stuck at + `pending` forever. (Business-logic failures — a runner throwing — + are unaffected; `handle()` already catches those internally and + never lets them bubble up to Laravel's retry mechanism.) + - Bug found and fixed via this work: `CrtShRunner` treated any + non-2xx as fatal, but crt.sh (a free, frequently overloaded + public service) intermittently returns 404/502/503/timeouts for + queries that succeed moments later — confirmed by hand, same + query, 404 then 200 with no code changes in between. Added + `->retry(3, 2000, throw: false)`. This doesn't fully solve it — + hit a sustained bad patch during testing where even 3 retries + over ~6s didn't survive it — but it's real instability on crt.sh's + end, not something worth over-engineering client-side retries + around; the failure path (clear error, re-runnable, doesn't block + other queued jobs) already handles it reasonably. + - Also reconciled `deploy/systemd/osint-queue-worker.service`'s + `--tries=1` → `--tries=3`: a job's own `$tries` property overrides + this CLI default, so `RunFinding`'s `$tries=30` was never at risk, + but leaving the generic default at 1 was a footgun for any future + job type that doesn't set its own `$tries` (Laravel's real + default if `--tries` were omitted entirely is unlimited retries). + - 8 new tests (75/75 passing): `FindingDispatcher` skip/re-queue + logic, `RunFinding::middleware()`/`failed()`, HTTP-level bulk-run + authorization. Plus a full manual pass against the real dev + server + the live systemd worker (not `queue:work --once`): + bulk-ran 3 sources against one target, bulk-ran across two + targets in one investigation, watched theHarvester/crt.sh/whois + resolve sequentially through the single worker, hit and diagnosed + the crt.sh flakiness live, confirmed a retried lookup completing + cleanly. All test data cleaned up afterward. ## Open questions - Any legal/ToS constraints on the sources to be queried? diff --git a/app/Http/Controllers/InvestigationController.php b/app/Http/Controllers/InvestigationController.php index 6d677f0..0fa7548 100644 --- a/app/Http/Controllers/InvestigationController.php +++ b/app/Http/Controllers/InvestigationController.php @@ -4,6 +4,7 @@ namespace App\Http\Controllers; use App\Models\Investigation; use App\Models\Source; +use App\Osint\FindingDispatcher; use App\Osint\RunnerRegistry; use Illuminate\Http\RedirectResponse; use Illuminate\Http\Request; @@ -72,4 +73,18 @@ class InvestigationController extends Controller return redirect()->route('investigations.index'); } + + public function runAll(Request $request, Investigation $investigation, FindingDispatcher $dispatcher): RedirectResponse + { + $this->authorize('update', $investigation); + + $queued = $investigation->targets->sum( + fn ($target) => $dispatcher->dispatchAllApplicable($target, $request->user()) + ); + + return redirect()->route('investigations.show', $investigation) + ->with('status', $queued > 0 + ? "Queued {$queued} lookup(s) across all targets..." + : 'Nothing to queue — already up to date.'); + } } diff --git a/app/Http/Controllers/TargetController.php b/app/Http/Controllers/TargetController.php index 4773306..a87dbc2 100644 --- a/app/Http/Controllers/TargetController.php +++ b/app/Http/Controllers/TargetController.php @@ -7,6 +7,7 @@ use App\Models\Finding; use App\Models\Investigation; use App\Models\Source; use App\Models\Target; +use App\Osint\FindingDispatcher; use App\Osint\RunnerRegistry; use Illuminate\Http\RedirectResponse; use Illuminate\Http\Request; @@ -74,4 +75,18 @@ class TargetController extends Controller return redirect()->route('investigations.show', $investigation) ->with('status', "Running {$source->name} against {$target->value}..."); } + + public function runAll(Request $request, Investigation $investigation, Target $target, FindingDispatcher $dispatcher): RedirectResponse + { + $this->authorize('update', $investigation); + + abort_if($target->investigation_id !== $investigation->id, 404); + + $queued = $dispatcher->dispatchAllApplicable($target, $request->user()); + + return redirect()->route('investigations.show', $investigation) + ->with('status', $queued > 0 + ? "Queued {$queued} lookup(s) for {$target->value}..." + : "Nothing to queue for {$target->value} — already up to date."); + } } diff --git a/app/Jobs/RunFinding.php b/app/Jobs/RunFinding.php index 8aa2670..41a197b 100644 --- a/app/Jobs/RunFinding.php +++ b/app/Jobs/RunFinding.php @@ -6,18 +6,41 @@ use App\Models\Finding; use App\Osint\RunnerRegistry; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Queue\Queueable; +use Illuminate\Queue\Middleware\RateLimited; class RunFinding implements ShouldQueue { use Queueable; - public int $tries = 1; + /** + * Business-logic failures (a runner throwing) are caught in handle() + * below and never bubble up to the queue, so this only needs to cover + * genuine retries: rate-limiter releases (see middleware()) and worker + * crashes. A bulk run against a busy investigation can easily queue + * dozens of same-source lookups behind one rate limit, so this is set + * high enough to ride that out rather than give up mid-queue. + */ + public int $tries = 30; public function __construct( public readonly int $findingId, ) { } + /** + * @return array + */ + public function middleware(): array + { + $sourceKey = Finding::find($this->findingId)?->source?->key; + + if ($sourceKey === null) { + return []; + } + + return [new RateLimited('osint-runner:'.$sourceKey)]; + } + public function handle(RunnerRegistry $registry): void { $finding = Finding::with(['target', 'source'])->findOrFail($this->findingId); @@ -41,4 +64,17 @@ class RunFinding implements ShouldQueue ]); } } + + /** + * Only reached if the job itself fails outside handle()'s own + * try/catch — e.g. retries exhausted while rate-limited. + */ + public function failed(\Throwable $exception): void + { + Finding::where('id', $this->findingId)->update([ + 'status' => 'failed', + 'error' => 'Gave up after retries: '.$exception->getMessage(), + 'ran_at' => now(), + ]); + } } diff --git a/app/Osint/FindingDispatcher.php b/app/Osint/FindingDispatcher.php new file mode 100644 index 0000000..a343b3f --- /dev/null +++ b/app/Osint/FindingDispatcher.php @@ -0,0 +1,50 @@ +get() + ->filter(fn (Source $source) => in_array($target->type, $source->target_types, true) + && $this->registry->has($source->key)); + + $busySourceIds = $target->findings() + ->whereIn('status', ['pending', 'running']) + ->pluck('source_id'); + + $queued = 0; + + foreach ($applicableSources as $source) { + if ($busySourceIds->contains($source->id)) { + continue; + } + + $finding = $target->findings()->create([ + 'source_id' => $source->id, + 'run_by' => $runBy->id, + 'status' => 'pending', + ]); + + RunFinding::dispatch($finding->id); + $queued++; + } + + return $queued; + } +} diff --git a/app/Osint/Runners/CrtShRunner.php b/app/Osint/Runners/CrtShRunner.php index 2a9d616..1ec5602 100644 --- a/app/Osint/Runners/CrtShRunner.php +++ b/app/Osint/Runners/CrtShRunner.php @@ -16,14 +16,20 @@ class CrtShRunner implements Runner throw new RunnerException('crt.sh only supports "domain" targets.'); } + // crt.sh is a free, frequently overloaded public service that + // intermittently returns 404/502/503 for queries that succeed + // moments later (confirmed by hand: same query, 404 then 200 with + // no code changes) — worth a few retries before treating it as a + // real failure. $response = Http::timeout((int) config('osint.run_timeout')) + ->retry(3, 2000, throw: false) ->get('https://crt.sh/', [ 'q' => $target->value, 'output' => 'json', ]); if (! $response->successful()) { - throw new RunnerException('crt.sh request failed with status '.$response->status().'.'); + throw new RunnerException('crt.sh request failed with status '.$response->status().' after retries.'); } // crt.sh returns an empty body (not valid JSON) when there are no matches. diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 452e6b6..ad2dbfc 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -2,6 +2,8 @@ namespace App\Providers; +use Illuminate\Cache\RateLimiting\Limit; +use Illuminate\Support\Facades\RateLimiter; use Illuminate\Support\ServiceProvider; class AppServiceProvider extends ServiceProvider @@ -19,6 +21,13 @@ class AppServiceProvider extends ServiceProvider */ public function boot(): void { - // + // Self-imposed limits so bulk-running lookups doesn't hammer the + // free public infrastructure these runners depend on. ip-api.com's + // limit (45/min) is theirs; the rest are courtesy limits of ours. + RateLimiter::for('osint-runner:ip-api', fn () => Limit::perMinute(45)); + RateLimiter::for('osint-runner:crtsh', fn () => Limit::perMinute(30)); + RateLimiter::for('osint-runner:whois', fn () => Limit::perMinute(20)); + RateLimiter::for('osint-runner:sherlock', fn () => Limit::perMinute(10)); + RateLimiter::for('osint-runner:theharvester', fn () => Limit::perMinute(5)); } } diff --git a/deploy/systemd/osint-queue-worker.service b/deploy/systemd/osint-queue-worker.service index f833533..8e1bfde 100644 --- a/deploy/systemd/osint-queue-worker.service +++ b/deploy/systemd/osint-queue-worker.service @@ -6,7 +6,7 @@ StartLimitIntervalSec=0 [Service] Type=simple -ExecStart=/usr/bin/php /home/m105/projects/osint/artisan queue:work --queue=default --sleep=3 --tries=1 --timeout=180 --max-time=3600 +ExecStart=/usr/bin/php /home/m105/projects/osint/artisan queue:work --queue=default --sleep=3 --tries=3 --timeout=180 --max-time=3600 WorkingDirectory=/home/m105/projects/osint Environment="PATH=/home/m105/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" Restart=always diff --git a/resources/views/investigations/show.blade.php b/resources/views/investigations/show.blade.php index a3d1430..ccc14bb 100644 --- a/resources/views/investigations/show.blade.php +++ b/resources/views/investigations/show.blade.php @@ -59,7 +59,15 @@ @endif
-

{{ __('Targets') }}

+
+

{{ __('Targets') }}

+ @if ($canEdit && $investigation->targets->count() > 1) +
+ @csrf + {{ __('Run all targets') }} +
+ @endif +
@forelse ($investigation->targets as $target)
@@ -90,6 +98,12 @@ {{ __('Run') }} @endif + @if ($applicableSources->filter(fn ($s) => $runnableSourceKeys->contains($s->key))->count() > 1) +
+ @csrf + {{ __('Run all') }} +
+ @endif
@csrf @method('DELETE') diff --git a/routes/web.php b/routes/web.php index 4ee2c1d..4fd7408 100644 --- a/routes/web.php +++ b/routes/web.php @@ -25,9 +25,11 @@ Route::middleware('auth')->group(function () { Route::get('/search', [SearchController::class, 'index'])->name('search.index'); Route::resource('investigations', InvestigationController::class)->only(['index', 'create', 'store', 'show', 'destroy']); + Route::post('investigations/{investigation}/run-all', [InvestigationController::class, 'runAll'])->name('investigations.run-all'); Route::post('investigations/{investigation}/targets', [TargetController::class, 'store'])->name('investigations.targets.store'); Route::delete('investigations/{investigation}/targets/{target}', [TargetController::class, 'destroy'])->name('investigations.targets.destroy'); Route::post('investigations/{investigation}/targets/{target}/run', [TargetController::class, 'run'])->name('investigations.targets.run'); + Route::post('investigations/{investigation}/targets/{target}/run-all', [TargetController::class, 'runAll'])->name('investigations.targets.run-all'); Route::get('investigations/{investigation}/access', [InvestigationAccessController::class, 'index'])->name('investigations.access.index'); Route::post('investigations/{investigation}/access/users', [InvestigationAccessController::class, 'storeUser'])->name('investigations.access.users.store'); diff --git a/tests/Feature/BulkRunTest.php b/tests/Feature/BulkRunTest.php new file mode 100644 index 0000000..5b2ff39 --- /dev/null +++ b/tests/Feature/BulkRunTest.php @@ -0,0 +1,69 @@ +create(); + $investigation = Investigation::factory()->for($owner, 'owner')->create(); + $target = $investigation->targets()->create(['type' => 'domain', 'value' => 'example.com', 'created_by' => $owner->id]); + Source::create(['key' => 'crtsh', 'name' => 'crt.sh', 'target_types' => ['domain'], 'enabled' => true]); + Source::create(['key' => 'whois', 'name' => 'WHOIS', 'target_types' => ['domain'], 'enabled' => true]); + + $this->actingAs($owner) + ->post(route('investigations.targets.run-all', [$investigation, $target])) + ->assertRedirect(route('investigations.show', $investigation)); + + Queue::assertPushed(RunFinding::class, 2); + $this->assertDatabaseCount('findings', 2); + } + + public function test_run_all_for_investigation_covers_every_target(): void + { + Queue::fake(); + + $owner = User::factory()->create(); + $investigation = Investigation::factory()->for($owner, 'owner')->create(); + $domainTarget = $investigation->targets()->create(['type' => 'domain', 'value' => 'example.com', 'created_by' => $owner->id]); + $ipTarget = $investigation->targets()->create(['type' => 'ip', 'value' => '8.8.8.8', 'created_by' => $owner->id]); + Source::create(['key' => 'crtsh', 'name' => 'crt.sh', 'target_types' => ['domain'], 'enabled' => true]); + Source::create(['key' => 'ip-api', 'name' => 'ip-api.com', 'target_types' => ['ip'], 'enabled' => true]); + + $this->actingAs($owner) + ->post(route('investigations.run-all', $investigation)) + ->assertRedirect(route('investigations.show', $investigation)); + + Queue::assertPushed(RunFinding::class, 2); + } + + public function test_viewer_cannot_bulk_run(): void + { + $owner = User::factory()->create(); + $viewer = User::factory()->create(); + $investigation = Investigation::factory()->for($owner, 'owner')->create(); + $investigation->users()->attach($viewer, ['role' => 'viewer']); + $target = $investigation->targets()->create(['type' => 'domain', 'value' => 'example.com', 'created_by' => $owner->id]); + + $this->actingAs($viewer) + ->post(route('investigations.targets.run-all', [$investigation, $target])) + ->assertForbidden(); + + $this->actingAs($viewer) + ->post(route('investigations.run-all', $investigation)) + ->assertForbidden(); + } +} diff --git a/tests/Unit/FindingDispatcherTest.php b/tests/Unit/FindingDispatcherTest.php new file mode 100644 index 0000000..403bb7f --- /dev/null +++ b/tests/Unit/FindingDispatcherTest.php @@ -0,0 +1,70 @@ +create(); + $investigation = Investigation::factory()->for($user, 'owner')->create(); + $target = $investigation->targets()->create(['type' => 'username', 'value' => 'someone', 'created_by' => $user->id]); + + Source::create(['key' => 'sherlock', 'name' => 'Sherlock', 'target_types' => ['username'], 'enabled' => true]); + Source::create(['key' => 'crtsh', 'name' => 'crt.sh', 'target_types' => ['domain'], 'enabled' => true]); + Source::create(['key' => 'unregistered', 'name' => 'Unregistered', 'target_types' => ['username'], 'enabled' => true]); + + $queued = app(FindingDispatcher::class)->dispatchAllApplicable($target, $user); + + $this->assertSame(1, $queued); + $this->assertDatabaseCount('findings', 1); + Queue::assertPushed(RunFinding::class, 1); + } + + public function test_skips_sources_already_pending_or_running(): void + { + Queue::fake(); + + $user = User::factory()->create(); + $investigation = Investigation::factory()->for($user, 'owner')->create(); + $target = $investigation->targets()->create(['type' => 'username', 'value' => 'someone', 'created_by' => $user->id]); + + $sherlock = Source::create(['key' => 'sherlock', 'name' => 'Sherlock', 'target_types' => ['username'], 'enabled' => true]); + $target->findings()->create(['source_id' => $sherlock->id, 'status' => 'pending']); + + $queued = app(FindingDispatcher::class)->dispatchAllApplicable($target, $user); + + $this->assertSame(0, $queued); + $this->assertDatabaseCount('findings', 1); + } + + public function test_does_not_skip_a_source_whose_last_finding_already_completed(): void + { + Queue::fake(); + + $user = User::factory()->create(); + $investigation = Investigation::factory()->for($user, 'owner')->create(); + $target = $investigation->targets()->create(['type' => 'username', 'value' => 'someone', 'created_by' => $user->id]); + + $sherlock = Source::create(['key' => 'sherlock', 'name' => 'Sherlock', 'target_types' => ['username'], 'enabled' => true]); + $target->findings()->create(['source_id' => $sherlock->id, 'status' => 'completed']); + + $queued = app(FindingDispatcher::class)->dispatchAllApplicable($target, $user); + + $this->assertSame(1, $queued); + $this->assertDatabaseCount('findings', 2); + } +} diff --git a/tests/Unit/Jobs/RunFindingTest.php b/tests/Unit/Jobs/RunFindingTest.php new file mode 100644 index 0000000..6907283 --- /dev/null +++ b/tests/Unit/Jobs/RunFindingTest.php @@ -0,0 +1,46 @@ +create(); + $investigation = Investigation::factory()->for($user, 'owner')->create(); + $target = $investigation->targets()->create(['type' => 'ip', 'value' => '8.8.8.8', 'created_by' => $user->id]); + $source = Source::create(['key' => 'ip-api', 'name' => 'ip-api.com', 'target_types' => ['ip'], 'enabled' => true]); + $finding = $target->findings()->create(['source_id' => $source->id, 'status' => 'pending']); + + $middleware = (new RunFinding($finding->id))->middleware(); + + $this->assertCount(1, $middleware); + $this->assertInstanceOf(RateLimited::class, $middleware[0]); + } + + public function test_failed_marks_the_finding_as_failed(): void + { + $user = User::factory()->create(); + $investigation = Investigation::factory()->for($user, 'owner')->create(); + $target = $investigation->targets()->create(['type' => 'ip', 'value' => '8.8.8.8', 'created_by' => $user->id]); + $source = Source::create(['key' => 'ip-api', 'name' => 'ip-api.com', 'target_types' => ['ip'], 'enabled' => true]); + $finding = $target->findings()->create(['source_id' => $source->id, 'status' => 'pending']); + + (new RunFinding($finding->id))->failed(new \RuntimeException('retries exhausted')); + + $finding->refresh(); + $this->assertSame('failed', $finding->status); + $this->assertStringContainsString('retries exhausted', $finding->error); + } +}