Add bulk-run actions and per-source rate limiting

Run-all buttons (per target and per investigation) queue every applicable,
not-already-in-flight source via a new FindingDispatcher, re-attempting
prior failures automatically. RunFinding now applies a per-source
RateLimited queue middleware so a bulk run can't hammer the free public
APIs (ip-api, crt.sh, whois, sherlock, theharvester) it depends on;
$tries raised to 30 to survive rate-limiter releases, with a failed()
handler so exhausted retries still resolve the Finding instead of leaving
it stuck pending.

Also fixes CrtShRunner treating crt.sh's intermittent 404/502 (a known
flaky free service) as fatal on the first try — added a bounded retry.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
cedry2k1992
2026-07-20 11:29:26 +03:00
co-authored by Claude Sonnet 5
parent 853686f3f3
commit e4fa57c3d6
13 changed files with 387 additions and 5 deletions
+70
View File
@@ -0,0 +1,70 @@
<?php
namespace Tests\Unit;
use App\Jobs\RunFinding;
use App\Models\Investigation;
use App\Models\Source;
use App\Models\User;
use App\Osint\FindingDispatcher;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Queue;
use Tests\TestCase;
class FindingDispatcherTest extends TestCase
{
use RefreshDatabase;
public function test_queues_only_applicable_registered_sources(): void
{
Queue::fake();
$user = User::factory()->create();
$investigation = Investigation::factory()->for($user, 'owner')->create();
$target = $investigation->targets()->create(['type' => 'username', 'value' => 'someone', 'created_by' => $user->id]);
Source::create(['key' => 'sherlock', 'name' => 'Sherlock', 'target_types' => ['username'], 'enabled' => true]);
Source::create(['key' => 'crtsh', 'name' => 'crt.sh', 'target_types' => ['domain'], 'enabled' => true]);
Source::create(['key' => 'unregistered', 'name' => 'Unregistered', 'target_types' => ['username'], 'enabled' => true]);
$queued = app(FindingDispatcher::class)->dispatchAllApplicable($target, $user);
$this->assertSame(1, $queued);
$this->assertDatabaseCount('findings', 1);
Queue::assertPushed(RunFinding::class, 1);
}
public function test_skips_sources_already_pending_or_running(): void
{
Queue::fake();
$user = User::factory()->create();
$investigation = Investigation::factory()->for($user, 'owner')->create();
$target = $investigation->targets()->create(['type' => 'username', 'value' => 'someone', 'created_by' => $user->id]);
$sherlock = Source::create(['key' => 'sherlock', 'name' => 'Sherlock', 'target_types' => ['username'], 'enabled' => true]);
$target->findings()->create(['source_id' => $sherlock->id, 'status' => 'pending']);
$queued = app(FindingDispatcher::class)->dispatchAllApplicable($target, $user);
$this->assertSame(0, $queued);
$this->assertDatabaseCount('findings', 1);
}
public function test_does_not_skip_a_source_whose_last_finding_already_completed(): void
{
Queue::fake();
$user = User::factory()->create();
$investigation = Investigation::factory()->for($user, 'owner')->create();
$target = $investigation->targets()->create(['type' => 'username', 'value' => 'someone', 'created_by' => $user->id]);
$sherlock = Source::create(['key' => 'sherlock', 'name' => 'Sherlock', 'target_types' => ['username'], 'enabled' => true]);
$target->findings()->create(['source_id' => $sherlock->id, 'status' => 'completed']);
$queued = app(FindingDispatcher::class)->dispatchAllApplicable($target, $user);
$this->assertSame(1, $queued);
$this->assertDatabaseCount('findings', 2);
}
}
+46
View File
@@ -0,0 +1,46 @@
<?php
namespace Tests\Unit\Jobs;
use App\Jobs\RunFinding;
use App\Models\Finding;
use App\Models\Investigation;
use App\Models\Source;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Queue\Middleware\RateLimited;
use Tests\TestCase;
class RunFindingTest extends TestCase
{
use RefreshDatabase;
public function test_middleware_rate_limits_by_the_findings_source_key(): void
{
$user = User::factory()->create();
$investigation = Investigation::factory()->for($user, 'owner')->create();
$target = $investigation->targets()->create(['type' => 'ip', 'value' => '8.8.8.8', 'created_by' => $user->id]);
$source = Source::create(['key' => 'ip-api', 'name' => 'ip-api.com', 'target_types' => ['ip'], 'enabled' => true]);
$finding = $target->findings()->create(['source_id' => $source->id, 'status' => 'pending']);
$middleware = (new RunFinding($finding->id))->middleware();
$this->assertCount(1, $middleware);
$this->assertInstanceOf(RateLimited::class, $middleware[0]);
}
public function test_failed_marks_the_finding_as_failed(): void
{
$user = User::factory()->create();
$investigation = Investigation::factory()->for($user, 'owner')->create();
$target = $investigation->targets()->create(['type' => 'ip', 'value' => '8.8.8.8', 'created_by' => $user->id]);
$source = Source::create(['key' => 'ip-api', 'name' => 'ip-api.com', 'target_types' => ['ip'], 'enabled' => true]);
$finding = $target->findings()->create(['source_id' => $source->id, 'status' => 'pending']);
(new RunFinding($finding->id))->failed(new \RuntimeException('retries exhausted'));
$finding->refresh();
$this->assertSame('failed', $finding->status);
$this->assertStringContainsString('retries exhausted', $finding->error);
}
}