Add bulk-run actions and per-source rate limiting
Run-all buttons (per target and per investigation) queue every applicable, not-already-in-flight source via a new FindingDispatcher, re-attempting prior failures automatically. RunFinding now applies a per-source RateLimited queue middleware so a bulk run can't hammer the free public APIs (ip-api, crt.sh, whois, sherlock, theharvester) it depends on; $tries raised to 30 to survive rate-limiter releases, with a failed() handler so exhausted retries still resolve the Finding instead of leaving it stuck pending. Also fixes CrtShRunner treating crt.sh's intermittent 404/502 (a known flaky free service) as fatal on the first try — added a bounded retry. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
853686f3f3
commit
e4fa57c3d6
@@ -0,0 +1,70 @@
|
||||
<?php
|
||||
|
||||
namespace Tests\Unit;
|
||||
|
||||
use App\Jobs\RunFinding;
|
||||
use App\Models\Investigation;
|
||||
use App\Models\Source;
|
||||
use App\Models\User;
|
||||
use App\Osint\FindingDispatcher;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
use Tests\TestCase;
|
||||
|
||||
class FindingDispatcherTest extends TestCase
|
||||
{
|
||||
use RefreshDatabase;
|
||||
|
||||
public function test_queues_only_applicable_registered_sources(): void
|
||||
{
|
||||
Queue::fake();
|
||||
|
||||
$user = User::factory()->create();
|
||||
$investigation = Investigation::factory()->for($user, 'owner')->create();
|
||||
$target = $investigation->targets()->create(['type' => 'username', 'value' => 'someone', 'created_by' => $user->id]);
|
||||
|
||||
Source::create(['key' => 'sherlock', 'name' => 'Sherlock', 'target_types' => ['username'], 'enabled' => true]);
|
||||
Source::create(['key' => 'crtsh', 'name' => 'crt.sh', 'target_types' => ['domain'], 'enabled' => true]);
|
||||
Source::create(['key' => 'unregistered', 'name' => 'Unregistered', 'target_types' => ['username'], 'enabled' => true]);
|
||||
|
||||
$queued = app(FindingDispatcher::class)->dispatchAllApplicable($target, $user);
|
||||
|
||||
$this->assertSame(1, $queued);
|
||||
$this->assertDatabaseCount('findings', 1);
|
||||
Queue::assertPushed(RunFinding::class, 1);
|
||||
}
|
||||
|
||||
public function test_skips_sources_already_pending_or_running(): void
|
||||
{
|
||||
Queue::fake();
|
||||
|
||||
$user = User::factory()->create();
|
||||
$investigation = Investigation::factory()->for($user, 'owner')->create();
|
||||
$target = $investigation->targets()->create(['type' => 'username', 'value' => 'someone', 'created_by' => $user->id]);
|
||||
|
||||
$sherlock = Source::create(['key' => 'sherlock', 'name' => 'Sherlock', 'target_types' => ['username'], 'enabled' => true]);
|
||||
$target->findings()->create(['source_id' => $sherlock->id, 'status' => 'pending']);
|
||||
|
||||
$queued = app(FindingDispatcher::class)->dispatchAllApplicable($target, $user);
|
||||
|
||||
$this->assertSame(0, $queued);
|
||||
$this->assertDatabaseCount('findings', 1);
|
||||
}
|
||||
|
||||
public function test_does_not_skip_a_source_whose_last_finding_already_completed(): void
|
||||
{
|
||||
Queue::fake();
|
||||
|
||||
$user = User::factory()->create();
|
||||
$investigation = Investigation::factory()->for($user, 'owner')->create();
|
||||
$target = $investigation->targets()->create(['type' => 'username', 'value' => 'someone', 'created_by' => $user->id]);
|
||||
|
||||
$sherlock = Source::create(['key' => 'sherlock', 'name' => 'Sherlock', 'target_types' => ['username'], 'enabled' => true]);
|
||||
$target->findings()->create(['source_id' => $sherlock->id, 'status' => 'completed']);
|
||||
|
||||
$queued = app(FindingDispatcher::class)->dispatchAllApplicable($target, $user);
|
||||
|
||||
$this->assertSame(1, $queued);
|
||||
$this->assertDatabaseCount('findings', 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
<?php
|
||||
|
||||
namespace Tests\Unit\Jobs;
|
||||
|
||||
use App\Jobs\RunFinding;
|
||||
use App\Models\Finding;
|
||||
use App\Models\Investigation;
|
||||
use App\Models\Source;
|
||||
use App\Models\User;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Queue\Middleware\RateLimited;
|
||||
use Tests\TestCase;
|
||||
|
||||
class RunFindingTest extends TestCase
|
||||
{
|
||||
use RefreshDatabase;
|
||||
|
||||
public function test_middleware_rate_limits_by_the_findings_source_key(): void
|
||||
{
|
||||
$user = User::factory()->create();
|
||||
$investigation = Investigation::factory()->for($user, 'owner')->create();
|
||||
$target = $investigation->targets()->create(['type' => 'ip', 'value' => '8.8.8.8', 'created_by' => $user->id]);
|
||||
$source = Source::create(['key' => 'ip-api', 'name' => 'ip-api.com', 'target_types' => ['ip'], 'enabled' => true]);
|
||||
$finding = $target->findings()->create(['source_id' => $source->id, 'status' => 'pending']);
|
||||
|
||||
$middleware = (new RunFinding($finding->id))->middleware();
|
||||
|
||||
$this->assertCount(1, $middleware);
|
||||
$this->assertInstanceOf(RateLimited::class, $middleware[0]);
|
||||
}
|
||||
|
||||
public function test_failed_marks_the_finding_as_failed(): void
|
||||
{
|
||||
$user = User::factory()->create();
|
||||
$investigation = Investigation::factory()->for($user, 'owner')->create();
|
||||
$target = $investigation->targets()->create(['type' => 'ip', 'value' => '8.8.8.8', 'created_by' => $user->id]);
|
||||
$source = Source::create(['key' => 'ip-api', 'name' => 'ip-api.com', 'target_types' => ['ip'], 'enabled' => true]);
|
||||
$finding = $target->findings()->create(['source_id' => $source->id, 'status' => 'pending']);
|
||||
|
||||
(new RunFinding($finding->id))->failed(new \RuntimeException('retries exhausted'));
|
||||
|
||||
$finding->refresh();
|
||||
$this->assertSame('failed', $finding->status);
|
||||
$this->assertStringContainsString('retries exhausted', $finding->error);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user