Add OpenCorporates collector for org targets, gated on an API token
Closes the last coverage gap for org (person remains intentionally unaddressed — no good free/non-scraping option exists). Unlike the other 8 sources, OpenCorporates now requires an API token even on its free tier (confirmed: keyless request gets 401). RunnerRegistry::has() checks services.opencorporates.token is configured before exposing the source as runnable, so it degrades to "not yet integrated" in the UI rather than a broken Run button, until OPENCORPORATES_API_TOKEN is set. Also fixes a real bug hit while building this: a RunnerRegistry::REQUIRES class constant mapping source key to a closure crashed on file load (PHP class constants can't hold closures) — inlined the one check instead of building an abstraction for a single case. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
86f73c46b2
commit
dc4388eabf
@@ -63,3 +63,6 @@ AWS_BUCKET=
|
||||
AWS_USE_PATH_STYLE_ENDPOINT=false
|
||||
|
||||
VITE_APP_NAME="${APP_NAME}"
|
||||
|
||||
# Free tier: register at https://opencorporates.com/api_accounts/new
|
||||
OPENCORPORATES_API_TOKEN=
|
||||
|
||||
@@ -52,8 +52,9 @@ commands).
|
||||
username, IP, org), scoped to one investigation
|
||||
- **Sources** — catalog of OSINT tools/techniques (seeded: Sherlock,
|
||||
theHarvester, crt.sh, WHOIS, ip-api.com, Holehe, Gravatar, DNS
|
||||
Records — all 8 now have a working runner). `sources.runner` is a
|
||||
display-only label; the real mapping is `App\Osint\RunnerRegistry`
|
||||
Records, OpenCorporates — 9 total, 8 runnable out of the box;
|
||||
OpenCorporates needs a free API token, see phase 12). `sources.runner`
|
||||
is a display-only label; the real mapping is `App\Osint\RunnerRegistry`
|
||||
- **Findings** — result of running a source against a target; raw JSON
|
||||
payload preserved plus a parsed summary
|
||||
|
||||
@@ -257,6 +258,42 @@ commands).
|
||||
(22 records across all 5 present types, each rendering as its
|
||||
own labeled section in the UI with correct counts and values).
|
||||
Test data cleaned up afterward.
|
||||
12. **OpenCorporates collector** (done) — 9th source, first attempt at
|
||||
closing the `org` gap. Correction to what was assumed when this
|
||||
was first suggested: OpenCorporates now requires an API token even
|
||||
on the free tier — confirmed by hand (keyless request → `401
|
||||
Invalid Api Token`). Free tier still exists (50 req/day, 200/month,
|
||||
register at opencorporates.com/api_accounts/new) but isn't fully
|
||||
keyless like the others.
|
||||
- `services.opencorporates.token` (`OPENCORPORATES_API_TOKEN` in
|
||||
`.env`) holds the token; left blank by default.
|
||||
- `RunnerRegistry::has('opencorporates')` additionally checks the
|
||||
token is configured — without one, the source is hidden from
|
||||
the "runnable" list (shows "not yet integrated", same as any
|
||||
other unbuilt runner) instead of exposing a "Run" button that
|
||||
always fails. Verified both layers by hand: the UI correctly
|
||||
grays it out, and a direct POST bypassing the UI still gets
|
||||
rejected server-side with a clear error and creates no `Finding`
|
||||
row.
|
||||
- Rate limit is their real quota (50/day) via `Limit::perDay(50)`,
|
||||
not a courtesy limit like the other sources — protects the
|
||||
configured token from getting exhausted or suspended.
|
||||
- Hit one real bug while wiring the token-gating: PHP class
|
||||
constants can't hold closures (`Constant expression contains
|
||||
invalid operations`) — an earlier `REQUIRES` map of source key →
|
||||
closure crashed every test process on file load. Fixed by
|
||||
inlining the one special case directly in `has()` instead of
|
||||
building a small closure-map abstraction for what's currently a
|
||||
single entry.
|
||||
- 8 new tests (93/93 passing) — `OpenCorporatesRunner` via
|
||||
`Http::fake()` (token missing, results found/not-found, wrong
|
||||
target type, HTTP failure), `RunnerRegistry` gating logic
|
||||
directly. Not runnable end-to-end without a real token (none
|
||||
configured in this environment) — verified what *can* be
|
||||
verified without one: the gating behavior in both the UI and
|
||||
the controller. **Whoever wants this source live needs to
|
||||
register for a free OpenCorporates account and set
|
||||
`OPENCORPORATES_API_TOKEN` in `.env`.**
|
||||
|
||||
## Open questions
|
||||
- Any legal/ToS constraints on the sources to be queried?
|
||||
|
||||
@@ -7,6 +7,7 @@ use App\Osint\Runners\DnsRunner;
|
||||
use App\Osint\Runners\GravatarRunner;
|
||||
use App\Osint\Runners\HoleheRunner;
|
||||
use App\Osint\Runners\IpApiRunner;
|
||||
use App\Osint\Runners\OpenCorporatesRunner;
|
||||
use App\Osint\Runners\SherlockRunner;
|
||||
use App\Osint\Runners\TheHarvesterRunner;
|
||||
use App\Osint\Runners\WhoisRunner;
|
||||
@@ -29,11 +30,23 @@ class RunnerRegistry
|
||||
'holehe' => HoleheRunner::class,
|
||||
'gravatar' => GravatarRunner::class,
|
||||
'dns' => DnsRunner::class,
|
||||
'opencorporates' => OpenCorporatesRunner::class,
|
||||
];
|
||||
|
||||
public function has(string $sourceKey): bool
|
||||
{
|
||||
return isset(self::RUNNERS[$sourceKey]);
|
||||
if (! isset(self::RUNNERS[$sourceKey])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// OpenCorporates additionally needs an API token configured before
|
||||
// it's actually runnable — otherwise it's hidden rather than shown
|
||||
// as a broken "Run" button that always fails.
|
||||
if ($sourceKey === 'opencorporates') {
|
||||
return filled(config('services.opencorporates.token'));
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public function resolve(string $sourceKey): Runner
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
<?php
|
||||
|
||||
namespace App\Osint\Runners;
|
||||
|
||||
use App\Models\Target;
|
||||
use App\Osint\Runner;
|
||||
use App\Osint\RunnerException;
|
||||
use App\Osint\RunnerResult;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
|
||||
class OpenCorporatesRunner implements Runner
|
||||
{
|
||||
public function run(Target $target): RunnerResult
|
||||
{
|
||||
if ($target->type !== 'org') {
|
||||
throw new RunnerException('OpenCorporates only supports "org" targets.');
|
||||
}
|
||||
|
||||
$token = config('services.opencorporates.token');
|
||||
if (blank($token)) {
|
||||
throw new RunnerException('OpenCorporates API token not configured (OPENCORPORATES_API_TOKEN).');
|
||||
}
|
||||
|
||||
$response = Http::timeout((int) config('osint.run_timeout'))
|
||||
->get('https://api.opencorporates.com/v0.4/companies/search', [
|
||||
'q' => $target->value,
|
||||
'api_token' => $token,
|
||||
'per_page' => 10,
|
||||
]);
|
||||
|
||||
if (! $response->successful()) {
|
||||
throw new RunnerException('OpenCorporates request failed with status '.$response->status().'.');
|
||||
}
|
||||
|
||||
$companies = $response->json('results.companies') ?? [];
|
||||
|
||||
$matches = collect($companies)->map(function (array $entry) {
|
||||
$company = $entry['company'];
|
||||
|
||||
$label = sprintf(
|
||||
'%s (%s%s)',
|
||||
$company['name'],
|
||||
$company['jurisdiction_code'] ?? 'unknown jurisdiction',
|
||||
$company['current_status'] ? ', '.$company['current_status'] : ''
|
||||
);
|
||||
|
||||
return ['name' => $label, 'url' => $company['opencorporates_url']];
|
||||
})->all();
|
||||
|
||||
$summary = count($matches) > 0
|
||||
? sprintf('Found %d compan%s matching "%s".', count($matches), count($matches) === 1 ? 'y' : 'ies', $target->value)
|
||||
: sprintf('No companies found matching "%s".', $target->value);
|
||||
|
||||
return new RunnerResult(raw: ['matches' => $matches], summary: $summary);
|
||||
}
|
||||
}
|
||||
@@ -32,5 +32,10 @@ class AppServiceProvider extends ServiceProvider
|
||||
RateLimiter::for('osint-runner:holehe', fn () => Limit::perMinute(15));
|
||||
RateLimiter::for('osint-runner:gravatar', fn () => Limit::perMinute(45));
|
||||
RateLimiter::for('osint-runner:dns', fn () => Limit::perMinute(60));
|
||||
|
||||
// OpenCorporates' free tier is 50 requests/day, 200/month — this
|
||||
// matches their real quota (not a courtesy limit) to protect the
|
||||
// configured token from getting exhausted or suspended.
|
||||
RateLimiter::for('osint-runner:opencorporates', fn () => Limit::perDay(50));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,4 +35,12 @@ return [
|
||||
],
|
||||
],
|
||||
|
||||
'opencorporates' => [
|
||||
// Free tier requires registering for an account at
|
||||
// opencorporates.com/api_accounts/new — 50 requests/day, 200/month.
|
||||
// Left blank, the source is hidden from "runnable" (see
|
||||
// RunnerRegistry::has()) rather than shown broken.
|
||||
'token' => env('OPENCORPORATES_API_TOKEN'),
|
||||
],
|
||||
|
||||
];
|
||||
|
||||
@@ -70,6 +70,13 @@ class SourceSeeder extends Seeder
|
||||
'target_types' => ['domain'],
|
||||
'runner' => 'dns:records',
|
||||
],
|
||||
[
|
||||
'key' => 'opencorporates',
|
||||
'name' => 'OpenCorporates',
|
||||
'description' => 'Company registration search by name. Requires a free OpenCorporates API token (OPENCORPORATES_API_TOKEN) — hidden from "runnable" sources until one is configured.',
|
||||
'target_types' => ['org'],
|
||||
'runner' => 'http:opencorporates',
|
||||
],
|
||||
];
|
||||
|
||||
foreach ($sources as $source) {
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
<?php
|
||||
|
||||
namespace Tests\Unit\Osint;
|
||||
|
||||
use App\Models\Target;
|
||||
use App\Osint\Runners\OpenCorporatesRunner;
|
||||
use App\Osint\RunnerException;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Tests\TestCase;
|
||||
|
||||
class OpenCorporatesRunnerTest extends TestCase
|
||||
{
|
||||
public function test_throws_when_no_token_configured(): void
|
||||
{
|
||||
config(['services.opencorporates.token' => null]);
|
||||
|
||||
$target = new Target(['type' => 'org', 'value' => 'Acme Inc']);
|
||||
|
||||
$this->expectException(RunnerException::class);
|
||||
$this->expectExceptionMessage('API token not configured');
|
||||
|
||||
(new OpenCorporatesRunner)->run($target);
|
||||
}
|
||||
|
||||
public function test_parses_search_results_into_matches(): void
|
||||
{
|
||||
config(['services.opencorporates.token' => 'test-token']);
|
||||
|
||||
Http::fake([
|
||||
'api.opencorporates.com/*' => Http::response([
|
||||
'results' => [
|
||||
'companies' => [
|
||||
[
|
||||
'company' => [
|
||||
'name' => 'Acme Inc',
|
||||
'jurisdiction_code' => 'us_de',
|
||||
'current_status' => 'Active',
|
||||
'opencorporates_url' => 'https://opencorporates.com/companies/us_de/123',
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
]),
|
||||
]);
|
||||
|
||||
$target = new Target(['type' => 'org', 'value' => 'Acme Inc']);
|
||||
|
||||
$result = (new OpenCorporatesRunner)->run($target);
|
||||
|
||||
$this->assertCount(1, $result->raw['matches']);
|
||||
$this->assertSame('Acme Inc (us_de, Active)', $result->raw['matches'][0]['name']);
|
||||
$this->assertSame('https://opencorporates.com/companies/us_de/123', $result->raw['matches'][0]['url']);
|
||||
$this->assertStringContainsString('Found 1 company', $result->summary);
|
||||
}
|
||||
|
||||
public function test_reports_no_companies_found(): void
|
||||
{
|
||||
config(['services.opencorporates.token' => 'test-token']);
|
||||
|
||||
Http::fake([
|
||||
'api.opencorporates.com/*' => Http::response(['results' => ['companies' => []]]),
|
||||
]);
|
||||
|
||||
$target = new Target(['type' => 'org', 'value' => 'Nonexistent Org XYZ']);
|
||||
|
||||
$result = (new OpenCorporatesRunner)->run($target);
|
||||
|
||||
$this->assertSame([], $result->raw['matches']);
|
||||
$this->assertStringContainsString('No companies found', $result->summary);
|
||||
}
|
||||
|
||||
public function test_rejects_non_org_targets(): void
|
||||
{
|
||||
config(['services.opencorporates.token' => 'test-token']);
|
||||
|
||||
$target = new Target(['type' => 'domain', 'value' => 'example.com']);
|
||||
|
||||
$this->expectException(RunnerException::class);
|
||||
|
||||
(new OpenCorporatesRunner)->run($target);
|
||||
}
|
||||
|
||||
public function test_throws_on_http_failure(): void
|
||||
{
|
||||
config(['services.opencorporates.token' => 'test-token']);
|
||||
|
||||
Http::fake(['api.opencorporates.com/*' => Http::response('', 401)]);
|
||||
|
||||
$target = new Target(['type' => 'org', 'value' => 'Acme Inc']);
|
||||
|
||||
$this->expectException(RunnerException::class);
|
||||
|
||||
(new OpenCorporatesRunner)->run($target);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
<?php
|
||||
|
||||
namespace Tests\Unit\Osint;
|
||||
|
||||
use App\Osint\RunnerRegistry;
|
||||
use Tests\TestCase;
|
||||
|
||||
class RunnerRegistryTest extends TestCase
|
||||
{
|
||||
public function test_opencorporates_is_hidden_without_a_configured_token(): void
|
||||
{
|
||||
config(['services.opencorporates.token' => null]);
|
||||
|
||||
$this->assertFalse((new RunnerRegistry)->has('opencorporates'));
|
||||
}
|
||||
|
||||
public function test_opencorporates_is_available_once_a_token_is_configured(): void
|
||||
{
|
||||
config(['services.opencorporates.token' => 'test-token']);
|
||||
|
||||
$this->assertTrue((new RunnerRegistry)->has('opencorporates'));
|
||||
}
|
||||
|
||||
public function test_unconditional_runners_are_always_available(): void
|
||||
{
|
||||
$this->assertTrue((new RunnerRegistry)->has('sherlock'));
|
||||
$this->assertTrue((new RunnerRegistry)->has('dns'));
|
||||
}
|
||||
|
||||
public function test_unknown_source_key_is_never_available(): void
|
||||
{
|
||||
$this->assertFalse((new RunnerRegistry)->has('not-a-real-source'));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user