Add Gravatar and DNS Records collectors
Two pure-PHP collectors, no Python/venv/subprocess needed:
- GravatarRunner (email): checks gravatar.com/avatar/{md5}?d=404 for a
200 vs 404. Scoped to that binary signal since Gravatar's profile
JSON endpoint now requires API auth even for real, public profiles
(confirmed against Gravatar's own creator's email).
- DnsRunner (domain): dns_get_record() for A/AAAA/MX/NS/TXT/CNAME. Raw
payload is naturally keyed by record type, which the existing
generic list-renderer on the investigation page turns into one
collapsible section per type with no UI changes needed.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
bb15f13bac
commit
86f73c46b2
@@ -51,9 +51,9 @@ commands).
|
||||
- **Targets** — the entity being investigated (person, domain, email,
|
||||
username, IP, org), scoped to one investigation
|
||||
- **Sources** — catalog of OSINT tools/techniques (seeded: Sherlock,
|
||||
theHarvester, crt.sh, WHOIS, ip-api.com, Holehe — all 6 now have a
|
||||
working runner). `sources.runner` is a display-only label; the real
|
||||
mapping is `App\Osint\RunnerRegistry`
|
||||
theHarvester, crt.sh, WHOIS, ip-api.com, Holehe, Gravatar, DNS
|
||||
Records — all 8 now have a working runner). `sources.runner` is a
|
||||
display-only label; the real mapping is `App\Osint\RunnerRegistry`
|
||||
- **Findings** — result of running a source against a target; raw JSON
|
||||
payload preserved plus a parsed summary
|
||||
|
||||
@@ -228,6 +228,35 @@ commands).
|
||||
live systemd worker, watched it fail with the exit-code-1 quirk
|
||||
before the fix and complete cleanly after, confirmed the summary
|
||||
rendered correctly in the UI. Test data cleaned up afterward.
|
||||
11. **Gravatar + DNS Records collectors** (done) — 7th and 8th
|
||||
sources, both pure PHP (no Python/venv, no external process at
|
||||
all for DNS).
|
||||
- `GravatarRunner` (email) — MD5-hashes the email, checks
|
||||
`gravatar.com/avatar/{hash}?d=404` for a 200 vs 404. Gravatar's
|
||||
public profile JSON endpoint (`{hash}.json`) turned out to now
|
||||
require API auth — confirmed by hand against a real profile
|
||||
(Matt Mullenweg's, Gravatar's own creator: avatar check returned
|
||||
200, `.json` returned 404 regardless) — so this stays scoped to
|
||||
the binary "does an avatar exist" signal rather than pulling
|
||||
profile details. Unit-tested with `Http::fake()` (avatar
|
||||
found/not-found/error, invalid email, wrong target type).
|
||||
- `DnsRunner` (domain) — pure PHP `dns_get_record()`, no HTTP
|
||||
client or subprocess. Looks up A/AAAA/MX/NS/TXT/CNAME. Its raw
|
||||
payload is naturally keyed by record type (`A`, `MX`, ...),
|
||||
which the existing generic list-renderer in the investigation
|
||||
show page turns into one collapsible section per record type
|
||||
automatically — no UI changes needed. Unit-tested against a real
|
||||
DNS lookup (`example.com`, IANA-controlled, stable for decades)
|
||||
plus a nonexistent-domain case, since `dns_get_record()` isn't
|
||||
mockable through `Http::fake()`.
|
||||
- Courtesy rate limits: Gravatar 45/min (cheap single GET), DNS
|
||||
60/min (local resolver, cheapest operation in the app).
|
||||
- 9 new tests (84/84 passing). Manually verified end-to-end
|
||||
through the live systemd worker: Gravatar against a real email
|
||||
with a known public avatar (found it), DNS against `google.com`
|
||||
(22 records across all 5 present types, each rendering as its
|
||||
own labeled section in the UI with correct counts and values).
|
||||
Test data cleaned up afterward.
|
||||
|
||||
## Open questions
|
||||
- Any legal/ToS constraints on the sources to be queried?
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
namespace App\Osint;
|
||||
|
||||
use App\Osint\Runners\CrtShRunner;
|
||||
use App\Osint\Runners\DnsRunner;
|
||||
use App\Osint\Runners\GravatarRunner;
|
||||
use App\Osint\Runners\HoleheRunner;
|
||||
use App\Osint\Runners\IpApiRunner;
|
||||
use App\Osint\Runners\SherlockRunner;
|
||||
@@ -25,6 +27,8 @@ class RunnerRegistry
|
||||
'ip-api' => IpApiRunner::class,
|
||||
'theharvester' => TheHarvesterRunner::class,
|
||||
'holehe' => HoleheRunner::class,
|
||||
'gravatar' => GravatarRunner::class,
|
||||
'dns' => DnsRunner::class,
|
||||
];
|
||||
|
||||
public function has(string $sourceKey): bool
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
<?php
|
||||
|
||||
namespace App\Osint\Runners;
|
||||
|
||||
use App\Models\Target;
|
||||
use App\Osint\Runner;
|
||||
use App\Osint\RunnerException;
|
||||
use App\Osint\RunnerResult;
|
||||
|
||||
class DnsRunner implements Runner
|
||||
{
|
||||
/**
|
||||
* @var array<string, int>
|
||||
*/
|
||||
private const RECORD_TYPES = [
|
||||
'A' => DNS_A,
|
||||
'AAAA' => DNS_AAAA,
|
||||
'MX' => DNS_MX,
|
||||
'NS' => DNS_NS,
|
||||
'TXT' => DNS_TXT,
|
||||
'CNAME' => DNS_CNAME,
|
||||
];
|
||||
|
||||
public function run(Target $target): RunnerResult
|
||||
{
|
||||
if ($target->type !== 'domain') {
|
||||
throw new RunnerException('DNS lookup only supports "domain" targets.');
|
||||
}
|
||||
|
||||
if (! preg_match('/^([a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$/', $target->value)) {
|
||||
throw new RunnerException('Domain contains characters that are not safe to look up.');
|
||||
}
|
||||
|
||||
$records = [];
|
||||
|
||||
foreach (self::RECORD_TYPES as $label => $type) {
|
||||
// dns_get_record() emits a warning (not an exception) when a
|
||||
// record type doesn't resolve for the domain — expected for
|
||||
// most domains on most types, not a real error.
|
||||
$found = @dns_get_record($target->value, $type);
|
||||
|
||||
if ($found === false || empty($found)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$records[$label] = array_values(array_unique(array_map(
|
||||
fn (array $record) => $this->formatRecord($label, $record),
|
||||
$found
|
||||
)));
|
||||
}
|
||||
|
||||
$total = array_sum(array_map('count', $records));
|
||||
|
||||
$summary = $total > 0
|
||||
? sprintf('Found %d DNS record%s across %d type%s (%s).',
|
||||
$total, $total === 1 ? '' : 's',
|
||||
count($records), count($records) === 1 ? '' : 's',
|
||||
implode(', ', array_keys($records)))
|
||||
: 'No DNS records found.';
|
||||
|
||||
return new RunnerResult(raw: $records, summary: $summary);
|
||||
}
|
||||
|
||||
private function formatRecord(string $label, array $record): string
|
||||
{
|
||||
return match ($label) {
|
||||
'A', 'AAAA' => $record['ip'] ?? '',
|
||||
'MX' => sprintf('%d %s', $record['pri'] ?? 0, $record['target'] ?? ''),
|
||||
'NS', 'CNAME' => $record['target'] ?? '',
|
||||
'TXT' => $record['txt'] ?? '',
|
||||
default => '',
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
<?php
|
||||
|
||||
namespace App\Osint\Runners;
|
||||
|
||||
use App\Models\Target;
|
||||
use App\Osint\Runner;
|
||||
use App\Osint\RunnerException;
|
||||
use App\Osint\RunnerResult;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
|
||||
class GravatarRunner implements Runner
|
||||
{
|
||||
public function run(Target $target): RunnerResult
|
||||
{
|
||||
if ($target->type !== 'email') {
|
||||
throw new RunnerException('Gravatar only supports "email" targets.');
|
||||
}
|
||||
|
||||
if (! filter_var($target->value, FILTER_VALIDATE_EMAIL)) {
|
||||
throw new RunnerException('Value is not a valid email address.');
|
||||
}
|
||||
|
||||
// Gravatar's public profile JSON endpoint now requires API auth
|
||||
// (returns 404 even for real profiles without a key), but the
|
||||
// plain avatar-existence check is still open and reliable.
|
||||
$hash = md5(strtolower(trim($target->value)));
|
||||
$avatarUrl = "https://www.gravatar.com/avatar/{$hash}";
|
||||
|
||||
$response = Http::timeout((int) config('osint.run_timeout'))
|
||||
->get($avatarUrl, ['d' => '404']);
|
||||
|
||||
if ($response->status() !== 200 && $response->status() !== 404) {
|
||||
throw new RunnerException('Gravatar request failed with status '.$response->status().'.');
|
||||
}
|
||||
|
||||
$hasAvatar = $response->status() === 200;
|
||||
|
||||
return new RunnerResult(
|
||||
raw: [
|
||||
'has_avatar' => $hasAvatar,
|
||||
'avatar_url' => $hasAvatar ? $avatarUrl : null,
|
||||
'profile_url' => "https://gravatar.com/{$hash}",
|
||||
],
|
||||
summary: $hasAvatar
|
||||
? 'Gravatar image found — this email is registered with Gravatar.'
|
||||
: 'No Gravatar found for this email.',
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -30,5 +30,7 @@ class AppServiceProvider extends ServiceProvider
|
||||
RateLimiter::for('osint-runner:sherlock', fn () => Limit::perMinute(10));
|
||||
RateLimiter::for('osint-runner:theharvester', fn () => Limit::perMinute(5));
|
||||
RateLimiter::for('osint-runner:holehe', fn () => Limit::perMinute(15));
|
||||
RateLimiter::for('osint-runner:gravatar', fn () => Limit::perMinute(45));
|
||||
RateLimiter::for('osint-runner:dns', fn () => Limit::perMinute(60));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,20 @@ class SourceSeeder extends Seeder
|
||||
'target_types' => ['email'],
|
||||
'runner' => 'python:holehe',
|
||||
],
|
||||
[
|
||||
'key' => 'gravatar',
|
||||
'name' => 'Gravatar',
|
||||
'description' => 'Checks whether an email address has a registered Gravatar image.',
|
||||
'target_types' => ['email'],
|
||||
'runner' => 'http:gravatar',
|
||||
],
|
||||
[
|
||||
'key' => 'dns',
|
||||
'name' => 'DNS Records',
|
||||
'description' => 'Looks up A, AAAA, MX, NS, TXT, and CNAME records for a domain.',
|
||||
'target_types' => ['domain'],
|
||||
'runner' => 'dns:records',
|
||||
],
|
||||
];
|
||||
|
||||
foreach ($sources as $source) {
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
<?php
|
||||
|
||||
namespace Tests\Unit\Osint;
|
||||
|
||||
use App\Models\Target;
|
||||
use App\Osint\Runners\DnsRunner;
|
||||
use App\Osint\RunnerException;
|
||||
use Tests\TestCase;
|
||||
|
||||
class DnsRunnerTest extends TestCase
|
||||
{
|
||||
public function test_rejects_non_domain_targets(): void
|
||||
{
|
||||
$target = new Target(['type' => 'ip', 'value' => '8.8.8.8']);
|
||||
|
||||
$this->expectException(RunnerException::class);
|
||||
|
||||
(new DnsRunner)->run($target);
|
||||
}
|
||||
|
||||
public function test_rejects_unsafe_domain_format(): void
|
||||
{
|
||||
$target = new Target(['type' => 'domain', 'value' => '../../etc/passwd']);
|
||||
|
||||
$this->expectException(RunnerException::class);
|
||||
|
||||
(new DnsRunner)->run($target);
|
||||
}
|
||||
|
||||
/**
|
||||
* example.com is IANA-controlled and has had stable NS/A records for
|
||||
* decades — safe to rely on for a real (non-mocked) DNS lookup.
|
||||
*/
|
||||
public function test_resolves_a_real_stable_domain(): void
|
||||
{
|
||||
$target = new Target(['type' => 'domain', 'value' => 'example.com']);
|
||||
|
||||
$result = (new DnsRunner)->run($target);
|
||||
|
||||
$this->assertArrayHasKey('NS', $result->raw);
|
||||
$this->assertNotEmpty($result->raw['NS']);
|
||||
$this->assertStringContainsString('DNS record', $result->summary);
|
||||
}
|
||||
|
||||
public function test_reports_no_records_for_a_nonexistent_domain(): void
|
||||
{
|
||||
$target = new Target(['type' => 'domain', 'value' => 'this-domain-should-not-exist-osint-test.invalid']);
|
||||
|
||||
$result = (new DnsRunner)->run($target);
|
||||
|
||||
$this->assertSame([], $result->raw);
|
||||
$this->assertSame('No DNS records found.', $result->summary);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
<?php
|
||||
|
||||
namespace Tests\Unit\Osint;
|
||||
|
||||
use App\Models\Target;
|
||||
use App\Osint\Runners\GravatarRunner;
|
||||
use App\Osint\RunnerException;
|
||||
use Illuminate\Support\Facades\Http;
|
||||
use Tests\TestCase;
|
||||
|
||||
class GravatarRunnerTest extends TestCase
|
||||
{
|
||||
public function test_reports_avatar_found(): void
|
||||
{
|
||||
Http::fake(['gravatar.com/avatar/*' => Http::response('', 200)]);
|
||||
|
||||
$target = new Target(['type' => 'email', 'value' => 'test@example.com']);
|
||||
|
||||
$result = (new GravatarRunner)->run($target);
|
||||
|
||||
$this->assertTrue($result->raw['has_avatar']);
|
||||
$this->assertNotNull($result->raw['avatar_url']);
|
||||
$this->assertStringContainsString('Gravatar image found', $result->summary);
|
||||
}
|
||||
|
||||
public function test_reports_no_avatar(): void
|
||||
{
|
||||
Http::fake(['gravatar.com/avatar/*' => Http::response('', 404)]);
|
||||
|
||||
$target = new Target(['type' => 'email', 'value' => 'test@example.com']);
|
||||
|
||||
$result = (new GravatarRunner)->run($target);
|
||||
|
||||
$this->assertFalse($result->raw['has_avatar']);
|
||||
$this->assertNull($result->raw['avatar_url']);
|
||||
$this->assertSame('No Gravatar found for this email.', $result->summary);
|
||||
}
|
||||
|
||||
public function test_rejects_non_email_targets(): void
|
||||
{
|
||||
$target = new Target(['type' => 'domain', 'value' => 'example.com']);
|
||||
|
||||
$this->expectException(RunnerException::class);
|
||||
|
||||
(new GravatarRunner)->run($target);
|
||||
}
|
||||
|
||||
public function test_rejects_invalid_email_format(): void
|
||||
{
|
||||
$target = new Target(['type' => 'email', 'value' => 'not-an-email']);
|
||||
|
||||
$this->expectException(RunnerException::class);
|
||||
|
||||
(new GravatarRunner)->run($target);
|
||||
}
|
||||
|
||||
public function test_throws_on_unexpected_status(): void
|
||||
{
|
||||
Http::fake(['gravatar.com/avatar/*' => Http::response('', 500)]);
|
||||
|
||||
$target = new Target(['type' => 'email', 'value' => 'test@example.com']);
|
||||
|
||||
$this->expectException(RunnerException::class);
|
||||
|
||||
(new GravatarRunner)->run($target);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user