From 86f73c46b211c063bb1d66aa7fd6d00187145a48 Mon Sep 17 00:00:00 2001 From: cedry2k1992 Date: Mon, 20 Jul 2026 11:59:20 +0300 Subject: [PATCH] Add Gravatar and DNS Records collectors Two pure-PHP collectors, no Python/venv/subprocess needed: - GravatarRunner (email): checks gravatar.com/avatar/{md5}?d=404 for a 200 vs 404. Scoped to that binary signal since Gravatar's profile JSON endpoint now requires API auth even for real, public profiles (confirmed against Gravatar's own creator's email). - DnsRunner (domain): dns_get_record() for A/AAAA/MX/NS/TXT/CNAME. Raw payload is naturally keyed by record type, which the existing generic list-renderer on the investigation page turns into one collapsible section per type with no UI changes needed. Co-Authored-By: Claude Sonnet 5 --- PLAN.md | 35 +++++++++++- app/Osint/RunnerRegistry.php | 4 ++ app/Osint/Runners/DnsRunner.php | 74 +++++++++++++++++++++++++ app/Osint/Runners/GravatarRunner.php | 49 ++++++++++++++++ app/Providers/AppServiceProvider.php | 2 + database/seeders/SourceSeeder.php | 14 +++++ tests/Unit/Osint/DnsRunnerTest.php | 54 ++++++++++++++++++ tests/Unit/Osint/GravatarRunnerTest.php | 67 ++++++++++++++++++++++ 8 files changed, 296 insertions(+), 3 deletions(-) create mode 100644 app/Osint/Runners/DnsRunner.php create mode 100644 app/Osint/Runners/GravatarRunner.php create mode 100644 tests/Unit/Osint/DnsRunnerTest.php create mode 100644 tests/Unit/Osint/GravatarRunnerTest.php diff --git a/PLAN.md b/PLAN.md index bc9d226..d4085d0 100644 --- a/PLAN.md +++ b/PLAN.md @@ -51,9 +51,9 @@ commands). - **Targets** — the entity being investigated (person, domain, email, username, IP, org), scoped to one investigation - **Sources** — catalog of OSINT tools/techniques (seeded: Sherlock, - theHarvester, crt.sh, WHOIS, ip-api.com, Holehe — all 6 now have a - working runner). `sources.runner` is a display-only label; the real - mapping is `App\Osint\RunnerRegistry` + theHarvester, crt.sh, WHOIS, ip-api.com, Holehe, Gravatar, DNS + Records — all 8 now have a working runner). `sources.runner` is a + display-only label; the real mapping is `App\Osint\RunnerRegistry` - **Findings** — result of running a source against a target; raw JSON payload preserved plus a parsed summary @@ -228,6 +228,35 @@ commands). live systemd worker, watched it fail with the exit-code-1 quirk before the fix and complete cleanly after, confirmed the summary rendered correctly in the UI. Test data cleaned up afterward. +11. **Gravatar + DNS Records collectors** (done) — 7th and 8th + sources, both pure PHP (no Python/venv, no external process at + all for DNS). + - `GravatarRunner` (email) — MD5-hashes the email, checks + `gravatar.com/avatar/{hash}?d=404` for a 200 vs 404. Gravatar's + public profile JSON endpoint (`{hash}.json`) turned out to now + require API auth — confirmed by hand against a real profile + (Matt Mullenweg's, Gravatar's own creator: avatar check returned + 200, `.json` returned 404 regardless) — so this stays scoped to + the binary "does an avatar exist" signal rather than pulling + profile details. Unit-tested with `Http::fake()` (avatar + found/not-found/error, invalid email, wrong target type). + - `DnsRunner` (domain) — pure PHP `dns_get_record()`, no HTTP + client or subprocess. Looks up A/AAAA/MX/NS/TXT/CNAME. Its raw + payload is naturally keyed by record type (`A`, `MX`, ...), + which the existing generic list-renderer in the investigation + show page turns into one collapsible section per record type + automatically — no UI changes needed. Unit-tested against a real + DNS lookup (`example.com`, IANA-controlled, stable for decades) + plus a nonexistent-domain case, since `dns_get_record()` isn't + mockable through `Http::fake()`. + - Courtesy rate limits: Gravatar 45/min (cheap single GET), DNS + 60/min (local resolver, cheapest operation in the app). + - 9 new tests (84/84 passing). Manually verified end-to-end + through the live systemd worker: Gravatar against a real email + with a known public avatar (found it), DNS against `google.com` + (22 records across all 5 present types, each rendering as its + own labeled section in the UI with correct counts and values). + Test data cleaned up afterward. ## Open questions - Any legal/ToS constraints on the sources to be queried? diff --git a/app/Osint/RunnerRegistry.php b/app/Osint/RunnerRegistry.php index 60ff5a7..b498b60 100644 --- a/app/Osint/RunnerRegistry.php +++ b/app/Osint/RunnerRegistry.php @@ -3,6 +3,8 @@ namespace App\Osint; use App\Osint\Runners\CrtShRunner; +use App\Osint\Runners\DnsRunner; +use App\Osint\Runners\GravatarRunner; use App\Osint\Runners\HoleheRunner; use App\Osint\Runners\IpApiRunner; use App\Osint\Runners\SherlockRunner; @@ -25,6 +27,8 @@ class RunnerRegistry 'ip-api' => IpApiRunner::class, 'theharvester' => TheHarvesterRunner::class, 'holehe' => HoleheRunner::class, + 'gravatar' => GravatarRunner::class, + 'dns' => DnsRunner::class, ]; public function has(string $sourceKey): bool diff --git a/app/Osint/Runners/DnsRunner.php b/app/Osint/Runners/DnsRunner.php new file mode 100644 index 0000000..0fa5c70 --- /dev/null +++ b/app/Osint/Runners/DnsRunner.php @@ -0,0 +1,74 @@ + + */ + private const RECORD_TYPES = [ + 'A' => DNS_A, + 'AAAA' => DNS_AAAA, + 'MX' => DNS_MX, + 'NS' => DNS_NS, + 'TXT' => DNS_TXT, + 'CNAME' => DNS_CNAME, + ]; + + public function run(Target $target): RunnerResult + { + if ($target->type !== 'domain') { + throw new RunnerException('DNS lookup only supports "domain" targets.'); + } + + if (! preg_match('/^([a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}$/', $target->value)) { + throw new RunnerException('Domain contains characters that are not safe to look up.'); + } + + $records = []; + + foreach (self::RECORD_TYPES as $label => $type) { + // dns_get_record() emits a warning (not an exception) when a + // record type doesn't resolve for the domain — expected for + // most domains on most types, not a real error. + $found = @dns_get_record($target->value, $type); + + if ($found === false || empty($found)) { + continue; + } + + $records[$label] = array_values(array_unique(array_map( + fn (array $record) => $this->formatRecord($label, $record), + $found + ))); + } + + $total = array_sum(array_map('count', $records)); + + $summary = $total > 0 + ? sprintf('Found %d DNS record%s across %d type%s (%s).', + $total, $total === 1 ? '' : 's', + count($records), count($records) === 1 ? '' : 's', + implode(', ', array_keys($records))) + : 'No DNS records found.'; + + return new RunnerResult(raw: $records, summary: $summary); + } + + private function formatRecord(string $label, array $record): string + { + return match ($label) { + 'A', 'AAAA' => $record['ip'] ?? '', + 'MX' => sprintf('%d %s', $record['pri'] ?? 0, $record['target'] ?? ''), + 'NS', 'CNAME' => $record['target'] ?? '', + 'TXT' => $record['txt'] ?? '', + default => '', + }; + } +} diff --git a/app/Osint/Runners/GravatarRunner.php b/app/Osint/Runners/GravatarRunner.php new file mode 100644 index 0000000..997ac06 --- /dev/null +++ b/app/Osint/Runners/GravatarRunner.php @@ -0,0 +1,49 @@ +type !== 'email') { + throw new RunnerException('Gravatar only supports "email" targets.'); + } + + if (! filter_var($target->value, FILTER_VALIDATE_EMAIL)) { + throw new RunnerException('Value is not a valid email address.'); + } + + // Gravatar's public profile JSON endpoint now requires API auth + // (returns 404 even for real profiles without a key), but the + // plain avatar-existence check is still open and reliable. + $hash = md5(strtolower(trim($target->value))); + $avatarUrl = "https://www.gravatar.com/avatar/{$hash}"; + + $response = Http::timeout((int) config('osint.run_timeout')) + ->get($avatarUrl, ['d' => '404']); + + if ($response->status() !== 200 && $response->status() !== 404) { + throw new RunnerException('Gravatar request failed with status '.$response->status().'.'); + } + + $hasAvatar = $response->status() === 200; + + return new RunnerResult( + raw: [ + 'has_avatar' => $hasAvatar, + 'avatar_url' => $hasAvatar ? $avatarUrl : null, + 'profile_url' => "https://gravatar.com/{$hash}", + ], + summary: $hasAvatar + ? 'Gravatar image found — this email is registered with Gravatar.' + : 'No Gravatar found for this email.', + ); + } +} diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index fe79998..367061d 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -30,5 +30,7 @@ class AppServiceProvider extends ServiceProvider RateLimiter::for('osint-runner:sherlock', fn () => Limit::perMinute(10)); RateLimiter::for('osint-runner:theharvester', fn () => Limit::perMinute(5)); RateLimiter::for('osint-runner:holehe', fn () => Limit::perMinute(15)); + RateLimiter::for('osint-runner:gravatar', fn () => Limit::perMinute(45)); + RateLimiter::for('osint-runner:dns', fn () => Limit::perMinute(60)); } } diff --git a/database/seeders/SourceSeeder.php b/database/seeders/SourceSeeder.php index 3c34a71..63382fe 100644 --- a/database/seeders/SourceSeeder.php +++ b/database/seeders/SourceSeeder.php @@ -56,6 +56,20 @@ class SourceSeeder extends Seeder 'target_types' => ['email'], 'runner' => 'python:holehe', ], + [ + 'key' => 'gravatar', + 'name' => 'Gravatar', + 'description' => 'Checks whether an email address has a registered Gravatar image.', + 'target_types' => ['email'], + 'runner' => 'http:gravatar', + ], + [ + 'key' => 'dns', + 'name' => 'DNS Records', + 'description' => 'Looks up A, AAAA, MX, NS, TXT, and CNAME records for a domain.', + 'target_types' => ['domain'], + 'runner' => 'dns:records', + ], ]; foreach ($sources as $source) { diff --git a/tests/Unit/Osint/DnsRunnerTest.php b/tests/Unit/Osint/DnsRunnerTest.php new file mode 100644 index 0000000..1069997 --- /dev/null +++ b/tests/Unit/Osint/DnsRunnerTest.php @@ -0,0 +1,54 @@ + 'ip', 'value' => '8.8.8.8']); + + $this->expectException(RunnerException::class); + + (new DnsRunner)->run($target); + } + + public function test_rejects_unsafe_domain_format(): void + { + $target = new Target(['type' => 'domain', 'value' => '../../etc/passwd']); + + $this->expectException(RunnerException::class); + + (new DnsRunner)->run($target); + } + + /** + * example.com is IANA-controlled and has had stable NS/A records for + * decades — safe to rely on for a real (non-mocked) DNS lookup. + */ + public function test_resolves_a_real_stable_domain(): void + { + $target = new Target(['type' => 'domain', 'value' => 'example.com']); + + $result = (new DnsRunner)->run($target); + + $this->assertArrayHasKey('NS', $result->raw); + $this->assertNotEmpty($result->raw['NS']); + $this->assertStringContainsString('DNS record', $result->summary); + } + + public function test_reports_no_records_for_a_nonexistent_domain(): void + { + $target = new Target(['type' => 'domain', 'value' => 'this-domain-should-not-exist-osint-test.invalid']); + + $result = (new DnsRunner)->run($target); + + $this->assertSame([], $result->raw); + $this->assertSame('No DNS records found.', $result->summary); + } +} diff --git a/tests/Unit/Osint/GravatarRunnerTest.php b/tests/Unit/Osint/GravatarRunnerTest.php new file mode 100644 index 0000000..1c69d81 --- /dev/null +++ b/tests/Unit/Osint/GravatarRunnerTest.php @@ -0,0 +1,67 @@ + Http::response('', 200)]); + + $target = new Target(['type' => 'email', 'value' => 'test@example.com']); + + $result = (new GravatarRunner)->run($target); + + $this->assertTrue($result->raw['has_avatar']); + $this->assertNotNull($result->raw['avatar_url']); + $this->assertStringContainsString('Gravatar image found', $result->summary); + } + + public function test_reports_no_avatar(): void + { + Http::fake(['gravatar.com/avatar/*' => Http::response('', 404)]); + + $target = new Target(['type' => 'email', 'value' => 'test@example.com']); + + $result = (new GravatarRunner)->run($target); + + $this->assertFalse($result->raw['has_avatar']); + $this->assertNull($result->raw['avatar_url']); + $this->assertSame('No Gravatar found for this email.', $result->summary); + } + + public function test_rejects_non_email_targets(): void + { + $target = new Target(['type' => 'domain', 'value' => 'example.com']); + + $this->expectException(RunnerException::class); + + (new GravatarRunner)->run($target); + } + + public function test_rejects_invalid_email_format(): void + { + $target = new Target(['type' => 'email', 'value' => 'not-an-email']); + + $this->expectException(RunnerException::class); + + (new GravatarRunner)->run($target); + } + + public function test_throws_on_unexpected_status(): void + { + Http::fake(['gravatar.com/avatar/*' => Http::response('', 500)]); + + $target = new Target(['type' => 'email', 'value' => 'test@example.com']); + + $this->expectException(RunnerException::class); + + (new GravatarRunner)->run($target); + } +}