From bb15f13bacac19883ef577c2014dc7ae72ca55bf Mon Sep 17 00:00:00 2001 From: cedry2k1992 Date: Mon, 20 Jul 2026 11:49:09 +0300 Subject: [PATCH] Add Holehe collector for email targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the email target-type gap (previously had zero collectors). Checks whether an email is registered on 120+ sites via the password-reset-oracle technique. Shares the Sherlock/whois venv. HoleheRunner gates success on the CSV file existing rather than the process exit code: holehe's CLI calls Python's exit("message") right after writing results on its normal success path, which always yields exit code 1 regardless of outcome — discovered while verifying this end-to-end against the real dev server and queue worker. Co-Authored-By: Claude Sonnet 5 --- PLAN.md | 24 +++++++- app/Osint/RunnerRegistry.php | 2 + app/Osint/Runners/HoleheRunner.php | 91 ++++++++++++++++++++++++++++ app/Providers/AppServiceProvider.php | 1 + database/seeders/SourceSeeder.php | 7 +++ scripts/python/README.md | 4 +- scripts/python/setup.sh | 4 +- 7 files changed, 126 insertions(+), 7 deletions(-) create mode 100644 app/Osint/Runners/HoleheRunner.php diff --git a/PLAN.md b/PLAN.md index 47627b2..bc9d226 100644 --- a/PLAN.md +++ b/PLAN.md @@ -51,9 +51,9 @@ commands). - **Targets** — the entity being investigated (person, domain, email, username, IP, org), scoped to one investigation - **Sources** — catalog of OSINT tools/techniques (seeded: Sherlock, - theHarvester, crt.sh, WHOIS, ip-api.com — all 5 now have a working - runner). `sources.runner` is a display-only label; the real mapping - is `App\Osint\RunnerRegistry` + theHarvester, crt.sh, WHOIS, ip-api.com, Holehe — all 6 now have a + working runner). `sources.runner` is a display-only label; the real + mapping is `App\Osint\RunnerRegistry` - **Findings** — result of running a source against a target; raw JSON payload preserved plus a parsed summary @@ -210,6 +210,24 @@ commands). resolve sequentially through the single worker, hit and diagnosed the crt.sh flakiness live, confirmed a retried lookup completing cleanly. All test data cleaned up afterward. +10. **Holehe collector** (done) — 6th source, closes the `email` + target-type gap (previously zero collectors for it). Checks + whether an email is registered on 120+ sites via the + password-reset-oracle technique; same venv as Sherlock/whois + (`scripts/python/setup.sh` updated). `HoleheRunner` follows the + Sherlock/CSV pattern but with one wrinkle discovered while + verifying it end-to-end: holehe's CLI calls Python's + `exit("message")` on its normal success path right after writing + the CSV (`core.py: export_csv`), which always yields exit code 1 — + a real successful run and a genuine crash are indistinguishable by + exit code alone. Fixed by gating on whether the CSV file actually + exists instead of `successful()`. Courtesy rate limit: 15/min + (`osint-runner:holehe`). Not unit-tested (real process + network, + same as Sherlock/whois/theHarvester); verified by hand instead: + registered a user, added an `email` target, ran Holehe through the + live systemd worker, watched it fail with the exit-code-1 quirk + before the fix and complete cleanly after, confirmed the summary + rendered correctly in the UI. Test data cleaned up afterward. ## Open questions - Any legal/ToS constraints on the sources to be queried? diff --git a/app/Osint/RunnerRegistry.php b/app/Osint/RunnerRegistry.php index 39893bf..60ff5a7 100644 --- a/app/Osint/RunnerRegistry.php +++ b/app/Osint/RunnerRegistry.php @@ -3,6 +3,7 @@ namespace App\Osint; use App\Osint\Runners\CrtShRunner; +use App\Osint\Runners\HoleheRunner; use App\Osint\Runners\IpApiRunner; use App\Osint\Runners\SherlockRunner; use App\Osint\Runners\TheHarvesterRunner; @@ -23,6 +24,7 @@ class RunnerRegistry 'whois' => WhoisRunner::class, 'ip-api' => IpApiRunner::class, 'theharvester' => TheHarvesterRunner::class, + 'holehe' => HoleheRunner::class, ]; public function has(string $sourceKey): bool diff --git a/app/Osint/Runners/HoleheRunner.php b/app/Osint/Runners/HoleheRunner.php new file mode 100644 index 0000000..fcbd61a --- /dev/null +++ b/app/Osint/Runners/HoleheRunner.php @@ -0,0 +1,91 @@ +type !== 'email') { + throw new RunnerException('holehe only supports "email" targets.'); + } + + if (! filter_var($target->value, FILTER_VALIDATE_EMAIL)) { + throw new RunnerException('Value is not a valid email address.'); + } + + $binary = config('osint.python_venv').'/bin/holehe'; + if (! is_executable($binary)) { + throw new RunnerException("holehe binary not found or not executable at [{$binary}]. Run scripts/python/setup.sh."); + } + + $runDir = storage_path('app/osint/runs/'.Str::uuid()); + File::ensureDirectoryExists($runDir); + + try { + // holehe's CLI calls Python's exit("message") on the success + // path after writing the CSV (see core.py: export_csv), which + // always yields exit code 1 — confirmed by hand, "successful" + // runs and real errors are otherwise indistinguishable by exit + // code alone. So: don't gate on successful(), gate on whether + // the CSV actually exists. + Process::path($runDir) + ->timeout((int) config('osint.run_timeout')) + ->run([ + $binary, + $target->value, + '--no-color', + '--no-clear', + '--csv', + '-T', '15', + ]); + + // Filename is holehe___results.csv — not + // predictable up front, so glob for it. + $csvFiles = File::glob($runDir.'/holehe_*_results.csv'); + if (empty($csvFiles)) { + throw new RunnerException('holehe did not produce the expected CSV output.'); + } + + $sites = $this->parseCsv($csvFiles[0]); + + $summary = count($sites) > 0 + ? sprintf('Found on %d site%s: %s', count($sites), count($sites) === 1 ? '' : 's', implode(', ', $sites)) + : 'No accounts found.'; + + return new RunnerResult(raw: ['sites' => $sites], summary: $summary); + } finally { + File::deleteDirectory($runDir); + } + } + + /** + * @return array + */ + private function parseCsv(string $path): array + { + $handle = fopen($path, 'r'); + $header = fgetcsv($handle); + $sites = []; + + while (($row = fgetcsv($handle)) !== false) { + $data = array_combine($header, $row); + + if (strcasecmp($data['exists'] ?? '', 'True') === 0) { + $sites[] = $data['domain'] ?: $data['name']; + } + } + + fclose($handle); + + return $sites; + } +} diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index ad2dbfc..fe79998 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -29,5 +29,6 @@ class AppServiceProvider extends ServiceProvider RateLimiter::for('osint-runner:whois', fn () => Limit::perMinute(20)); RateLimiter::for('osint-runner:sherlock', fn () => Limit::perMinute(10)); RateLimiter::for('osint-runner:theharvester', fn () => Limit::perMinute(5)); + RateLimiter::for('osint-runner:holehe', fn () => Limit::perMinute(15)); } } diff --git a/database/seeders/SourceSeeder.php b/database/seeders/SourceSeeder.php index a6aaaa9..3c34a71 100644 --- a/database/seeders/SourceSeeder.php +++ b/database/seeders/SourceSeeder.php @@ -49,6 +49,13 @@ class SourceSeeder extends Seeder 'target_types' => ['ip'], 'runner' => 'http:ip-api', ], + [ + 'key' => 'holehe', + 'name' => 'Holehe', + 'description' => 'Checks whether an email address is registered on 120+ sites via the password-reset-oracle technique.', + 'target_types' => ['email'], + 'runner' => 'python:holehe', + ], ]; foreach ($sources as $source) { diff --git a/scripts/python/README.md b/scripts/python/README.md index e26e754..e6c710e 100644 --- a/scripts/python/README.md +++ b/scripts/python/README.md @@ -7,8 +7,8 @@ scripts/python/setup.sh ``` Sets up: -- `scripts/python/.venv` — Sherlock (`sherlock-project`) and `python-whois`, - both lightweight and dependency-compatible. +- `scripts/python/.venv` — Sherlock (`sherlock-project`), `python-whois`, + and Holehe, all lightweight and dependency-compatible. - `scripts/python/.venv-theharvester` — theHarvester, isolated because it pulls in a much heavier dependency set (playwright, censys, shodan, fastapi). Vendored from `theharvester-src/` at tag `4.6.0` (the last diff --git a/scripts/python/setup.sh b/scripts/python/setup.sh index a15b2ff..aefb313 100755 --- a/scripts/python/setup.sh +++ b/scripts/python/setup.sh @@ -3,10 +3,10 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")" -# Sherlock + python-whois share one lightweight venv. +# Sherlock, python-whois, and Holehe share one lightweight venv. python3 -m venv .venv .venv/bin/pip install --upgrade pip -q -.venv/bin/pip install sherlock-project python-whois -q +.venv/bin/pip install sherlock-project python-whois holehe -q # theHarvester (>=4.7 needs Python 3.12; pinned to 4.6.0 for 3.9+ compat) # pulls in a much heavier dependency set (playwright, censys, shodan, fastapi),