Add Holehe collector for email targets
Closes the email target-type gap (previously had zero collectors).
Checks whether an email is registered on 120+ sites via the
password-reset-oracle technique. Shares the Sherlock/whois venv.
HoleheRunner gates success on the CSV file existing rather than the
process exit code: holehe's CLI calls Python's exit("message") right
after writing results on its normal success path, which always yields
exit code 1 regardless of outcome — discovered while verifying this
end-to-end against the real dev server and queue worker.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e4fa57c3d6
commit
bb15f13bac
@@ -51,9 +51,9 @@ commands).
|
||||
- **Targets** — the entity being investigated (person, domain, email,
|
||||
username, IP, org), scoped to one investigation
|
||||
- **Sources** — catalog of OSINT tools/techniques (seeded: Sherlock,
|
||||
theHarvester, crt.sh, WHOIS, ip-api.com — all 5 now have a working
|
||||
runner). `sources.runner` is a display-only label; the real mapping
|
||||
is `App\Osint\RunnerRegistry`
|
||||
theHarvester, crt.sh, WHOIS, ip-api.com, Holehe — all 6 now have a
|
||||
working runner). `sources.runner` is a display-only label; the real
|
||||
mapping is `App\Osint\RunnerRegistry`
|
||||
- **Findings** — result of running a source against a target; raw JSON
|
||||
payload preserved plus a parsed summary
|
||||
|
||||
@@ -210,6 +210,24 @@ commands).
|
||||
resolve sequentially through the single worker, hit and diagnosed
|
||||
the crt.sh flakiness live, confirmed a retried lookup completing
|
||||
cleanly. All test data cleaned up afterward.
|
||||
10. **Holehe collector** (done) — 6th source, closes the `email`
|
||||
target-type gap (previously zero collectors for it). Checks
|
||||
whether an email is registered on 120+ sites via the
|
||||
password-reset-oracle technique; same venv as Sherlock/whois
|
||||
(`scripts/python/setup.sh` updated). `HoleheRunner` follows the
|
||||
Sherlock/CSV pattern but with one wrinkle discovered while
|
||||
verifying it end-to-end: holehe's CLI calls Python's
|
||||
`exit("message")` on its normal success path right after writing
|
||||
the CSV (`core.py: export_csv`), which always yields exit code 1 —
|
||||
a real successful run and a genuine crash are indistinguishable by
|
||||
exit code alone. Fixed by gating on whether the CSV file actually
|
||||
exists instead of `successful()`. Courtesy rate limit: 15/min
|
||||
(`osint-runner:holehe`). Not unit-tested (real process + network,
|
||||
same as Sherlock/whois/theHarvester); verified by hand instead:
|
||||
registered a user, added an `email` target, ran Holehe through the
|
||||
live systemd worker, watched it fail with the exit-code-1 quirk
|
||||
before the fix and complete cleanly after, confirmed the summary
|
||||
rendered correctly in the UI. Test data cleaned up afterward.
|
||||
|
||||
## Open questions
|
||||
- Any legal/ToS constraints on the sources to be queried?
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
namespace App\Osint;
|
||||
|
||||
use App\Osint\Runners\CrtShRunner;
|
||||
use App\Osint\Runners\HoleheRunner;
|
||||
use App\Osint\Runners\IpApiRunner;
|
||||
use App\Osint\Runners\SherlockRunner;
|
||||
use App\Osint\Runners\TheHarvesterRunner;
|
||||
@@ -23,6 +24,7 @@ class RunnerRegistry
|
||||
'whois' => WhoisRunner::class,
|
||||
'ip-api' => IpApiRunner::class,
|
||||
'theharvester' => TheHarvesterRunner::class,
|
||||
'holehe' => HoleheRunner::class,
|
||||
];
|
||||
|
||||
public function has(string $sourceKey): bool
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
<?php
|
||||
|
||||
namespace App\Osint\Runners;
|
||||
|
||||
use App\Models\Target;
|
||||
use App\Osint\Runner;
|
||||
use App\Osint\RunnerException;
|
||||
use App\Osint\RunnerResult;
|
||||
use Illuminate\Support\Facades\File;
|
||||
use Illuminate\Support\Facades\Process;
|
||||
use Illuminate\Support\Str;
|
||||
|
||||
class HoleheRunner implements Runner
|
||||
{
|
||||
public function run(Target $target): RunnerResult
|
||||
{
|
||||
if ($target->type !== 'email') {
|
||||
throw new RunnerException('holehe only supports "email" targets.');
|
||||
}
|
||||
|
||||
if (! filter_var($target->value, FILTER_VALIDATE_EMAIL)) {
|
||||
throw new RunnerException('Value is not a valid email address.');
|
||||
}
|
||||
|
||||
$binary = config('osint.python_venv').'/bin/holehe';
|
||||
if (! is_executable($binary)) {
|
||||
throw new RunnerException("holehe binary not found or not executable at [{$binary}]. Run scripts/python/setup.sh.");
|
||||
}
|
||||
|
||||
$runDir = storage_path('app/osint/runs/'.Str::uuid());
|
||||
File::ensureDirectoryExists($runDir);
|
||||
|
||||
try {
|
||||
// holehe's CLI calls Python's exit("message") on the success
|
||||
// path after writing the CSV (see core.py: export_csv), which
|
||||
// always yields exit code 1 — confirmed by hand, "successful"
|
||||
// runs and real errors are otherwise indistinguishable by exit
|
||||
// code alone. So: don't gate on successful(), gate on whether
|
||||
// the CSV actually exists.
|
||||
Process::path($runDir)
|
||||
->timeout((int) config('osint.run_timeout'))
|
||||
->run([
|
||||
$binary,
|
||||
$target->value,
|
||||
'--no-color',
|
||||
'--no-clear',
|
||||
'--csv',
|
||||
'-T', '15',
|
||||
]);
|
||||
|
||||
// Filename is holehe_<unix-timestamp>_<email>_results.csv — not
|
||||
// predictable up front, so glob for it.
|
||||
$csvFiles = File::glob($runDir.'/holehe_*_results.csv');
|
||||
if (empty($csvFiles)) {
|
||||
throw new RunnerException('holehe did not produce the expected CSV output.');
|
||||
}
|
||||
|
||||
$sites = $this->parseCsv($csvFiles[0]);
|
||||
|
||||
$summary = count($sites) > 0
|
||||
? sprintf('Found on %d site%s: %s', count($sites), count($sites) === 1 ? '' : 's', implode(', ', $sites))
|
||||
: 'No accounts found.';
|
||||
|
||||
return new RunnerResult(raw: ['sites' => $sites], summary: $summary);
|
||||
} finally {
|
||||
File::deleteDirectory($runDir);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<int, string>
|
||||
*/
|
||||
private function parseCsv(string $path): array
|
||||
{
|
||||
$handle = fopen($path, 'r');
|
||||
$header = fgetcsv($handle);
|
||||
$sites = [];
|
||||
|
||||
while (($row = fgetcsv($handle)) !== false) {
|
||||
$data = array_combine($header, $row);
|
||||
|
||||
if (strcasecmp($data['exists'] ?? '', 'True') === 0) {
|
||||
$sites[] = $data['domain'] ?: $data['name'];
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
|
||||
return $sites;
|
||||
}
|
||||
}
|
||||
@@ -29,5 +29,6 @@ class AppServiceProvider extends ServiceProvider
|
||||
RateLimiter::for('osint-runner:whois', fn () => Limit::perMinute(20));
|
||||
RateLimiter::for('osint-runner:sherlock', fn () => Limit::perMinute(10));
|
||||
RateLimiter::for('osint-runner:theharvester', fn () => Limit::perMinute(5));
|
||||
RateLimiter::for('osint-runner:holehe', fn () => Limit::perMinute(15));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +49,13 @@ class SourceSeeder extends Seeder
|
||||
'target_types' => ['ip'],
|
||||
'runner' => 'http:ip-api',
|
||||
],
|
||||
[
|
||||
'key' => 'holehe',
|
||||
'name' => 'Holehe',
|
||||
'description' => 'Checks whether an email address is registered on 120+ sites via the password-reset-oracle technique.',
|
||||
'target_types' => ['email'],
|
||||
'runner' => 'python:holehe',
|
||||
],
|
||||
];
|
||||
|
||||
foreach ($sources as $source) {
|
||||
|
||||
@@ -7,8 +7,8 @@ scripts/python/setup.sh
|
||||
```
|
||||
|
||||
Sets up:
|
||||
- `scripts/python/.venv` — Sherlock (`sherlock-project`) and `python-whois`,
|
||||
both lightweight and dependency-compatible.
|
||||
- `scripts/python/.venv` — Sherlock (`sherlock-project`), `python-whois`,
|
||||
and Holehe, all lightweight and dependency-compatible.
|
||||
- `scripts/python/.venv-theharvester` — theHarvester, isolated because it
|
||||
pulls in a much heavier dependency set (playwright, censys, shodan,
|
||||
fastapi). Vendored from `theharvester-src/` at tag `4.6.0` (the last
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
set -euo pipefail
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")"
|
||||
|
||||
# Sherlock + python-whois share one lightweight venv.
|
||||
# Sherlock, python-whois, and Holehe share one lightweight venv.
|
||||
python3 -m venv .venv
|
||||
.venv/bin/pip install --upgrade pip -q
|
||||
.venv/bin/pip install sherlock-project python-whois -q
|
||||
.venv/bin/pip install sherlock-project python-whois holehe -q
|
||||
|
||||
# theHarvester (>=4.7 needs Python 3.12; pinned to 4.6.0 for 3.9+ compat)
|
||||
# pulls in a much heavier dependency set (playwright, censys, shodan, fastapi),
|
||||
|
||||
Reference in New Issue
Block a user