Add Holehe collector for email targets
Closes the email target-type gap (previously had zero collectors).
Checks whether an email is registered on 120+ sites via the
password-reset-oracle technique. Shares the Sherlock/whois venv.
HoleheRunner gates success on the CSV file existing rather than the
process exit code: holehe's CLI calls Python's exit("message") right
after writing results on its normal success path, which always yields
exit code 1 regardless of outcome — discovered while verifying this
end-to-end against the real dev server and queue worker.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e4fa57c3d6
commit
bb15f13bac
@@ -51,9 +51,9 @@ commands).
|
|||||||
- **Targets** — the entity being investigated (person, domain, email,
|
- **Targets** — the entity being investigated (person, domain, email,
|
||||||
username, IP, org), scoped to one investigation
|
username, IP, org), scoped to one investigation
|
||||||
- **Sources** — catalog of OSINT tools/techniques (seeded: Sherlock,
|
- **Sources** — catalog of OSINT tools/techniques (seeded: Sherlock,
|
||||||
theHarvester, crt.sh, WHOIS, ip-api.com — all 5 now have a working
|
theHarvester, crt.sh, WHOIS, ip-api.com, Holehe — all 6 now have a
|
||||||
runner). `sources.runner` is a display-only label; the real mapping
|
working runner). `sources.runner` is a display-only label; the real
|
||||||
is `App\Osint\RunnerRegistry`
|
mapping is `App\Osint\RunnerRegistry`
|
||||||
- **Findings** — result of running a source against a target; raw JSON
|
- **Findings** — result of running a source against a target; raw JSON
|
||||||
payload preserved plus a parsed summary
|
payload preserved plus a parsed summary
|
||||||
|
|
||||||
@@ -210,6 +210,24 @@ commands).
|
|||||||
resolve sequentially through the single worker, hit and diagnosed
|
resolve sequentially through the single worker, hit and diagnosed
|
||||||
the crt.sh flakiness live, confirmed a retried lookup completing
|
the crt.sh flakiness live, confirmed a retried lookup completing
|
||||||
cleanly. All test data cleaned up afterward.
|
cleanly. All test data cleaned up afterward.
|
||||||
|
10. **Holehe collector** (done) — 6th source, closes the `email`
|
||||||
|
target-type gap (previously zero collectors for it). Checks
|
||||||
|
whether an email is registered on 120+ sites via the
|
||||||
|
password-reset-oracle technique; same venv as Sherlock/whois
|
||||||
|
(`scripts/python/setup.sh` updated). `HoleheRunner` follows the
|
||||||
|
Sherlock/CSV pattern but with one wrinkle discovered while
|
||||||
|
verifying it end-to-end: holehe's CLI calls Python's
|
||||||
|
`exit("message")` on its normal success path right after writing
|
||||||
|
the CSV (`core.py: export_csv`), which always yields exit code 1 —
|
||||||
|
a real successful run and a genuine crash are indistinguishable by
|
||||||
|
exit code alone. Fixed by gating on whether the CSV file actually
|
||||||
|
exists instead of `successful()`. Courtesy rate limit: 15/min
|
||||||
|
(`osint-runner:holehe`). Not unit-tested (real process + network,
|
||||||
|
same as Sherlock/whois/theHarvester); verified by hand instead:
|
||||||
|
registered a user, added an `email` target, ran Holehe through the
|
||||||
|
live systemd worker, watched it fail with the exit-code-1 quirk
|
||||||
|
before the fix and complete cleanly after, confirmed the summary
|
||||||
|
rendered correctly in the UI. Test data cleaned up afterward.
|
||||||
|
|
||||||
## Open questions
|
## Open questions
|
||||||
- Any legal/ToS constraints on the sources to be queried?
|
- Any legal/ToS constraints on the sources to be queried?
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
namespace App\Osint;
|
namespace App\Osint;
|
||||||
|
|
||||||
use App\Osint\Runners\CrtShRunner;
|
use App\Osint\Runners\CrtShRunner;
|
||||||
|
use App\Osint\Runners\HoleheRunner;
|
||||||
use App\Osint\Runners\IpApiRunner;
|
use App\Osint\Runners\IpApiRunner;
|
||||||
use App\Osint\Runners\SherlockRunner;
|
use App\Osint\Runners\SherlockRunner;
|
||||||
use App\Osint\Runners\TheHarvesterRunner;
|
use App\Osint\Runners\TheHarvesterRunner;
|
||||||
@@ -23,6 +24,7 @@ class RunnerRegistry
|
|||||||
'whois' => WhoisRunner::class,
|
'whois' => WhoisRunner::class,
|
||||||
'ip-api' => IpApiRunner::class,
|
'ip-api' => IpApiRunner::class,
|
||||||
'theharvester' => TheHarvesterRunner::class,
|
'theharvester' => TheHarvesterRunner::class,
|
||||||
|
'holehe' => HoleheRunner::class,
|
||||||
];
|
];
|
||||||
|
|
||||||
public function has(string $sourceKey): bool
|
public function has(string $sourceKey): bool
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Osint\Runners;
|
||||||
|
|
||||||
|
use App\Models\Target;
|
||||||
|
use App\Osint\Runner;
|
||||||
|
use App\Osint\RunnerException;
|
||||||
|
use App\Osint\RunnerResult;
|
||||||
|
use Illuminate\Support\Facades\File;
|
||||||
|
use Illuminate\Support\Facades\Process;
|
||||||
|
use Illuminate\Support\Str;
|
||||||
|
|
||||||
|
class HoleheRunner implements Runner
|
||||||
|
{
|
||||||
|
public function run(Target $target): RunnerResult
|
||||||
|
{
|
||||||
|
if ($target->type !== 'email') {
|
||||||
|
throw new RunnerException('holehe only supports "email" targets.');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (! filter_var($target->value, FILTER_VALIDATE_EMAIL)) {
|
||||||
|
throw new RunnerException('Value is not a valid email address.');
|
||||||
|
}
|
||||||
|
|
||||||
|
$binary = config('osint.python_venv').'/bin/holehe';
|
||||||
|
if (! is_executable($binary)) {
|
||||||
|
throw new RunnerException("holehe binary not found or not executable at [{$binary}]. Run scripts/python/setup.sh.");
|
||||||
|
}
|
||||||
|
|
||||||
|
$runDir = storage_path('app/osint/runs/'.Str::uuid());
|
||||||
|
File::ensureDirectoryExists($runDir);
|
||||||
|
|
||||||
|
try {
|
||||||
|
// holehe's CLI calls Python's exit("message") on the success
|
||||||
|
// path after writing the CSV (see core.py: export_csv), which
|
||||||
|
// always yields exit code 1 — confirmed by hand, "successful"
|
||||||
|
// runs and real errors are otherwise indistinguishable by exit
|
||||||
|
// code alone. So: don't gate on successful(), gate on whether
|
||||||
|
// the CSV actually exists.
|
||||||
|
Process::path($runDir)
|
||||||
|
->timeout((int) config('osint.run_timeout'))
|
||||||
|
->run([
|
||||||
|
$binary,
|
||||||
|
$target->value,
|
||||||
|
'--no-color',
|
||||||
|
'--no-clear',
|
||||||
|
'--csv',
|
||||||
|
'-T', '15',
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Filename is holehe_<unix-timestamp>_<email>_results.csv — not
|
||||||
|
// predictable up front, so glob for it.
|
||||||
|
$csvFiles = File::glob($runDir.'/holehe_*_results.csv');
|
||||||
|
if (empty($csvFiles)) {
|
||||||
|
throw new RunnerException('holehe did not produce the expected CSV output.');
|
||||||
|
}
|
||||||
|
|
||||||
|
$sites = $this->parseCsv($csvFiles[0]);
|
||||||
|
|
||||||
|
$summary = count($sites) > 0
|
||||||
|
? sprintf('Found on %d site%s: %s', count($sites), count($sites) === 1 ? '' : 's', implode(', ', $sites))
|
||||||
|
: 'No accounts found.';
|
||||||
|
|
||||||
|
return new RunnerResult(raw: ['sites' => $sites], summary: $summary);
|
||||||
|
} finally {
|
||||||
|
File::deleteDirectory($runDir);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return array<int, string>
|
||||||
|
*/
|
||||||
|
private function parseCsv(string $path): array
|
||||||
|
{
|
||||||
|
$handle = fopen($path, 'r');
|
||||||
|
$header = fgetcsv($handle);
|
||||||
|
$sites = [];
|
||||||
|
|
||||||
|
while (($row = fgetcsv($handle)) !== false) {
|
||||||
|
$data = array_combine($header, $row);
|
||||||
|
|
||||||
|
if (strcasecmp($data['exists'] ?? '', 'True') === 0) {
|
||||||
|
$sites[] = $data['domain'] ?: $data['name'];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fclose($handle);
|
||||||
|
|
||||||
|
return $sites;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -29,5 +29,6 @@ class AppServiceProvider extends ServiceProvider
|
|||||||
RateLimiter::for('osint-runner:whois', fn () => Limit::perMinute(20));
|
RateLimiter::for('osint-runner:whois', fn () => Limit::perMinute(20));
|
||||||
RateLimiter::for('osint-runner:sherlock', fn () => Limit::perMinute(10));
|
RateLimiter::for('osint-runner:sherlock', fn () => Limit::perMinute(10));
|
||||||
RateLimiter::for('osint-runner:theharvester', fn () => Limit::perMinute(5));
|
RateLimiter::for('osint-runner:theharvester', fn () => Limit::perMinute(5));
|
||||||
|
RateLimiter::for('osint-runner:holehe', fn () => Limit::perMinute(15));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,6 +49,13 @@ class SourceSeeder extends Seeder
|
|||||||
'target_types' => ['ip'],
|
'target_types' => ['ip'],
|
||||||
'runner' => 'http:ip-api',
|
'runner' => 'http:ip-api',
|
||||||
],
|
],
|
||||||
|
[
|
||||||
|
'key' => 'holehe',
|
||||||
|
'name' => 'Holehe',
|
||||||
|
'description' => 'Checks whether an email address is registered on 120+ sites via the password-reset-oracle technique.',
|
||||||
|
'target_types' => ['email'],
|
||||||
|
'runner' => 'python:holehe',
|
||||||
|
],
|
||||||
];
|
];
|
||||||
|
|
||||||
foreach ($sources as $source) {
|
foreach ($sources as $source) {
|
||||||
|
|||||||
@@ -7,8 +7,8 @@ scripts/python/setup.sh
|
|||||||
```
|
```
|
||||||
|
|
||||||
Sets up:
|
Sets up:
|
||||||
- `scripts/python/.venv` — Sherlock (`sherlock-project`) and `python-whois`,
|
- `scripts/python/.venv` — Sherlock (`sherlock-project`), `python-whois`,
|
||||||
both lightweight and dependency-compatible.
|
and Holehe, all lightweight and dependency-compatible.
|
||||||
- `scripts/python/.venv-theharvester` — theHarvester, isolated because it
|
- `scripts/python/.venv-theharvester` — theHarvester, isolated because it
|
||||||
pulls in a much heavier dependency set (playwright, censys, shodan,
|
pulls in a much heavier dependency set (playwright, censys, shodan,
|
||||||
fastapi). Vendored from `theharvester-src/` at tag `4.6.0` (the last
|
fastapi). Vendored from `theharvester-src/` at tag `4.6.0` (the last
|
||||||
|
|||||||
@@ -3,10 +3,10 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
cd "$(dirname "${BASH_SOURCE[0]}")"
|
cd "$(dirname "${BASH_SOURCE[0]}")"
|
||||||
|
|
||||||
# Sherlock + python-whois share one lightweight venv.
|
# Sherlock, python-whois, and Holehe share one lightweight venv.
|
||||||
python3 -m venv .venv
|
python3 -m venv .venv
|
||||||
.venv/bin/pip install --upgrade pip -q
|
.venv/bin/pip install --upgrade pip -q
|
||||||
.venv/bin/pip install sherlock-project python-whois -q
|
.venv/bin/pip install sherlock-project python-whois holehe -q
|
||||||
|
|
||||||
# theHarvester (>=4.7 needs Python 3.12; pinned to 4.6.0 for 3.9+ compat)
|
# theHarvester (>=4.7 needs Python 3.12; pinned to 4.6.0 for 3.9+ compat)
|
||||||
# pulls in a much heavier dependency set (playwright, censys, shodan, fastapi),
|
# pulls in a much heavier dependency set (playwright, censys, shodan, fastapi),
|
||||||
|
|||||||
Reference in New Issue
Block a user