diff --git a/PLAN.md b/PLAN.md index 79c18bc..b4e5e44 100644 --- a/PLAN.md +++ b/PLAN.md @@ -95,17 +95,17 @@ A self-hosted error tracking application (inspired by [Bugsink](https://www.bugs Implement a `/api/{project_id}/store/` endpoint that accepts the Sentry event payload format. This allows using existing Sentry SDKs (for PHP, JS, Python, etc.) as clients without any custom SDK work. -- [ ] `POST /api/{project_id}/store/` route -- [ ] Validate DSN public key via `sentry_key` query param or `X-Sentry-Auth` header -- [ ] `AuthenticateDsn` middleware -- [ ] `EventIngestController` +- [x] `POST /api/{project_id}/store/` route +- [x] Validate DSN public key via `sentry_key` query param or `X-Sentry-Auth` header +- [x] `AuthenticateDsn` middleware +- [x] `EventIngestController` ### 2.2 Event Processing Pipeline -- [ ] `ProcessEventJob` (queued) -- [ ] `PayloadNormalizer` service -- [ ] `EventProcessor` service (find or create Issue, store Event, update counters) -- [ ] Regression detection (re-open resolved issues) +- [x] `ProcessEventJob` (queued) +- [x] `PayloadNormalizer` service +- [x] `EventProcessor` service (find or create Issue, store Event, update counters) +- [x] Regression detection (re-open resolved issues) ``` Ingest Request @@ -122,7 +122,7 @@ Ingest Request ### 2.3 Fingerprinting / Grouping Strategy -- [ ] `Fingerprinter` service +- [x] `Fingerprinter` service Group events into issues using a fingerprint hash built from: 1. Exception type (class name) @@ -238,8 +238,8 @@ Clients configure this DSN in their Sentry SDK — no changes needed on the SDK - [x] 1. Install Breeze, set up auth - [x] 2. Create migrations for `projects`, `issues`, `events`, `project_user` - [x] 3. Build models with relationships -- [ ] 4. Build the ingest endpoint (`POST /api/{project_id}/store/`) -- [ ] 5. Implement fingerprinting and the `ProcessEventJob` +- [x] 4. Build the ingest endpoint (`POST /api/{project_id}/store/`) +- [x] 5. Implement fingerprinting and the `ProcessEventJob` - [ ] 6. Build a minimal issues list page and issue detail page - [ ] 7. Test with a real Sentry SDK (e.g. `sentry/sentry-laravel` on another app pointing at this tracker) diff --git a/app/Http/Controllers/Api/EventIngestController.php b/app/Http/Controllers/Api/EventIngestController.php new file mode 100644 index 0000000..83407da --- /dev/null +++ b/app/Http/Controllers/Api/EventIngestController.php @@ -0,0 +1,28 @@ +json()->all(); + + if (empty($payload)) { + return response()->json(['error' => 'Empty payload'], 400); + } + + ProcessEventJob::dispatch($project, $payload); + + return response()->json( + ['id' => $payload['event_id'] ?? null], + 202, + ); + } +} diff --git a/app/Http/Middleware/AuthenticateDsn.php b/app/Http/Middleware/AuthenticateDsn.php new file mode 100644 index 0000000..0afd9eb --- /dev/null +++ b/app/Http/Middleware/AuthenticateDsn.php @@ -0,0 +1,48 @@ +route('project'); + + if (! $project instanceof Project) { + abort(404, 'Project not found.'); + } + + $sentryKey = $this->extractSentryKey($request); + + if (! $sentryKey || $sentryKey !== $project->dsn_public_key) { + abort(401, 'Invalid DSN key.'); + } + + return $next($request); + } + + /** + * Extract the sentry_key from the query string or X-Sentry-Auth header. + * + * Header format: Sentry sentry_version=7, sentry_client=..., sentry_key= + */ + private function extractSentryKey(Request $request): ?string + { + if ($key = $request->query('sentry_key')) { + return $key; + } + + $header = $request->header('X-Sentry-Auth', ''); + + if (preg_match('/sentry_key=([a-f0-9\-]+)/i', $header, $matches)) { + return $matches[1]; + } + + return null; + } +} diff --git a/app/Jobs/ProcessEventJob.php b/app/Jobs/ProcessEventJob.php new file mode 100644 index 0000000..13cbce4 --- /dev/null +++ b/app/Jobs/ProcessEventJob.php @@ -0,0 +1,40 @@ +process($this->project, $this->rawPayload); + } + + public function failed(\Throwable $exception): void + { + Log::error('Failed to process event', [ + 'project_id' => $this->project->id, + 'event_id' => $this->rawPayload['event_id'] ?? 'unknown', + 'error' => $exception->getMessage(), + ]); + } +} diff --git a/app/Services/EventProcessor.php b/app/Services/EventProcessor.php new file mode 100644 index 0000000..9936015 --- /dev/null +++ b/app/Services/EventProcessor.php @@ -0,0 +1,150 @@ +normalizer->normalize($rawPayload); + $fingerprint = $this->fingerprinter->compute($rawPayload); + $now = Carbon::now(); + + $issue = $this->findOrCreateIssue($project, $data, $fingerprint, $now); + + $event = Event::create([ + 'issue_id' => $issue->id, + 'project_id' => $project->id, + 'event_id' => $data['event_id'], + 'level' => $data['level'], + 'message' => $data['message'], + 'platform' => $data['platform'], + 'timestamp' => $data['timestamp'], + 'server_name' => $data['server_name'], + 'release' => $data['release'], + 'environment' => $data['environment'], + 'exception' => $data['exception'], + 'breadcrumbs' => $data['breadcrumbs'], + 'contexts' => $data['contexts'], + 'tags' => $data['tags'], + 'extra' => $data['extra'], + 'user_context' => $data['user_context'], + 'request_context' => $data['request_context'], + 'sdk' => $data['sdk'], + 'raw_payload' => $rawPayload, + ]); + + return $event; + } + + private function findOrCreateIssue( + Project $project, + array $data, + string $fingerprint, + Carbon $now, + ): Issue { + $issue = Issue::where('project_id', $project->id) + ->where('fingerprint', $fingerprint) + ->first(); + + if ($issue) { + $this->updateExistingIssue($issue, $data, $now); + + return $issue; + } + + return $this->createNewIssue($project, $data, $fingerprint, $now); + } + + private function createNewIssue( + Project $project, + array $data, + string $fingerprint, + Carbon $now, + ): Issue { + $exception = $this->fingerprinter->extractPrimaryException( + ['exception' => $data['exception']], + ); + + $title = $this->buildTitle($exception, $data['message']); + $culprit = $this->buildCulprit($exception); + + return Issue::create([ + 'project_id' => $project->id, + 'fingerprint' => $fingerprint, + 'title' => $title, + 'culprit' => $culprit, + 'level' => $data['level'], + 'status' => 'unresolved', + 'first_seen_at' => $now, + 'last_seen_at' => $now, + 'times_seen' => 1, + 'is_regression' => false, + 'metadata' => array_filter([ + 'platform' => $data['platform'], + 'sdk' => $data['sdk'], + ]), + ]); + } + + private function updateExistingIssue(Issue $issue, array $data, Carbon $now): void + { + $wasResolved = $issue->status === 'resolved'; + + $issue->increment('times_seen'); + $issue->update(['last_seen_at' => $now]); + + if ($wasResolved) { + $issue->markRegression(); + } + } + + private function buildTitle(?array $exception, ?string $message): string + { + if ($exception) { + $type = $exception['type'] ?? 'Error'; + $value = $exception['value'] ?? ''; + + return str($value) + ->whenEmpty(fn () => str($type)) + ->prepend("{$type}: ") + ->limit(255) + ->toString(); + } + + return str($message ?? 'Unknown Error')->limit(255)->toString(); + } + + private function buildCulprit(?array $exception): ?string + { + if (! $exception) { + return null; + } + + $frames = $exception['stacktrace']['frames'] ?? []; + $last = end($frames); + + if (! $last) { + return null; + } + + $file = $last['filename'] ?? $last['abs_path'] ?? '?'; + $function = $last['function'] ?? null; + + return $function ? "{$file} in {$function}" : $file; + } +} diff --git a/app/Services/Fingerprinter.php b/app/Services/Fingerprinter.php new file mode 100644 index 0000000..3e5d77f --- /dev/null +++ b/app/Services/Fingerprinter.php @@ -0,0 +1,85 @@ +hash($payload['fingerprint']); + } + + $parts = []; + + $exception = $this->extractPrimaryException($payload); + + if ($exception) { + $parts[] = $exception['type'] ?? ''; + $parts[] = $this->sanitizeMessage($exception['value'] ?? ''); + $parts = array_merge($parts, $this->topFrames($exception, 5)); + } else { + $parts[] = $payload['message'] ?? $payload['logentry']['message'] ?? 'unknown'; + } + + return $this->hash($parts); + } + + public function extractPrimaryException(array $payload): ?array + { + $exceptions = $payload['exception']['values'] ?? []; + + return end($exceptions) ?: null; + } + + /** + * Strip variable parts (numbers, UUIDs, hex strings) so that + * "User 42 not found" and "User 99 not found" group together. + */ + private function sanitizeMessage(string $message): string + { + $message = preg_replace( + '/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i', + '{uuid}', + $message, + ); + + $message = preg_replace('/\b[0-9a-f]{24,}\b/i', '{hash}', $message); + + $message = preg_replace('/\b\d+\b/', '{n}', $message); + + return $message; + } + + /** + * Return file+function strings from the top N stacktrace frames. + */ + private function topFrames(array $exception, int $limit): array + { + $frames = $exception['stacktrace']['frames'] ?? []; + + $frames = array_reverse($frames); + $frames = array_slice($frames, 0, $limit); + + $parts = []; + foreach ($frames as $frame) { + $file = $frame['filename'] ?? $frame['abs_path'] ?? ''; + $function = $frame['function'] ?? ''; + $parts[] = "{$file}:{$function}"; + } + + return $parts; + } + + private function hash(array $parts): string + { + return md5(implode("\x00", $parts)); + } +} diff --git a/app/Services/PayloadNormalizer.php b/app/Services/PayloadNormalizer.php new file mode 100644 index 0000000..daf4ec8 --- /dev/null +++ b/app/Services/PayloadNormalizer.php @@ -0,0 +1,111 @@ + $raw['event_id'] ?? Str::uuid()->toString(), + 'level' => $this->normalizeLevel($raw['level'] ?? 'error'), + 'message' => $this->extractMessage($raw), + 'platform' => $raw['platform'] ?? null, + 'timestamp' => $this->normalizeTimestamp($raw['timestamp'] ?? null), + 'server_name' => $raw['server_name'] ?? null, + 'release' => $raw['release'] ?? null, + 'environment' => $raw['environment'] ?? null, + 'exception' => $raw['exception'] ?? ['values' => []], + 'breadcrumbs' => $raw['breadcrumbs'] ?? null, + 'contexts' => $raw['contexts'] ?? null, + 'tags' => $this->normalizeTags($raw['tags'] ?? null), + 'extra' => $raw['extra'] ?? null, + 'user_context' => $raw['user'] ?? null, + 'request_context' => $raw['request'] ?? null, + 'sdk' => $raw['sdk'] ?? null, + 'fingerprint' => $raw['fingerprint'] ?? null, + ]; + } + + private function normalizeLevel(string $level): string + { + $map = [ + 'fatal' => 'fatal', + 'critical' => 'fatal', + 'error' => 'error', + 'warning' => 'warning', + 'warn' => 'warning', + 'info' => 'info', + 'log' => 'info', + 'debug' => 'debug', + ]; + + return $map[strtolower($level)] ?? 'error'; + } + + private function extractMessage(array $raw): ?string + { + if (! empty($raw['message'])) { + return Str::limit($raw['message'], 2000); + } + + if (! empty($raw['logentry']['message'])) { + return Str::limit($raw['logentry']['message'], 2000); + } + + $exceptions = $raw['exception']['values'] ?? []; + $last = end($exceptions); + if ($last) { + $type = $last['type'] ?? 'Error'; + $value = $last['value'] ?? ''; + + return Str::limit("{$type}: {$value}", 2000); + } + + return null; + } + + private function normalizeTimestamp(mixed $timestamp): Carbon + { + if ($timestamp === null) { + return Carbon::now(); + } + + if (is_numeric($timestamp)) { + return Carbon::createFromTimestamp((float) $timestamp); + } + + return Carbon::parse($timestamp); + } + + /** + * Tags can arrive as key-value pairs or as an array of {key, value} objects. + * Normalize to a plain associative array. + */ + private function normalizeTags(mixed $tags): ?array + { + if ($tags === null) { + return null; + } + + if (array_is_list($tags)) { + $normalized = []; + foreach ($tags as $tag) { + if (isset($tag['key'], $tag['value'])) { + $normalized[$tag['key']] = $tag['value']; + } + } + + return $normalized ?: null; + } + + return $tags; + } +} diff --git a/bootstrap/app.php b/bootstrap/app.php index c183276..98fe6dc 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -7,6 +7,8 @@ use Illuminate\Foundation\Configuration\Middleware; return Application::configure(basePath: dirname(__DIR__)) ->withRouting( web: __DIR__.'/../routes/web.php', + api: __DIR__.'/../routes/api.php', + apiPrefix: 'api', commands: __DIR__.'/../routes/console.php', health: '/up', ) diff --git a/routes/api.php b/routes/api.php new file mode 100644 index 0000000..07bbec2 --- /dev/null +++ b/routes/api.php @@ -0,0 +1,9 @@ +middleware(AuthenticateDsn::class) + ->name('api.events.store');