$textLength, 'max_length' => $maxLength, ]); return $fullText; } Log::info('TextProcessor: Extracting excerpt', [ 'original_length' => $textLength, 'max_length' => $maxLength, 'strategy' => $strategy, ]); $excerpt = match ($strategy) { 'beginning' => self::extractBeginning($fullText, $maxLength), 'middle' => self::extractMiddle($fullText, $maxLength), 'end' => self::extractEnd($fullText, $maxLength), 'distributed' => self::extractDistributed($fullText, $maxLength), default => self::extractBeginning($fullText, $maxLength), }; Log::info('TextProcessor: Excerpt extracted', [ 'excerpt_length' => strlen($excerpt), 'reduction_percent' => round((1 - strlen($excerpt) / $textLength) * 100, 2), ]); return $excerpt; } /** * Extract from the beginning of the text. * * @param string $text Full text * @param int $maxLength Maximum length * @return string */ protected static function extractBeginning(string $text, int $maxLength): string { $excerpt = substr($text, 0, $maxLength); return self::trimToLastSentence($excerpt); } /** * Extract from the middle of the text. * * @param string $text Full text * @param int $maxLength Maximum length * @return string */ protected static function extractMiddle(string $text, int $maxLength): string { $textLength = strlen($text); $startPos = intval(($textLength - $maxLength) / 2); $excerpt = substr($text, $startPos, $maxLength); return self::trimToCompleteSentences($excerpt); } /** * Extract from the end of the text. * * @param string $text Full text * @param int $maxLength Maximum length * @return string */ protected static function extractEnd(string $text, int $maxLength): string { $textLength = strlen($text); $startPos = $textLength - $maxLength; $excerpt = substr($text, $startPos); return self::trimToFirstSentence($excerpt); } /** * Extract distributed chunks from throughout the text. * * @param string $text Full text * @param int $maxLength Maximum length * @return string */ protected static function extractDistributed(string $text, int $maxLength): string { $textLength = strlen($text); $numChunks = 5; // Extract 5 chunks from different parts $chunkSize = intval($maxLength / $numChunks); $stepSize = intval($textLength / $numChunks); $chunks = []; for ($i = 0; $i < $numChunks; $i++) { $startPos = $i * $stepSize; $chunk = substr($text, $startPos, $chunkSize); $chunk = self::trimToCompleteSentences($chunk); if (!empty($chunk)) { $chunks[] = $chunk; } } return implode("\n\n[...]\n\n", $chunks); } /** * Clean text by removing formatting artifacts and normalizing whitespace. * * @param string $text The text to clean * @return string Cleaned text */ public static function cleanText(string $text): string { Log::info('TextProcessor: Cleaning text', [ 'original_length' => strlen($text), ]); // Remove Project Gutenberg header and footer $text = self::removeGutenbergBoilerplate($text); // Normalize line endings to Unix style $text = str_replace(["\r\n", "\r"], "\n", $text); // Remove excessive whitespace $text = preg_replace('/\n{3,}/', "\n\n", $text); $text = preg_replace('/[ \t]+/', ' ', $text); // Remove page numbers (common patterns) $text = preg_replace('/\n\s*\d+\s*\n/', "\n", $text); // Remove common formatting artifacts $text = preg_replace('/_{3,}/', '', $text); // Underscores $text = preg_replace('/={3,}/', '', $text); // Equals signs $text = preg_replace('/-{3,}/', '', $text); // Dashes // Trim $text = trim($text); Log::info('TextProcessor: Text cleaned', [ 'cleaned_length' => strlen($text), ]); return $text; } /** * Remove Project Gutenberg boilerplate text. * * @param string $text Full text * @return string Text without boilerplate */ protected static function removeGutenbergBoilerplate(string $text): string { // Remove header (everything before "START OF THE PROJECT GUTENBERG") $startMarkers = [ '*** START OF THE PROJECT GUTENBERG', '*** START OF THIS PROJECT GUTENBERG', '*END*THE SMALL PRINT', ]; foreach ($startMarkers as $marker) { $pos = stripos($text, $marker); if ($pos !== false) { // Find the end of the line after the marker $endOfLine = strpos($text, "\n", $pos); if ($endOfLine !== false) { $text = substr($text, $endOfLine + 1); break; } } } // Remove footer (everything after "END OF THE PROJECT GUTENBERG") $endMarkers = [ '*** END OF THE PROJECT GUTENBERG', '*** END OF THIS PROJECT GUTENBERG', ]; foreach ($endMarkers as $marker) { $pos = stripos($text, $marker); if ($pos !== false) { $text = substr($text, 0, $pos); break; } } return $text; } /** * Truncate text to fit within a token limit (approximate). * * @param string $text The text to truncate * @param int $tokenLimit Maximum number of tokens (approximate) * @return string Truncated text */ public static function truncateToTokenLimit(string $text, int $tokenLimit = 4000): string { // Rough approximation: 1 token ≈ 4 characters for English text $charLimit = $tokenLimit * 4; if (strlen($text) <= $charLimit) { return $text; } Log::info('TextProcessor: Truncating to token limit', [ 'original_length' => strlen($text), 'token_limit' => $tokenLimit, 'char_limit' => $charLimit, ]); $truncated = substr($text, 0, $charLimit); return self::trimToLastSentence($truncated); } /** * Trim text to the last complete sentence. * * @param string $text Text to trim * @return string Trimmed text */ protected static function trimToLastSentence(string $text): string { $sentenceEndings = ['. ', '! ', '? ', '."', '!"', '?"']; $lastPos = 0; foreach ($sentenceEndings as $ending) { $pos = strrpos($text, $ending); if ($pos !== false && $pos > $lastPos) { $lastPos = $pos + strlen($ending) - 1; } } if ($lastPos > 0) { return substr($text, 0, $lastPos + 1); } return $text; } /** * Trim text to the first complete sentence. * * @param string $text Text to trim * @return string Trimmed text */ protected static function trimToFirstSentence(string $text): string { $sentenceEndings = ['. ', '! ', '? ']; $firstPos = strlen($text); foreach ($sentenceEndings as $ending) { $pos = strpos($text, $ending); if ($pos !== false && $pos < $firstPos) { $firstPos = $pos + strlen($ending); } } if ($firstPos < strlen($text)) { return substr($text, $firstPos); } return $text; } /** * Trim text to complete sentences at both ends. * * @param string $text Text to trim * @return string Trimmed text */ protected static function trimToCompleteSentences(string $text): string { $text = self::trimToFirstSentence($text); $text = self::trimToLastSentence($text); return $text; } /** * Count approximate tokens in text. * * @param string $text Text to analyze * @return int Approximate token count */ public static function countTokens(string $text): int { // Rough approximation: 1 token ≈ 4 characters for English text return intval(strlen($text) / 4); } /** * Get text statistics. * * @param string $text Text to analyze * @return array Statistics array */ public static function getStatistics(string $text): array { return [ 'character_count' => strlen($text), 'word_count' => str_word_count($text), 'line_count' => substr_count($text, "\n") + 1, 'sentence_count' => self::countSentences($text), 'approximate_tokens' => self::countTokens($text), ]; } /** * Count sentences in text. * * @param string $text Text to analyze * @return int Number of sentences */ protected static function countSentences(string $text): int { $count = preg_match_all('/[.!?]+/', $text); return $count ?: 1; } /** * Split text into chunks of approximately equal size. * * @param string $text Text to split * @param int $chunkSize Approximate size of each chunk in characters * @return array Array of text chunks */ public static function splitIntoChunks(string $text, int $chunkSize = 10000): array { $textLength = strlen($text); $numChunks = ceil($textLength / $chunkSize); $chunks = []; for ($i = 0; $i < $numChunks; $i++) { $start = $i * $chunkSize; $chunk = substr($text, $start, $chunkSize); // Trim to complete sentences except for the last chunk if ($i < $numChunks - 1) { $chunk = self::trimToLastSentence($chunk); } $chunks[] = $chunk; } Log::info('TextProcessor: Text split into chunks', [ 'original_length' => $textLength, 'chunk_size' => $chunkSize, 'num_chunks' => count($chunks), ]); return array_filter($chunks); // Remove empty chunks } }