373 lines
11 KiB
PHP
373 lines
11 KiB
PHP
<?php
|
|
|
|
namespace App\Helpers;
|
|
|
|
use Illuminate\Support\Facades\Log;
|
|
|
|
class TextProcessor
|
|
{
|
|
/**
|
|
* Extract a representative excerpt from the full text.
|
|
*
|
|
* @param string $fullText The complete book text
|
|
* @param int $maxLength Maximum character length of the excerpt
|
|
* @param string $strategy Extraction strategy: 'beginning', 'middle', 'end', 'distributed'
|
|
* @return string The extracted excerpt
|
|
*/
|
|
public static function extractExcerpt(
|
|
string $fullText,
|
|
int $maxLength = 50000,
|
|
string $strategy = 'beginning'
|
|
): string {
|
|
$textLength = strlen($fullText);
|
|
|
|
// If text is already short enough, return as is
|
|
if ($textLength <= $maxLength) {
|
|
Log::info('TextProcessor: Text within limit, no extraction needed', [
|
|
'length' => $textLength,
|
|
'max_length' => $maxLength,
|
|
]);
|
|
return $fullText;
|
|
}
|
|
|
|
Log::info('TextProcessor: Extracting excerpt', [
|
|
'original_length' => $textLength,
|
|
'max_length' => $maxLength,
|
|
'strategy' => $strategy,
|
|
]);
|
|
|
|
$excerpt = match ($strategy) {
|
|
'beginning' => self::extractBeginning($fullText, $maxLength),
|
|
'middle' => self::extractMiddle($fullText, $maxLength),
|
|
'end' => self::extractEnd($fullText, $maxLength),
|
|
'distributed' => self::extractDistributed($fullText, $maxLength),
|
|
default => self::extractBeginning($fullText, $maxLength),
|
|
};
|
|
|
|
Log::info('TextProcessor: Excerpt extracted', [
|
|
'excerpt_length' => strlen($excerpt),
|
|
'reduction_percent' => round((1 - strlen($excerpt) / $textLength) * 100, 2),
|
|
]);
|
|
|
|
return $excerpt;
|
|
}
|
|
|
|
/**
|
|
* Extract from the beginning of the text.
|
|
*
|
|
* @param string $text Full text
|
|
* @param int $maxLength Maximum length
|
|
* @return string
|
|
*/
|
|
protected static function extractBeginning(string $text, int $maxLength): string
|
|
{
|
|
$excerpt = substr($text, 0, $maxLength);
|
|
return self::trimToLastSentence($excerpt);
|
|
}
|
|
|
|
/**
|
|
* Extract from the middle of the text.
|
|
*
|
|
* @param string $text Full text
|
|
* @param int $maxLength Maximum length
|
|
* @return string
|
|
*/
|
|
protected static function extractMiddle(string $text, int $maxLength): string
|
|
{
|
|
$textLength = strlen($text);
|
|
$startPos = intval(($textLength - $maxLength) / 2);
|
|
$excerpt = substr($text, $startPos, $maxLength);
|
|
return self::trimToCompleteSentences($excerpt);
|
|
}
|
|
|
|
/**
|
|
* Extract from the end of the text.
|
|
*
|
|
* @param string $text Full text
|
|
* @param int $maxLength Maximum length
|
|
* @return string
|
|
*/
|
|
protected static function extractEnd(string $text, int $maxLength): string
|
|
{
|
|
$textLength = strlen($text);
|
|
$startPos = $textLength - $maxLength;
|
|
$excerpt = substr($text, $startPos);
|
|
return self::trimToFirstSentence($excerpt);
|
|
}
|
|
|
|
/**
|
|
* Extract distributed chunks from throughout the text.
|
|
*
|
|
* @param string $text Full text
|
|
* @param int $maxLength Maximum length
|
|
* @return string
|
|
*/
|
|
protected static function extractDistributed(string $text, int $maxLength): string
|
|
{
|
|
$textLength = strlen($text);
|
|
$numChunks = 5; // Extract 5 chunks from different parts
|
|
$chunkSize = intval($maxLength / $numChunks);
|
|
$stepSize = intval($textLength / $numChunks);
|
|
|
|
$chunks = [];
|
|
for ($i = 0; $i < $numChunks; $i++) {
|
|
$startPos = $i * $stepSize;
|
|
$chunk = substr($text, $startPos, $chunkSize);
|
|
$chunk = self::trimToCompleteSentences($chunk);
|
|
if (!empty($chunk)) {
|
|
$chunks[] = $chunk;
|
|
}
|
|
}
|
|
|
|
return implode("\n\n[...]\n\n", $chunks);
|
|
}
|
|
|
|
/**
|
|
* Clean text by removing formatting artifacts and normalizing whitespace.
|
|
*
|
|
* @param string $text The text to clean
|
|
* @return string Cleaned text
|
|
*/
|
|
public static function cleanText(string $text): string
|
|
{
|
|
Log::info('TextProcessor: Cleaning text', [
|
|
'original_length' => strlen($text),
|
|
]);
|
|
|
|
// Remove Project Gutenberg header and footer
|
|
$text = self::removeGutenbergBoilerplate($text);
|
|
|
|
// Normalize line endings to Unix style
|
|
$text = str_replace(["\r\n", "\r"], "\n", $text);
|
|
|
|
// Remove excessive whitespace
|
|
$text = preg_replace('/\n{3,}/', "\n\n", $text);
|
|
$text = preg_replace('/[ \t]+/', ' ', $text);
|
|
|
|
// Remove page numbers (common patterns)
|
|
$text = preg_replace('/\n\s*\d+\s*\n/', "\n", $text);
|
|
|
|
// Remove common formatting artifacts
|
|
$text = preg_replace('/_{3,}/', '', $text); // Underscores
|
|
$text = preg_replace('/={3,}/', '', $text); // Equals signs
|
|
$text = preg_replace('/-{3,}/', '', $text); // Dashes
|
|
|
|
// Trim
|
|
$text = trim($text);
|
|
|
|
Log::info('TextProcessor: Text cleaned', [
|
|
'cleaned_length' => strlen($text),
|
|
]);
|
|
|
|
return $text;
|
|
}
|
|
|
|
/**
|
|
* Remove Project Gutenberg boilerplate text.
|
|
*
|
|
* @param string $text Full text
|
|
* @return string Text without boilerplate
|
|
*/
|
|
protected static function removeGutenbergBoilerplate(string $text): string
|
|
{
|
|
// Remove header (everything before "START OF THE PROJECT GUTENBERG")
|
|
$startMarkers = [
|
|
'*** START OF THE PROJECT GUTENBERG',
|
|
'*** START OF THIS PROJECT GUTENBERG',
|
|
'*END*THE SMALL PRINT',
|
|
];
|
|
|
|
foreach ($startMarkers as $marker) {
|
|
$pos = stripos($text, $marker);
|
|
if ($pos !== false) {
|
|
// Find the end of the line after the marker
|
|
$endOfLine = strpos($text, "\n", $pos);
|
|
if ($endOfLine !== false) {
|
|
$text = substr($text, $endOfLine + 1);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Remove footer (everything after "END OF THE PROJECT GUTENBERG")
|
|
$endMarkers = [
|
|
'*** END OF THE PROJECT GUTENBERG',
|
|
'*** END OF THIS PROJECT GUTENBERG',
|
|
];
|
|
|
|
foreach ($endMarkers as $marker) {
|
|
$pos = stripos($text, $marker);
|
|
if ($pos !== false) {
|
|
$text = substr($text, 0, $pos);
|
|
break;
|
|
}
|
|
}
|
|
|
|
return $text;
|
|
}
|
|
|
|
/**
|
|
* Truncate text to fit within a token limit (approximate).
|
|
*
|
|
* @param string $text The text to truncate
|
|
* @param int $tokenLimit Maximum number of tokens (approximate)
|
|
* @return string Truncated text
|
|
*/
|
|
public static function truncateToTokenLimit(string $text, int $tokenLimit = 4000): string
|
|
{
|
|
// Rough approximation: 1 token ≈ 4 characters for English text
|
|
$charLimit = $tokenLimit * 4;
|
|
|
|
if (strlen($text) <= $charLimit) {
|
|
return $text;
|
|
}
|
|
|
|
Log::info('TextProcessor: Truncating to token limit', [
|
|
'original_length' => strlen($text),
|
|
'token_limit' => $tokenLimit,
|
|
'char_limit' => $charLimit,
|
|
]);
|
|
|
|
$truncated = substr($text, 0, $charLimit);
|
|
return self::trimToLastSentence($truncated);
|
|
}
|
|
|
|
/**
|
|
* Trim text to the last complete sentence.
|
|
*
|
|
* @param string $text Text to trim
|
|
* @return string Trimmed text
|
|
*/
|
|
protected static function trimToLastSentence(string $text): string
|
|
{
|
|
$sentenceEndings = ['. ', '! ', '? ', '."', '!"', '?"'];
|
|
|
|
$lastPos = 0;
|
|
foreach ($sentenceEndings as $ending) {
|
|
$pos = strrpos($text, $ending);
|
|
if ($pos !== false && $pos > $lastPos) {
|
|
$lastPos = $pos + strlen($ending) - 1;
|
|
}
|
|
}
|
|
|
|
if ($lastPos > 0) {
|
|
return substr($text, 0, $lastPos + 1);
|
|
}
|
|
|
|
return $text;
|
|
}
|
|
|
|
/**
|
|
* Trim text to the first complete sentence.
|
|
*
|
|
* @param string $text Text to trim
|
|
* @return string Trimmed text
|
|
*/
|
|
protected static function trimToFirstSentence(string $text): string
|
|
{
|
|
$sentenceEndings = ['. ', '! ', '? '];
|
|
|
|
$firstPos = strlen($text);
|
|
foreach ($sentenceEndings as $ending) {
|
|
$pos = strpos($text, $ending);
|
|
if ($pos !== false && $pos < $firstPos) {
|
|
$firstPos = $pos + strlen($ending);
|
|
}
|
|
}
|
|
|
|
if ($firstPos < strlen($text)) {
|
|
return substr($text, $firstPos);
|
|
}
|
|
|
|
return $text;
|
|
}
|
|
|
|
/**
|
|
* Trim text to complete sentences at both ends.
|
|
*
|
|
* @param string $text Text to trim
|
|
* @return string Trimmed text
|
|
*/
|
|
protected static function trimToCompleteSentences(string $text): string
|
|
{
|
|
$text = self::trimToFirstSentence($text);
|
|
$text = self::trimToLastSentence($text);
|
|
return $text;
|
|
}
|
|
|
|
/**
|
|
* Count approximate tokens in text.
|
|
*
|
|
* @param string $text Text to analyze
|
|
* @return int Approximate token count
|
|
*/
|
|
public static function countTokens(string $text): int
|
|
{
|
|
// Rough approximation: 1 token ≈ 4 characters for English text
|
|
return intval(strlen($text) / 4);
|
|
}
|
|
|
|
/**
|
|
* Get text statistics.
|
|
*
|
|
* @param string $text Text to analyze
|
|
* @return array Statistics array
|
|
*/
|
|
public static function getStatistics(string $text): array
|
|
{
|
|
return [
|
|
'character_count' => strlen($text),
|
|
'word_count' => str_word_count($text),
|
|
'line_count' => substr_count($text, "\n") + 1,
|
|
'sentence_count' => self::countSentences($text),
|
|
'approximate_tokens' => self::countTokens($text),
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Count sentences in text.
|
|
*
|
|
* @param string $text Text to analyze
|
|
* @return int Number of sentences
|
|
*/
|
|
protected static function countSentences(string $text): int
|
|
{
|
|
$count = preg_match_all('/[.!?]+/', $text);
|
|
return $count ?: 1;
|
|
}
|
|
|
|
/**
|
|
* Split text into chunks of approximately equal size.
|
|
*
|
|
* @param string $text Text to split
|
|
* @param int $chunkSize Approximate size of each chunk in characters
|
|
* @return array Array of text chunks
|
|
*/
|
|
public static function splitIntoChunks(string $text, int $chunkSize = 10000): array
|
|
{
|
|
$textLength = strlen($text);
|
|
$numChunks = ceil($textLength / $chunkSize);
|
|
$chunks = [];
|
|
|
|
for ($i = 0; $i < $numChunks; $i++) {
|
|
$start = $i * $chunkSize;
|
|
$chunk = substr($text, $start, $chunkSize);
|
|
|
|
// Trim to complete sentences except for the last chunk
|
|
if ($i < $numChunks - 1) {
|
|
$chunk = self::trimToLastSentence($chunk);
|
|
}
|
|
|
|
$chunks[] = $chunk;
|
|
}
|
|
|
|
Log::info('TextProcessor: Text split into chunks', [
|
|
'original_length' => $textLength,
|
|
'chunk_size' => $chunkSize,
|
|
'num_chunks' => count($chunks),
|
|
]);
|
|
|
|
return array_filter($chunks); // Remove empty chunks
|
|
}
|
|
}
|