gutendex integration phase 3
This commit is contained in:
Doar Mihai
@@ -0,0 +1,372 @@
|
||||
<?php
|
||||
|
||||
namespace App\Helpers;
|
||||
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class TextProcessor
|
||||
{
|
||||
/**
|
||||
* Extract a representative excerpt from the full text.
|
||||
*
|
||||
* @param string $fullText The complete book text
|
||||
* @param int $maxLength Maximum character length of the excerpt
|
||||
* @param string $strategy Extraction strategy: 'beginning', 'middle', 'end', 'distributed'
|
||||
* @return string The extracted excerpt
|
||||
*/
|
||||
public static function extractExcerpt(
|
||||
string $fullText,
|
||||
int $maxLength = 50000,
|
||||
string $strategy = 'beginning'
|
||||
): string {
|
||||
$textLength = strlen($fullText);
|
||||
|
||||
// If text is already short enough, return as is
|
||||
if ($textLength <= $maxLength) {
|
||||
Log::info('TextProcessor: Text within limit, no extraction needed', [
|
||||
'length' => $textLength,
|
||||
'max_length' => $maxLength,
|
||||
]);
|
||||
return $fullText;
|
||||
}
|
||||
|
||||
Log::info('TextProcessor: Extracting excerpt', [
|
||||
'original_length' => $textLength,
|
||||
'max_length' => $maxLength,
|
||||
'strategy' => $strategy,
|
||||
]);
|
||||
|
||||
$excerpt = match ($strategy) {
|
||||
'beginning' => self::extractBeginning($fullText, $maxLength),
|
||||
'middle' => self::extractMiddle($fullText, $maxLength),
|
||||
'end' => self::extractEnd($fullText, $maxLength),
|
||||
'distributed' => self::extractDistributed($fullText, $maxLength),
|
||||
default => self::extractBeginning($fullText, $maxLength),
|
||||
};
|
||||
|
||||
Log::info('TextProcessor: Excerpt extracted', [
|
||||
'excerpt_length' => strlen($excerpt),
|
||||
'reduction_percent' => round((1 - strlen($excerpt) / $textLength) * 100, 2),
|
||||
]);
|
||||
|
||||
return $excerpt;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract from the beginning of the text.
|
||||
*
|
||||
* @param string $text Full text
|
||||
* @param int $maxLength Maximum length
|
||||
* @return string
|
||||
*/
|
||||
protected static function extractBeginning(string $text, int $maxLength): string
|
||||
{
|
||||
$excerpt = substr($text, 0, $maxLength);
|
||||
return self::trimToLastSentence($excerpt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract from the middle of the text.
|
||||
*
|
||||
* @param string $text Full text
|
||||
* @param int $maxLength Maximum length
|
||||
* @return string
|
||||
*/
|
||||
protected static function extractMiddle(string $text, int $maxLength): string
|
||||
{
|
||||
$textLength = strlen($text);
|
||||
$startPos = intval(($textLength - $maxLength) / 2);
|
||||
$excerpt = substr($text, $startPos, $maxLength);
|
||||
return self::trimToCompleteSentences($excerpt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract from the end of the text.
|
||||
*
|
||||
* @param string $text Full text
|
||||
* @param int $maxLength Maximum length
|
||||
* @return string
|
||||
*/
|
||||
protected static function extractEnd(string $text, int $maxLength): string
|
||||
{
|
||||
$textLength = strlen($text);
|
||||
$startPos = $textLength - $maxLength;
|
||||
$excerpt = substr($text, $startPos);
|
||||
return self::trimToFirstSentence($excerpt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract distributed chunks from throughout the text.
|
||||
*
|
||||
* @param string $text Full text
|
||||
* @param int $maxLength Maximum length
|
||||
* @return string
|
||||
*/
|
||||
protected static function extractDistributed(string $text, int $maxLength): string
|
||||
{
|
||||
$textLength = strlen($text);
|
||||
$numChunks = 5; // Extract 5 chunks from different parts
|
||||
$chunkSize = intval($maxLength / $numChunks);
|
||||
$stepSize = intval($textLength / $numChunks);
|
||||
|
||||
$chunks = [];
|
||||
for ($i = 0; $i < $numChunks; $i++) {
|
||||
$startPos = $i * $stepSize;
|
||||
$chunk = substr($text, $startPos, $chunkSize);
|
||||
$chunk = self::trimToCompleteSentences($chunk);
|
||||
if (!empty($chunk)) {
|
||||
$chunks[] = $chunk;
|
||||
}
|
||||
}
|
||||
|
||||
return implode("\n\n[...]\n\n", $chunks);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean text by removing formatting artifacts and normalizing whitespace.
|
||||
*
|
||||
* @param string $text The text to clean
|
||||
* @return string Cleaned text
|
||||
*/
|
||||
public static function cleanText(string $text): string
|
||||
{
|
||||
Log::info('TextProcessor: Cleaning text', [
|
||||
'original_length' => strlen($text),
|
||||
]);
|
||||
|
||||
// Remove Project Gutenberg header and footer
|
||||
$text = self::removeGutenbergBoilerplate($text);
|
||||
|
||||
// Normalize line endings to Unix style
|
||||
$text = str_replace(["\r\n", "\r"], "\n", $text);
|
||||
|
||||
// Remove excessive whitespace
|
||||
$text = preg_replace('/\n{3,}/', "\n\n", $text);
|
||||
$text = preg_replace('/[ \t]+/', ' ', $text);
|
||||
|
||||
// Remove page numbers (common patterns)
|
||||
$text = preg_replace('/\n\s*\d+\s*\n/', "\n", $text);
|
||||
|
||||
// Remove common formatting artifacts
|
||||
$text = preg_replace('/_{3,}/', '', $text); // Underscores
|
||||
$text = preg_replace('/={3,}/', '', $text); // Equals signs
|
||||
$text = preg_replace('/-{3,}/', '', $text); // Dashes
|
||||
|
||||
// Trim
|
||||
$text = trim($text);
|
||||
|
||||
Log::info('TextProcessor: Text cleaned', [
|
||||
'cleaned_length' => strlen($text),
|
||||
]);
|
||||
|
||||
return $text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove Project Gutenberg boilerplate text.
|
||||
*
|
||||
* @param string $text Full text
|
||||
* @return string Text without boilerplate
|
||||
*/
|
||||
protected static function removeGutenbergBoilerplate(string $text): string
|
||||
{
|
||||
// Remove header (everything before "START OF THE PROJECT GUTENBERG")
|
||||
$startMarkers = [
|
||||
'*** START OF THE PROJECT GUTENBERG',
|
||||
'*** START OF THIS PROJECT GUTENBERG',
|
||||
'*END*THE SMALL PRINT',
|
||||
];
|
||||
|
||||
foreach ($startMarkers as $marker) {
|
||||
$pos = stripos($text, $marker);
|
||||
if ($pos !== false) {
|
||||
// Find the end of the line after the marker
|
||||
$endOfLine = strpos($text, "\n", $pos);
|
||||
if ($endOfLine !== false) {
|
||||
$text = substr($text, $endOfLine + 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove footer (everything after "END OF THE PROJECT GUTENBERG")
|
||||
$endMarkers = [
|
||||
'*** END OF THE PROJECT GUTENBERG',
|
||||
'*** END OF THIS PROJECT GUTENBERG',
|
||||
];
|
||||
|
||||
foreach ($endMarkers as $marker) {
|
||||
$pos = stripos($text, $marker);
|
||||
if ($pos !== false) {
|
||||
$text = substr($text, 0, $pos);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return $text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate text to fit within a token limit (approximate).
|
||||
*
|
||||
* @param string $text The text to truncate
|
||||
* @param int $tokenLimit Maximum number of tokens (approximate)
|
||||
* @return string Truncated text
|
||||
*/
|
||||
public static function truncateToTokenLimit(string $text, int $tokenLimit = 4000): string
|
||||
{
|
||||
// Rough approximation: 1 token ≈ 4 characters for English text
|
||||
$charLimit = $tokenLimit * 4;
|
||||
|
||||
if (strlen($text) <= $charLimit) {
|
||||
return $text;
|
||||
}
|
||||
|
||||
Log::info('TextProcessor: Truncating to token limit', [
|
||||
'original_length' => strlen($text),
|
||||
'token_limit' => $tokenLimit,
|
||||
'char_limit' => $charLimit,
|
||||
]);
|
||||
|
||||
$truncated = substr($text, 0, $charLimit);
|
||||
return self::trimToLastSentence($truncated);
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim text to the last complete sentence.
|
||||
*
|
||||
* @param string $text Text to trim
|
||||
* @return string Trimmed text
|
||||
*/
|
||||
protected static function trimToLastSentence(string $text): string
|
||||
{
|
||||
$sentenceEndings = ['. ', '! ', '? ', '."', '!"', '?"'];
|
||||
|
||||
$lastPos = 0;
|
||||
foreach ($sentenceEndings as $ending) {
|
||||
$pos = strrpos($text, $ending);
|
||||
if ($pos !== false && $pos > $lastPos) {
|
||||
$lastPos = $pos + strlen($ending) - 1;
|
||||
}
|
||||
}
|
||||
|
||||
if ($lastPos > 0) {
|
||||
return substr($text, 0, $lastPos + 1);
|
||||
}
|
||||
|
||||
return $text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim text to the first complete sentence.
|
||||
*
|
||||
* @param string $text Text to trim
|
||||
* @return string Trimmed text
|
||||
*/
|
||||
protected static function trimToFirstSentence(string $text): string
|
||||
{
|
||||
$sentenceEndings = ['. ', '! ', '? '];
|
||||
|
||||
$firstPos = strlen($text);
|
||||
foreach ($sentenceEndings as $ending) {
|
||||
$pos = strpos($text, $ending);
|
||||
if ($pos !== false && $pos < $firstPos) {
|
||||
$firstPos = $pos + strlen($ending);
|
||||
}
|
||||
}
|
||||
|
||||
if ($firstPos < strlen($text)) {
|
||||
return substr($text, $firstPos);
|
||||
}
|
||||
|
||||
return $text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Trim text to complete sentences at both ends.
|
||||
*
|
||||
* @param string $text Text to trim
|
||||
* @return string Trimmed text
|
||||
*/
|
||||
protected static function trimToCompleteSentences(string $text): string
|
||||
{
|
||||
$text = self::trimToFirstSentence($text);
|
||||
$text = self::trimToLastSentence($text);
|
||||
return $text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count approximate tokens in text.
|
||||
*
|
||||
* @param string $text Text to analyze
|
||||
* @return int Approximate token count
|
||||
*/
|
||||
public static function countTokens(string $text): int
|
||||
{
|
||||
// Rough approximation: 1 token ≈ 4 characters for English text
|
||||
return intval(strlen($text) / 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get text statistics.
|
||||
*
|
||||
* @param string $text Text to analyze
|
||||
* @return array Statistics array
|
||||
*/
|
||||
public static function getStatistics(string $text): array
|
||||
{
|
||||
return [
|
||||
'character_count' => strlen($text),
|
||||
'word_count' => str_word_count($text),
|
||||
'line_count' => substr_count($text, "\n") + 1,
|
||||
'sentence_count' => self::countSentences($text),
|
||||
'approximate_tokens' => self::countTokens($text),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Count sentences in text.
|
||||
*
|
||||
* @param string $text Text to analyze
|
||||
* @return int Number of sentences
|
||||
*/
|
||||
protected static function countSentences(string $text): int
|
||||
{
|
||||
$count = preg_match_all('/[.!?]+/', $text);
|
||||
return $count ?: 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split text into chunks of approximately equal size.
|
||||
*
|
||||
* @param string $text Text to split
|
||||
* @param int $chunkSize Approximate size of each chunk in characters
|
||||
* @return array Array of text chunks
|
||||
*/
|
||||
public static function splitIntoChunks(string $text, int $chunkSize = 10000): array
|
||||
{
|
||||
$textLength = strlen($text);
|
||||
$numChunks = ceil($textLength / $chunkSize);
|
||||
$chunks = [];
|
||||
|
||||
for ($i = 0; $i < $numChunks; $i++) {
|
||||
$start = $i * $chunkSize;
|
||||
$chunk = substr($text, $start, $chunkSize);
|
||||
|
||||
// Trim to complete sentences except for the last chunk
|
||||
if ($i < $numChunks - 1) {
|
||||
$chunk = self::trimToLastSentence($chunk);
|
||||
}
|
||||
|
||||
$chunks[] = $chunk;
|
||||
}
|
||||
|
||||
Log::info('TextProcessor: Text split into chunks', [
|
||||
'original_length' => $textLength,
|
||||
'chunk_size' => $chunkSize,
|
||||
'num_chunks' => count($chunks),
|
||||
]);
|
||||
|
||||
return array_filter($chunks); // Remove empty chunks
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user