<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library\processors;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\processors\JpgProcessor;
use seekquarry\yioop\library\UrlParser;
use seekquarry\yioop\library\summarizers\CentroidSummarizer;
use seekquarry\yioop\library\summarizers\GraphBasedSummarizer;
use seekquarry\yioop\library\summarizers\ScrapeSummarizer;
use seekquarry\yioop\library\summarizers\CentroidWeightedSummarizer;
/**
* To try to guess locale's from string samples
*/
require_once __DIR__."/../LocaleFunctions.php";
/**
* Parent class common to all processors used to create crawl summary
* information that involves basically text data
*
* @author Chris Pollett
*/
class TextProcessor extends PageProcessor
{
/**
* Set-ups the any indexing plugins associated with this page
* processor
*
* @param array $plugins an array of indexing plugins which might
* do further processing on the data handles by this page
* processor
* @param int $max_description_len maximal length of a page summary
* @param int $max_links_to_extract maximum number of links to extract
* from a single document
* @param string $summarizer_option CRAWL_CONSTANT specifying what kind
* of summarizer to use self::BASIC_SUMMARIZER,
* self::GRAPH_BASED_SUMMARIZER and self::CENTROID_SUMMARIZER
* self::CENTROID_WEIGHTED_SUMMARIZER
*/
public function __construct($plugins = [], $max_description_len = null,
$max_links_to_extract = null,
$summarizer_option = self::BASIC_SUMMARIZER)
{
parent::__construct($plugins, $max_description_len,
$max_links_to_extract, $summarizer_option);
/** Register File Types We Handle*/
$add_extensions = ["csv", "tab", "tsv", "txt"];
self::$indexed_file_types = array_merge(self::$indexed_file_types,
$add_extensions);
self::$mime_processor["text/plain"] = "TextProcessor";
self::$mime_processor["text/csv"] = "TextProcessor";
self::$mime_processor["text/x-java-source"] = "TextProcessor";
self::$mime_processor["text/tab-separated-values"] = "TextProcessor";
}
/**
* Computes a summary based on a text string of a document
*
* @param string $page text string of a document
* @param string $url location the document came from, not used by
* TextProcessor at this point. Some of its subclasses override
* this method and use url to produce complete links for
* relative links within a document
*
* @return array a summary of (title, description,links, and content) of
* the information in $page
*/
public function process($page, $url)
{
$summary = null;
if (is_string($page)) {
$remove_styles_page = preg_replace(
'@<style[^>]*?>.*?</style>@si', ' ', $page);
$dom = self::dom($remove_styles_page);
$summary[self::TITLE] = "";
$summary[self::LANG] = self::calculateLang($remove_styles_page);
$summary[self::PAGE] = "<html><body><div><pre>" .
strip_tags($remove_styles_page) . "</pre></div></body></html>";
list($summary[self::DESCRIPTION], $summary[self::WORD_CLOUD],
$summary[self::DESCRIPTION_SCORES]) =
$this->summarizer->getSummary($dom, $summary[self::PAGE],
$summary[self::LANG]);
$summary[self::LINKS] = self::extractHttpHttpsUrls(
$remove_styles_page);
}
return $summary;
}
/**
* Tries to determine the language of the document by looking at the
* $sample_text and $url provided
* the language
* @param string $sample_text sample text to try guess the language from
* @param string $url url of web-page as a fallback look at the country
* to figure out language
*
* @return string language tag for guessed language
*/
public static function calculateLang($sample_text = null, $url = null)
{
if ($url != null) {
$lang = UrlParser::getLang($url);
if ($lang && !in_array($lang, ["en", "en-US"])) {
return $lang;
}
}
if ($sample_text != null) {
$lang = L\guessLocaleFromString($sample_text);
} else {
$lang = null;
}
return $lang;
}
/**
* Gets the text between two tags in a document starting at the current
* position.
*
* @param string $string document to extract text from
* @param int $cur_pos current location to look if can extract text
* @param string $start_tag starting tag that we want to extract after
* @param string $end_tag ending tag that we want to extract until
* @return array pair consisting of when in the document we are after
* the end tag, together with the data between the two tags
*/
public static function getBetweenTags($string, $cur_pos, $start_tag,
$end_tag)
{
$len = strlen($string);
if (($between_start = strpos($string, $start_tag, $cur_pos)) ===
false ) {
return [$len, ""];
}
$between_start += strlen($start_tag);
if (($between_end = strpos($string, $end_tag, $between_start)) ===
false ) {
$between_end = $len;
}
$cur_pos = $between_end + strlen($end_tag);
$between_string = substr($string, $between_start,
$between_end - $between_start);
return [$cur_pos, $between_string];
}
/**
* Tries to extract http or https links from a string of text.
* Does this by a very approximate regular expression.
*
* @param string $page text string of a document
* @return array a set of http or https links that were extracted from
* the document
*/
public static function extractHttpHttpsUrls($page)
{
$pattern =
'@((http|https)://([^ \t\r\n\v\f\'\"\;\,<>\{\}])*)@i';
$page ??= "";
$sites = [];
preg_match_all($pattern, $page, $matches);
$i = 0;
foreach ($matches[0] as $url) {
if (!isset($sites[$url]) && strlen($url) < C\MAX_URL_LEN &&
strlen($url) > 4) {
$sites[$url] = UrlParser::extractTextFromUrl($url);
$i++;
if (self::$max_links_to_extract > 0 &&
$i >= self::$max_links_to_extract) {
break;
}
}
}
return $sites;
}
/**
* If an end of file is reached before closed tags are seen, this methods
* closes these tags in the correct order.
*
* @param string &$page a reference to an xml or html document
*/
public static function closeDanglingTags(&$page)
{
$l_pos = strrpos($page, "<");
$g_pos = strrpos($page, ">");
if ($g_pos && $l_pos > $g_pos) {
$page = substr($page, 0, $l_pos);
}
// put all opened tags into an array
preg_match_all("#<([a-z]+)( .*)?(?!/)>#iU", $page, $result);
$openedtags = $result[1];
// put all closed tags into an array
preg_match_all("#</([a-z]+)>#iU", $page, $result);
$closedtags=$result[1];
$len_opened = count($openedtags);
// all tags are closed
if (count($closedtags) == $len_opened){
return;
}
$openedtags = array_reverse($openedtags);
// close tags
for ($i=0;$i < $len_opened;$i++) {
if (!in_array($openedtags[$i],$closedtags)){
$page .= '</'.$openedtags[$i].'>';
} else {
unset($closedtags[array_search($openedtags[$i],$closedtags)]);
}
}
}
/**
* Return a document object based on a string containing the contents of
* a web page
*
* @param string $page a web page
*
* @return object document object
*/
public static function dom($page)
{
/*
first do a crude check to see if we have at least an <html> tag
otherwise try to make a simplified html document from what we got
*/
if (stristr($page, "<html") === false) {
$head_tags = "<title><meta><base>";
$head = strip_tags($page, $head_tags);
$body_tags = "<frameset><frame><noscript><img><span><b><i><em>".
"<strong><h1><h2><h3><h4><h5><h6><p><div>".
"<a><table><tr><td><th><dt><dir><dl><dd>";
$body = "\n\n" . strip_tags($page, $body_tags). "\n\n";
$body = preg_replace("/\n\n(.+)\n\n/s", '<div>\n$1\n</div>\n',
$body);
$page = "<html><head>$head</head><body>$body</body></html>";
}
$dom = L\getDomFromString($page);
return $dom;
}
/**
* Used to create a thumbnail file in a thumb folder from a book, an
* html page, or a text file. A book's cover is taken out of the book
* itself and scaled; anything else has its opening words drawn.
* Neither needs any program outside Yioop.
*
* @param string $folder with file in it
* @param string $thumb_folder folder to generate
* @param string $file_name of file file in $folder
* @param int $width = width in pixels of thumb
* @param int $height = height in pixels of thumb
*/
public static function createThumb($folder, $thumb_folder, $file_name,
$width = C\THUMB_DIM, $height = C\THUMB_DIM)
{
if (!function_exists("imagecreatetruecolor")) {
return;
}
$thumb_path = "$thumb_folder/$file_name.jpg";
if (file_exists($thumb_path)) {
@unlink($thumb_path);
}
$path = "$folder/$file_name";
$words = @file_get_contents($path, false, null, 0,
C\MAX_DESCRIPTION_LEN);
if ($words === false) {
return;
}
self::drawText(strip_tags($words), $thumb_path, $width,
$height, "jpeg");
clearstatcache(true, $thumb_path);
}
/**
* How many lines of a document's words are drawn on a picture made
* from its text.
*/
const DRAWN_LINES = 14;
/**
* How many lines of a page's words are laid over its picture.
*/
const LEAST_DRAWN = 4;
/**
* How dark a place in a drawn word must be to count as part of a
* letter rather than the space around it.
*/
const LETTER_EDGE = 140;
/**
* How many numbers a shape must have to be worth filling, being three
* corners of two numbers each.
*/
const LEAST_CORNERS = 6;
/**
* Scales a picture already in hand down to the wanted size, keeping
* its shape, and writes it where asked.
*
* @param string $bytes the picture as it was found
* @param string $path where to write the smaller one
* @param int $width the most it may be across
* @param int $height the most it may be down
* @param string $format either webp or jpeg
* @param array $over runs of words to lay over the picture, for a
* page whose words are drawn on top of its picture
* @param array $page how large the page the words came from is
* @return bool whether a picture was written
*/
public static function scaleTo($bytes, $path, $width, $height,
$format = "webp", $over = [], $page = null)
{
/* Asked outright whether these bytes are a picture, since being
handed something that is not one is ordinary here: a file saved
under the wrong name, or one that arrived damaged. */
if (@getimagesizefromstring($bytes) === false) {
return false;
}
/* Through the jpeg reader rather than PHP's own, since a picture
written in the four colors ink uses comes back black from
PHP's and has to be worked out instead. */
$found = JpgProcessor::toImage($bytes);
if ($found === false) {
return false;
}
$was_wide = imagesx($found);
$was_high = imagesy($found);
if ($was_wide < 1 || $was_high < 1) {
return false;
}
$scale = min($width / $was_wide, $height / $was_high, 1);
$now_wide = max(1, (int)round($was_wide * $scale));
$now_high = max(1, (int)round($was_high * $scale));
$smaller = imagecreatetruecolor($now_wide, $now_high);
imagefill($smaller, 0, 0,
imagecolorallocate($smaller, 255, 255, 255));
imagecopyresampled($smaller, $found, 0, 0, 0, 0, $now_wide,
$now_high, $was_wide, $was_high);
if (!empty($over)) {
self::layWordsOver($smaller, $over, $page);
}
$written = ($format == "jpeg") ? @imagejpeg($smaller, $path, 85) :
@imagewebp($smaller, $path);
return $written;
}
/**
* Lays a few words over a picture, on a band pale enough to read them
* against. A magazine cover carries its title and headlines drawn on
* top of its picture rather than beside it, and a thumbnail showing
* only the picture says less than the cover does.
*
* @param object $picture the picture to lay words over
* @param array $runs each run of words with its place and size
* @param array $page how large the page they came from is
*/
public static function layWordsOver($picture, $runs, $page = null)
{
if ($page === null || empty($runs)) {
return;
}
$wide = imagesx($picture);
$high = imagesy($picture);
$shrink = $wide / max(1, $page["wide"]);
self::fillShapesOver($picture, $page, $shrink);
foreach ($runs as $run) {
$tall = (int)round($run["size"] * $shrink);
if ($tall < self::LEAST_DRAWN || trim($run["words"]) === "") {
continue;
}
/* A page may carry more than it shows, and what it shows
begins at a corner of its own, so places are counted from
there. A page also counts up from its foot where a picture
counts down from its head. */
$from_across = $page["from_across"] ?? 0;
$from_down = $page["from_down"] ?? 0;
$across = (int)round(($run["across"] - $from_across) *
$shrink);
$down = (int)round(($page["high"] -
($run["down"] - $from_down)) * $shrink) - $tall;
if ($down > $high || $across > $wide) {
continue;
}
$said = $run["ink"] ?? [0, 0, 0];
self::drawWordsAt($picture, $run["words"], $across, $down,
$tall, $said);
}
}
/**
* Fills the shapes a page draws onto a picture. A nameplate set as an
* outline rather than as type is drawn this way, so a thumbnail shows
* it as the page has it.
*
* @param object $picture the picture to draw on
* @param array $page the page, with its shapes and its size
* @param float $shrink how much smaller the picture is than the page
*/
public static function fillShapesOver($picture, $page, $shrink)
{
if (empty($page["shapes"])) {
return;
}
$from_across = $page["from_across"] ?? 0;
$from_down = $page["from_down"] ?? 0;
$high = imagesy($picture);
foreach ($page["shapes"] as $shape) {
$corners = [];
foreach ($shape["corners"] as $corner) {
$corners[] = (int)round(($corner[0] - $from_across) *
$shrink);
$corners[] = (int)round(($page["high"] -
($corner[1] - $from_down)) * $shrink);
}
if (count($corners) < self::LEAST_CORNERS) {
continue;
}
$ink = imagecolorallocate($picture, $shape["ink"][0],
$shape["ink"][1], $shape["ink"][2]);
imagefilledpolygon($picture, $corners, $ink);
}
}
/**
* Draws one run of words at a place on a picture, at a given height.
* The letters Yioop can draw come in a few fixed sizes, so the words
* are drawn at the largest of those and the drawing is then stretched
* to the height the page set them at.
*
* @param object $picture the picture to draw on
* @param string $words the words to draw
* @param int $across where they begin across the picture
* @param int $down where their top sits down the picture
* @param int $tall how tall they should be
* @param array $said the color the page set them in
*/
public static function drawWordsAt($picture, $words, $across, $down,
$tall, $said)
{
$font = 5;
$letter = imagefontwidth($font);
$flat_wide = max(1, $letter * strlen($words));
$flat_high = imagefontheight($font);
$flat = imagecreatetruecolor($flat_wide, $flat_high);
/* The letters are drawn in black on white here and the white is
then dropped, so what lands on the picture is the letters alone
in the color the page set them, with nothing behind them. */
$behind = imagecolorallocate($flat, 255, 255, 255);
imagefill($flat, 0, 0, $behind);
imagestring($flat, $font, 0, 0, $words,
imagecolorallocate($flat, 0, 0, 0));
$stretched = max(1, (int)round($flat_wide * $tall / $flat_high));
$shaped = imagecreatetruecolor($stretched, $tall);
imagefill($shaped, 0, 0, imagecolorallocate($shaped, 255, 255,
255));
imagecopyresampled($shaped, $flat, 0, 0, 0, 0, $stretched, $tall,
$flat_wide, $flat_high);
$ink = imagecolorallocate($picture, $said[0], $said[1], $said[2]);
$wide = imagesx($picture);
$high = imagesy($picture);
for ($step = 0; $step < $stretched; $step++) {
for ($line = 0; $line < $tall; $line++) {
$shade = imagecolorat($shaped, $step, $line);
if ((($shade >> 16) & 255) > self::LETTER_EDGE) {
continue;
}
$at_across = $across + $step;
$at_down = $down + $line;
if ($at_across < 0 || $at_across >= $wide ||
$at_down < 0 || $at_down >= $high) {
continue;
}
imagesetpixel($picture, $at_across, $at_down, $ink);
}
}
}
/**
* Draws a picture of the opening words of a document, laid out on a
* white page the shape of the thumbnail, so a file with no picture in
* it still shows something that says what it is.
*
* @param string $text the document's words
* @param string $path where to write the picture
* @param int $width how wide the picture should be
* @param int $height how tall the picture should be
* @param string $format either webp or jpeg
* @return bool whether a picture was written
*/
public static function drawText($text, $path, $width, $height,
$format = "webp")
{
$page = imagecreatetruecolor($width, $height);
$paper = imagecolorallocate($page, 255, 255, 255);
$ink = imagecolorallocate($page, 40, 40, 40);
$edge = imagecolorallocate($page, 190, 190, 190);
imagefill($page, 0, 0, $paper);
imagerectangle($page, 0, 0, $width - 1, $height - 1, $edge);
$font = 2;
$letter = imagefontwidth($font);
$line_height = imagefontheight($font) + 2;
$margin = max(4, (int)round($width / 16));
$across = max(1, (int)(($width - 2 * $margin) / $letter));
$words = preg_replace('/\s+/u', " ", trim($text));
$lines = ($words === "") ? [] :
explode("\n", wordwrap($words, $across, "\n", true));
$down = $margin;
$drawn = 0;
foreach ($lines as $line) {
if ($drawn >= self::DRAWN_LINES ||
$down + $line_height > $height - $margin) {
break;
}
imagestring($page, $font, $margin, $down, $line, $ink);
$down += $line_height;
$drawn++;
}
$written = ($format == "jpeg") ? @imagejpeg($page, $path, 85) :
@imagewebp($page, $path);
return $written;
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software