<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Ravi Dhillon ravi.dhillon@yahoo.com, Chris Pollett
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library;
use seekquarry\yioop\configs as C;
/** For Yioop global defines */
require_once __DIR__."/../configs/Config.php";
/**
* Library of functions used to create and extract n word grams
*
* @author Ravi Dhillon (Bigram Version), Chris Pollett (ngrams + rewrite +
* support for page count dumps)
*/
class NWordGrams
{
/**
* Static copy of n-grams files
* @var object
*/
protected static $ngrams = null;
/**
* How many bytes to read in one go from wiki file when creating filter
*/
const BLOCK_SIZE = 8192;
/**
* Suffix appended to language tag to create the
* filter file name containing bigrams.
*/
const FILTER_SUFFIX = "_word_grams.ftr";
/**
* Suffix appended to language tag to create the
* text file name containing bigrams.
*/
const TEXT_SUFFIX = "_word_grams.txt";
/**
* Auxiliary suffice file ngrams to add to filter
*/
const AUX_SUFFIX = "_aux_grams.txt";
const WIKI_DUMP_REDIRECT = 0;
const WIKI_DUMP_TITLE = 1;
const PAGE_COUNT_WIKIPEDIA = 2;
const PAGE_COUNT_WIKTIONARY = 3;
/**
* Says whether or not phrase exists in the N word gram Bloom Filter
*
* @param $phrase what to check if is a bigram
* @param string $lang language of bigrams file
* @param string $filter_prefix either the word "segment", "all", or
* number n of the number of words in an ngram in filter.
* @return true or false
*/
public static function ngramsContains($phrase, $lang, $filter_prefix = 2)
{
$lang = str_replace("-", "_", $lang);
if ($lang == 'en' || $lang == 'en_GB') {
$lang = 'en_US';
}
if (empty(self::$ngrams)) {
self::$ngrams = [];
}
if (empty(self::$ngrams[$lang])) {
self::$ngrams[$lang] = [];
}
if (empty(self::$ngrams[$lang][$filter_prefix])) {
$filter_path = C\LOCALE_DIR . "/$lang/resources/" .
"{$filter_prefix}" . self::FILTER_SUFFIX;
if (file_exists($filter_path)) {
self::$ngrams[$lang][$filter_prefix] =
BloomFilterFile::load($filter_path);
} else {
return false;
}
}
return self::$ngrams[$lang][$filter_prefix]->contains(
mb_strtolower($phrase));
}
/**
* Creates a bloom filter file from a n word gram text file. The
* path of n word gram text file used is based on the input $lang.
* The name of output filter file is based on the $lang and the
* number n. Size is based on input number of n word grams .
* The n word grams are read from text file, stemmed if a stemmer
* is available for $lang and then stored in filter file.
*
* @param string $lang locale to be used to stem n grams.
* @param string $num_gram value of n in n-gram (how many words in sequence
* should constitute a gram)
* @param int $num_ngrams_found count of n word grams in text file.
* @param int $max_gram_len value n of longest n gram to be added.
* @return none
*/
public static function makeNWordGramsFilterFile($lang, $num_gram,
$num_ngrams_found, $max_gram_len = 2)
{
$lang = str_replace("-", "_", $lang);
$filter_path = C\LOCALE_DIR . "/$lang/resources/" .
"{$num_gram}" . self::FILTER_SUFFIX;
if (file_exists($filter_path)) {
unlink($filter_path); //build again from scratch
}
$ngrams = new BloomFilterFile($filter_path, $num_ngrams_found);
$input_file_path = C\LOCALE_DIR . "/$lang/resources/" .
"{$num_gram}" . self::TEXT_SUFFIX;
$fp = fopen($input_file_path, 'r') or
die("Can't open ngrams text file");
while ( ($ngram = fgets($fp)) !== false) {
$ngram = trim(mb_strtolower($ngram));
$ngrams->add($ngram);
}
fclose($fp);
$input_file_path = C\LOCALE_DIR . "/$lang/resources/" .
"{$num_gram}" . self::AUX_SUFFIX;
if (file_exists($input_file_path)) {
$fp = fopen($input_file_path, 'r') or
die("Can't open ngrams text file");
while ( ($ngram = fgets($fp)) !== false) {
$ngram = trim(mb_strtolower($ngram));
$ngrams->add($ngram);
}
fclose($fp);
}
$ngrams->max_gram_len = $max_gram_len;
$ngrams->save();
}
/**
* Used to create a filter file suitable for use in word segmentation
* (splitting text like "thiscontainsnospaces" into
* "this contains no spaces"). Used by @see token_tool.php
*
* @param string $dict_file file to use as a dictionary to make filter from
* @param string $lang locale tag of locale we are building the filter for
*/
public static function makeSegmentFilterFile($dict_file, $lang)
{
$lang = str_replace("-", "_", $lang);
$filter_path = C\LOCALE_DIR . "/$lang/resources/" .
"segment" . self::FILTER_SUFFIX;
if (file_exists($filter_path)) {
unlink($filter_path); //build again from scratch
}
$words = file($dict_file);
$filter = new BloomFilterFile($filter_path, count($words));
foreach ($words as $word) {
$tmp = trim($word);
$len = mb_strlen($tmp);
$filter->add(mb_strtolower($tmp));
for ($i = 1; $i < $len; $i++) {
$tmp2 = "*" . mb_substr($tmp, $i, $len, "UTF-8");
if ($tmp2 == "*") {
continue;
}
$filter->add(mb_strtolower($tmp2));
}
}
$filter->save();
}
/**
* Generates a n word grams text file from input wikipedia xml file.
* The input file can be a bz2 compressed or uncompressed.
* The input XML file is parsed line by line and pattern for
* n word gram is searched. If a n word gram is found it is added to the
* array. After the complete file is parsed we remove the duplicate
* n word grams and sort them. The resulting array is written to the
* text file. The function returns the number of bigrams stored in
* the text file.
*
* @param string $wiki_file compressed or uncompressed wikipedia
* XML file path to be used to extract bigrams. This can also
* be a folder containing such files
* @param string $lang Language to be used to create n grams.
* @param string $locale Locale to be used to store results.
* @param int $num_gram number of words in grams we are looking for
* @param int $ngram_type where in Wiki Dump to extract grams from
* @param int $max_terms maximum number of n-grams to compute and put in
* file
* @return int $num_ngrams_found count of n-grams in text file.
*/
public static function makeNWordGramsTextFile($wiki_file, $lang,
$locale, $num_gram = 2, $ngram_type = self::PAGE_COUNT_WIKIPEDIA,
$max_terms = -1)
{
$output_message_threshold = self::BLOCK_SIZE * self::BLOCK_SIZE;
[$pattern, $pattern_end, $replace_array, $is_count_type] =
match ($ngram_type) {
self::WIKI_DUMP_TITLE => [
'/<title>[^\p{P}]+', '<\/title>/u',
['<title>', '</title>'], false],
self::WIKI_DUMP_REDIRECT => [
'/#redirect\s\[\[[^\p{P}]+', '\]\]/u',
['#redirect [[', ']]'], false],
self::PAGE_COUNT_WIKIPEDIA => [
'/^' . $lang . "\.wikipedia",
'\s\d+\s[a-z]+\s(\d+)/u', [], true],
self::PAGE_COUNT_WIKTIONARY => [
'/^' . $lang . "\.wiktionary",
'\s\d+\s[a-z]+\s(\d+)/u', [], true],
};
$is_all = false;
$repeat_pattern = "[\s|_][^\s\,\$\+\d\=\:\/\_\!\"\%]+";
if ($num_gram == "all" || $is_count_type) {
$pattern .= "((?:$repeat_pattern)+)";
if ($num_gram == "all") {
$is_all = true;
}
$max_gram_len = -1;
} else {
$pattern .= "(";
for ($i = 1; $i < $num_gram; $i++) {
$pattern .= $repeat_pattern;
}
$pattern .= ")";
$max_gram_len = $num_gram;
}
$pattern .= $pattern_end;
$replace_types = [self::WIKI_DUMP_TITLE, self::WIKI_DUMP_REDIRECT];
if (is_dir(C\PREP_DIR . "/$wiki_file") ) {
$folder_files = glob(C\PREP_DIR . "/$wiki_file/*.{gz,bz2}",
GLOB_BRACE);
} else {
$folder_files = [C\PREP_DIR . "/$wiki_file"];
}
$ngrams_file_path
= C\LOCALE_DIR . "/$locale/resources/" . "{$num_gram}" .
self::TEXT_SUFFIX;
$ngrams = [];
foreach ($folder_files as $wiki_file_path) {
if (str_contains($wiki_file_path, "bz2")) {
$fr = bzopen($wiki_file_path, 'r') or
die ("Can't open compressed file");
$read = "bzread";
$close = "bzclose";
} else if (str_contains($wiki_file_path, "gz")) {
$fr = gzopen($wiki_file_path, 'r') or
die ("Can't open compressed file");
$read = "gzread";
$close = "gzclose";
} else {
$fr = fopen($wiki_file_path, 'r') or die("Can't open file");
$read = "fread";
$close = "fclose";
}
$input_buffer = "";
$time = time();
echo "Reading wiki file ...$wiki_file_path...\n";
$bytes = 0;
$bytes_since_last_output = 0;
while (!feof($fr)) {
$input_text = $read($fr, self::BLOCK_SIZE);
$len = strlen($input_text);
if ($len == 0) {
break;
}
$bytes += $len;
$bytes_since_last_output += $len;
if ($bytes_since_last_output > $output_message_threshold) {
echo "Have now read " . $bytes . " many bytes." .
" Peak memory so far: ".memory_get_peak_usage().
".\n Number of word grams so far: ".count($ngrams).
". Elapsed time so far: ".(time() - $time)."s\n";
$bytes_since_last_output = 0;
}
$input_buffer .= mb_strtolower($input_text);
$lines = explode("\n", $input_buffer);
$input_buffer = array_pop($lines);
foreach ($lines as $line) {
preg_match($pattern, $line, $matches);
if (count($matches) > 0) {
if ($is_count_type) {
if (!empty($matches[1]) &&
!empty($matches[2])) {
list(, $ngram, $ngram_count) = $matches;
$ngram = trim(preg_replace('/_/u', " ",
$ngram));
$ngram = preg_split("/\s\(/", $ngram)[0];
PhraseParser::canonicalizePunctuatedTerms(
$ngram, $locale);
$ngram = preg_replace('/\./u', "", $ngram);
if ($char_grams =
PhraseParser::getCharGramsTerm(
[$ngram], $locale)) {
$ngram = implode(" ", $char_grams);
}
$ngram_num_words =
mb_substr_count($ngram, " ") + 1;
if ($lang == 'en' && preg_match(
'/^(a\s|the\s|of\s|if\s)/', $ngram)) {
$ngram_num_words--;
}
if (($is_all && $ngram_num_words > 1) ||
(!$is_all &&
$ngram_num_words == $num_gram)) {
$ngrams[$ngram] = intval($ngram_count);
}
}
} else {
$ngram = mb_ereg_replace(
$replace_array, "", $matches[0]);
$ngram = trim(preg_replace('/_/u', " ",
$ngram));
PhraseParser::canonicalizePunctuatedTerms(
$ngram, $locale);
$ngram = preg_replace('/\./u', "", $ngram);
$ngrams[] = $ngram;
}
if ($is_all && isset($ngram)) {
$ngram_num_words = mb_substr_count($ngram, " ") + 1;
$max_gram_len =
max($max_gram_len, $ngram_num_words);
}
}
if ($is_count_type && count($ngrams) > 4 * $max_terms
&& $max_terms > 0) {
echo "..pruning results to $max_terms terms.\n";
arsort($ngrams);
$ngrams = array_slice($ngrams, 0, $max_terms);
}
}
}
$close($fr);
}
if ($is_count_type) {
arsort($ngrams);
$ngrams = array_keys($ngrams);
}
$ngrams = array_unique($ngrams);
$num_ngrams_found = count($ngrams);
if ($max_terms > 0 && $num_ngrams_found > $max_terms) {
$ngrams = array_slice($ngrams, 0, $max_terms);
}
$num_ngrams_found = count($ngrams);
// in is_all case add prefix*'s for (n >= 3)-grams
if ($is_all) {
for ($i = 0; $i < $num_ngrams_found; $i++) {
$ngram_in_word = mb_substr_count($ngrams[$i], " ") + 1;
if ($ngram_in_word >= 3) {
$ngram_parts = explode(" ", $ngrams[$i]);
$ngram = $ngram_parts[0];
for ($j = 1; $j < $ngram_in_word - 1; $j++ ) {
$ngram .= " " . $ngram_parts[$j];
$ngrams[] = $ngram . "*";
}
}
}
$ngrams = array_unique($ngrams);
$num_ngrams_found = count($ngrams);
}
sort($ngrams);
$ngrams_string = implode("\n", $ngrams);
file_put_contents($ngrams_file_path, $ngrams_string);
return [$num_ngrams_found, $max_gram_len];
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software