<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\library;

/**
 * This file contains global functions connected to localization that
 * are used throughout the web site part of Yioop!
 */
use seekquarry\yioop\configs as C;
use seekquarry\yioop\models\LocaleModel;
use seekquarry\yioop\library\RequestScope;
use seekquarry\yioop\library\language_processing\PhraseParser;
/**
 * For Yioop global defines
 */
require_once __DIR__."/../configs/Config.php";
/**
 * Returns an array of locales that have a stop words list and a stop words
 * remover method
 * @return array list of locales that have a stopwords list;
 */
function localesWithStopwordsList()
{
    return ['ar', 'bn', 'de', 'el-GR', 'en-US', 'es', 'fa', 'fr-FR', 'he', 'hi',
        'id', 'it', 'ja', 'kn', 'ko', 'nl', 'pl', 'pt', 'ru', 'te', 'th', 'tl',
        'tr', 'vi-VN', 'zh-CN'];
}
/**
 * Converts a $locale_tag (major-minor) to an Iso 632-2 language name
 * @param string $locale_tag want to convert
 * @return string corresponding Iso 632-2 language tag
 */
function localeTagToIso639_2Tag($locale_tag)
{
    $lang_map = ["ar" => "ara", "bn" => "ben", "de" => "deu",
        "en" => "eng", "es" => "spa", "fa" => "fas", "fr" => "fra",
        "he" => "heb", "hi" => "hin", "id" => "ind", "it" => "ita",
        "ja" => "jpn+jpn_vert", "kn" => "kan", "ko" => "kor", "nl" => "nld",
        "pl" => "pol", "pt" => "por", "ru" => "rus", "te" => "tel",
        "th" => "tha", "tl"=> "tgl", "tr"=> "tur", "vi" => "vie",
        "zh" => "chi_sim+chi_tra+chi_sim_vert+chi_tra_vert"];
    $lookup_tag = preg_split("/\-|\_/", $locale_tag ?? "")[0];
    return $lang_map[$lookup_tag] ?? C\p('DEFAULT_LOCALE');
}
/**
 * How many characters long a locale tag may be. A tag such as en-US or
 * zh-CN is short, so anything longer did not come from the list of
 * locales and is not used to pick one.
 */
const LOCALE_TAG_LENGTH = 10;
/**
 * The name the locale a request settled on is kept under, among the
 * values that belong to one request rather than to the process.
 */
const LOCALE_KEPT_UNDER = "locale_tag";
/**
 * guessLocale works out which locale a reader wants, from the l in the
 * address, then the session, then the language the browser asks for. A
 * tag is used only where a folder of that name stands among the
 * locales, so a made-up tag falls through to the site's own.
 *
 * @return string the language tag of the locale a reader wants, such as
 *     en-US, or an empty string where none of the three names one
 */
function guessLocale()
{
    /* the request variable l and the browser's HTTP_ACCEPT_LANGUAGE
       are used to determine the locale */
    if (isset($_SERVER['HTTP_ACCEPT_LANGUAGE'])) {
        $l_parts = explode(",", $_SERVER['HTTP_ACCEPT_LANGUAGE']);
        if (count($l_parts) > 0) {
            $guess_l = $l_parts[0];
        }
        $guess_map = [
            "cn" => "zh-CN",
            "en" => "en-US",
            "en-us" => "en-US",
            "en-US" => "en-US",
            "fr" => "fr-FR",
            "ko" => "ko",
            "in" => "in-ID",
            "ja" => "ja",
            "vi" => "vi-VN",
            "vi-vn" => "vi-VN",
            "vi-VN" => "vi-VN",
            "zh" => "zh-CN",
            "zh-CN" => "zh-CN",
            "zh-cn" => "zh-CN",
        ];
        if (isset($guess_map[$guess_l])) {
            $guess_l = $guess_map[$guess_l];
        }
    }
    if (isset($_SESSION['l']) || isset($_REQUEST['l']) || isset($guess_l)) {
        $asked_for = (isset($_REQUEST['l'])) ? $_REQUEST['l'] :
            ((isset($_SESSION['l'])) ? $_SESSION['l'] : $guess_l);
        if (strlen($asked_for) < LOCALE_TAG_LENGTH) {
            $asked_for = addslashes($asked_for);
            if (is_dir(C\LOCALE_DIR . "/" .
                str_replace("-", "_", $asked_for))) {
                $locale_tag = $asked_for;
            }
        }
    }
    if (!isset($locale_tag)) {
        $locale_tag = C\p('DEFAULT_LOCALE');
    }
    return $locale_tag;
}
/**
 * Attempts to guess the user's locale based on a string sample
 *
 * @param string $phrase_string used to make guess
 * @param string $locale_tag language tag to use if can't guess -- if not
 *     provided uses current locale's value
 * @param int threshold number of chars to guess a particular encoding
 * @return string IANA language tag of the guessed locale
 */
function guessLocaleFromString($phrase_string, $locale_tag = null)
{
    static $cache;
    $phrase_string = trim($phrase_string);
    if (!empty($cache[$phrase_string])) {
        return $cache[trim($phrase_string)];
    }
    if (!$locale_tag) {
        $locale_tag = getLocaleTag();
    }
    if (preg_match("/\blang\:(\w\w(\-\w\w)?)\b/", $phrase_string, $matches)) {
        return $matches[1];
    }
    $guess_string = preg_replace("/\b[^\s]+\:[^\s]+\b/", " ",
        $phrase_string);
    $guess_string = trim(preg_replace("/[\s\-]+/", " ", $guess_string));
    $len = strlen($guess_string);
    if ($len >= C\NAME_LEN) {
        foreach (localesWithStopwordsList() as $lang) {
            $tokenizer = PhraseParser::getTokenizer($lang);
            if ($tokenizer) {
                /* keep next line as stopWordRemover uses preg_replace
                   while will return null if $guess_string is not UTF-8
                 */
                $guess_string = mb_convert_encoding($guess_string, "UTF-8");
                $compressed_string =
                    $tokenizer->stopwordsRemover($guess_string);
                /* if regex fails might get null. Usually caused by bad encoding
                   of $guess_string
                 */
                if (!is_string($compressed_string)) {
                    break;
                }
                $test_len = strlen($compressed_string);
                if ($test_len < $len) {
                    $len = $test_len;
                    $locale_tag = $lang;
                }
            }
        }
    }
    $cache[$phrase_string] = $locale_tag;
    if (count($cache) > 100) {
        array_shift($cache);
    }
    return $locale_tag;
}
/**
 * Tries to guess at a language tag based on the name of a character
 * encoding
 *
 * @param string $encoding a character encoding name
 *
 * @return string guessed language tag
 */
function guessLangEncoding($encoding)
{
    $lang = ["EUC-JP", "Shift_JIS", "JIS", "ISO-2022-JP"];
    if (in_array($encoding, $lang)) {
        return "ja";
    }
    $lang = ["EUC-CN", "GBK", "GB2312", "EUC-TW", "HZ", "CP936",
        "BIG-5", "CP950"];
    if (in_array($encoding, $lang)) {
        return "zh-CN";
    }
    $lang = ["EUC-KR", "UHC", "CP949", "ISO-2022-KR"];
    if (in_array($encoding, $lang)) {
        return "ko";
    }
    $lang = ["Windows-1251", "CP1251", "CP866", "IBM866", "KOI8-R"];
    if (in_array($encoding, $lang)) {
        return "ru";
    }
    return 'en';
}
/**
 * Tries to guess the encoding used for an Html document
 *
 * @param string $html a character encoding name
 * @param string $return_loc_info if meta http-equiv info was used to
 *     find the encoding, then if $return_loc_info is true, we
 *     return the location of charset substring. This allows converting to
 *     UTF-8 later so cached pages will display correctly and
 *     redirects without char encoding won't be given a different hash.
 *
 * @return mixed either string or array if string then guessed encoding,
 *     if array guessed encoding, start_pos of where charset info came from,
 *     length
 */
function guessEncodingHtmlXml($html, $return_loc_info = false)
{
    /* sometimes the page will say it is other than UTF-8 even if it is
       so do a quick check first
     */
    if (mb_check_encoding($html, 'UTF-8')) {
        return 'UTF-8';
    }
    /* first try for XML encoding info */
    preg_match("/\<\?xml[^\?]+encoding\=[\'\"]\s*(\S+)\s*[\'\"][^\?]+\?\>/",
        $html, $matches, PREG_OFFSET_CAPTURE);
    if (!empty($matches[1][1])) {
        $encoding = strtoupper($matches[1][0]);
        $start_charset = $matches[1][1];
        $len_c = strlen($encoding);
        if ($return_loc_info) {
            return [$encoding, $start_charset, $len_c];
        }
        return $encoding;
    }
     /*
       If the doc is HTML and it uses a http-equiv to set the encoding
       then we override what the server says (if anything). As we
       are going to convert to UTF-8 we remove the charset info
       from the meta tag so cached pages will display correctly and
       redirects without char encoding won't be given a different hash.
     */
    $end_head = stripos($html, "</head");
    if ($end_head !== false) {
        $reg = "/charset(\s*)\=(\s*)(\'|\")?((\w|\-)+)(\'|\")?/i";
        $is_match = preg_match($reg, $html, $match);
        if (!$is_match) {
            $reg = "charset(\s*)\=(\s*)(\'|\")?((\w|\-)+)(\'|\")?";
            mb_regex_encoding("UTF-8");
            mb_ereg_search_init($html);
            mb_ereg_search($reg, "i");
            $match = mb_ereg_search_getregs();
            if (isset($match[0])) {
                $is_match = true;
            }
        }
        if ($is_match && isset($match[6])) {
            $len_c = strlen($match[0]);
            if (($match[6] == "'" || $match[6] == '"') &&
               $match[3] != $match[6]) {
                $len_c--;
            }
            $start_charset = strpos($html, $match[0]);
            if ($start_charset + $len_c < $end_head) {
                if (isset($match[4])) {
                    $encoding = strtoupper($match[4]);
                    if ($return_loc_info) {
                        return [$encoding, $start_charset, $len_c];
                    }
                    return $encoding;
                }
            }
        }
    }
    return mb_detect_encoding($html, 'auto');
}
/**
 * Converts page data in a site associative array to UTF-8 if it is not
 * already in UTF-8
 *
 * @param array &$site an associative of info about a web site
 * @param string $page_field the field in the associative array that
 *  contains the $site's web page as a string.
 * @param string $encoding_field the  field in the associative array that
 *  contains the character encoding the page is currently in
 * @param function $log_function a callback function used to write log
 *  messages with, if desired.
 */
function convertUtf8IfNeeded(&$site, $page_field, $encoding_field,
    $log_function = "")
{
    if ($log_function == "") {
        $log_function = function($msg) {
        };
    }
    if (empty($site[$encoding_field])) {
        $site[$encoding_field] = guessEncodingHtmlXml($site[$page_field]);
    }
    if (!empty($site[$encoding_field]) && stristr($site[$encoding_field],
        "UTF-8") === false) {
        set_error_handler(null);
        $known_encoding = in_array($site[$encoding_field], mb_list_encodings());
        if (!$known_encoding || ($known_encoding &&
            !@mb_check_encoding($site[$page_field], $site[$encoding_field]))) {
            $log_function("  MB_CHECK_ENCODING FAILED!!");
        } else {
            $log_function("  Converting from encoding ".
                $site[$encoding_field]."...");
            /* if HEBREW WINDOWS-1255 use ISO-8859 instead */
            if (stristr($site[$encoding_field], "1255")) {
                $site[$encoding_field]= "ISO-8859-8";
                $log_function("  using encoding " .
                    $site[$encoding_field]."...");
            }
            if (stristr($site[$encoding_field], "1256")) {
                $site[$page_field] = w1256ToUTF8($site[$page_field]);
                $log_function("  using Yioop hack encoding ...");
            } else {
                $site[$page_field] = @mb_convert_encoding($site[$page_field],
                    "UTF-8", $site[$encoding_field]);
            }
        }
        restore_error_handler();
    } else if (!empty($site[$encoding_field]) &&
        $site[$encoding_field] == "UTF-8") {
        $log_function("   UTF-8 data detected!");
    }
}
/**
 * Translate the supplied arguments into the current locale.
 * This function takes a variable number of arguments. The first
 * being an identifier to translate. Additional arguments
 * are used to interpolate values in for %s's in the translation.
 *
 * @param string string_identifier  identifier to be translated
 * @param mixed additional_args  used for interpolation in translated string
 * @return string  translated string
 */
function tl()
{
    $locale = LocaleModel::$current_locale;
    if (!is_object($locale)) {
        return false;
    }
    $args = func_get_args();
    $translation = $locale->translate($args);
    if (!trim($translation)) {
        $translation = $args[0];
    }
    return $translation;
}
/**
 * Sets the language to be used for locale settings
 *
 * @param string $locale_tag the tag of the language to use to determine
 *     locale settings
 */
function setLocaleObject($locale_tag)
{
    $locale_model = C\NS_MODELS . "LocaleModel";
    $locale = new $locale_model();
    $locale->initialize($locale_tag);
    LocaleModel::$current_locale = $locale;
    RequestScope::set(LOCALE_KEPT_UNDER, $locale_tag);
}
/**
 * Gets the language tag (for instance, en_US for American English) of the
 * locale that is currently being used. This function has the side
 * effect of setting Yioop's current locale.
 *
 * @return string  the tag of the language currently being used for locale
 *     settings
 */
function getLocaleTag()
{
    $kept = RequestScope::get(LOCALE_KEPT_UNDER);
    if ($kept !== null) {
        return $kept;
    }
    $locale_tag = guessLocale();
    if (empty($locale_tag)) {
        $locale_tag = C\p('DEFAULT_LOCALE');
    }
    setLocaleObject($locale_tag);
    return $locale_tag;
}
/**
 * Returns the current language directions.
 *
 * @return string ltr or rtl depending on if the language is left-to-right
 * or right-to-left
 */
function getLocaleDirection()
{
    $locale = LocaleModel::$current_locale;
    return $locale->getLocaleDirection();
}
/**
 * Returns the query statistics info for the current llocalt.
 *
 * @return array consisting of queries and elapses times for locale computations
 */
function getLocaleQueryStatistics()
{
    $locale = LocaleModel::$current_locale;
    $query_info = [];
    $query_info['QUERY_LOG'] = $locale->db->query_log;
    $query_info['TOTAL_ELAPSED_TIME'] = $locale->db->total_time;
    return $query_info;
}
/**
 * Returns the current locales method of writing blocks (things like divs or
 * paragraphs).A language like English puts blocks one after another from the
 * top of the page to the bottom. Other languages like classical Chinese list
 * them from right to left.
 *
 * @return string  tb lr rl depending on the current locales block progression
 */
function getBlockProgression()
{
    $locale = LocaleModel::$current_locale;
    return $locale->getBlockProgression();

}
/**
 * Returns the writing mode of the current locale. This is a combination of the
 * locale direction and the block progression. For instance, for English the
 * writing mode is lr-tb (left-to-right top-to-bottom).
 *
 * @return string   the locales writing mode
 */
function getWritingMode()
{
    $locale = LocaleModel::$current_locale;
    return $locale->getWritingMode();

}
/**
 * Convert the string $str encoded in Windows-1256 into UTF-8
 *
 * @param string $str Windows-1256 string to convert
 * @return string the UTF-8 equivalent
 */
function w1256ToUTF8($str)
{
    static $conv = [
        0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008,
        0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, 0x0010, 0x0011,
        0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001A,
        0x001B, 0x001C, 0x001D, 0x001E, 0x001F, 0x0020, 0x0021, 0x0022, 0x0023,
        0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002A, 0x002B, 0x002C,
        0x002D, 0x002E, 0x002F, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035,
        0x0036, 0x0037, 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E,
        0x003F, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
        0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050,
        0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059,
        0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, 0x0060, 0x0061, 0x0062,
        0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B,
        0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074,
        0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D,
        0x007E, 0x007F, 0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020,
        0x2021, 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688,
        0x06AF, 0x2018, 0x2020, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x06A9,
        0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA, 0x00A0, 0x060C,
        0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x06BE,
        0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1, 0x00B2, 0x00B3,
        0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC,
        0x00BD, 0x00BE, 0x061F, 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625,
        0x0626, 0x0627, 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E,
        0x062F, 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7,
        0x0637, 0x0638, 0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, 0x00E0,
        0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, 0x00E8, 0x00E9,
        0x00EA, 0x00EB, 0x0649, 0x064A, 0x00EE, 0x00EF, 0x064B, 0x064C, 0x064D,
        0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, 0x0651, 0x00F9, 0x0652, 0x00FB,
        0x00FC, 0x200E, 0x200F, 0x06D2
    ];
    $len = strlen($str);
    $out = "";
    for ($i = 0; $i < $len; $i++) {
        $out .= utf8chr($conv[ord($str[$i])]);
    }
    return $out;
}
/**
 * Given a unicode codepoint convert it to UTF-8
 *
 * @param int $code  the codepoint to convert
 * @return string the corresponding UTF-8 string
 */
function utf8chr($code)
{
    if ($code <= 0x7F)
        return chr($code);
    if ($code <= 0x7FF)
        return pack("C*", ($code >> 6)+192, ($code & 63) + 128);
    if ($code <= 0xFFFF)
            return pack("C*", ($code >> 12)+224, (($code>>6) & 63) + 128,
                ($code&63)+128);
    if ($code <= 0x1FFFFF)
        return pack("C*", ($code >> 18) + 240, (($code >> 12) & 63) + 128,
            (($code >> 6) & 63) + 128, ($code & 63) + 128);
    return '';
}
/**
 * Function for formatting a date string based on the locale.
 * @param $timestamp is the crawl time
 * @param $locale_tag is the tag for locale
 * @return string formatted date string
 */
function formatDateByLocale($timestamp, $locale_tag)
{
    $date_formatter = datefmt_create($locale_tag, \IntlDateFormatter::FULL,
        \IntlDateFormatter::FULL, date_default_timezone_get(),
        \IntlDateFormatter::GREGORIAN, "LLLL dd yyyy H:mm:ss");
    return datefmt_format($date_formatter, $timestamp);
}
