<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */

namespace seekquarry\yioop\library\language_processing;

use seekquarry\yioop\configs as C;
/**
 * Abstract, base context tagger class.
 * A context tagger is used to apply a sequence of labels to a sequence terms
 * or characters of text based on a surrounding context. Context Taggers
 * typically make use of n-gram context of a term such as the n/2 - terms
 * before and after the term and maybe the earlier tags from a same phrase or
 * sentence to make prediction
 *
 * @author Chris Pollett
 */
abstract class ContextTagger
{
    /**
     * Locale tag of language this recognizer is for
     * @var string
     */
    public $lang;
    /**
     * The name of the file where the tagging model should be stored and read
     * from
     * @var string
     */
    public $tagger_file = "tagger.txt.gz";
    /**
     * Complete file system path to the file where the tagging model should be
     * stored and read from
     * @var string
     */
    public $tagger_path = "";
    /**
     * 2D weights for features involving the prior two words to the
     * current word and the next two words after the current word
     * For a given word position, one has vector, that gives te
     * value for each term in the complete training term set, unknown term set,
     * and rule based tag term set, what its weight is
     * Determined during training
     * @var array
     */
    public $word_feature;
    /**
     * The bias vector for features we are training
     *
     * Determined during training
     * @var array
     */
    public $bias;
    /**
     * The weights for features involving the prior two tags to the
     * current word whose tag we are trying to determine
     * Determined during training
     * @var array
     */
    public $tag_feature;
    /**
     * Array of strings for each possible tag for a term
     * associated as [tag => tag index]
     * @var array
     */
    public $tag_set;
    /**
     * Minimum allowed value for a weight component
     * @var float
     */
    public $min_w;
    /**
     * Maximum allowed value for a weight component
     * @var float
     */
    public $max_w;
    /**
     * Tokenizer for the language this tagger tags for
     * @var Tokenizer
     */
    public $tokenizer;
    /**
     * Constructor for the ContextTagger.
     * Sets the language this tagger tags for and sets up the path for
     * where it should be stored
     * @param string $lang locale tag of the language this tagger tags is for
     */
    public function __construct($lang)
    {
        $lang = str_replace("-", "_", $lang);
        $this->lang = $lang;
        $this->tagger_path = C\LOCALE_DIR . "/$lang/resources/" .
            $this->tagger_file;
        $this->tokenizer = PhraseParser::getTokenizer($lang);
    }
    /**
     * Converts training data from the format tagged sentence with terms of the
     * form term_tag into a pair of arrays [[terms_in_sentence],
     *  [tags_in_sentence]]
     * @param mixed $text_files can be a file or an array of file names
     * @param string $term_tag_separator separator used to separate term and tag
     *  for terms in input sentence
     * @param function $term_callback callback function applied to a term
     *  before adding term to sentence term array
     * @param function $tag_callback callback function applied to a part of
     *  speech tag  before adding tag to sentence tag array
     * @param bool $tag_on_array_chars for some kinds of text processing
     *  it better to assume the tags are applied to each char within a term
     *  rather than at the term level. For example, we might want to use
     *  char within a term for name entity tagging. THis flag if true
     *  says to do this; otherwise don't
     * @return array of separated sentences, each sentence having the format of
     *  [[terms...], [tags...]]
     *  Currently, the training data needs to fit Chinese Treebank format:
     *  term followed by a underscore and followed by the tag
     *  e.g. "新_VA 的_DEC 南斯拉夫_NR 会国_NN"
     *  To adapt to other language, some modifications are needed
     */
    public static function processTexts($text_files, $term_tag_separator = "_",
        $term_callback = null, $tag_callback = null,
        $tag_on_array_chars = false)
    {
        $out = [];
        foreach($text_files as $text_file) {
            if (file_exists($text_file)) {
                $fh = fopen($text_file, "r");
                while (!feof($fh))  {
                    $line = fgets($fh);
                    if(str_contains($line, '<')) {
                        continue;
                    }
                    $term_tag_pairs = preg_split("/[\s　]+/u", $line);
                    if (!count($term_tag_pairs)) {
                        continue;
                    }
                    $out[] = [];
                    $last_out = count($out) - 1;
                    $out[$last_out][0] = [];
                    $out[$last_out][1] = [];
                    foreach ($term_tag_pairs as $term_tag_pair) {
                        $t = explode($term_tag_separator, $term_tag_pair);
                        if (count($t) == 2) {
                            $tag = $tag_callback ? $tag_callback($t[1]) : $t[1];
                            if ($tag_on_array_chars) {
                                $to_tags = preg_split('//u', $t[0], null,
                                    PREG_SPLIT_NO_EMPTY);
                            } else {
                                $to_tags = [$t[0]];
                            }
                            foreach($to_tags as $to_tag) {
                                $out[$last_out][0][] = $term_callback ?
                                    $term_callback($to_tag) : $to_tag;
                                $out[$last_out][1][] = $tag;
                            }
                        }
                    }
                }
                fclose($fh);
            }
        }
        return $out;
    }
    /**
     * Maps a term to a corresponding key if the term matches some simple
     * pattern such as being a number
     * @param string $term is the term to be checked
     * @return mixed either the int key for those matrices of just the term
     *  itself if the tokenizer does not ave the method getPosKey for the
     *  current language
     */
    public function getKey($term)
    {
        if (!empty($this->tokenizer) && method_exists($this->tokenizer,
            "getPosKey")) {
            return $this->tokenizer::getPosKey($term);
        }
        return $term;
    }
    /**
     * Given a sentence (array $terms), find the key for the term at position
     * $index
     * @param int $index position of term to get key for
     * @param array $terms an array of terms typically from and in the order of
     *  a sentence
     * @return mixed key position in word_feature weights and bias arrays
     *  could be either an int, or the term itself, or the simple rule
     *  based part of speec it belongs to
     */
    public function getIndex($index, $terms)
    {
        if ($index < 0) {
            $k = $index - 2;
        } else if ($index >= count($terms)) {
            $k = $index - count($terms) - 2;
        } else {
            $k = $this->getKey($terms[$index]);
        }
        return $k;
    }
    /**
     * Save the trained weight to disk
     */
    public function saveWeights()
    {
        $out = [];
        $out["min_w"] = $this->min_w;
        $out["max_w"] = $this->max_w;
        $out["w"] = [];
        foreach(array_keys($this->word_feature) as $key) {
            $out["w"][$key] = $this->packW($key);
        }
        foreach(array_keys($this->tag_feature) as $key) {
            $out["t"][$key] = (pack("f*", ...$this->tag_feature[$key]));
        }
        $out["b"] = (pack("f*", ...$this->bias));
        $out["tag_set"] = $this->tag_set;
        echo "Saving...";
        file_put_contents($this->tagger_path,
            gzencode(serialize($out), 9));
        echo " ok\n";
    }
    /**
     * Load the trained data from disk
     * @param bool $for_training whether we are continuing to train (true) or
     *  whether we are using the loaded data for prediction
     */
    public function loadWeights($for_training = false)
    {
        if (!file_exists($this->tagger_path)) {
            echo "$this->tagger_path does not exist!";
            exit();
        }
        $f = unserialize(gzdecode(file_get_contents($this->tagger_path)),
            ['allowed_classes' => false]);
        $this->word_feature = $f["w"];
        $this->tag_feature = $f["t"] ?? [];
        $this->bias = $f["b"];
        $this->min_w = $f["min_w"];
        $this->max_w = $f["max_w"];
        $this->tag_set = $f["tag_set"];
        if ($for_training) {
            foreach(array_keys($this->word_feature) as $key) {
                $this->word_feature[$key] = $this->unpackW($key);
            }
            foreach(array_keys($this->tag_feature) as $key) {
                $this->tag_feature[$key] = (array_merge(unpack("f" .
                    strval(count($this->tag_set)), $this->tag_feature[$key])));
            }
            $this->bias = (array_merge(unpack("f" .
                strval(count($this->tag_set)), $this->bias)));
        }
    }
    /**
     * Pack the weights matrix to a string for a particular part of speech key
     * @param int $key index corresponding to a part of speech according to
     *  $this->tag_set
     * @return string the packed weights matrix
     */
    public function packW($key)
    {
        $bin_str = "";
        foreach ($this->word_feature[$key] as $i => $t) {
            foreach ($t as $u) {
                $v = 65535 * ($u - $this->min_w) /
                    ($this->max_w - $this->min_w);
                $bin_str .= pack("S", intval($v));
            }
        }
        return $bin_str;
    }
    /**
     * Unpack the weight matrix for a given part of speech key. This
     * is a 5 x term_set_size matrix the 5 rows corresponds to
     * -2, -1, 0, 1, 2, locations in a 5-gram.
     * An (i, j) entry roughly gives the probability of the j term in location i
     * having the part of speech given by $key
     * @param int $key in word_feature set corresponding to a part of speech
     * @return array of weights corresponding to that key
     */
    public function unpackW($key)
    {
        $weights = [];
        $size = count($this->tag_set);
        for ($i = 0; $i < 5; $i++) {
            $weights[$i - 2] = array_merge(unpack("S" . strval($size),
                $this->word_feature[$key], 2 * $i * count($this->tag_set)));
            for($j = 0; $j < $size; $j++) {
                $weights[$i - 2][$j] = ($weights[$i - 2][$j] / 65535) *
                    ($this->max_w - $this->min_w) + $this->min_w;
            }
        }
        return $weights;
    }
    /**
     * Get the weight value for term at position for tag
     * @param string $term to get weight of
     * @param int $position of term within the current 5-gram
     * @param int $tag_index index of the particular tag we are trying to see
     *  the term's weight for
     * @return float weight that the trained model assigns to ($term,
     *  $position, $tag_index), de-quantised from the 16-bit
     *  word_feature table back into [min_w, max_w]
     */
    public function getW($term, $position, $tag_index)
    {
        $t = unpack("S", $this->word_feature[$term], 2 * ($position + 2) *
            count($this->tag_set) + $tag_index * 2)[1] / 65535 *
            ($this->max_w - $this->min_w) + $this->min_w;;
        return $t;
    }
    /**
     * Tags a sequence of strings according to this tagger's predict method
     * returning the tagged result as a string.
     * This function is mainly used to facilitate unit testing of taggers.
     * @param string $text to be tagged
     * @param string $tag_separator terms in the output string will
     *  be the terms from the input texts followed by $tag_separator
     *  followed by their tag. So if $tag_separator == "_", then a term
     *  中国 in the input texts might be 中国_NR in the output string
     * @return string single string where terms in the input texts
     *  have been tagged. For example output might look like:
     *  中国_NR 人民_NN 将_AD 满怀信心_VV
     *  地_DEV 开创_VV 新_VA 的_DEC 业绩_NN 。_PU
     */
    public function tag($text, $tag_separator = "_")
    {
        $tagged_text = "";
        $lines = preg_split('/\r\n|\r|\n/u', $text);
        foreach($lines as $line) {
            $line_vector = explode(" ", trim($line));
            $tag_vector = $this->predict($line_vector);
            $tagged_term_vector = [];
            for($i = 0; $i < count($tag_vector); $i++) {
                if (is_array($tag_vector[$i])) {
                    list($term_element, $tag_element) = $tag_vector[$i];
                } else {
                    $term_element = $line_vector[$i];
                    $tag_element = $tag_vector[$i];
                }
                $tagged_term_vector[$i] = $term_element . $tag_separator .
                    $tag_element;
            }
            $tagged_text .= join(" ", $tagged_term_vector);
        }
        return $tagged_text;
    }
    /**
     * Uses text files to train a tagger for terms or chars in a  document
     * @param mixed $text_files with training data. These can be a file or
     *  an array of file names.
     * @param string $term_tag_separator separator used to separate term and tag
     *  for terms in input sentence
     * @param float $learning_rate learning rate when cycling over data trying
     *  to minimize the cross-entropy loss in the prediction of the tag of the
     *  middle term.
     * @param int $num_epoch number of times to cycle through the
     *  complete data set. Default value of 1200 seems to avoid overfitting
     * @param function $term_callback callback function applied to a term
     *  before adding term to sentence term array as part of processing and
     *  training with a sentence.
     * @param function $tag_callback callback function applied to a part of
     *  speech tag  before adding tag to sentence tag array as part of
     *  processing and training with a sentence.
     * @param bool $resume when true, continue training from a
     *  previously-saved checkpoint instead of reinitialising the
     *  model from scratch
     */
    public abstract function train($text_files, $term_tag_separator = "-",
        $learning_rate = 0.1, $num_epoch = 1200, $term_callback = null,
        $tag_callback = null, $resume = false);
    /**
     * Predicts a tagging for all elements of $sentence
     *
     * @param mixed $sentence is an array of segmented terms/chars
     *  or a string that will be split on white space
     * @return array predicted tags. The ith entry in the returned results
     *  is the tag of ith element of $sentence
     */
    public abstract function predict($sentence);
}
