<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\library;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\models\ParallelModel;
use seekquarry\yioop\library\storage_formats\IndexShard;

/**
 * For crawlHash
 */
require_once __DIR__ . "/Utility.php";
/**
     * IndexManager holds the index bundles a search has open, so that while
     * performing
 * a query. Ensures an easy place to obtain references to these bundles
 * and ensures only one object per bundle is instantiated in a Singleton-esque
 * way.
 *
 * @author Chris Pollett
 */
class IndexManager implements CrawlConstants
{
    /**
     * $indexes holds the index bundles this manager
     * @var array
     */
    public static $indexes = [];
    /**
     * $index_times holds, for each bundle name, the time when cached
     * @var array
     */
    public static $index_times = [];
    /**
     * $urls_cache holds, for each address, the doc_map info when cached
     * @var array
     */
    public static $urls_cache = [];
    /**
     * $info_cache holds what was looked up about a word, kept under a
     * hash of the query parameters.
     * Kept as a class property (rather than a local static of
     * getWordInfo) so the periodic memory log can report how large it has
     * grown.
     * @var array
     */
    public static $info_cache = [];
    /**
     * How many indexes are kept open in memory at once. Yioop reads from a
     * handful of indexes at most, and an index is held on disk rather than
     * in memory, so the count that matters is small and fixed. The index
     * used least recently is dropped once a sixth is opened.
     */
    const MAX_OPEN_INDEXES = 5;
    /**
     * How many lookups are remembered, for words and for addresses alike.
     * A lookup is a small record and a busy query screen makes many of
     * them, so this is far larger than the number of indexes.
     */
    const MAX_REMEMBERED_LOOKUPS = 1000;
    /**
     * getIndex gives back the copy this class holds of an IndexArchiveBundle
     * object
     * with a given timestamp or feed (for handling media feeds)
     *
     * @param string $index_name timestamp of desired IndexArchiveBundle
     * @return object the desired IndexArchiveBundle reference
     */
    public static function getIndex($index_name)
    {
        $index_name = trim($index_name ?? ""); //trim to fix postgres quirkiness
        $cache_dir = C\CACHE_DIR . '/';
        $index_archive_full_path = $cache_dir . self::index_data_base_name .
            $index_name;
        $handled = false;
        if ($index_name == "feed" || $index_name == self::FEED_CRAWL_TIME) {
            $index_archive_name = self::feed_index_data_base_name;
            $index_name = "feed";
            $handled = true;
        } else if (is_numeric($index_name) &&
            file_exists($index_archive_full_path)) {
            $index_archive_name = self::index_data_base_name . $index_name;
            $handled = true;
        }
        if (!$handled && is_numeric($index_name) ) {
            $index_name = $cache_dir . self::double_index_base_name .
                $index_name;
        }
        if (!$handled && file_exists($index_name) ) {
            $is_repeating = file_exists("$index_name/bundle0");
            $serve_archive = "0";
            if ($is_repeating) {
                $status_file = "$index_name/status.txt";
                if (file_exists($status_file)) {
                    $status = unserialize(file_get_contents($status_file));
                    $active_archive = (empty($status["swap_count"])) ? 1 :
                        $status["swap_count"] % 2;
                    $serve_archive = 1 - $active_archive;
                }
            }
            $sub_folder = ($is_repeating) ? "/bundle$serve_archive" : "";
            $is_old = ($is_repeating) ? (file_exists($index_name.
                "/bundle0/summaries")) : (file_exists($index_name .
                    "/summaries"));
            $bundle_class_name = (
                $is_old) ? C\NS_LIB . "storage_formats\\IndexArchiveBundle"
                : C\NS_LIB . "storage_formats\\IndexDocumentBundle";
            $tmp = new $bundle_class_name($index_name . $sub_folder);
        } else if (!$handled) {
            return false;
        }
        if (empty(self::$indexes[$index_name]) ||
            (!empty(self::$index_times[$index_name]) &&
            ($index_name == 'feed' && php_sapi_name() == 'cli') &&
            (time() - self::$index_times[$index_name])
            > C\MIN_QUERY_CACHE_TIME) ) {
            if (!isset($tmp)) {
                $index_filename = $cache_dir . $index_archive_name;
                if (file_exists($index_filename)) {
                    $is_old = (file_exists($index_filename . "/summaries"));
                    $bundle_class_name = ($is_old) ?
                        C\NS_LIB . "storage_formats\\IndexArchiveBundle"
                        : C\NS_LIB .
                        "storage_formats\\IndexDocumentBundle";
                    $tmp = new $bundle_class_name($cache_dir .
                        $index_archive_name);
                    if (!$tmp) {
                        return false;
                    }
                } else {
                    $tmp = false;
                    $use_name = $index_name;
                    $serve_archive = -1;
                    if (preg_match("/\-\d$/", $index_name)) {
                        $serve_archive = substr($index_name, -1);
                        $use_name = substr($index_name, 0, -2);
                    }
                    $index_archive_name = self::double_index_base_name .
                        $use_name;
                    $status_file = $cache_dir . $index_archive_name .
                        "/status.txt";
                    if ($serve_archive < 0 && file_exists($status_file)) {
                        $status = unserialize(file_get_contents($status_file));
                        $active_archive = (empty($status["swap_count"])) ? 1 :
                            $status["swap_count"] % 2;
                        $serve_archive = 1 - $active_archive;
                    }
                    $is_old = (file_exists($index_filename .
                        "/bundle0/summaries") ||
                        file_exists($index_filename . "/bundle1/summaries"));
                    $bundle_class_name = ($is_old) ?
                        C\NS_LIB . "storage_formats\\IndexArchiveBundle"
                        : C\NS_LIB . "storage_formats\\IndexDocumentBundle";
                    $tmp = new $bundle_class_name($cache_dir .
                        $index_archive_name . "/bundle$serve_archive");
                    if (!$tmp) {
                        $serve_archive = ($serve_archive == 0) ? 1 : 0;
                        $tmp = new $bundle_class_name($cache_dir .
                            $index_archive_name . "/bundle$serve_archive");
                    }
                    if (!$tmp) {
                        return false;
                    }
                }
            }
            self::$indexes[$index_name] = $tmp;
            if ($is_old) {
                self::$indexes[$index_name]->setCurrentShard(0, true);
            }
            self::$index_times[$index_name] = time();
            while (count(self::$indexes) > self::MAX_OPEN_INDEXES) {
                $least_recent = array_key_first(self::$index_times);
                unset(self::$index_times[$least_recent],
                    self::$indexes[$least_recent]);
            }
        }
        if (isset(self::$indexes[$index_name]) &&
            isset(self::$index_times[$index_name])) {
            $keep_time = self::$index_times[$index_name];
            unset(self::$index_times[$index_name]);
            self::$index_times[$index_name] = $keep_time;
        }
        return self::$indexes[$index_name];
    }
    /**
     * clearCache empties the places caches of read in indexes
     *  and dictionary info is stored.
     */
    public static function clearCache()
    {
        self::$indexes = [];
        self::$index_times = [];
    }
    /**
     * getVersion gives the version of the index, so that Yioop can determine
     * how to do word lookup.The only major change to the format was
     * when word_id's went from 8 to 20 bytes which happened around Unix
     * time 1369754208.
     *
     * @param string $index_name unix timestamp of index
     * @return int 0 - if the original format for Yioop indexes; 1 -if 20 byte
     *     word_id format
     */
    public static function getVersion($index_name)
    {
        $index_name = (string) $index_name;
        $index_name = (empty($index_name) || $index_name[0] != '-') ?
            $index_name : substr($index_name, 1);
        $index_time = intval($index_name);
        if (!is_numeric($index_name)) {
            $description_file = $index_name . "/summaries/description.txt";
            if (file_exists($description_file)) {
                $description = unserialize(
                    file_get_contents($description_file));
                if (!empty($description['DESCRIPTION'])) {
                    $description = unserialize($description['DESCRIPTION']);
                }
                if (!empty($description[self::CRAWL_TIME])) {
                    if (intval($description[self::CRAWL_TIME]) <
                        C\VERSION_0_TIMESTAMP &&
                        intval($description[self::CRAWL_TIME]) !=
                        self::FEED_CRAWL_TIME) {
                        return 0;
                    }
                }
            }
        } else if ($index_time != self::FEED_CRAWL_TIME &&
            $index_time < C\VERSION_0_TIMESTAMP) {
            return 0;
        }
        $tmp_index = self::getIndex($index_name);
        if (isset($tmp_index->version)) {
            return $tmp_index->version;
        } else if (isset($tmp_index->archive_info['VERSION'])) {
            return $tmp_index->archive_info['VERSION'];
        }
        return C\DEFAULT_CRAWL_FORMAT;
    }
    /**
     * getWordInfo gives the posting list positions for each shard in the
     * bundle $index_name for the word id $term_id
     *
     * @param string $index_name bundle to look for $term_id in
     * @param string $term_id id of phrase or word to look up in bundle
     *     dictionary
     * @param int $threshold after the number of results exceeds this amount
     *     stop looking for more dictionary entries.
     * @param int $start_generation what generation in the index to start
     *      finding occurrence of phrase from
     * @param int $num_distinct_generations from $start_generation how
     *      many generation to search forward to
     * @param bool $with_remaining_total whether to total number of
     *      postings found as well or not
     * @return array either [total, sequence of four tuples]
    *       or sequence of four tuples:
     *      (index_shard generation, posting_list_offset, length, exact id
     *      that match $term_id)
     */
    public static function getWordInfo($index_name, $term_id, $threshold = -1,
        $start_generation = -1, $num_distinct_generations = -1,
        $with_remaining_total = false)
    {
        $lookup_hash = crawlHash($index_name . $term_id . $threshold .
            $start_generation . $num_distinct_generations .
            $with_remaining_total);
        if (isset(self::$info_cache[$lookup_hash])) {
            $tmp = self::$info_cache[$lookup_hash];
            unset(self::$info_cache[$lookup_hash]);
            self::$info_cache[$lookup_hash] = $tmp;
            return $tmp;
        }
        $index = self::getIndex($index_name);
        $start_generation = ($start_generation < 0) ? 0 : $start_generation;
        /* An index made before the current layout is served by an older
           bundle, which does not look words up itself; its dictionary and
           its newest shard are asked instead. Newer indexes answer for
           themselves, so they are asked directly. */
        if (method_exists($index, "getWordInfo")) {
            $word_info = $index->getWordInfo($term_id,
                $threshold, $start_generation, $num_distinct_generations,
                $with_remaining_total);
        } else {
            $word_info = self::getWordInfoOlderIndex($index, $term_id,
                $threshold, $start_generation, $num_distinct_generations,
                $with_remaining_total);
        }
        self::$info_cache[$lookup_hash] = $word_info;
        if (count(self::$info_cache) >= self::MAX_REMEMBERED_LOOKUPS) {
            array_shift(self::$info_cache);
        }
        return $word_info;
    }
    /**
     * getWordInfoOlderIndex looks a word up in an index made before
     * the current layout. Such an
     * index keeps a dictionary of the words its finished generations
     * hold, and may have a newest generation whose shard has not been
     * folded into that dictionary yet, so both are asked and what they
     * say is put together. A newer index answers a look up itself and
     * never reaches here.
     *
     * @param object $index bundle holding the index being read
     * @param string $term_id what to look up, as the index names it
     * @param int $threshold stop looking after this many documents; -1
     *      for no limit
     * @param int $start_generation which generation to look from
     * @param int $num_distinct_generations how many generations to look
     *      across; negative for all of them
     * @param bool $with_remaining_total whether to answer with the total
     *      number of documents alongside the list of them
     * @return array what the index knows about the word: a list of
     *      (generation, offset, count, ...) rows, or that list together
     *      with a total when one was asked for
     */
    private static function getWordInfoOlderIndex($index, $term_id,
        $threshold, $start_generation, $num_distinct_generations,
        $with_remaining_total)
    {
        $pre_info = [];
        if (!empty($index->dictionary)) {
            $pre_info = $index->dictionary->getWordInfo($term_id, true,
                $threshold, $start_generation, $num_distinct_generations,
                true);
        }
        $last_desired_generation = $start_generation +
            $num_distinct_generations;
        if (isset($index->generation_info['ACTIVE'])) {
            $active_generation = $index->generation_info['ACTIVE'];
            if ((empty($index->generation_info['LAST_DICTIONARY_SHARD']) ||
                $index->generation_info['LAST_DICTIONARY_SHARD'] <
                $active_generation) && ($active_generation <
                $last_desired_generation || $last_desired_generation < 0)) {
                $active_shard_file = $index->dir_name .
                    "/posting_doc_shards/index" . $active_generation;
                if (file_exists($active_shard_file)) {
                    if (!empty($index->non_merged_shard) &&
                        !empty($index->non_merged_generation) &&
                        $index->non_merged_generation == $active_generation) {
                        $active_shard = $index->non_merged_shard;
                    } else {
                        $active_shard = new IndexShard($active_shard_file, 0,
                            C\NUM_DOCS_PER_PARTITION, true);
                        $index->non_merged_shard = $active_shard;
                        $index->non_merged_generation = $active_generation;
                    }
                    $active_info = $active_shard->getWordInfo($term_id, true);
                    if (is_array($active_info)) {
                        if (empty($pre_info)) {
                            $pre_info[0] = 0;
                            $pre_info[1] = [];
                        }
                        $pre_info[1][] = [$active_generation,
                            $active_info[0], $active_info[1], $active_info[2],
                            $active_info[3]];
                        $pre_info[0] += $active_info[2];
                    }
                }
            }
        }
        if (!empty($pre_info[1])) {
            list($total, $info) = $pre_info;
        } else {
            $total = 0;
            $info = [];
        }
        return ($with_remaining_total) ? [$total, $info] : $info;
    }
    /**
     * lookupLatestVersionPage finds posting info about the most recent version
     * of a URL in the given index
     *
     * @param string $url_hash hash of the URL to be looked up
     * @param string $index_name current index
     * @return array of posting info | null
     */
    public static function lookupLatestVersionPage($url_hash, $index_name)
    {
        // Check if the url hash exists in the cache
        if (array_key_exists($url_hash, self::$urls_cache)) {
            return self::$urls_cache[$url_hash];
        }
        $model_for_url_hash_lookup = new ParallelModel();
        $page_versions = $model_for_url_hash_lookup->
            lookupSummaryOffsetGeneration(L\base64Hash($url_hash),
            $index_name, false, true);
        if (!empty($page_versions['ROWS']) &&
            count($page_versions['ROWS']) > 0) {
            $latest_postings_info =
                end($page_versions['ROWS'])['POSTINGS'] ?? null;
            $latest_partition =
                end($page_versions['ROWS'])['PARTITION'] ?? null;
            if (is_array($latest_postings_info) &&
                count($latest_postings_info) > 0) {
                $latest_posting = end($latest_postings_info);
                if (count(self::$urls_cache) >=
                    self::MAX_REMEMBERED_LOOKUPS) {
                    self::$urls_cache = [];
                }
                self::$urls_cache[$url_hash] = array($latest_partition,
                    $latest_posting);
                return self::$urls_cache[$url_hash];
            }
        }
        return null;
    }
}
