<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\library\index_bundle_iterators;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\CrawlConstants;
use seekquarry\yioop\library\storage_formats\IndexDocumentBundle;
use seekquarry\yioop\library\language_processing\PhraseParser;
use seekquarry\yioop\library\storage_formats\PartitionDocumentBundle;



/** For toHexString and Yioop constants*/
require_once __DIR__."/../Utility.php";
/**
 * Abstract classed used to model iterating documents indexed in
 * an IndexArchiveBundle or set of such bundles.
 *
 *
 * @author Chris Pollett
 * @see IndexArchiveBundle
 */
abstract class IndexBundleIterator implements CrawlConstants
{
    /**
     * Default number of documents returned for each block (at most)
     * @var int
     */
    const RESULTS_PER_BLOCK = 200;
    /**
     * Estimate of the number of documents that this iterator can return
     * @var int
     */
    public $num_docs;
    /**
     *
     * @var int
     */
    public $total_num_docs;
    /**
     * The number of documents already iterated over
     * @var int
     */
    public $seen_docs;
    /**
     * The number of documents in the current block
     * @var int
     */
    public $count_block;
    /**
     * Cache of what currentDocsWithWord returns
     * @var array
     */
    public $pages;
    /**
     * Says whether the value in $this->count_block is up to date
     * @var bool
     */
    public $current_block_fresh;
    /**
     * Number of documents returned for each block (at most)
     * @var int
     */
    public $results_per_block = self::RESULTS_PER_BLOCK;
    /**
     * Returns the iterators to the first document block that it could iterate
     * over
     */
    abstract function reset();
    /**
     * Forwards the iterator one group of docs
     * @param array $gen_doc_offset a generation, doc_offset pair. If set,
     *     the must be of greater than or equal generation, and if equal the
     *     next block must all have $doc_offsets larger than or equal to
     *     this value
     */
    public abstract function advance($gen_doc_offset = null);
    /**
     * Gets the doc_offset and generation for the next document that
     * would be return by this iterator
     *
     * @return mixed an array with the desired document offset
     * and generation; -1 on fail
     */
    public abstract function currentGenDocOffsetWithWord();
    /**
     * Hook function used by currentDocsWithWord to return the current block
     * of docs if it is not cached
     *
     * @return mixed doc ids and score if there are docs left, -1 otherwise
     */
     public abstract function findDocsWithWord();
    /**
     * Returns a string representation of a plan by which the current iterator
     * finds its results
     *
     * @return string a representation of the current iterator and its
     *      subiterators, useful for determining how a query will be processed
     */
    public function plan()
    {
        $out = " ".get_class($this) . "\n===============\n";
        if (isset($this->word_key)) {
            $out .= "Word Key: " . L\toHexString($this->word_key)."\n";
        }
        if (isset($this->index_name)) {
            $out .= "Index Name: " . $this->index_name . "\n";
        }
        if (isset($this->results_per_block)) {
            $out .= "Results Per Block: " . $this->results_per_block . "\n";
        }
        $out .= "Number of Docs: ".$this->num_docs;
        if (isset($this->index_bundle_iterator)) {
            $out .= "\n   " . str_replace("\n", "\n   ",
                $this->index_bundle_iterator->plan());
        } else if (isset($this->index_bundle_iterators)) {
            $out .= "\nNumber of Child Iterators: " .
                $this->num_iterators . "\n";
            for ($i = 0; $i < $this->num_iterators; $i++) {
                $out .= "\n  Child $i\n  ========\n";
                $out .= "   ". str_replace("\n", "\n    ",
                    $this->index_bundle_iterators[$i]->plan());
            }
        }
        return $out;
    }
    /**
     * Compares two arrays each containing a (generation, offset) pair.
     *
     * @param array $gen_doc1  first ordered pair
     * @param array $gen_doc2  second ordered pair
     * @param int $direction whether the comparison should be done for
     *  a self::ASCEDNING or a self::DESCENDING search
     * @return int -1,0,1 depending on which is bigger
     */
    public function genDocOffsetCmp($gen_doc1, $gen_doc2, $direction =
        self::ASCENDING)
    {
        $diff1 = $gen_doc1[0] - $gen_doc2[0];
        $diff2 = $gen_doc1[1] - $gen_doc2[1];
        if ($diff2 == 0 && $diff1 == $diff2) {
            return 0;
        }
        if ($direction == self::ASCENDING) {
            return ($diff1 != 0) ? $diff1 : $diff2;
        } else {
            return ($diff1 != 0) ? -$diff1 : -$diff2;
        }
    }
    /**
     * Returns the direction of a IndexBundleIterator. Depending on the
     * iterator could be either forward from the start of an index
     * (self::ASCENDING) or backward from the end of the index
     * (self::DESCENDING). For this base class, the function always returns
     * self::ASCENDING, but subclasses might return different values.
     * @return int either CrawlConstants::ASCENDING or
     *  CrawlConstants::DESCENDING
     */
    public function getDirection()
    {
        return self::ASCENDING;
    }
    /**
     * Gets the current block of doc ids and score associated with the
     * this iterators word
     *
     * @return mixed doc ids and score if there are docs left, -1 otherwise
     */
    public function currentDocsWithWord()
    {
        if ($this->current_block_fresh == true) {
            return $this->pages;
        }
        $this->current_block_fresh = true;
        return $this->findDocsWithWord();
    }
    /**
     * Gets the summaries associated with the keys provided the keys
     * can be found in the current block of docs returned by this iterator
     * @param array $keys keys to try to find in the current block of returned
     *     results
     * @return array doc summaries that match provided keys
     */
    public function getCurrentDocsForKeys($keys = null)
    {
        if ($this->current_block_fresh == false) {
            $pages = $this->currentDocsWithWord();
            if (!is_array($pages)) {
                return $pages;
            }
        } else {
            $pages = & $this->pages;
        }
        if ($keys == null) {
            if (is_array($pages)) {
                return $pages;
            } else {
                return null;
            }
        }
        $out_pages = [];
        foreach ($keys as $doc_key) {
            if (!isset($pages[$doc_key])) {
                continue;
            } else {
                $doc_info = $pages[$doc_key];
            }
            if (isset($doc_info[self::SUMMARY_OFFSET]) &&
                isset($doc_info[self::GENERATION])) {
                $out_pages[$doc_key] = $doc_info;
            }
        }
        return $out_pages;
    }
    /**
     * Get the current block of doc summaries for the word iterator and advances
     * the current pointer to the next block of documents. If a doc index is
     * the next block must be of docs after this doc_index
     *
     * @param $doc_offset if set the next block must all have $doc_offsets
     *     equal to or larger than this value
     * @return array doc summaries matching the $this->restrict_phrases
     */
    public function nextDocsWithWord($doc_offset = null)
    {
        $doc_block = $this->getCurrentDocsForKeys();
        if ($doc_block == -1 || !is_array($doc_block) ) {
            return null;
        }
        $this->advance($doc_offset);
        return $doc_block;
    }
    /**
     * Updates the seen_docs count during an advance() call
     */
    public function advanceSeenDocs()
    {
        if ($this->current_block_fresh != true) {
            $doc_block = $this->currentDocsWithWord();
            if ($doc_block == -1 || !is_array($doc_block) ) {
                return;
            }
        }
        $this->current_block_fresh = false;
        $this->seen_docs += $this->count_block;
    }
    /**
     * Sets the value of the result_per_block field. This field controls
     * the maximum number of results that can be returned in one go by
     * currentDocsWithWord()
     *
     * @param int $num the maximum number of results that can be returned by
     *     a block
     */
     public function setResultsPerBlock($num)
     {
        $this->results_per_block = $num;
     }
     /**
      * Computes the DOC_RANK quality score for the document identified
      * by $doc_key. Combines freshness (or add-order) signals with
      * URL-shape signals (CLD vs HOST, number of path slashes) and
      * wiki-page bonuses.
      *
      * @param string $doc_key the doc-id binary key (DOCID_LEN bytes)
      *      identifying the document whose rank is being computed
      * @param int $doc_map_index index of the doc within the
      *      partition's doc_map
      * @param int $num_seen_partitions how many partitions have been
      *      walked before the one this doc lives in (used for
      *      add-order freshness)
      * @param int $number_of_partitions total partitions in the bundle
      * @param int $num_doc_keys number of doc keys in the partition
      *      this doc lives in
      * @param float $avg_items_per_partition average partition density
      *      across the bundle; used to normalize add-order signals
      * @param int $max_items_per_partition partition-size cap
      *      configured for the bundle
      * @param array $ranking_factors per-signal weights; an empty
      *      array falls back to the bundle's stored defaults
      * @param bool $is_ascending direction the iterator walks; when
      *      false the add-order signal is inverted so newer documents
      *      still score higher
      * @return float the computed DOC_RANK score combining freshness or
      *      add-order signals with URL-shape and page-type signals
      */
    public function computeDocRank($doc_key, $doc_map_index = 1,
        $num_seen_partitions = 0, $number_of_partitions = 1,
        $num_doc_keys = PartitionDocumentBundle::MAX_ITEMS_PER_FILE,
        $avg_items_per_partition = PartitionDocumentBundle::MAX_ITEMS_PER_FILE,
        $max_items_per_partition = PartitionDocumentBundle::MAX_ITEMS_PER_FILE,
        $ranking_factors = [], $is_ascending = true)
    {
        /*
           DOC_RANK calculate is a computes a document quality measure
           either based on time item was added (freshness) or a
           sum of signals (how early or late it was added to index),
           whether the url was a CLD or HOST, whether page was a wiki
           page, and the number of slashes in the url path
         */
        $cld_bonus = $ranking_factors["CLD_URL_BONUS"] ?? 1;
        $host_bonus = $ranking_factors["HOST_URL_BONUS"] ?? 1;
        $wiki_bonus = $ranking_factors["WIKI_BONUS"] ?? 1;
        $num_slashes_bonus = $ranking_factors["NUM_SLASHES_BONUS"] ?? 1;
        $max_pre_rank_and_bonuses = $cld_bonus + $host_bonus +
            $wiki_bonus + $wiki_bonus + 1;
        $last_partition_pos =  ($is_ascending) ?
            $num_doc_keys - $doc_map_index :
            $doc_map_index;
        $remaining_partitions =  ($is_ascending) ?
            $number_of_partitions - $num_seen_partitions :
            $num_seen_partitions - 1;
        $rank_and_bonuses = ($remaining_partitions *
            $avg_items_per_partition)/
            (($number_of_partitions + 1) *
            ($avg_items_per_partition + 1)) +
            $last_partition_pos / $max_items_per_partition;
        if (IndexDocumentBundle::isAHostDocId($doc_key)) {
            $rank_and_bonuses +=
                ((ord($doc_key[IndexDocumentBundle::DOCID_PART_LEN << 1] ??
                chr(0)) & 128) > 0) ? $cld_bonus : $host_bonus;
        }
        /**
         * For backward compatibility: new bonuses should only be added
         * for  doc_ids following the new letter_code format. Since all
         * old formats use letters (b, t, etc.) to denote the doc
         * type, the ASCII values for these letters are all > 96 (i.e.,
         * bits 6 and 7 of the doc_id's 9th byte are both true).
         * Since all new letter_code formats use bits 4, 5, 6, 7 to
         * represent  the doc type as int values mapped between 0-8,
         * there is no value in a doc_id's 9th byte that can have both
         * bits 6 and 7  set to true.
         * This difference can be used to check whether $doc_key follows
         * the old or new letter_code format.
         */
        $doc_id_format = ord($doc_key[
            IndexDocumentBundle::DOCID_PART_LEN << 1] ?? 0) & 96;
        if ($doc_id_format != 96) {
            if ((ord($doc_key[IndexDocumentBundle::DOCID_PART_LEN << 1] ??
                chr(0)) & 4) > 0) {
                $rank_and_bonuses += $wiki_bonus;
            }
            $rank_and_bonuses  += $num_slashes_bonus /
                ((ord($doc_key[IndexDocumentBundle::DOCID_PART_LEN << 1] ??
                chr(0)) & 3) + 1);
        }
        return $rank_and_bonuses;
    }
}
