<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* This file contains unit tests of the PackedTableTools class
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\tests;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\models\Model;
use seekquarry\yioop\library\UnitTest;
/**
* Used to test the PackedTableTools class. PackedTableTools are used for
* reading and storing rows with respect to some signature
*/
class PackedTableToolsTest extends UnitTest
{
/**
* Prefix of folders for testing PackedTable's
*/
const TEST_DIR = C\TEST_DIR . '/test_files/packed_test';
/**
* Creates test directory for the tests
*/
public function setUp()
{
if (!file_exists(self::TEST_DIR)) {
mkdir(self::TEST_DIR);
}
}
/**
* After the tests delete test directory
*/
public function tearDown()
{
$model = new Model();
$model->db->unlinkRecursive(self::TEST_DIR);
}
/**
* Auxiliary function used during test that takes a PackedTableTools
* object instantiated according to some row signature and an array
* rows in that signature and checks that for all rows packing the row
* followed by unpacking it returns the original row. IF a row
* fails a testCase message is output
*
* @param PackedTableTools $table_factory used for pack and unpacking rows
* @param array $rows rows according to $table_factory's signature
*/
public function checkPackUnpackTableRows($table_factory, $rows)
{
foreach($rows as $row) {
$packed_row = $table_factory->pack($row);
$pack_unpack_row = $table_factory->unpack($packed_row);
$id = $row['ID'];
unset($row['ID']);
$this->assertEqual($row, $pack_unpack_row[0],
"Row $id pack unpack produces same.");
}
}
/**
* This tests packing and unpacking rows according to a PackedTableTools
* signature in the case that there is only one record associated with the
* primary key (no-clustering).
*/
public function packUnpackSingleRowTestCase()
{
$table_factory = new L\PackedTableTools(
["PRIMARY KEY" => "ID", "A" => "INT", "B" => "TEXT",
"C" => "INT"]);
$rows = [
["ID" => 1, "A" => 5, "B" => "Hello World", "C"=> 256],
["ID" => 2, "A" => 65000, "B" => "yo", "C"=> 2000000]
];
$this->checkPackUnpackTableRows($table_factory, $rows);
$table_factory = new L\PackedTableTools(
["PRIMARY KEY" => "ID", "A" => "DOUBLE", "B" => "TEXT",
"C" => "TEXT", "D" => "BOOL", "E" => "INT"]);
$rows = [
["ID" => 1, "A" => 1.59, "B" => "Bong", "C" => "yockledoo",
"D" => true, "E" => 1000],
["ID" => 2, "A" => 100.4, "B" => "Blah", "C" => "doodoo",
"D" => false, "E" => 9990],
];
$this->checkPackUnpackTableRows($table_factory, $rows);
}
/**
* This tests packing and unpacking rows according to a PackedTableTools
* signature in the case that where more than one record can be associated
* with the primary key (clustering).
*/
public function packUnpackMultiRowTestCase()
{
$table_factory = new L\PackedTableTools(
["PRIMARY KEY" => "ID", "A" => "INT", "B" => "TEXT",
"C" => "INT"]);
$rows = [
["A" => 5, "B" => "Hello World", "C"=> 256],
["A" => 20000, "B" => "laladida", "C"=> 5600]
];
$packed_rows = $table_factory->pack($rows);
$pack_unpack_rows = $table_factory->unpack($packed_rows);
$this->assertEqual($rows, $pack_unpack_rows,
"Two row pack unpack produces same.");
}
/**
* This tests if a set of rows packed according to a PackedTableTools record
* is saved that it can be reloaded and unpacked and still get the same
* results
*/
public function saveAddLoadTestCase()
{
$table_factory = new L\PackedTableTools(
["PRIMARY KEY" => "ID", "A" => "INT", "B" => "TEXT",
"C" => "INT"]);
$table_factory2 = new L\PackedTableTools(
["PRIMARY KEY" => "ID", "A" => "INT", "B" => "TEXT",
"C" => "INT"], C\NS_COMPRESSORS . "GzipCompressor");
$table = [];
$hash_key1 = md5("1", true);
$table_factory->add($table, $hash_key1,
$table_factory->pack(["A" => 5, "B" => "Hello World", "C"=> 256]),
mode: L\PackedTableTools::APPEND_MODE);
$table_factory->add($table, $hash_key1,
$table_factory->pack(["A" => 6, "B" => "Hello World2", "C"=> 257]),
mode: L\PackedTableTools::APPEND_MODE);
$hash_key2 = md5("2", true);
$table_factory->add($table, $hash_key2,
$table_factory->pack(["A" => 20000, "B" => "laladida",
"C"=> 5600]));
$table_factory->save(self::TEST_DIR . "/save.txt", $table);
$loaded_table = $table_factory->load(self::TEST_DIR . "/save.txt");
$this->assertEqual($table, $loaded_table,
"Add two rows, save, load table gives same result.");
$table_factory2->save(self::TEST_DIR . "/save2.txt", $table);
$loaded_table2 = $table_factory2->load(self::TEST_DIR . "/save2.txt");
$this->assertEqual($table, $loaded_table2,
"Add two rows, save, load compressed table gives same result.");
$entry1 = $loaded_table[$hash_key1];
$start = 0;
$num_entries = L\vByteDecode($entry1, $start);
$this->assertEqual($num_entries, 2,
"Row one has correct number of sub-entries.");
}
/**
* Tests that the in-memory table cache stays within its byte budget,
* drops the tables used longest ago first, spares a table that was just
* used, and drops a table's entry markers in step with the table. This
* guards the fix for the slow web-server memory growth that came from
* the cache holding every partition it was ever asked for.
*/
public function tableCacheBoundTestCase()
{
$signature = ["PRIMARY KEY" => "ID", "A" => "INT", "B" => "TEXT"];
$tools = new L\PackedTableTools($signature);
$paths = [];
for ($i = 1; $i <= 4; $i++) {
$one_table = [];
$tools->add($one_table, md5((string)$i, true),
$tools->pack(["A" => $i, "B" => "row for cache table $i"]));
$paths[$i] = self::TEST_DIR . "/cache_$i.txt";
$tools->save($paths[$i], $one_table);
}
$sample = $tools->load($paths[1],
L\PackedTableTools::AS_STRING_MODE, true);
$entry_size = strlen($sample);
$tools = new L\PackedTableTools($signature);
$tools->max_table_cache_bytes = 2 * $entry_size + 1;
foreach ([1, 2, 3] as $number) {
$tools->load($paths[$number],
L\PackedTableTools::AS_STRING_MODE, true);
}
$this->assertTrue(
$tools->table_cache_bytes <= $tools->max_table_cache_bytes,
"Cache bytes stay within the budget after loading three tables.");
$this->assertEqual(count($tools->table_cache), 2,
"Only the two most recent tables stay cached.");
$this->assertTrue(!isset($tools->table_cache[L\crawlHash($paths[1])]),
"The least recently used table was evicted.");
$tools->load($paths[2], L\PackedTableTools::AS_STRING_MODE, true);
$tools->load($paths[4], L\PackedTableTools::AS_STRING_MODE, true);
$this->assertTrue(!isset($tools->table_cache[L\crawlHash($paths[3])]),
"Using a table spares it and the next oldest goes instead.");
$this->assertTrue(isset($tools->table_cache[L\crawlHash($paths[2])]),
"The recently used table is kept.");
$tight = new L\PackedTableTools($signature);
$tight->max_table_cache_bytes = $entry_size + 1;
$tight->getEntryMarkers($paths[1]);
$this->assertTrue(
isset($tight->table_entry_markers[L\crawlHash($paths[1])]),
"Markers are cached when a table is read.");
$tight->getEntryMarkers($paths[2]);
$this->assertTrue(
!isset($tight->table_entry_markers[L\crawlHash($paths[1])]),
"Evicting a table drops its markers too.");
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software