<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Sarika Padmashali padmashalisarika@gmail.com
* (Reworked so could scale for yioop.com by Chris Pollett)
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library\media_jobs;
use seekquarry\yioop\configs as C;
use seekquarry\yioop\library as L;
use seekquarry\yioop\library\LinearAlgebra as LinearAlgebra;
use seekquarry\yioop\library\LRUCache as LRUCache;
use seekquarry\yioop\library\PhraseParser as PhraseParser;
use seekquarry\yioop\models\CronModel;
/**
* Recommendation Job recommends the trending threads as well
* as threads and groups which are relevant based on the
* users viewing history
*/
class RecommendationJob extends MediaJob
{
/**
* Datasource object used to run db queries related to recommendation items
* (for storing and updating them)
* @var object
*/
public $db;
/**
* Model used for timing when things were computed
* @var object
*/
public $cron_model;
/**
* Time in current epoch when analytics last updated
* @var int
*/
public $update_time;
/**
* Used to track what is the active recommendation timestamp
* @var int
*/
public $active_time;
/**
* Associative array of the number of items a term appears in
* @var array
*/
public $item_idf;
/**
* Associative array of the number of user views a term appears in
* @var array
*/
public $user_idf;
/**
* LRUCache for term embeddings
* @var \seekquarry\yioop\library\LRUCache
*/
public $lru_cache;
/**
* Maximum number of group items used in making recommendations
*/
const MAX_GROUP_ITEMS = 50000;
/**
* Maximum number of terms used in making recommendations
*/
const MAX_TERMS = 20000;
/**
* File containing paths to description folders of wiki page resources
* that should be used to create data corpus for computing recommendations
*/
const RECOMMENDATION_FILE = C\APP_DIR . "/resources/recommendation.txt";
/**
* Length of context window for calculating term embeddings
*/
const CONTEXT_WINDOW_LENGTH = 5;
/**
* Update period to consider for fetching the records from
* ITEM_IMPRESSION_SUMMARY table
*/
const UPDATE_PERIOD = C\ONE_MONTH;
/**
* Stop words to exclude from the descriptions fetched by DescriptionUpdate
* media job
*/
const DESCRIPTION_STOP_WORDS = ["author", "authors", "plot", "genre",
"genres", "star", "stars", "credits", "rating", "ratings", "year",
"director", "cast", "runtime"];
/**
* Hash algorithm to be used for calculating hash in Hash2Vec embedding
*/
const HASH_ALGORITHM = "md5";
/**
* Hash algorithm to be used for calculating sign in Hash2Vec term embedding
*/
const SIGN_HASH_ALGORITHM = "crc32";
/**
* MAX term embeddings fetched from database to initialize LRUCache
*/
const MAX_TERM_EMBEDDINGS = 500;
/**
* Maximum number of resources used in making resource recommendations/
* Maximum number of group items to hold in memory in one go
*/
const MAX_BATCH_SIZE = 200;
/**
* Sets up the database connection so can access tables related
* to recommendations. Initialize timing info related to job.
*/
public function init()
{
$this->update_time = 0;
$this->active_time = 0;
$this->name_server_does_client_tasks = true;
$this->name_server_does_client_tasks_only = true;
$this->cron_model = new CronModel();
$db_class = C\NS_DATASOURCES . ucfirst(C\p('DBMS')). "Manager";
$this->db = new $db_class();
$this->db->connect();
}
/**
* Only update if its been more than an hour since the last update
*
* @return bool whether its been an hour since the last update
*/
public function checkPrerequisites()
{
$time = time();
$delta = $time - $this->update_time;
if ($delta > C\ONE_DAY) {
$this->update_time = $time;
L\crawlLog("Prerequisites for Recommendation Media Job met");
return true;
}
L\crawlLog("Time since last update not exceeded, skipping".
" Recommendation MediaJob $delta");
return false;
}
/**
* For now analytics update is only done on name server as Yioop
* currently only supports one DBMS at a time.
*/
public function nondistributedTasks()
{
L\crawlLog("Performing the Recommendation Media Job");
$this->active_time = $this->cron_model->getCronTime(
"item_group_recommendations");
L\crawlLog("Current Active Recommendation Timestamp: ".
$this->active_time);
L\crawlLog("...Start computing similarity-based group and item ".
"recommendations...");
$this->computeThreadGroupRecommendations();
L\crawlLog("...Finished computing similarity-based group and item ".
"recommendations.");
L\crawlLog("...Start computing similarity-based wiki resource " .
"recommendations...");
$this->computeWikiResourceRecommendations();
L\crawlLog("...Finished computing similarity-based wiki" .
"resource recommendations...");
L\crawlLog("...Start computing new user recommendations...");
$this->initializeNewUserRecommendations();
L\crawlLog("...Finished computing new user recommendations...");
$this->cron_model->updateCronTime(
"item_group_recommendations", $this->update_time);
}
/**
* Computes recommendations for users who have yet to receive any
* recommendation of the given type based on what is the most
* most popular recommendation
*/
public function initializeNewUserRecommendations()
{
$db = $this->db;
$popular_recommendations = [
C\THREAD_RECOMMENDATION => [], C\GROUP_RECOMMENDATION => []];
$sql = "SELECT ITEM_ID, SUM(SCORE) AS TOTAL_SCORE FROM " .
"GROUP_ITEM_RECOMMENDATION WHERE ITEM_TYPE = ? " .
"GROUP BY ITEM_ID ORDER BY TOTAL_SCORE DESC ".
$db->limitOffset(C\MAX_RECOMMENDATIONS);
foreach ($popular_recommendations as $type => $recommendation) {
$results = $db->execute($sql, [$type]);
while ($row = $db->fetchArray($results)) {
$popular_recommendations[$type][] = $row;
}
}
$new_user_sql = "SELECT USER_ID AS USER_ID ".
"FROM USERS WHERE USER_ID NOT IN ".
"(SELECT USER_ID FROM GROUP_ITEM_RECOMMENDATION)";
$new_user_results = $db->execute($new_user_sql);
$base_recommend_sql = "INSERT INTO GROUP_ITEM_RECOMMENDATION VALUES ";
$insert_recommend_sql = $base_recommend_sql;
$comma = "";
$insert_count = 0;
$i = 0;
while($row = $db->fetchArray($new_user_results)) {
$user_id = $row['USER_ID'];
foreach ($popular_recommendations as $type => $recommendations) {
foreach ($recommendations as $recommendation) {
$insert_recommend_sql .=
"$comma ($user_id, {$recommendation['ITEM_ID']}, ".
"$type, {$recommendation['TOTAL_SCORE']}," .
$this->update_time . ")";
$comma = ",";
$insert_count++;
}
if ($insert_count > C\BATCH_SQL_INSERT_NUM) {
$db->execute($insert_recommend_sql);
$insert_recommend_sql = $base_recommend_sql;
$insert_count = 0;
$comma = "";
}
}
}
if ($insert_count > 0) {
$db->execute($insert_recommend_sql);
}
$sql = "SELECT GROUP_ID, PAGE_ID, RESOURCE_PATH, RESOURCE_ID," .
" SUM(SCORE) AS TOTAL_SCORE FROM" .
" GROUP_RESOURCE_RECOMMENDATION GROUP BY GROUP_ID," .
" PAGE_ID, RESOURCE_PATH, RESOURCE_ID ORDER BY TOTAL_SCORE DESC";
$results = $db->execute($sql);
$popular_recommendations[C\RESOURCE_RECOMMENDATION] = [];
while ($row = $db->fetchArray($results)) {
$popular_recommendations[C\RESOURCE_RECOMMENDATION][] = $row;
}
$base_recommend_sql = "INSERT INTO GROUP_RESOURCE_RECOMMENDATION" .
" VALUES ";
$insert_recommend_sql = $base_recommend_sql;
$comma = "";
$insert_count = 0;
$new_user_sql = "SELECT USER_ID FROM USERS WHERE USER_ID NOT IN" .
"(SELECT USER_ID FROM GROUP_RESOURCE_RECOMMENDATION)";
$new_user_results = $db->execute($new_user_sql);
while ($row = $db->fetchArray($new_user_results)) {
$user_id = $row['USER_ID'];
$timestamp = time();
foreach ($popular_recommendations[C\RESOURCE_RECOMMENDATION] as
$recommendation) {
$insert_recommend_sql .=
"$comma ($user_id, {$recommendation['GROUP_ID']}, ".
"{$recommendation['PAGE_ID']}, " .
"'{$recommendation['RESOURCE_PATH']}', ".
"{$recommendation['TOTAL_SCORE']}, {$this->update_time}, ".
"{$recommendation['RESOURCE_ID']})";
$comma = ",";
$insert_count++;
if ($insert_count > C\BATCH_SQL_INSERT_NUM) {
$db->execute($insert_recommend_sql);
$insert_recommend_sql = $base_recommend_sql;
$insert_count = 0;
$comma = "";
}
}
}
if ($insert_count > 0) {
$db->execute($insert_recommend_sql);
}
}
/**
* Manages the whole process of computing thread and group recommendations
* for users. Makes a series of calls to handle parts of this computation
* before synthesizing the result
*/
public function computeThreadGroupRecommendations()
{
L\crawlLog("...Start computing Item Term Embeddings...");
$item_terms = $this->computeItemTermEmbeddings();
L\crawlLog("...Finished computing Item Term Embeddings...");
L\crawlLog("...Start computing Item Embeddings...");
$item_embeddings = $this->computeItemEmbeddings($item_terms);
L\crawlLog("...Finished computing Item Embeddings...");
L\crawlLog("...Start write back term embeddings from cache to db");
$this->saveTermEmbeddingsCacheToDb(C\THREAD_RECOMMENDATION);
L\crawlLog("...Finished write back term embeddings from cache to db");
L\crawlLog("...Start computing Item User Embeddings...");
[$item_user_embeddings, $user_items] = $this->
computeItemUserEmbeddings($item_embeddings);
L\crawlLog("...Finshed computing Item User Embeddings...");
L\crawlLog("...Start computing Item User Recommendations...");
$user_groups = $this->computeItemUserRecommendations($item_embeddings,
$item_user_embeddings, $user_items);
L\crawlLog("...Finished computing Item User Recommendations...");
unset($item_user_embeddings);
unset($user_items);
L\crawlLog("...Start computing Group Embeddings...");
$group_embeddings = $this->computeGroupEmbeddings($item_embeddings);
L\crawlLog("...Finished computing Group Embeddings...");
unset($item_embedding);
L\crawlLog("...Start computing Group User Embeddings...");
[$group_user_embeddings, $user_group_impression] =
$this->computeGroupUserEmbeddings($group_embeddings);
L\crawlLog("...Finished computing Group User Embeddings...");
L\crawlLog("...Start computing Group User Recommendations...");
$this->computeGroupUserRecommendations($group_embeddings,
$group_user_embeddings, $user_groups, $user_group_impression);
L\crawlLog("...Finished computing Group User Recommendations...");
unset($group_embeddings);
unset($group_user_embeddings);
unset($user_group_impression);
unset($user_groups);
}
/**
* Computes the term embeddings for individual items (main thread only and
* not comments) in groups feeds for the terms in their title and
* description text. Processes only MAX_GROUP_ITEMS which are either newly
* created or recently edited
*
* @return array $item_terms terms in each item
*/
public function computeItemTermEmbeddings()
{
$db = $this->db;
$this->lru_cache = new LRUCache(self::MAX_TERM_EMBEDDINGS);
$select_sql = "SELECT * FROM RECOMMENDATION_TERM_EMBEDDING WHERE" .
" ITEM_TYPE = ? " . $db->limitOffset(self::MAX_TERM_EMBEDDINGS);
$results = $db->execute($select_sql, [C\THREAD_RECOMMENDATION]);
$term_embeddings = [];
$item_terms = [];
L\crawlLog("Start Populating LRUCache of Embeddings...");
while ($row = $db->fetchArray($results)) {
if (is_string($row['VECTOR'])) {
$this->lru_cache->put($row['ID'],
base64_decode($row['VECTOR'], true));
} else {
var_dump($row['VECTOR']);
}
}
L\crawlLog("Finish Populating LRUCache of Embeddings");
$context_distance_sum = (self::CONTEXT_WINDOW_LENGTH *
(self::CONTEXT_WINDOW_LENGTH + 1)) / 2.0;
$mean = $context_distance_sum / self::CONTEXT_WINDOW_LENGTH;
$carry = 0.0;
for ($i = 1; $i <= self::CONTEXT_WINDOW_LENGTH; $i++) {
$difference = $i - $mean;
$carry += $difference * $difference;
}
$std_deviation = sqrt($carry / self::CONTEXT_WINDOW_LENGTH);
$item_count_sql = "SELECT COUNT(*) AS NUM_ITEMS FROM GROUP_ITEM ".
"WHERE ID = PARENT_ID AND TITLE NOT LIKE '%Page%'";
$results = $db->execute($item_count_sql);
$num_items = 0;
if ($results) {
$row = $db->fetchArray($results);
$num_items = min($row['NUM_ITEMS'], self::MAX_GROUP_ITEMS) ?? 0;
}
$num_batches = ceil($num_items/self::MAX_BATCH_SIZE);
L\crawlLog("Number of group items will consider: " . $num_items);
$item_count = 0;
for ($item_batch = 0; $item_batch < $num_batches; $item_batch++) {
$group_item_sql = "SELECT * FROM GROUP_ITEM WHERE ID = PARENT_ID" .
" AND TITLE NOT LIKE '%Page%' ORDER BY EDIT_DATE DESC " .
$db->limitOffset($item_batch * self::MAX_BATCH_SIZE,
self::MAX_BATCH_SIZE);
L\crawlTimeoutLog("Have processed $item_count many group items");
$results = $db->execute($group_item_sql);
$batch_items = [];
while ($row = $db->fetchArray($results)) {
$item_id = $row['ID'];
$text_corpus = $row['TITLE'] . " " . $row['DESCRIPTION'];
$text_corpus = mb_strtolower($text_corpus);
$terms = $this->cleanRemoveStopWords($text_corpus);
$batch_items[$item_id] = [$terms, $row['GROUP_ID']];
}
foreach ($batch_items as $item_id => $term_data) {
$item_terms[$item_id] = $term_data;
$terms = $term_data[0];
$num_terms = count($terms);
for ($i = 0; $i < $num_terms; $i++) {
L\crawlTimeoutLog("Have processed $i of $num_terms terms");
[$term_id, $term] = $terms[$i];
$term_hash = unpack('N',
hash(self::HASH_ALGORITHM, $term, true))[1] %
C\EMBEDDING_VECTOR_SIZE + 1;
$term_sign_hash =
hash(self::SIGN_HASH_ALGORITHM, $term, true);
$term_sign =
unpack('N', $term_sign_hash)[1] % 2 == 0 ? -1 : 1;
$term_embedding = $this->getTermEmbedding($term_id,
C\THREAD_RECOMMENDATION);
$term_embedding = unpack("E*", $term_embedding);
for ($j = $i - 1; $j >= 0 &&
$j >= $i - self::CONTEXT_WINDOW_LENGTH; $j--) {
[$context_term_id, $context_term] = $terms[$j];
$context_term_embedding = $this->getTermEmbedding(
$context_term_id, C\THREAD_RECOMMENDATION);
$context_term_embedding = unpack("E*",
$context_term_embedding);
$weight = exp(-1 * pow(($i - $j) / $std_deviation, 2));
$context_term_hash =
unpack('N', hash(self::HASH_ALGORITHM,
$context_term, true))[1] %
C\EMBEDDING_VECTOR_SIZE + 1;
$context_term_sign_hash =
hash(self::SIGN_HASH_ALGORITHM,$context_term, true);
$context_term_sign =
unpack('N', $context_term_sign_hash)[1] % 2 == 0 ?
-1 : 1;
$term_embedding[$context_term_hash] +=
$context_term_sign * $weight;
$context_term_embedding[$term_hash] +=
$term_sign * $weight;
$context_term_embedding = pack("E*",
...$context_term_embedding);
$this->updateTermEmbeddingCache($context_term_id,
$context_term_embedding, C\THREAD_RECOMMENDATION,
"thread_context_term_update");
}
$term_embedding = pack("E*", ...$term_embedding);
$this->updateTermEmbeddingCache($term_id, $term_embedding,
C\THREAD_RECOMMENDATION, "thread_term_update");
}
}
$item_count++;
}
return $item_terms;
}
/**
* Computes the item embeddings for individual items (main thread only and
* not comments) in groups feeds using the term embeddings for their terms.
* Additionally fetches the existing item embeddings from database and
* updates them if the term embeddings are updated for their terms
*
* @param array $item_terms terms in each item
* @return array $updated_item_embeddings containing embeddings for items
*/
public function computeItemEmbeddings($item_terms)
{
$db = $this->db;
$updated_item_embeddings = [];
$item_count = 0;
foreach ($item_terms as $item_id => [$terms, $group_id]) {
L\crawlTimeoutLog("Have done $item_count many group items");
$item_embedding = array_fill(1, C\EMBEDDING_VECTOR_SIZE, 0);
foreach ($terms as [$term_id, $term]) {
$term_embedding = $this->getTermEmbedding($term_id,
C\THREAD_RECOMMENDATION, true);
$term_embedding = unpack("E*", $term_embedding);
$item_embedding = LinearAlgebra::add($item_embedding,
$term_embedding);
}
$item_embedding = LinearAlgebra::normalize($item_embedding);
$item_embedding = pack("E*", ...$item_embedding);
$updated_item_embeddings[$item_id] = [$item_embedding, $group_id];
$item_count++;
}
$base_delete_sql = "DELETE FROM RECOMMENDATION_ITEM_EMBEDDING" .
" WHERE ITEM_TYPE = ? AND ID IN (";
$delete_sql = $base_delete_sql;
$base_insert_sql = "INSERT INTO RECOMMENDATION_ITEM_EMBEDDING VALUES ";
$insert_sql = $base_insert_sql;
$comma = "";
$insert_count = 0;
$total_insert = 0;
$item_type = C\THREAD_RECOMMENDATION;
foreach ($updated_item_embeddings as
$item_id => [$embedding, $parent_id]) {
L\crawlTimeoutLog("Have inserted $total_insert many group items");
$embedding = base64_encode($embedding);
$insert_sql .= "$comma($item_id, $item_type, " .
"'$embedding', $parent_id)";
$delete_sql .= "$comma $item_id";
$comma = ",";
$insert_count++;
$total_insert++;
if ($insert_count == C\BATCH_SQL_INSERT_NUM) {
$delete_sql .= ")";
$db->execute($delete_sql, [C\THREAD_RECOMMENDATION]);
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
$insert_count = 0;
$comma = "";
$delete_sql = $base_delete_sql;
$insert_sql = $base_insert_sql;
}
}
if ($insert_count > 0) {
$delete_sql .= ")";
$db->execute($delete_sql, [C\THREAD_RECOMMENDATION]);
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
}
return $updated_item_embeddings;
}
/**
* Computes the user embeddings based on the item embeddings which user have
* impression in ITEM_IMPRESSION_SUMMARY table for defined UPDATE_PERIOD
*
* @param array $item_embeddings embedding vectors of items
* @return array [$item_user_embedding, $user_items] user embeddings for
* items and the items id user have impression
*/
public function computeItemUserEmbeddings($item_embeddings)
{
$db = $this->db;
//SQLITE and MYSQL use GROUP_CONCAT, Postgres uses STRING_AGG
$db_list_function = in_array($db->to_upper_dbms, ["SQLITE3", "MYSQL"]) ?
"GROUP_CONCAT" : "STRING_AGG";
$timestamp = floor(time() / self::UPDATE_PERIOD ) * self::UPDATE_PERIOD;
$condition = "ITEM_TYPE = ? AND USER_ID <> 2 AND" .
" ((UPDATE_PERIOD = ? AND UPDATE_TIMESTAMP = ?) OR" .
" (UPDATE_PERIOD = ?))";
$impression_sql = "SELECT USER_ID, ".
"$db_list_function(CAST(ITEM_ID AS VARCHAR), ',') AS " .
"ITEM_IDS FROM ITEM_IMPRESSION_SUMMARY WHERE $condition " .
"GROUP BY USER_ID";
$results = $db->execute($impression_sql,
[C\THREAD_IMPRESSION, self::UPDATE_PERIOD, $timestamp,
C\MOST_RECENT_VIEW]);
$item_user_embeddings = [];
$user_items = [];
$user_count = 0;
while ($row = $db->fetchArray($results)) {
L\crawlTimeoutLog("Have done $user_count many user embeddings");
$user_id = $row['USER_ID'];
$item_ids = explode(",", $row['ITEM_IDS']);
$item_ids = array_unique($item_ids);
$item_user_embeddings[$user_id] = array_fill(1,
C\EMBEDDING_VECTOR_SIZE, 0);
$user_items[$user_id] = [];
foreach ($item_ids as $item_id) {
if (array_key_exists($item_id, $item_embeddings)) {
$item_embedding = unpack("E*",
$item_embeddings[$item_id][0]);
$item_user_embeddings[$user_id] = LinearAlgebra::add(
$item_user_embeddings[$user_id], $item_embedding);
$user_items[$user_id][] = $item_id;
}
}
$item_user_embeddings[$user_id] = LinearAlgebra::normalize(
$item_user_embeddings[$user_id]);
$item_user_embeddings[$user_id] = pack("E*",
...$item_user_embeddings[$user_id]);
$user_count++;
}
return [$item_user_embeddings, $user_items];
}
/**
* Computes the items recommendation for user based on the cosine similarity
* between user embeddings and item embeddings. Recommendations are
* calculated for the items user have not interacted with yet and items
* should be from the groups where the user is already a memeber
*
* @param array $item_embeddings embeddings vectors for items
* @param array $item_user_embeddings embeddings vectors for user
* @param array $user_items items id for user in impression table
* @return array $user_groups group ids where the user is a member
*/
public function computeItemUserRecommendations($item_embeddings,
$item_user_embeddings, $user_items)
{
L\crawlLog("...Computing User Item Similarity Scores.");
$db = $this->db;
//SQLITE and MYSQL use GROUP_CONCAT, Postgres uses STRING_AGG
$db_list_function = in_array($db->to_upper_dbms, ["SQLITE3", "MYSQL"]) ?
"GROUP_CONCAT" : "STRING_AGG";
$user_group_sql = "SELECT USER_ID, $db_list_function(" .
"CAST(GROUP_ID AS VARCHAR), ',') " .
"AS GROUP_IDS FROM USER_GROUP GROUP BY USER_ID";
$results = $db->execute($user_group_sql);
$user_groups = [];
while ($row = $db->fetchArray($results)) {
$user_id = $row['USER_ID'];
$group_ids = explode(",", $row['GROUP_IDS']);
$user_groups[$user_id] = $group_ids;
}
$item_user_recommendations = [];
$user_count = 0;
foreach ($item_user_embeddings as $user_id => $embedding) {
L\crawlTimeoutLog("Have done $user_count many user recommendation");
$embedding = unpack("E*", $embedding);
if (array_key_exists($user_id, $user_groups)) {
$item_count = 0;
foreach ($item_embeddings as
$item_id => [$item_embedding, $parent_id]) {
L\crawlTimeoutLog("Have done $item_count many items");
if (in_array($item_id, $user_items[$user_id]) ||
!in_array($parent_id, $user_groups[$user_id])) {
continue;
}
$item_embedding = unpack("E*", $item_embedding);
$similarity = LinearAlgebra::similarity(
$item_embedding, $embedding);
$item_user_recommendations[] = [$user_id,
$item_id, $similarity];
$item_count++;
}
}
$user_count++;
}
$delete_sql = "DELETE FROM GROUP_ITEM_RECOMMENDATION WHERE" .
" ITEM_TYPE = ?";
$db->execute($delete_sql, [C\THREAD_RECOMMENDATION]);
$base_insert_sql = "INSERT INTO GROUP_ITEM_RECOMMENDATION VALUES ";
$insert_sql = $base_insert_sql;
$comma = "";
$insert_count = 0;
$total_insert = 0;
$item_type = C\THREAD_RECOMMENDATION;
foreach ($item_user_recommendations as $recommendation) {
[$user_id, $item_id, $similarity] = $recommendation;
L\crawlTimeoutLog("Have inserted $total_insert recommendations");
$insert_sql .= "$comma($user_id, $item_id" .
", $item_type, $similarity, {$this->update_time})";
$comma = ",";
$insert_count++;
$total_insert++;
if ($insert_count == C\BATCH_SQL_INSERT_NUM) {
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
$insert_count = 0;
$comma = "";
$insert_sql = $base_insert_sql;
}
}
if ($insert_count > 0) {
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
}
return $user_groups;
}
/**
* Computes the group embeddings using the item embeddings for the items in
* a group. Additionally fetches the existing group embeddings from database
* and updates them if the item embeddings are updated
*
* @param array $item_embeddings embedding for the items
* @return array $updated_group_embeddings containing embeddings for groups
*/
public function computeGroupEmbeddings($item_embeddings)
{
$db = $this->db;
$updated_group_embeddings = [];
$group_count = 0;
foreach ($item_embeddings as $item_id => [$embedding, $parent_id]) {
L\crawlTimeoutLog("Have done $group_count many groups");
if (array_key_exists($parent_id, $updated_group_embeddings)) {
$embedding = unpack("E*", $embedding);
$group_embedding = unpack("E*",
$updated_group_embeddings[$parent_id]);
$updated_group_embeddings[$parent_id] = pack("E*",
...LinearAlgebra::add($embedding, $group_embedding));
} else {
$updated_group_embeddings[$parent_id] = $embedding;
}
$group_count++;
}
foreach ($updated_group_embeddings as $group_id => $embedding) {
$embedding = unpack("E*", $embedding);
$embedding = LinearAlgebra::normalize($embedding);
$updated_group_embeddings[$group_id] = pack("E*",
...$embedding);
}
$base_delete_sql = "DELETE FROM RECOMMENDATION_ITEM_EMBEDDING" .
" WHERE ITEM_TYPE = ? AND ID IN (";
$delete_sql = $base_delete_sql;
$base_insert_sql = "INSERT INTO RECOMMENDATION_ITEM_EMBEDDING VALUES ";
$insert_sql = $base_insert_sql;
$comma = "";
$insert_count = 0;
$total_insert = 0;
$item_type = C\GROUP_RECOMMENDATION;
foreach ($updated_group_embeddings as $group_id => $embedding) {
L\crawlTimeoutLog("Have inserted $total_insert group embeddings");
$embedding = serialize(unpack("E*", $embedding));
$insert_sql .= "$comma($group_id, $item_type, " .
"'$embedding', $group_id)";
$delete_sql .= "$comma $group_id";
$comma = ",";
$insert_count++;
$total_insert++;
if ($insert_count == C\BATCH_SQL_INSERT_NUM) {
$delete_sql .= ")";
$db->execute($delete_sql, [C\GROUP_RECOMMENDATION]);
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
$insert_count = 0;
$comma = "";
$delete_sql = $base_delete_sql;
$insert_sql = $base_insert_sql;
}
}
if ($insert_count > 0) {
$delete_sql .= ")";
$db->execute($delete_sql, [C\GROUP_RECOMMENDATION]);
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
}
return $updated_group_embeddings;
}
/**
* Computes the user embeddings based on the group embeddings which user
* have impression in ITEM_IMPRESSION_SUMMARY table for defined
* UPDATE_PERIOD or are a member in the group
*
* @param array $group_embeddings embedding vectors of groups
* @return array [$group_user_embedding, $user_groups] user embeddings for
* groups and the groups id user have membership
*/
public function computeGroupUserEmbeddings($group_embeddings)
{
$db = $this->db;
//SQLITE and MYSQL use GROUP_CONCAT, Postgres uses STRING_AGG
$db_list_function = in_array($db->to_upper_dbms, ["SQLITE3", "MYSQL"]) ?
"GROUP_CONCAT" : "STRING_AGG";
$timestamp = floor(time() / self::UPDATE_PERIOD ) * self::UPDATE_PERIOD;
$condition = "ITEM_TYPE = ? AND USER_ID <> 2 AND" .
" ((UPDATE_PERIOD = ? AND UPDATE_TIMESTAMP = ?) OR" .
" (UPDATE_PERIOD = ?))";
$impression_sql = "SELECT USER_ID, $db_list_function( ".
"CAST(ITEM_ID AS VARCHAR), ',') AS " .
"ITEM_IDS FROM ITEM_IMPRESSION_SUMMARY WHERE $condition " .
"GROUP BY USER_ID";
$results = $db->execute($impression_sql,
[C\GROUP_IMPRESSION, self::UPDATE_PERIOD, $timestamp,
C\MOST_RECENT_VIEW]);
$group_user_embeddings = [];
$user_groups = [];
$user_count = 0;
while ($row = $db->fetchArray($results)) {
L\crawlTimeoutLog("Have done $user_count many user embeddings");
$user_id = $row['USER_ID'];
$group_ids = explode(",", $row['ITEM_IDS']);
$group_ids = array_unique($group_ids);
$group_user_embeddings[$user_id] = array_fill(1,
C\EMBEDDING_VECTOR_SIZE, 0);
$user_groups[$user_id] = [];
$group_count = 0;
foreach ($group_ids as $group_id) {
L\crawlTimeoutLog("Have done $group_count many groups");
if (array_key_exists($group_id, $group_embeddings)) {
$embedding = unpack("E*",
$group_embeddings[$group_id]);
$group_user_embeddings[$user_id] = LinearAlgebra::add(
$group_user_embeddings[$user_id], $embedding);
$user_groups[$user_id][] = $group_id;
}
$group_count++;
}
$group_user_embeddings[$user_id] = pack("E*",
...LinearAlgebra::normalize($group_user_embeddings[$user_id]));
$user_count++;
}
return [$group_user_embeddings, $user_groups];
}
/**
* Computes the group recommendation for user based on the cosine similarity
* between user embeddings and group embeddings. Recommendations are
* calculated for the groups whic user has not interacted with yet and
* they are not member of that group
*
* @param array $group_embeddings embeddings vector for groups
* @param array $group_user_embeddings embeddings vector for users
* @param array $user_groups groups id for user having membership
* @param array $user_group_impression map user_id => list of group ids
* the user has already interacted with; used to exclude
* already-seen groups from the recommendations
*/
public function computeGroupUserRecommendations($group_embeddings,
$group_user_embeddings, $user_groups, $user_group_impression)
{
$db = $this->db;
$invite_groups_sql = "SELECT GROUP_ID FROM SOCIAL_GROUPS" .
" WHERE REGISTER_TYPE = ?";
$results = $db->execute($invite_groups_sql, [C\INVITE_ONLY_JOIN]);
$exclude_group_ids = [];
while ($row = $db->fetchArray($results)) {
$exclude_group_ids[] = $row['GROUP_ID'];
}
$group_user_recommendations = [];
$user_count = 0;
foreach ($group_user_embeddings as $user_id => $embedding) {
L\crawlTimeoutLog("Have done $user_count many user");
$embedding = unpack("E*", $embedding);
$group_count = 0;
foreach ($group_embeddings as $group_id => $group_embedding) {
L\crawlTimeoutLog("Have done $group_count many groups");
if (in_array($group_id, $exclude_group_ids) ||
in_array($group_id, ($user_groups[$user_id] ?? [])) ||
in_array($group_id,
($user_group_impression[$user_id] ?? []))) {
continue;
}
$group_embedding = unpack("E*", $group_embedding);
$similarity = LinearAlgebra::similarity($embedding,
$group_embedding);
$group_user_recommendations[] = [$user_id, $group_id,
$similarity];
$group_count++;
}
$user_count++;
}
$delete_sql = "DELETE FROM GROUP_ITEM_RECOMMENDATION WHERE" .
" ITEM_TYPE = ?";
$db->execute($delete_sql, [C\GROUP_RECOMMENDATION]);
$base_insert_sql = "INSERT INTO GROUP_ITEM_RECOMMENDATION VALUES ";
$insert_sql = $base_insert_sql;
$comma = "";
$insert_count = 0;
$total_insert = 0;
$item_type = C\GROUP_RECOMMENDATION;
foreach ($group_user_recommendations as $recommendation) {
L\crawlTimeoutLog("Have inserted $total_insert recommendations");
[$user_id, $group_id, $similarity] = $recommendation;
$insert_sql .= "$comma($user_id, $group_id" .
", $item_type, $similarity, {$this->update_time})";
$comma = ",";
$insert_count++;
$total_insert++;
if ($insert_count == C\BATCH_SQL_INSERT_NUM) {
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
$insert_count = 0;
$comma = "";
$insert_sql = $base_insert_sql;
}
}
if ($insert_count > 0) {
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
}
}
/**
* Manages the whole process of computing wiki resource recommendations
* for users. Makes a series of calls to handle parts of this computation
* before synthesizing the result
*/
public function computeWikiResourceRecommendations()
{
L\crawlLog("...Start fetching descriptions for the wiki page " .
"resources...");
[$descriptions, $resource_metadata] = $this->
getWikiResourceDescriptions();
L\crawlLog("...Finished fetching descriptions for the wiki page " .
"resources...");
L\crawlLog("...Start computing wiki term embeddings...");
[$resource_terms, $meta_details_terms] =
$this->computeWikiTermEmbeddings($descriptions);
L\crawlLog("...Finished computing wiki term embeddings...");
L\crawlLog("...Start computing wiki resource embeddings...");
$item_embeddings = $this->computeWikiResourceEmbeddings($resource_terms,
$meta_details_terms);
L\crawlLog("...Finished computing wiki resource embeddings...");
unset($resource_terms);
unset($meta_details_terms);
L\crawlLog("...Start write back term embeddings from cache to db");
$this->saveTermEmbeddingsCacheToDb(C\RESOURCE_RECOMMENDATION);
L\crawlLog("...Finished write back term embeddings from cache to db");
L\crawlLog("...Start computing wiki user embeddings...");
[$user_embeddings, $user_items] = $this->computeWikiUserEmbeddings(
$item_embeddings);
L\crawlLog("...Finished computing wiki user embeddings...");
L\crawlLog("...Start computing wiki resource recommendations...");
$this->computeWikiUserRecommendations($item_embeddings,
$user_embeddings, $user_items, $resource_metadata);
L\crawlLog("...Done computing wiki resource recommendations...");
unset($user_embeddings);
unset($user_items);
unset($item_embeddings);
unset($resource_metadata);
}
/**
* Fetches the description for the eligible wiki resources having the root
* folder path captured in RECOMMENDATION_FILE
*
* @return array $descriptions of resources
*/
public function getWikiResourceDescriptions()
{
$thumb_folders = [];
if (file_exists(self::RECOMMENDATION_FILE)) {
$thumb_folders = explode("\n",
file_get_contents(self::RECOMMENDATION_FILE));
}
$thumb_folders = array_unique($thumb_folders);
$thumb_folders_copy = $thumb_folders;
$descriptions = [];
$resource_metadata = [];
foreach ($thumb_folders as $thumb_folder) {
array_shift($thumb_folders_copy);
if (empty($thumb_folder)) {
continue;
}
list($group_id, $page_id, $folder) = explode("###", $thumb_folder);
$folder = trim($folder, " \n\r\t\v\x00");
$files = $this->getDescriptionFiles($folder);
foreach ($files as $file) {
$resource_file = substr($file, 0, strlen($file) - 4);
$resource_id = unpack('n', md5($group_id . $page_id .
$resource_file, true))[1];
if (array_key_exists($resource_id, $descriptions)) {
continue;
}
$description = file_get_contents($file);
if (strcmp($description, "Description search sources".
" failed to find description.") == 0) {
continue;
}
$descriptions[$resource_id] = $description;
$resource_metadata[$resource_id] = [$group_id,
$page_id, $resource_file];
if (count($descriptions) >= self::MAX_BATCH_SIZE) {
L\crawlLog("Reached max resources limit");
file_put_contents(self::RECOMMENDATION_FILE,
implode(PHP_EOL, $thumb_folders_copy));
return [$descriptions, $resource_metadata];
}
}
}
return [$descriptions, $resource_metadata];
}
/**
* Returns all the resource description files in a given thumb folder and
* also recursively scan through subfolders if any
*
* @param string $thumb_folder path of a thumb folder
* @return array $files list of description files path in given folder
*/
public function getDescriptionFiles($thumb_folder)
{
if (!is_dir($thumb_folder)) {
return [];
}
$exclude_files = [".", "..", "needs_description.txt",
"subfolder_counts.txt", ".DS_Store"];
$files = scandir($thumb_folder);
$file_paths = [];
foreach ($files as $file) {
if (in_array($file, $exclude_files)) {
continue;
}
$to_process = $thumb_folder . "/" . $file;
if (is_dir($to_process)) {
L\crawlLog("...$to_process is a folder," .
" looking files inside it...");
$sub_file_paths = $this->getDescriptionFiles($to_process);
$file_paths = array_merge($file_paths, $sub_file_paths);
} else {
$file_paths[] = $to_process;
}
}
return $file_paths;
}
/**
* Computes the embedding for new terms in the description of wiki
* resources and updates the embedding of existing terms using Hash2Vec
* approach
*
* @param array $descriptions of resources
* @return array [$resource_terms, $meta_details_term]
*/
public function computeWikiTermEmbeddings($descriptions)
{
$db = $this->db;
$this->lru_cache = new LRUCache(self::MAX_TERM_EMBEDDINGS);
$select_sql = "SELECT * FROM RECOMMENDATION_TERM_EMBEDDING WHERE " .
"ITEM_TYPE = ? " . $db->limitOffset(self::MAX_TERM_EMBEDDINGS);
$results = $db->execute($select_sql, [C\RESOURCE_RECOMMENDATION]);
$resource_terms = [];
$meta_details_terms = [];
while ($row = $db->fetchArray($results)) {
if (is_string($row['VECTOR'])) {
$this->lru_cache->put($row['ID'],
base64_decode($row['VECTOR'], true));
} else {
var_dump($row);
}
}
$context_distance_sum = (self::CONTEXT_WINDOW_LENGTH *
(self::CONTEXT_WINDOW_LENGTH + 1)) / 2.0;
$mean = $context_distance_sum / self::CONTEXT_WINDOW_LENGTH;
$carry = 0.0;
for ($i = 1; $i <= self::CONTEXT_WINDOW_LENGTH; $i++) {
$difference = $i - $mean;
$carry += $difference * $difference;
}
$std_deviation = sqrt($carry / self::CONTEXT_WINDOW_LENGTH);
$resource_count = 0;
foreach ($descriptions as $resource_id => $description) {
L\crawlTimeoutLog("Have processed $resource_count many resources");
$resource_terms[$resource_id] = [];
$meta_details_terms[$resource_id] = [];
$description_parts = explode("\n", $description);
foreach ($description_parts as $description_part) {
$description_part = mb_strtolower($description_part);
$terms = $this->cleanRemoveStopWords($description_part, true);
if (count($terms) < self::CONTEXT_WINDOW_LENGTH) {
$meta_details_terms[$resource_id] = array_merge($terms,
$meta_details_terms[$resource_id]);
} else {
$resource_terms[$resource_id] = array_merge($terms,
$resource_terms[$resource_id]);
}
}
if (count($resource_terms[$resource_id]) > 0) {
$terms = $resource_terms[$resource_id];
$num_terms = count($terms);
for ($i = 0; $i < $num_terms; $i++) {
L\crawlTimeoutLog("Have processed $i of $num_terms terms");
[$term_id, $term] = $terms[$i];
$term_hash = unpack('N', hash(self::HASH_ALGORITHM, $term,
true))[1] % C\EMBEDDING_VECTOR_SIZE + 1;
$term_sign_hash = hash(self::SIGN_HASH_ALGORITHM,
$term, true);
$term_sign = unpack('N', $term_sign_hash)[1]
% 2 == 0 ? -1 : 1;
$term_embedding = $this->getTermEmbedding($term_id,
C\RESOURCE_RECOMMENDATION);
$term_embedding = unpack("E*", $term_embedding);
for ($j = $i - 1; $j >= 0 &&
$j >= $i - self::CONTEXT_WINDOW_LENGTH; $j--) {
[$context_term_id, $context_term] = $terms[$j];
$context_term_embedding = $this->getTermEmbedding(
$context_term_id, C\RESOURCE_RECOMMENDATION);
$context_term_embedding = unpack("E*",
$context_term_embedding);
$weight = exp(-1 * pow(($i - $j) / $std_deviation, 2));
$context_term_hash = unpack('N', hash(
self::HASH_ALGORITHM, $context_term, true))[1] %
C\EMBEDDING_VECTOR_SIZE + 1;
$context_term_sign_hash = hash(
self::SIGN_HASH_ALGORITHM, $context_term, true);
$context_term_sign = unpack('N',
$context_term_sign_hash)[1] % 2 == 0 ? -1 : 1;
$term_embedding[$context_term_hash] +=
$context_term_sign * $weight;
$context_term_embedding[$term_hash] +=
$term_sign * $weight;
$context_term_embedding = pack("E*",
...$context_term_embedding);
$this->updateTermEmbeddingCache($context_term_id,
$context_term_embedding, C\RESOURCE_RECOMMENDATION,
"resource_context_term_update");
}
$term_embedding = pack("E*", ...$term_embedding);
$this->updateTermEmbeddingCache($term_id, $term_embedding,
C\RESOURCE_RECOMMENDATION, "resource_term_update");
}
$resource_count++;
}
}
return [$resource_terms, $meta_details_terms];
}
/**
* Split the given text into terms, clean the terms by removing non
* alphanumeric characters and remove the stop terms in order to reduce the
* noise while calculating the embeddings
*
* @param string $text which needs to be processed
* @param boolean $description_stop_word_flag to remove
* words present in DESCRIPTION_STOP_WORDS
* @return array $terms [term_id, term] term_id calculated using md5 hash
* for the term
*/
public function cleanRemoveStopWords($text,
$description_stop_word_flag = false)
{
$raw_terms = preg_split("/[\s,\/\._-]+/", $text);
$terms = [];
foreach ($raw_terms as $term) {
$term = preg_replace("/\W/", "", $term);
$term = preg_replace("/&rsquo/", "'", $term);
$term = str_replace(['"', "'"], "", $term);
if (strlen($term) > 0) {
$terms[] = $term;
}
}
$text_locale = L\guessLocaleFromString($text);
$stop_obj = PhraseParser::getTokenizer($text_locale);
if ($stop_obj && method_exists($stop_obj, "stoptermsRemover")) {
$terms = $stop_obj->stoptermsRemover($terms);
}
$term_ids = [];
foreach ($terms as $term) {
if ($description_stop_word_flag &&
in_array($term, self::DESCRIPTION_STOP_WORDS)) {
continue;
}
$term_id = L\canonicalTerm($term);
$term_ids[] = [$term_id, $term];
}
return $term_ids;
}
/**
* Computes the embeddings for wiki page resources using the calculated
* term embeddings and add the metadata details separately to the embeddings
*
* @param array $resource_terms of processed terms from resource description
* @param array $meta_details_terms of raw resource descriptions
* @return array $updated_item_embeddings array of updated wiki resource
* embeddings
*/
public function computeWikiResourceEmbeddings($resource_terms,
$meta_details_terms)
{
$db = $this->db;
$updated_item_embeddings = [];
$resource_count = 0;
foreach ($resource_terms as $resource_id => $terms) {
L\crawlTimeoutLog("Have processed $resource_count many resources");
$item_embedding = array_fill(1, C\EMBEDDING_VECTOR_SIZE, 0);
foreach ($terms as [$term_id, $term]) {
$term_embedding = $this->getTermEmbedding($term_id,
C\RESOURCE_RECOMMENDATION, true);
$term_embedding = unpack("E*", $term_embedding);
$item_embedding = LinearAlgebra::add($item_embedding,
$term_embedding);
}
$updated_item_embeddings[$resource_id] = pack("E*",
...$item_embedding);
$resource_count++;
}
foreach ($meta_details_terms as $resource_id => $meta_terms) {
if (!array_key_exists($resource_id, $updated_item_embeddings)) {
$item_embedding = array_fill(1, C\EMBEDDING_VECTOR_SIZE, 0);
} else {
$item_embedding = unpack("E*",
$updated_item_embeddings[$resource_id]);
}
foreach ($meta_terms as [$meta_term_id, $meta_term]) {
if (strlen($meta_term) <= 1) {
continue;
}
$meta_term_hash = unpack('N', hash(self::HASH_ALGORITHM,
$meta_term, true))[1] % C\EMBEDDING_VECTOR_SIZE + 1;
$sign_hash = hash(self::SIGN_HASH_ALGORITHM, $meta_term, true);
$sign = unpack('N', $sign_hash)[1] % 2 == 0 ? -1 : 1;
$item_embedding[$meta_term_hash] += $sign * 1.0;
}
$updated_item_embeddings[$resource_id] = pack("E*",
...$item_embedding);
}
foreach ($updated_item_embeddings as $item_id => $embedding) {
$embedding = unpack("E*", $embedding);
$updated_item_embeddings[$item_id] = pack("E*",
...LinearAlgebra::normalize($embedding));
}
$delete_sql = "DELETE FROM RECOMMENDATION_ITEM_EMBEDDING WHERE" .
" ITEM_TYPE = ?";
$db->execute($delete_sql, [C\RESOURCE_RECOMMENDATION]);
$base_insert_sql = "INSERT INTO RECOMMENDATION_ITEM_EMBEDDING VALUES ";
$insert_sql = $base_insert_sql;
$comma = "";
$insert_count = 0;
$total_insert = 0;
$item_type = C\RESOURCE_RECOMMENDATION;
foreach ($updated_item_embeddings as $resource_id => $embedding) {
L\crawlTimeoutLog("Have inserted $total_insert many resources");
$embedding = base64_encode($embedding);
$insert_sql .= "$comma($resource_id, $item_type," .
" '$embedding', $resource_id)";
$comma = ",";
$insert_count++;
$total_insert++;
if ($insert_count == C\BATCH_SQL_INSERT_NUM) {
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
$insert_count = 0;
$comma = "";
$insert_sql = $base_insert_sql;
}
}
if ($insert_count > 0) {
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
}
return $updated_item_embeddings;
}
/**
* Computes user embeddings for wiki resources based on the user's resources
* impression logged in ITEM_IMPRESSION_SUMMARY table for the defined update
* period
*
* @param array $item_embeddings of wiki page resources embedding
* @return array [$user_embeddings, $user_items] of user embeddings
* for wiki resources and the user resource impression
*/
public function computeWikiUserEmbeddings($item_embeddings)
{
$db = $this->db;
//SQLITE and MYSQL use GROUP_CONCAT, Postgres uses STRING_AGG
$db_list_function = in_array($db->to_upper_dbms, ["SQLITE3", "MYSQL"]) ?
"GROUP_CONCAT" : "STRING_AGG";
$timestamp = floor(time() / self::UPDATE_PERIOD ) * self::UPDATE_PERIOD;
$condition = "ITEM_TYPE = ? AND USER_ID <> 2 AND" .
" ((UPDATE_PERIOD = ? AND UPDATE_TIMESTAMP = ?) OR" .
" (UPDATE_PERIOD = ?))";
$impression_sql = "SELECT USER_ID, $db_list_function( " .
"CAST(ITEM_ID AS VARCHAR), ',') AS " .
"ITEM_IDS FROM ITEM_IMPRESSION_SUMMARY WHERE $condition " .
"GROUP BY USER_ID";
$results = $db->execute($impression_sql,
[C\RESOURCE_IMPRESSION, self::UPDATE_PERIOD, $timestamp,
C\MOST_RECENT_VIEW]);
$user_embeddings = [];
$user_items = [];
$user_count = 0;
while ($row = $db->fetchArray($results)) {
L\crawlTimeoutLog("Have processed $user_count many users");
$user_id = $row['USER_ID'];
$item_ids = explode(",", $row['ITEM_IDS']);
$item_ids = array_unique($item_ids);
$user_embeddings[$user_id] = array_fill(1,
C\EMBEDDING_VECTOR_SIZE, 0);
$user_items[$user_id] = [];
foreach ($item_ids as $item_id) {
if (array_key_exists($item_id, $item_embeddings)) {
$embedding = unpack("E*",
$item_embeddings[$item_id]);
$user_embeddings[$user_id] = LinearAlgebra::add(
$user_embeddings[$user_id], $embedding);
$user_items[$user_id][] = $item_id;
}
}
$user_embeddings[$user_id] = pack("E*",
...LinearAlgebra::normalize($user_embeddings[$user_id]));
$user_count++;
}
return [$user_embeddings, $user_items];
}
/**
* Computes the wiki resource recommendations based on cosine similarity
* between resource embeddings and user embeddings
*
* @param array $item_embeddings of wiki resources embeddings
* @param array $user_embeddings of users consumed wiki resources
* embeddings
* @param array $user_items of users consumed wiki resources
* @param array $resource_metadata map resource_id => [group_id,
* page_id, sub_path] for resolving the resource back to its
* source page when recording a recommendation
*/
public function computeWikiUserRecommendations($item_embeddings,
$user_embeddings, $user_items, $resource_metadata)
{
$db = $this->db;
$recommendations = [];
$user_count = 0;
foreach ($user_embeddings as $user_id => $user_embedding) {
L\crawlTimeoutLog("Have processed $user_count many users");
$user_embedding = unpack("E*", $user_embedding);
$resource_count = 0;
foreach ($item_embeddings as $item_id => $item_embedding) {
L\crawlTimeoutLog("Have processed $resource_count resources");
if (in_array($item_id, $user_items[$user_id]) ||
!array_key_exists($item_id, $resource_metadata)) {
continue;
}
$item_embedding = unpack("E*", $item_embedding);
$similarity = LinearAlgebra::similarity($user_embedding,
$item_embedding);
list($group_id, $page_id, $resource_path) =
$resource_metadata[$item_id];
unset($resource_metadata[$item_id]);
$recommendations[] = [$user_id, $group_id, $page_id,
$resource_path, $similarity, $item_id];
$resource_count++;
}
$user_count++;
}
$delete_sql = "DELETE FROM GROUP_RESOURCE_RECOMMENDATION";
$db->execute($delete_sql);
$base_insert_sql = "INSERT INTO GROUP_RESOURCE_RECOMMENDATION " .
"VALUES ";
$insert_sql = $base_insert_sql;
$comma = "";
$insert_count = 0;
$total_insert = 0;
foreach ($recommendations as $recommendation) {
L\crawlTimeoutLog("Have inserted $total_insert recommendations");
list($user_id, $group_id, $page_id, $resource_path,
$score, $item_id) = $recommendation;
$time = $this->update_time;
$insert_sql .= "$comma($user_id, $group_id, $page_id, " .
"'$resource_path', $score, $time, $item_id)";
$comma = ",";
$insert_count++;
$total_insert++;
if ($insert_count == C\BATCH_SQL_INSERT_NUM) {
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
$insert_count = 0;
$comma = "";
$insert_sql = $base_insert_sql;
}
}
if ($insert_count > 0) {
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
}
}
/**
* Returns the term embedding either from LRU cache or database.
* A stored vector that is shorter than the configured embedding
* size (for example one written before the size was increased,
* or one that failed to decode) is padded with zeros up to the
* current size, so callers always receive a full-length vector
* and never index past its end.
*
* @param int $term_id ID of the term whose embedding vector
* should be returned
* @param int $item_type ITEM_TYPE the term belongs to (group item,
* wiki resource, etc.); namespaces $term_id in the
* RECOMMENDATION_TERM_EMBEDDING table
* @param boolean $update indicates whether to update the cache
* @return string $term_embedding
*/
public function getTermEmbedding($term_id, $item_type, $update = false)
{
$db = $this->db;
$term_embedding = $this->lru_cache->get($term_id);
if (!isset($term_embedding)) {
$sql = "SELECT VECTOR FROM RECOMMENDATION_TERM_EMBEDDING " .
"WHERE ITEM_TYPE = ? AND ID = ? " . $db->limitOffset(1);
$result = $db->execute($sql, [$item_type, $term_id]);
$row = null;
if ($result) {
$row = $db->fetchArray($result);
}
if (!$row || !is_string($row['VECTOR'])) {
$term_embedding = pack("E*", ...array_fill(1,
C\EMBEDDING_VECTOR_SIZE, 0.0));
} else {
$db->closeCursor($result);
$decoded = base64_decode($row['VECTOR'], true);
$values = ($decoded === false) ? [] :
unpack("E*", $decoded);
$values += array_fill(1, C\EMBEDDING_VECTOR_SIZE, 0.0);
$term_embedding = pack("E*", ...$values);
}
}
if ($update) {
$this->updateTermEmbeddingCache($term_id, $term_embedding,
$item_type, "get_term_embedding");
}
return $term_embedding;
}
/**
* Updates LRU cache of term embeddings and save the evicted
* embedding back to database
*
* @param int $term_id ID column for the term whose embedding is
* being inserted into the cache
* @param string $term_embedding raw vector bytes for the term being
* inserted; will be base64-encoded before being written to the DB
* @param int $item_type ITEM_TYPE column value associated with this
* embedding (e.g. group, thread, post)
* @param string $message optional log-message prefix prepended to
* the eviction message that gets attached to $db->pre_message
*/
public function updateTermEmbeddingCache($term_id, $term_embedding,
$item_type, $message = "")
{
$db = $this->db;
$evicted_item = $this->lru_cache->put($term_id, $term_embedding);
if (isset($evicted_item)) {
$on_conflict = in_array($db->to_upper_dbms, ["MYSQL"]) ?
" ON DUPLICATE KEY " :
" ON CONFLICT (ITEM_TYPE, ID) DO UPDATE ";
$sql = "INSERT INTO RECOMMENDATION_TERM_EMBEDDING VALUES ".
"(?, ?, ?) $on_conflict SET VECTOR = ?";
$vector = base64_encode($evicted_item[1]);
$db->pre_message = "$message {$evicted_item[0]} was evicted";
$db->execute($sql, [$evicted_item[0], $item_type,
$vector, $vector]);
}
}
/**
* Writes back the term embeddings in cache to database and free up memory
*
* @param int $item_type value for ITEM_TYPE column
*/
public function saveTermEmbeddingsCacheToDb($item_type)
{
L\crawlLog("Doing final persistence flush of LRU cache for ".
"$item_type");
$db = $this->db;
$base_delete_sql = "DELETE FROM RECOMMENDATION_TERM_EMBEDDING" .
" WHERE ITEM_TYPE = ? AND ID IN (";
$delete_sql = $base_delete_sql;
$base_insert_sql = "INSERT INTO RECOMMENDATION_TERM_EMBEDDING VALUES ";
$insert_sql = $base_insert_sql;
$comma = "";
$insert_count = 0;
$total_insert = 0;
foreach ($this->lru_cache->getAll() as $id => $embedding) {
L\crawlTimeoutLog("Have inserted $total_insert many embeddings");
$embedding = base64_encode($embedding);
$insert_sql .= "$comma('$id', $item_type, '$embedding')";
$delete_sql .= "$comma '$id'";
$comma = ",";
$insert_count++;
$total_insert++;
if ($insert_count == C\BATCH_SQL_INSERT_NUM) {
$delete_sql .= ")";
$db->execute($delete_sql, [$item_type]);
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
$insert_count = 0;
$comma = "";
$delete_sql = $base_delete_sql;
$insert_sql = $base_insert_sql;
}
}
if ($insert_count > 0) {
$delete_sql .= ")";
$db->execute($delete_sql, [$item_type]);
$insert_sql = $db->insertIgnore($insert_sql);
$db->execute($insert_sql);
}
unset($this->lru_cache);
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software