<?php
/**
* SeekQuarry/Yioop --
* Open Source Pure PHP Search Engine, Crawler, and Indexer
*
* Copyright (C) 2009 - 2026 Chris Pollett chris@pollett.org
*
* LICENSE:
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
* END LICENSE
*
* @author Chris Pollett chris@pollett.org
* @license https://www.gnu.org/licenses/ GPL3
* @link https://www.seekquarry.com/
* @copyright 2009 - 2026
* @filesource
*/
namespace seekquarry\yioop\library;
use seekquarry\yioop\library\RequestScope;
use seekquarry\yioop\configs as C;
/** For Yioop global defines */
require_once __DIR__."/../configs/Config.php";
/**
* Library of functions used to manipulate and to extract components from urls
*
* @author Chris Pollett
*/
class UrlParser
{
/**
* The name the list of names this machine answers to is kept under,
* among the values that belong to one request. The list holds the
* name the request arrived on, which differs between requests where
* a site serves several domains.
*/
const LOCALHOSTS_KEPT_UNDER = "localhosts";
/**
* Checks if the url scheme is either http, https, or gopher (old protocol
* but somewhat geeky-cool to still support).
*
* @param string $url the url to check
* @return bool returns true if it is either http,https, or gopher and false
* otherwise
*/
public static function isSchemeCrawlable($url)
{
$url_parts = @parse_url($url);
if (isset($url_parts['scheme']) && !in_array($url_parts['scheme'],
["http", "https", "gopher"])) {
return false;
}
return true;
}
/**
* Converts a url with a scheme into one without. Also removes trailing
* slashes from url. Shortens url to desired length by inserting ellipsis
* for part of it if necessary
*
* @param string $url the url to trim
* @param int $max_len length to shorten url to, 0 = no shortening
* @return string the trimmed url
*/
public static function simplifyUrl($url, $max_len = 0)
{
$url_parts = @parse_url($url);
if (isset($url_parts['scheme'])) {
$scheme_len = strlen($url_parts['scheme']);
$url = mb_substr($url, $scheme_len + 3);
}
$len = strlen($url);
if (isset($url[$len - 1]) && $url[$len - 1] == "/") {
$url = mb_substr($url, 0, $len - 1);
}
if ($len > 0 && $len > $max_len && $max_len > 0) {
$front_len = ceil($max_len * 0.8);
$end_len = ceil($max_len * 0.2);
$url = mb_substr($url, 0, $front_len)."...".
mb_substr($url, -$end_len);
}
$url = urldecode($url);
$url = mb_ereg_replace(" ", "%20", $url);
return $url;
}
/**
* Get the port number of a url if present; if not return 80
*
* @param string $url the url to extract port number from
* @return int a port number
*/
public static function getPort($url)
{
$url_parts = @parse_url($url);
if (isset($url_parts['port'])) {
return $url_parts['port'];
}
if (isset($url_parts['scheme']) &&
$url_parts['scheme'] == 'https') {
return 443;
}
return 80;
}
/**
* Get the scheme of a url if present; if not return http
*
* @param string $url the url to extract scheme from
* @return int a port number
*/
public static function getScheme($url)
{
$scheme = substr($url, 0, strpos($url, ":"));
if ($scheme) {
return $scheme;
}
return "http";
}
/**
* Attempts to guess the language tag based on url
*
* @param string $url the url to parse
* @return the top level domain if present; false otherwise
*/
public static function getLang($url)
{
$LANG = [
'us' => 'en',
"uk" => 'en',
"ca" => 'en',
"au" => 'en',
"bz" => 'en',
"ie" => 'en',
"jm" => 'en',
"nz" => 'en',
"za" => 'en',
"zw" => 'en',
"tt" => 'en',
"eg" => 'ar',
"dz" => 'ar',
"bh" => 'ar',
"jo" => 'ar',
"kw" => 'ar',
"lb" => 'ar',
"iq" => 'ar',
"ma" => 'ar',
"om" => 'ar',
"qa" => 'ar',
"sa" => 'ar',
"sy" => 'ar',
"tn" => 'ar',
"ae" => 'ar',
"ye" => 'ar',
"de" => 'de',
"at" => "de",
"es" => 'es',
"ar" => 'es',
"bo" => 'es',
"cl" => 'es',
"co" => 'es',
"cr" => 'es',
"dr" => 'es',
"ec" => 'es',
"sv" => 'es',
"gt" => 'es',
"hn" => 'es',
"mx" => 'es',
"ni" => 'es',
"pa" => 'es',
"py" => 'es',
"pe" => 'es',
"pr" => 'es',
"uy" => 'es',
"ve" => 'es',
"fr" => 'fr-FR',
"be" => 'fr-FR',
"lu" => 'fr-FR',
"hk" => 'zh-CN',
"id" => 'in-ID',
"il" => 'he',
"ir" => 'fa',
"it" => 'it',
"jp" => 'ja',
"kp" => 'ko',
"kr" => 'ko',
"pl" => 'pl',
"br" => 'pt',
"pt" => 'pt',
"qc" => 'fr',
"ru" => 'ru',
"sg" => 'zh-CN',
"th" => 'th',
"tr" => 'tr',
"tw" => 'zh-CN',
"vi" => 'vi-VN',
"vn" => 'vi-VN',
"cn" => 'zh-CN',
];
$host = @parse_url($url, \PHP_URL_HOST);
if (!$host) {
return false;
}
$host_parts = explode(".", $host);
$count = count($host_parts);
if ($count > 0) {
$tld = $host_parts[$count - 1];
if ($tld == 'ca' && isset($host_parts[$count - 2]) &&
$host_parts[$count - 2] == 'qc') {
$tld = 'qc';
}
if (isset($LANG[$tld])) {
return $LANG[$tld];
}
}
/* this is to guess language from domains like:
it.wikipedia.org
*/
if (in_array($tld, ["com", "edu", "gov", "mil", "org", "net"])) {
if (in_array($host_parts[0], ['az', 'ar', 'ba', 'be', 'bg', 'bn',
'bo', 'br', 'bs', 'ca', 'co', 'cs', 'cy', 'da', 'de', 'ds',
'dv', 'el', 'en', 'es', 'eu', 'fa', 'fi', 'fil', 'fo', 'fr',
'fy', 'ga', 'gd', 'gl', 'gsw', 'gu', 'ha', 'he', 'hi', 'hr',
'hsb', 'hu', 'hy', 'id', 'ig', 'ii', 'is', 'it', 'iu', 'ja',
'ka', 'kk', 'kl', 'km', 'kn', 'kok', 'ko', 'ky', 'lb', 'lo',
'lt', 'lv', 'mi', 'mk', 'ml', 'mn', 'moh', 'mr', 'ms', 'mt',
'nb', 'ne', 'nl', 'nn', 'nso', 'oc', 'or', 'pa', 'pl', 'prs',
'ps', 'pt', 'qut', 'quz', 'rm', 'ro', 'ru', 'rw', 'sah', 'sa',
'se', 'si', 'sk','sl','sma','smj','smn','sms','sq','sr','sv',
'sw','syr','ta', 'te','tg','th','tk','tn', 'tr', 'tt', 'tzm',
'ug', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh','yo', 'zh', 'zu'])) {
return $host_parts[0];
} else {
return "en";
}
}
return null;
}
/**
* Used to guess the file size in bytes of the file that a url is pointed at
* based on its file type.
*
* @param string $url to estimate the size of
* @return int estimated number of bytes
*/
public static function guessFileSizeFromUrl($url)
{
$size_map = [
"aac" => 3000000,
"asp" => 300000,
"aspx" => 300000,
"bmp" => 1000000,
"bz" => 10000000,
"c" => 450000,
"cc" => 450000,
"cfm" => 300000,
"cfml" => 300000,
"cgi" => 300000,
"cs" => 300000,
"css" => 300000,
"csv" => 3000000,
"doc" => 3000000,
"docx" => 3000000,
"epub" => 3000000,
"gif" => 1000000,
"gz" => 10000000,
"html" => 300000,
"ico" => 1000000,
"jpg" => 1000000,
"js" => 450000,
"jsp" => 300000,
"mp3" => 3000000,
"mp4" => 50000000,
"png" => 1000000,
"pdf" => 3000000,
"php" => 300000,
"pl" => 300000,
"ppt" => 3000000,
"pptx" => 3000000,
"py" => 300000,
"rss" => 300000,
"rtf" => 3000000,
"shtml" => 300000,
"svg" => 1000000,
"tab" => 3000000,
"tsv" => 3000000,
"txt" => 300000,
"webp" => 1000000,
"xlsx" => 3000000,
"xlsx" => 3000000,
"xml" => 5000000,
];
$file_type = UrlParser::getDocumentType($url);
return $size_map[$file_type] ?? 1000000;
}
/**
* Get the host name portion of a url if present; if not return false
*
* @param string $url the url to parse
* @param bool $with_login_and_port whether to include user,password,port
* if present
* @return the host portion of the url if present; false otherwise
*/
public static function getHost($url, $with_login_and_port = true)
{
if (empty($url)) {
$url_parts = [];
} else {
$url_parts = @parse_url($url);
}
if (!isset($url_parts['scheme']) ) {
return false;
}
$host_url = $url_parts['scheme'].'://';
//handles common typo http:/yahoo.com rather than http://yahoo.com
if (!isset($url_parts['host'])) {
if (isset($url_parts['path'])) {
$url_parts = @parse_url($url_parts['scheme'].":/".
$url_parts['path']);
if (!isset($url_parts['host'])) {
return false;
}
} else {
return false;
}
}
if ($with_login_and_port &&
isset($url_parts['user']) && isset($url_parts['pass'])) {
$host_url .= $url_parts['user'].":".$url_parts['pass']."@";
}
if (strlen($url_parts['host']) <= 0) {
return false;
}
$host_url .= $url_parts['host'];
if ($with_login_and_port && isset($url_parts['port'])) {
$host_url .= ":".$url_parts['port'];
}
return $host_url;
}
/**
* Gets the domain of a url less any leading www
* @param string $url to get domain of
* @return string the base domain as defined above
*/
public static function getBaseDomain($url)
{
$host = UrlParser::getHost($url, false);
$scheme = UrlParser::getScheme($url);
$domain = substr($host, strlen($scheme) + 3);
$start_four = substr($domain, 0 , 4);
if ($start_four == "www.") {
return substr($domain, 4);
}
return $domain;
}
/**
* Get the path portion of a url if present; if not return null
*
* @param string $url the url to parse
* @param bool $with_query_string (whether to also include the query
* string at the end of the path)
* @return the host portion of the url if present; null otherwise
*/
public static function getPath($url, $with_query_string = false)
{
if (empty($url)) {
$url_parts = [];
} else {
$url_parts = @parse_url($url);
}
if (!isset($url_parts['path'])) {
return null;
}
// windows hack
$url_parts['path'] = str_replace("\/", "/", $url_parts['path']);
$path = $url_parts['path'];
$len = strlen($url);
if ($len < 1) {
return null;
}
if ($with_query_string && isset($url_parts['query'])) {
$path .= "?".$url_parts['query'];
} else if ($with_query_string && $url[$len - 1] == "?") {
$path .= "?"; //handle blank query string case
}
return $path;
}
/**
* Returns as a two element array the host and path of a url
*
* @param string $url initial url to get host and path of
* @param bool $with_login_and_port controls whether the host should
* should contain login and port info
* @param bool $with_query_string says whether the path should contain
* the query string as well
* @return array host and the path as a pair
*/
public static function getHostAndPath($url, $with_login_and_port = true,
$with_query_string = false)
{
$url_parts = @parse_url($url);
$path = (isset($url_parts['path'])) ? $url_parts['path'] : false;
if (!isset($url_parts['scheme']) ) {return [false, false];}
$host_url = $url_parts['scheme'].'://';
//handles common typo http:/yahoo.com rather than http://yahoo.com
if (!isset($url_parts['host'])) {
if ($path) {
$url_parts = @parse_url($url_parts['scheme'].":/".
$path);
if (!isset($url_parts['host'])) {
return [false, false];
}
$path = (isset($url_parts['path'])) ? $url_parts['path'] :false;
} else {
return [false, false];
}
}
if ($with_login_and_port &&
isset($url_parts['user']) && isset($url_parts['pass'])) {
$host_url .= $url_parts['user'].":".$url_parts['pass']."@";
}
if (strlen($url_parts['host']) <= 0) { return [false, false]; }
$host_url .= $url_parts['host'];
if ($with_login_and_port && isset($url_parts['port'])) {
$host_url .= ":".$url_parts['port'];
}
if (!$path) {
return [$host_url, false];
}
// windows hack
$path = str_replace("\/", "/", $path);
$len = strlen($url);
if ($len < 1) {
return [$host_url, false];
}
if ($with_query_string && isset($url_parts['query'])) {
$path .= "?".$url_parts['query'];
} else if ($with_query_string && $url[$len - 1] == "?") {
$path .= "?"; //handle blank query string case
}
return [$host_url, $path];
}
/**
* Gets an array of prefix urls from a given url. Each prefix contains at
* least the the hostname of the the start url
*
* http://host.com/b/c/ would yield http://host.com/ , http://host.com/b,
* http://host.com/b/, http://host.com/b/c, http://host.com/b/c/
*
* @param string $url the url to extract prefixes from
* @return array the array of url prefixes
*/
public static function getHostPaths($url)
{
$host_paths = [$url];
list($host, $path) = self::getHostAndPath($url);
if (!$host) {return $host_paths;}
$host_paths[] = $host;
$path_parts = explode("/", $path);
$url = $host;
foreach ($path_parts as $part) {
if ($part != "") {
$url .="/$part";
$host_paths[] = $url;
}
$host_paths[] = $url."/";
}
$host_paths = array_unique($host_paths);
return $host_paths;
}
/**
* Gets the subdomains of the host portion of a url. So
*
* http://a.b.c/d/f/
* will return a.b.c, .a.b.c, b.c, .b.c, c, .c
*
* @param string $url the url to extract prefixes from
* @return array the array of url prefixes
*/
public static function getHostSubdomains($url)
{
$subdomains = [];
$url_parts = @parse_url($url);
if (!isset($url_parts['host']) || strlen($url_parts['host']) <= 0) {
return $subdomains;
}
$host = $url_parts['host'];
$host_parts = explode(".", $host);
$num_parts = count($host_parts);
$domain = "";
for ($i = $num_parts - 1; $i >= 0; $i--) {
$domain = $host_parts[$i].$domain;
$subdomains[] = $domain;
$domain = ".$domain";
$subdomains[] = $domain;
}
return $subdomains;
}
/**
* Checks if $path matches against any of the Robots.txt style regex
* paths in $paths
*
* @param string $path a path component of a url
* @param array $robot_paths in format of robots.txt regex paths
* @return bool whether it is a member or not
*/
public static function isPathMemberRegexPaths($path, $robot_paths)
{
$is_member = false;
$len = strlen($path);
foreach ($robot_paths as $robot_path) {
$rlen = strlen($robot_path);
if ($rlen == 0) {
continue;
}
$end_match = false;
$end = ($robot_path[$rlen - 1] == "$") ? 1 : 0;
$path_string = substr($robot_path, 0, $rlen - $end);
$path_parts = explode("*", $path_string);
$offset = -1;
$old_part_len = 0;
$is_match = true;
foreach ($path_parts as $part) {
$offset += 1 + $old_part_len;
$old_part_len = strlen($part);
if ($part == "") {
continue;
} else if ($offset >= $len) {
$is_match = false;
break;
}
$new_offset = stripos($path, $part, $offset);
if ($new_offset === false ||
($offset == 0 && $new_offset > 0)) {
$is_match = false;
break;
}
$offset = $new_offset;
}
if ($is_match) {
if ($end == 0 || strlen($part) + $offset == $len) {
$is_member = true;
}
}
}
return $is_member;
}
/**
* Given a url, extracts the words in the host part of the url
* provided the url does not have a path part more than / .
* Ignores a leading www and also ignore tld.
*
* For example, "http://www.yahoo.com/" returns " yahoo "
*
* @param string $url a url to figure out the file type for
*
* @return string space separated words extracted.
*
*/
public static function getWordsInHostUrl($url)
{
$words = [];
$url_parts = @parse_url($url);
if (!isset($url_parts['host']) || strlen($url_parts['host']) <= 0
|| (isset($url_parts['path']) && $url_parts['path'] != "/")||
isset($url_parts['query'])
|| isset($url_parts['fragment'])) {
// if no host or has a query string bail
return "";
}
$host = $url_parts['host'];
$host_parts = preg_split("/\.|\-/u", $host);
if (!is_array($host_parts) || count($host_parts) <= 1) {
return "";
}
array_pop($host_parts); // get rid of tld
if (stristr($host_parts[0], "www")) {
array_shift($host_parts);
}
$words = array_merge($words, $host_parts);
$word_string = " " . implode(" ", $words). " ";
return $word_string;
}
/**
* Given a url, extracts the words in the last path part of the url
* For example,
* http://us3.php.net/manual/en/function.array-filter.php
* yields " function array filter "
*
* @param string $url a url to figure out the file type for
*
* @return string space separated words extracted.
*
*/
public static function getWordsLastPathPartUrl($url)
{
$words = [];
$url_parts = @parse_url($url);
$path_info = (empty($url_parts['path'])) ? [] :
@pathinfo($url_parts['path']);
$path = "";
if (isset($path_info['dirname'])) {
$path .= $path_info['dirname']."/";
}
if (isset($path_info['filename'])) {
$path .= $path_info['filename'];
}
$pre_path_parts = explode("/", $path);
$count = count($pre_path_parts);
if ($count < 1 ) {
return "";
}
$last_path = $pre_path_parts[$count - 1];
$path_parts = preg_split("/(_|-|\ |\+|\.)/", urldecode($last_path));
foreach ($path_parts as $part) {
if (strlen($part) > 0 ) {
$words[] = $part;
}
}
$word_string = " ".implode(" ", $words). " ";
return $word_string;
}
/**
* Given a url, makes a guess at the file type of the file it points to
*
* @param string $url a url to figure out the file type for
* @param string $default default type to be returned in the case that
* document type cannot be determined from the url, defaults to
* html
*
* @return string the guessed file type.
*
*/
public static function getDocumentType($url, $default = "html")
{
if ($url == "" || !is_string($url)) {
return $default;
}
$url = str_replace("%3F", "?", urlencode($url));
$url = str_replace("%2F", "/", $url);
$url_parts = @parse_url($url);
if (!isset($url_parts['path'])) {
return $default;
} else if ($url[strlen($url) - 1] == "/" || $url[strlen($url) - 1]
== "\\") {
return $default;
} else {
$path_parts = pathinfo($url_parts['path']);
if (empty($path_parts["extension"]) ||
str_contains($path_parts["extension"], "/")) {
return $default;
}
return urldecode($path_parts["extension"]);
}
}
/**
* Gets the filename portion of a url if present;
* otherwise returns "Some File"
*
* @param string $url a url to parse
* @return string the filename portion of this url
*/
public static function getDocumentFilename($url)
{
$url_parts = @parse_url($url);
if (!isset($url_parts['path'])) {
return "html"; //we default to html
} else {
$path_parts = pathinfo($url_parts['path']);
if (empty($path_parts["filename"]) ) {
return "Some File";
}
return urldecode($path_parts["filename"]);
}
}
/**
* Get the query string component of a url
*
* @param string $url a url to get the query string out of
* @return string the query string if present; null otherwise
*/
public static function getQuery($url)
{
$url_parts = @parse_url($url);
if (isset($url_parts['query'])) {
$out = $url_parts['query'];
} else {
$out = null;
}
return $out;
}
/**
* Get the url fragment string component of a url
*
* @param string $url a url to get the url fragment string out of
* @return string the url fragment string if present; null otherwise
*/
public static function getFragment($url)
{
$url_parts = @parse_url($url);
if (isset($url_parts['fragment'])) {
$out = $url_parts['fragment'];
} else {
$out = null;
}
return $out;
}
/**
* Given a $link that was obtained from a website $site, returns
* a complete URL for that link.
* For example, the $link
* some_dir/test.html
* on the $site
* http://www.somewhere.com/bob
* would yield the complete url
* http://www.somewhere.com/bob/some_dir/test.html
*
* @param string $link a relative or complete url
* @param string $site a base url
* @param string $no_fragment if false then if the url had a fragment
* (#link_within_page) then the fragment will be included
*
* @return string a complete url based on these two pieces of information
*
*/
public static function canonicalLink($link, $site, $no_fragment = true)
{
$link = trim($link);
if (!self::isSchemeCrawlable($link)) {
return "";
}
if (isset($link[0]) &&
$link[0] == "/" && isset($link[1]) && $link[1] == "/") {
$http = ($site[4] == 's') ? "https:" : "http:";
$link = $http . $link;
}
$link_parts = @parse_url($link);
if (isset($link_parts['host'])) {
list($host, $path) = self::getHostAndPath($link);
$query = self::getQuery($link);
$fragment = self::getFragment($link);
} else {
$host = self::getHost($site);
if ($link != null && $link[0] == "/") {
$path = $link;
} else {
$site_path = self::getPath($site) ?? "";
$site_path_parts = pathinfo($site_path);
if (isset($site_path_parts['dirname'])) {
$pre_path = $site_path_parts['dirname'];
} else {
$pre_path = "";
}
if (isset($site_path_parts['basename']) &&
!isset($site_path_parts['extension'])) {
$pre_path .= "/" . $site_path_parts['basename'];
}
if (strlen($link) > 0) {
$pre_path = ($link[0] != "#") ? $pre_path . "/" . $link :
$pre_path . $link;
}
$path = $pre_path;
$path2 = $path;
do {
$path = $path2;
$path2 = str_replace("//", "/", $path);
} while($path != $path2);
$path = self::getPath($path);
$so_far_link = $host . $pre_path;
$query = self::getQuery($so_far_link);
$fragment = self::getFragment($so_far_link);
}
}
// take a stab at paths containing ..
$path = preg_replace('/(\/\w+\/\.\.\/)+/', "/", $path ?? "");
// if still has .. give up
if (stristr($path, "../"))
{
return "";
}
// handle paths with dot in it
$path = preg_replace('/(\.\/)+/', "", $path);
$path = str_replace(" ", "%20", $path);
$link_path_parts = pathinfo($path);
$path2 = $path;
do {
$path = $path2;
$path2 = str_replace("//", "/", $path);
} while($path != $path2);
$path = str_replace("/./", "/", $path);
if ($path == "." || str_ends_with($path, "/.")) {
$path = "/";
}
if ($path == "" && !(isset($fragment) && $fragment !== "")) {
$path = "/";
}
$url = $host.$path;
if (isset($query) && $query !== "") {
$url .= "?" . $query;
}
if (!empty($fragment) && !$no_fragment) {
$url .= "#".$fragment;
}
return $url;
}
/**
* Checks if a url has a repeated set of subdirectories, and if the number
* of repeats occurs more than some threshold number of times
*
* A pattern like bob/.../bob counts as own repetition.
* bob/.../alice/.../bob/.../alice would count as two (... should be read
* as ellipsis, not a directory name).If the threshold is three and there
* are at least three repeated matches this function return true; it returns
* false otherwise.
*
* @param string $url the url to check
* @param int $repeat_threshold the number of repeats of a subdir name to
* trigger a true response
* @return bool whether a repeated subdirectory name with more matches than
* the threshold was found
*
*/
public static function checkRecursiveUrl($url, $repeat_threshold = 3)
{
if (!is_string($url)) {
return false;
}
$url_parts = explode("/", $url);
$count= count($url_parts);
$flag = 0;
for ($i = 0; $i < $count; $i++) {
for ($j = 0; $j < $i; $j++) {
if ($url_parts[$j] == $url_parts[$i]) {
$flag++;
}
}
}
if ($flag > $repeat_threshold) {
return true;
}
return false;
}
/**
* Checks if a $url is on localhost
*
* @param string $url the url to check
* @return bool whether or not it is on localhost
*/
public static function isLocalhostUrl($url)
{
$host = UrlParser::getHost($url, false);
$localhosts = RequestScope::get(self::LOCALHOSTS_KEPT_UNDER);
if ($localhosts === null) {
/* The names this machine answers to include the name the
request came in on. A site may serve several domains, so
that name differs between requests, and the list is worked
out once for each request rather than once for the
process. Looking the name's address up is the costly part,
which is why the list is kept at all. */
$localhosts = ["localhost", "127.0.0.1", "::1"];
if (isset($_SERVER["SERVER_NAME"])) {
$localhosts[] = $_SERVER["SERVER_NAME"];
$localhosts[] = gethostbyname($_SERVER["SERVER_NAME"]);
}
if (isset($_SERVER["SERVER_ADDR"])) {
$localhosts[] = $_SERVER["SERVER_ADDR"];
}
RequestScope::set(self::LOCALHOSTS_KEPT_UNDER, $localhosts);
}
foreach ($localhosts as $localhost) {
if (stristr($host, $localhost)) {
return true;
}
}
return false;
}
/**
* Checks if the url belongs to one of the sites listed in site_array
* Sites can be either given in the form domain:host or
* in the form of a url in which case it is check that the site url
* is a substring of the passed url.
*
* @param string $url url to check
* @param array $site_array sites to check against
* @param string $name identifier to store $site_array with in this
* public function's cache
* @param bool $return_rule whether when a match is found to return true or
* to return the matching site rule
* @return mixed whether the url belongs to one of the sites
*/
public static function urlMemberSiteArray($url, $site_array,
$name, $return_rule = false)
{
static $cache = [];
if (!is_array($site_array)) {
return false;
}
if (!isset($cache[$name])) {
if (count($cache) > 2 * C\p('MIN_RESULTS_TO_GROUP')) {
$cache = [];
}
$i = 0;
$cache[$name]["domains"] = [];
$cache[$name]["regexes"] = [];
$cache[$name]["hosts"] = [];
$cache[$name]["paths"] = [];
$cache[$name]["sites"] = [];
foreach ($site_array as $site) {
if (strncmp($site, "domain:", 7) == 0) {
$cache[$name]["domains"][] = substr($site, 7);
continue;
}
if (strncmp($site, "regex:", 6) == 0) {
$cache[$name]["regexes"][] = substr($site, 6);
continue;
}
list($site_host, $site_path) =
UrlParser::getHostAndPath($site, true, true);
$cache[$name]["hosts"][] = $site_host;
$cache[$name]["paths"][] = $site_path;
$cache[$name]["sites"][] = $site;
$i++;
}
$cache[$name]["domains"] = array_values(array_unique(
$cache[$name]["domains"]));
}
$flag = false;
$domains = & $cache[$name]["domains"];
$regexes = & $cache[$name]["regexes"];
$hosts = & $cache[$name]["hosts"];
$paths = & $cache[$name]["paths"];
$sites = & $cache[$name]["sites"];
foreach ($regexes as $regex) {
if (preg_match($regex, $url)) {
if ($return_rule) {
return "regex:$regex";
}
return true;
}
}
list($host, $path) = UrlParser::getHostAndPath($url, true, true);
foreach ($domains as $domain) {
$pos = strrpos($host, $domain);
if ($pos !== false &&
$pos + strlen($domain) == strlen($host) ) {
if ($return_rule) {
return "domain:$domain";
}
return true;
}
}
$count = count($sites);
for ($i = 0; $i < $count; $i++) {
$flag = UrlParser::isPathMemberRegexPaths($host, [$hosts[$i]]);
if (!$flag) {
continue;
}
$flag = UrlParser::isPathMemberRegexPaths($path, [$paths[$i]]);
if ($flag) {
break;
}
}
if ($return_rule && $flag) {
$flag = $sites[$i];
}
return $flag;
}
/**
* Used to delete links from array of links $links based on whether
* they are the same as the site they came from (or otherwise judged
* irrelevant)
*
* @param array $links pairs of the form $link =>$link_info
* @param string $parent_url a site that the links were found on
* @return array just those links which pass the relevancy test
*/
public static function cleanRedundantLinks($links, $parent_url)
{
$out_links = [];
foreach ($links as $url => $url_info) {
//ignore links back to oneself (too easy to spam)
if (strcmp($parent_url, $url) != 0) {
$out_links[$url] = $url_info;
}
}
return $out_links;
}
/**
* Prunes a list of url => text pairs down to max_link many pairs
* by choosing those whose text has the most information. Information
* crudely measured by the effective number of terms in the text.
* To compute this, we count the number of terms by splitting on white
* space. We then multiply this by the ratio of the compressed length
* of the text divided by its uncompressed length.
*
* @param array $links list of pairs $url=>$text
* @param int $max_links maximum number of links from $links to return
* @return array $out_links extracted from $links according to the
* description above.
*/
public static function pruneLinks($links, $max_links =
C\MAX_LINKS_TO_EXTRACT)
{
$consonants = "bcdfghjklmnpqrstvyz";
$vowels = "aeiouy";
$digit_consonants = "([$consonants]|\d)";
$digit_vowels = "([$vowels]|\d)";
if ($max_links <= 0 || count($links) <= $max_links) {
return $links;
}
$info_link = [];
// choose the $max_links many pages with most info (crude)
foreach ($links as $url => $text) {
$text = (is_string($text)) ? $text : "";
$terms = preg_split("/\s+|\-|\_|\~/", $text);
$num_terms = count($terms);
$len_text = strlen($text) + 1;
$compressed_len = strlen(gzcompress($text)) + 1;
$effective_num_terms = $num_terms *
(min($compressed_len/$len_text, 1));
if (!isset($info_link[$url])) {
$info_link[$url] = $effective_num_terms;
} else {
$info_link[$url] += $effective_num_terms;
}
}
arsort($info_link);
$link_urls = array_keys(array_slice($info_link, 0,
$max_links));
$out_links = [];
foreach ($link_urls as $url) {
$out_links[$url] = $links[$url];
}
return $out_links;
}
/**
* Returns the number of links in the array $links which
* which share the same company level domain (cld) as $url
* For www.yahoo.com the cld is yahoo.com, for
* www.theregister.co.uk it is theregister.co.uk. It is
* similar for organizations. It also tries to determine
* if a $url is potentially part of a link farm. To do this
* it checks (1) if the number of distinct, not sub-locale domains with a
* shared company domain is high > $threshold/2. This suggest a lot of bogus
* outgoing links that are all under one company's control. For
* example, a site
* www.foo.com linking to md5_hash.foo.com for many different md5 hashes.
* If this is detected this method returns -1. This method also returns -1
* if (2) there seem to be lots of links ($threshold) from the current
* domain to a single domain that shares the same company domain. This
* might indicate a domain md5_hash.foo.com with lots of links to a domain
* www.foo.com
*
* @param string $url the url to compare against $links
* @param array $links an array of urls
* @param int $threshold number above which if either situation (1) or (2)
* above happens then deem site spam
* @return int the number of times $url shares the cld with a
* link in $links. If thinks part of link farm returns -1
*/
public static function countCompanyLevelDomainsInCommonDetectFarm($url,
$links, $threshold = 200)
{
$cld = UrlParser::getCompanyLevelDomain($url);
$cnt = 0;
$dom_cnt = 0;
$host = parse_url($url, \PHP_URL_HOST);
$seen_hosts = [];
foreach ($links as $link_url) {
$link_cld = UrlParser::getCompanyLevelDomain($link_url);
$link_host = parse_url($link_url, \PHP_URL_HOST) ?? "";
// count number of shared company domains
if (strcmp($cld, $link_cld) == 0) {
$cnt++;
/* count distinct subdomains which are not likely to be locale
sub-domain
*/
if ($link_host != $host) {
if (empty($seen_hosts[$link_host])) {
$dom_cnt++;
$seen_hosts[$link_host] = 0;
}
$seen_hosts[$link_host]++;
}
}
}
$maximal_host = (empty($seen_hosts)) ? 0 : max($seen_hosts);
if (($dom_cnt > $threshold/2) || ($maximal_host > $threshold)) {
if (!in_array($cld, ['wikipedia.org', 'wiktionary.org',
'wikimedia.org', 'wikiquote.org', 'wikiversity.org',
'wikimediafoundation.org', 'wikinews.org'])) {
return -1;
}
}
return $cnt;
}
/**
* Calculates the company level domain for the given url
*
* For www.yahoo.com the cld is yahoo.com, for
* www.theregister.co.uk it is theregister.co.uk. It is
* similar for organizations.
*
* @param string $url url to determine cld for
* @return string the cld of $url
*/
public static function getCompanyLevelDomain($url)
{
if (preg_match("/^https?\:\/\/localhost\//", $url)) {
return "localhost";
}
$subdomains = UrlParser::getHostSubdomains($url);
if (!isset($subdomains[0]) || !isset($subdomains[2])) {
return "";
}
/*
if $url is www.yahoo.com
$subdomains[0] == com, $subdomains[1] == .com,
$subdomains[2] == yahoo.com,$subdomains[3] == .yahoo.com
etc.
*/
if (strlen($subdomains[0]) == 2 && strlen($subdomains[2]) <= 6
&& isset($subdomains[4])) {
return $subdomains[4];
}
return $subdomains[2];
}
/**
* Checks if a url's host is either a company level domain (a cld) or
* is of the form www.cld or has as its cld a domain that is in one of
* the supplied BloomFilterFile objects
* @param string $url url to check if in above form
* @param array $filters array of BloomFilterFile objects
* @return bool whether or not url has above form
*/
public static function cullByDomainFilter($url, $filters)
{
$url = mb_strtolower($url);
$cld = self::getCompanyLevelDomain($url);
$host = parse_url($url, \PHP_URL_HOST);
$host = (empty($host)) ? "" : $host;
$host_less_cld = mb_strtolower(
preg_replace('/' . preg_quote($cld) . '$/u', '', $host));
if (in_array($host_less_cld, ["", "www."])) {
return false;
}
foreach ($filters as $filter) {
if ($filter->contains($cld)) {
return false;
}
}
return true;
}
/**
* Guess mime type based on extension of the file
*
* @param string $file_name name of the file
* @param string $default what mime type to return if mime type
* couldn't be determined
* @return string $mime_type for the given file name
*/
public static function guessMimeTypeFromFileName($file_name,
$default = 'text/plain')
{
$mime_type_map = [
"bmp" => 'image/bmp',
"doc" => 'application/msword',
"epub" => 'application/epub+zip',
"gif" => 'image/gif',
"asp" => 'text/asp',
"aspx" => 'text/asp',
'cgi' => 'text/html',
"cfm" => 'text/html',
"cfml" => 'text/html',
"do" => 'text/html',
"htm" => 'text/html',
"html" => 'text/html',
"jsp" => 'text/html',
"php" => 'text/html',
"pl" => 'text/html',
"java" => 'text/java',
"py" => 'text/py',
"shtml" => 'text/html',
"jpg" => 'image/jpeg',
"jpeg" => 'image/jpeg',
"pdf" => 'application/pdf',
"png" => 'image/png',
"ppt" => 'application/vnd.ms-powerpoint',
"pptx" => 'application/vnd.openxmlformats-officedocument.'.
'presentationml.presentation',
"rss" => 'application/rss+xml',
"rtf" => 'text/rtf',
"svg" => 'image/svg+xml',
"csv" => 'text/csv',
"tab" => 'text/tab-separated-values',
"tsv" => 'text/x-java-source',
"txt" => 'text/plain',
"xlsx" => 'application/vnd.openxmlformats-officedocument.'.
'spreadsheetml.sheet',
"xml" => 'text/xml',
"js" => 'text/plain',
"c" => 'text/plain',
"cc" => 'text/plain',
"cs" => 'text/plain'
];
$extension = UrlParser::getDocumentType($file_name);
if (isset($mime_type_map[$extension])) {
$mime_type = $mime_type_map[$extension];
} else {
$mime_type = $default;
}
return $mime_type;
}
/**
* Extracts text from a url. Similar to @see getWordsInHostUrl
* and @see getWordsLastPathPartUrl except operates on whole
* url. This function is mainly used on link documents, the
* previous two are mainly used with standard documents
*
* @param string $url to find text that might say what link is about
* @return string heuristically derived text.
*/
public static function extractTextFromUrl($url)
{
$text = preg_replace("/(\:\d*)|\/|\-|\#/", " ", strip_tags($url));
$text = preg_replace("/www|https?|".
"\.(com|gov|org|mil|edu|net|biz|io|ca|au|uk|to|us|ly|".
"fr|gk|dk|es|pt|vn|cn|tw|in|jp|is|de|id|it|kp|kr|th|tr|sg|".
"uy|ni|mx|ar|ye|ae|za|zw|be|ir|pr|py|pa|il|sv|bo|at|ma|qc|".
"dr|ec|gt|pl|vi|s?html|php\d?|cgi|asp|jsp|gif|png|jpg|mp4|".
"mp3|txt|ftp)/", "", $text);
$text = preg_replace("/\.|\%[a-f0-9]+/i", " ", $text);
$text = preg_replace("/\s+/", " ", $text);
return $text;
}
}
(c) 2026 Seekquarry - Open Source Search Engine Software