<?php
/**
 * SeekQuarry/Yioop --
 * Open Source Pure PHP Search Engine, Crawler, and Indexer
 *
 * Copyright (C) 2009 - 2026  Chris Pollett chris@pollett.org
 *
 * LICENSE:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *
 * END LICENSE
 *
 * @author Chris Pollett chris@pollett.org
 * @license https://www.gnu.org/licenses/ GPL3
 * @link https://www.seekquarry.com/
 * @copyright 2009 - 2026
 * @filesource
 */
namespace seekquarry\yioop\tests;

use seekquarry\yioop\configs as C;
use seekquarry\yioop\library\data_structures\BloomFilterFile;
use seekquarry\yioop\library\UrlParser;
use seekquarry\yioop\library\UnitTest;

/**
 * Used to test that the UrlParser class. For now, want to see that the
 * method canonicalLink is working correctly and that
 * isPathMemberRegexPaths (used in robot_processor.php) works
 *
 * @author Chris Pollett
 */
class UrlParserTest extends UnitTest
{
    /**
     * UrlParser uses static methods so doesn't do anything right now
     */
    public function setUp()
    {
    }
    /**
     * UrlParser uses static methods so doesn't do anything right now
     */
    public function tearDown()
    {
    }
    /**
     * Check if can go from a relative link, base link to a complete link
     * in various different ways
     */
    public function canonicalLinkTestCase()
    {
        $test_links = [
             [".", "http://www.example.com/",
                "http://www.example.com/", "root dir0"],
            ["/bob.html", "http://www.example.com/",
                "http://www.example.com/bob.html", "root dir1"],
            ["bob.html", "http://www.example.com/",
                "http://www.example.com/bob.html", "root dir2"],
            ["bob", "http://www.example.com/",
                "http://www.example.com/bob", "root dir3"],
            ["bob", "http://www.example.com",
                "http://www.example.com/bob", "root dir4"],
            ["http://print.bob.com/bob", "http://www.example.com",
                "http://print.bob.com/bob", "root dir5"],
            ["/.", "http://www.example.com/",
                "http://www.example.com/", "root dir6"],
            ["//slashdot.org", "http://www.slashdot.org",
                "http://slashdot.org/", "slashdot dir"],
            ["bob", "http://www.example.com/a",
                "http://www.example.com/a/bob", "sub dir1"],
            ["../bob", "http://www.example.com/a",
                "http://www.example.com/bob", "sub dir2"],
            ["../../bob", "http://www.example.com/a",
                null, "sub dir3"],
            ["./bob", "http://www.example.com/a",
                "http://www.example.com/a/bob", "sub dir4"],
            ["bob.html?a=1", "http://www.example.com/a",
                "http://www.example.com/a/bob.html?a=1", "query 1"],
            ["bob?a=1&b=2", "http://www.example.com/a",
                "http://www.example.com/a/bob?a=1&b=2", "query 2"],
            ["/?a=1&b=2", "http://www.example.com/a",
                "http://www.example.com/?a=1&b=2", "query 3"],
            ["?a=1&b=2", "http://www.example.com/a",
                "http://www.example.com/a/?a=1&b=2", "query 4"],
            ["b/b.html?a=1&b=2", "http://www.example.com/a/c",
                "http://www.example.com/a/c/b/b.html?a=1&b=2", "query 5"],
            ["b/b.html?a=1&b=2?c=4", "http://www.example.com/a/c",
                "http://www.example.com/a/c/b/b.html?a=1&b=2?c=4", "query 6"],
            ["b#1", "http://www.example.com/",
                "http://www.example.com/b#1", "fragment 1"],
            ["b?a=1#1", "http://www.example.com/",
                "http://www.example.com/b?a=1#1", "fragment 2"],
            ["b?a=1#1#2", "http://www.example.com/",
                "http://www.example.com/b?a=1#1#2", "fragment 3"],
            ["#a", "http://www.example.com/c:d",
                "http://www.example.com/c:d#a", "fragment 4"],
        ];
        foreach ($test_links as $test_link) {
            $result = UrlParser::canonicalLink($test_link[0],
                $test_link[1], false);
            $this->assertEqual($result, $test_link[2], $test_link[3]);
        }
    }
    /**
     * Check is a path matches with a list of paths presumably coming from
     * a robots.txt file
     */
    public function isPathMemberRegexPathsTestCase()
    {
        $path = [];
        $robot_paths = [];
        $results = [];
        $tests = [
            ["/bobby", ["/bob"], true, "Substring Positive"],
            ["/bobby", ["/alice", "/f/g/h/d"], false,
                "Substring Negative 1"],
            ["/bobby/", ["/bobby/bay", "/f/g/h/d", "/yo"], false,
                "Substring Negative 2"],
            ["/bay/bobby/", ["/bobby/", "/f/g/h/d", "/yo"], false,
                "Substring Negative 3 (should match start)"],
            ["http://test.com/bay/bobby/",
                ["/bobby/", "/f/g/h/d", "/yo"], false,
                "Substring Negative 4 (should match start)"],
            ["/a/bbbb/c/", ["/bobby/bay", "/a/*/c/", "/yo"], true,
                "Star Positive 1"],
            ["/a/bbbb/d/", ["/bobby/bay", "/a/*/c/", "/yo"], false,
                "Star Negative 1"],
            ["/test.html?a=b", ["/bobby/bay", "/*?", "/yo"], true,
                "Star Positive 2"],
            ["/test.html", ["/bobby/bay", "/*.html$", "/yo"], true,
                "Dollar Positive 1"],
            ["/test.htmlish", ["/bobby/bay", "/*.html$", "/yo"], false,
                "Dollar Negative 1"],
            ["/test.htmlish", ["/bobby/bay", "*", "/yo"], true,
                "Degenerate 1"],
            ["/test.html", ["/bobby/bay", "/**.html$", "/yo"], true,
                "Degenerate 2"],
            ["/videos/search?q=Angelina+Jolie",
                  ["/videos/search?"], true, "End With Question Regex Case 1"],
        ];
        foreach ($tests as $test) {
            list($path, $robot_paths, $result, $description) = $test;
            $this->assertEqual(UrlParser::isPathMemberRegexPaths($path,
                $robot_paths), $result, $description);
        }
    }
    /**
     * Tests simplifyUrl function used on SERP pages
     */
    public function simplifyUrlTestCase()
    {
        $test_urls = [
            ["http://www.example.com/", 100,
                "www.example.com", "HTTP Domain only"],
            ["https://www.example.com/", 100,
                "www.example.com", "HTTPS Domain only"],
            ["http://www.superreallylongexample.com/", 25,
                "www.superreallylonge...e.com", "Domain truncate"],
            ["http://www.example.com/word1/word2/word3/word4", 25,
                "www.example.com/word...word4", "Path truncate"],
        ];

        foreach ($test_urls as $test_url) {
            $result = UrlParser::simplifyUrl($test_url[0], $test_url[1]);
            $this->assertEqual($result, $test_url[2], $test_url[3]);
        }
    }
    /**
     * urlMemberSiteArray is a function called by both allowedToCrawlSite
     * disallowedToCrawlSite to test if a url belongs to alist of
     * regex's of urls or domain. This test function tests this functionality
     */
    public function urlMemberSiteArrayTestCase()
    {
        $sites = ["http://www.example.com/",
            "http://www.cs.sjsu.edu/faculty/pollett/*/*/",
            "http://www.bing.com/video/search?*&*&",
            "http://*.cool.*/a/*/", "domain:ucla.edu",
            "domain:foodnetwork.com",
            "domain:.ottawa.ca",
            "domain:.ottawa2.ca",
            "http://ottawa2.ca/"];
        $test_urls = [
            ["http://www.cs.sjsu.edu/faculty/pollett/", false,
                "regex url negative 1"],
            ["http://www.bing.com/video/search?", false,"regex url negative 2"],
            ["http://www.cool.edu/a", false, "regex url negative 3"],
            ["http://ucla.edu.com", false, "domain test negative"],
            ["http://www.cs.sjsu.edu/faculty/pollett/a/b/c", true,
                "regex url positive 1"],
            ["http://www.bing.com/video/search?a&b&c", true,
                "regex url positive 2"],
            ["http://www.cool.bob.edu/a/b/c", true, "regex url positive 3"],
            ["http://test.ucla.edu", true, "domain test positive"],
            ["https://test.ucla.edu", true, "domain https test positive"],
            ["gopher://test.ucla.edu", true, "domain gopher stest positive"],
            ["http://www.foodnetworkstore.com/small-appliances/", false,
                "domain test negative"],
            ["http://a.ottawa.ca/", true,
                "domain starting dot test positive 2"],
            ["http://ottawa.ca/", false, "domain starting dot test negative"],
            ["http://a.ottawa2.ca/", true,
                "domain starting dot test positive 2"],
            ["http://ottawa2.ca/", true, "domain starting dot test positive 3"],
        ];
        foreach ($test_urls as $test_url) {
            $result = UrlParser::urlMemberSiteArray($test_url[0], $sites,
                "s");
            $this->assertEqual($result, $test_url[1], $test_url[2]);
        }
    }
    /**
     * Checks if getScheme is working okay
     */
    public function getSchemeTestCase()
    {
        $test_links = [
            ["http://www.example.com/", "http", "Simple HTTP 1"],
            ["https://www.example.com/", "https", "Simple HTTPS 1"],
            ["gopher://www.example.com/", "gopher", "Simple GOPHER 1"],
            ["./", "http", "Simple HTTP 2"],
        ];
        foreach ($test_links as $test_link) {
            $result = UrlParser::getScheme($test_link[0]);
            $this->assertEqual($result, $test_link[1], $test_link[2]);
        }
    }
    /**
     * Checks if getHost is working okay
     */
    public function getHostTestCase()
    {
        $test_links = [
            ["https://somewhere.com:80/la/de/da", "https://somewhere.com:80",
                "Host with por"],
            ["https://10.1.10.10/?dfas=aga/", "https://10.1.10.10",
                "IP based host with query"],
            ["https://www.yioop.com/###@?woohoo", "https://www.yioop.com",
                "Host with fragment"],
        ];
        foreach ($test_links as $test_link) {
            $result = UrlParser::getHost($test_link[0]);
            $this->assertEqual($result, $test_link[1], $test_link[2]);
        }
    }
    /**
     * Checks if getCompanyLevelDomain is working okay
     */
    public function getCompanyLevelDomainTestCase()
    {
        $test_links = [
            ["https://www.somewhere.com/", "somewhere.com",
                "US URL Test 1"],
            ["https://aaa.bbb.cc.edu/", "cc.edu",
                "US URL Test 2"],
            ["https://somewhere.else.jp/", "else.jp", "Japanese URL Test 1"],
            ["https://somewhere.co.jp/", "somewhere.co.jp",
                "Japanese URL Test 2"],
            ["https://somewhere.com.jp/", "somewhere.com.jp",
                "Japanese URL Test 3"],
            ["https://somewhere.co.uk/", "somewhere.co.uk",
                "UK URL Test 1"],
        ];
        foreach ($test_links as $test_link) {
            $result = UrlParser::getCompanyLevelDomain($test_link[0]);
            $this->assertEqual($result, $test_link[1], $test_link[2]);
        }
    }
    /**
     * Checks if cullByDomainFilter is working okay. Only works if
     * there is a BloomFilterFile called C\WORK_DIRECTORY .
     *     "/data/domain_filters/top10000000.ftr" that at least contains
     * yahoo.com
     */
    public function cullByDomainFilterTestCase()
    {
        $filter_name = C\WORK_DIRECTORY .
            "/data/domain_filters/top10000000.ftr";
        if (!file_exists($filter_name)) {
            return;
        }
        $filter = BloomFilterFile::load($filter_name);
        $filters = [$filter];
        $test_links = [
            ["https://www.yahoo.com/foo", false,
                "Top Ten Million Site Test 1"],
            ["https://aaa.bbb.ccc.yahoo.com/foo/goo", false,
                "Top Ten Million Site Test 2"],
            ["https://pollett.org/foo/goo", false,
                "Not Top Ten Million Site Test 1"],
            ["https://www.pollett.org/foo/goo", false,
                "Not Top Ten Million Site Test 1"],
            ["https://aaa.bbb.ccc.pollett.org/foo/goo", true,
                "Not Top Ten Million Site Test 3"],
        ];
        foreach ($test_links as $test_link) {
            $result = UrlParser::cullByDomainFilter($test_link[0], $filters);
            $this->assertEqual($result, $test_link[1], $test_link[2]);
        }
    }
}
