Code Forge

<?php

/**

*

* @package search

* @copyright (c) 2005 phpBB Group

* @license http://opensource.org/licenses/gpl-2.0.php GNU General Public License v2

*

*/

/**

* @ignore

*/

if (!defined('IN_PHPBB'))

{

        exit;

}

/**

* fulltext_native

* phpBB's own db driven fulltext search, version 2

* @package search

*/

class phpbb_search_fulltext_native extends phpbb_search_base

{

        var $stats = array();

        var $word_length = array();

        var $search_query;

        var $common_words = array();

        var $must_contain_ids = array();

        var $must_not_contain_ids = array();

        var $must_exclude_one_ids = array();

        /**

        * Initialises the fulltext_native search backend with min/max word length and makes sure the UTF-8 normalizer is loaded.

        *

        * @param        boolean|string        &$error        is passed by reference and should either be set to false on success or an error message on failure.

        */

        public function __construct(&$error)

        {

                global $phpbb_root_path, $phpEx, $config;

                $this->word_length = array('min' => $config['fulltext_native_min_chars'], 'max' => $config['fulltext_native_max_chars']);

                /**

                * Load the UTF tools

                */

                if (!class_exists('utf_normalizer'))

                {

                        include($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx);

                }

                $error = false;

        }

        /**

        * Returns the name of this search backend to be displayed to administrators

        *

        * @return string Name

        */

        public function get_name()

        {

                return 'phpBB Native Fulltext';

        }

        /**

        * This function fills $this->search_query with the cleaned user search query.

        *

        * If $terms is 'any' then the words will be extracted from the search query

        * and combined with | inside brackets. They will afterwards be treated like

        * an standard search query.

        *

        * Then it analyses the query and fills the internal arrays $must_not_contain_ids,

        * $must_contain_ids and $must_exclude_one_ids which are later used by keyword_search().

        *

        * @param        string        $keywords        contains the search query string as entered by the user

        * @param        string        $terms                is either 'all' (use search query as entered, default words to 'must be contained in post')

        *         or 'any' (find all posts containing at least one of the given words)

        * @return        boolean                                false if no valid keywords were found and otherwise true

        *

        * @access        public

        */

        function split_keywords($keywords, $terms)

        {

                global $db, $user, $config;

                $tokens = '+-|()*';

                $keywords = trim($this->cleanup($keywords, $tokens));

                // allow word|word|word without brackets

                if ((strpos($keywords, ' ') === false) && (strpos($keywords, '|') !== false) && (strpos($keywords, '(') === false))

                {

                        $keywords = '(' . $keywords . ')';

                }

                $open_bracket = $space = false;

                for ($i = 0, $n = strlen($keywords); $i < $n; $i++)

                {

                        if ($open_bracket !== false)

                        {

                                switch ($keywords[$i])

                                {

                                        case ')':

                                                if ($open_bracket + 1 == $i)

                                                {

                                                        $keywords[$i - 1] = '|';

                                                        $keywords[$i] = '|';

                                                }

                                                $open_bracket = false;

                                        break;

                                        case '(':

                                                $keywords[$i] = '|';

                                        break;

                                        case '+':

                                        case '-':

                                        case ' ':

                                                $keywords[$i] = '|';

                                        break;

                                        case '*':

                                                if ($i === 0 || ($keywords[$i - 1] !== '*' && strcspn($keywords[$i - 1], $tokens) === 0))

                                                {

                                                        if ($i === $n - 1 || ($keywords[$i + 1] !== '*' && strcspn($keywords[$i + 1], $tokens) === 0))

                                                        {

                                                                $keywords = substr($keywords, 0, $i) . substr($keywords, $i + 1);

                                                        }

                                                }

                                        break;

                                }

                        }

                        else

                        {

                                switch ($keywords[$i])

                                {

                                        case ')':

                                                $keywords[$i] = ' ';

                                        break;

                                        case '(':

                                                $open_bracket = $i;

                                                $space = false;

                                        break;

                                        case '|':

                                                $keywords[$i] = ' ';

                                        break;

                                        case '-':

                                        case '+':

                                                $space = $keywords[$i];

                                        break;

                                        case ' ':

                                                if ($space !== false)

                                                {

                                                        $keywords[$i] = $space;

                                                }

                                        break;

                                        default:

                                                $space = false;

                                }

                        }

                }

                if ($open_bracket)

                {

                        $keywords .= ')';

                }

                $match = array(

                        '#  +#',

                        '#\|\|+#',

                        '#(\+|\-)(?:\+|\-)+#',

                        '#\(\|#',

                        '#\|\)#',

                );

                $replace = array(

                        ' ',

                        '|',

                        '$1',

                        '(',

                        ')',

                );

                $keywords = preg_replace($match, $replace, $keywords);

                $num_keywords = sizeof(explode(' ', $keywords));

                // We limit the number of allowed keywords to minimize load on the database

                if ($config['max_num_search_keywords'] && $num_keywords > $config['max_num_search_keywords'])

                {

                        trigger_error($user->lang('MAX_NUM_SEARCH_KEYWORDS_REFINE', $config['max_num_search_keywords'], $num_keywords));

                }

                // $keywords input format: each word separated by a space, words in a bracket are not separated

                // the user wants to search for any word, convert the search query

                if ($terms == 'any')

                {

                        $words = array();

                        preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $words);

                        if (sizeof($words[1]))

                        {

                                $keywords = '(' . implode('|', $words[1]) . ')';

                        }

                }

                // set the search_query which is shown to the user

                $this->search_query = $keywords;

                $exact_words = array();

                preg_match_all('#([^\\s+\\-|*()]+)(?:$|[\\s+\\-|()])#u', $keywords, $exact_words);

                $exact_words = $exact_words[1];

                $common_ids = $words = array();

                if (sizeof($exact_words))

                {

                        $sql = 'SELECT word_id, word_text, word_common

                                FROM ' . SEARCH_WORDLIST_TABLE . '

                                WHERE ' . $db->sql_in_set('word_text', $exact_words) . '

                                ORDER BY word_count ASC';

                        $result = $db->sql_query($sql);

                        // store an array of words and ids, remove common words

                        while ($row = $db->sql_fetchrow($result))

                        {

                                if ($row['word_common'])

                                {

                                        $this->common_words[] = $row['word_text'];

                                        $common_ids[$row['word_text']] = (int) $row['word_id'];

                                        continue;

                                }

                                $words[$row['word_text']] = (int) $row['word_id'];

                        }

                        $db->sql_freeresult($result);

                }

                unset($exact_words);

                // now analyse the search query, first split it using the spaces

                $query = explode(' ', $keywords);

                $this->must_contain_ids = array();

                $this->must_not_contain_ids = array();

                $this->must_exclude_one_ids = array();

                $mode = '';

                $ignore_no_id = true;

                foreach ($query as $word)

                {

                        if (empty($word))

                        {

                                continue;

                        }

                        // words which should not be included

                        if ($word[0] == '-')

                        {

                                $word = substr($word, 1);

                                // a group of which at least one may not be in the resulting posts

                                if ($word[0] == '(')

                                {

                                        $word = array_unique(explode('|', substr($word, 1, -1)));

                                        $mode = 'must_exclude_one';

                                }

                                // one word which should not be in the resulting posts

                                else

                                {

                                        $mode = 'must_not_contain';

                                }

                                $ignore_no_id = true;

                        }

                        // words which have to be included

                        else

                        {

                                // no prefix is the same as a +prefix

                                if ($word[0] == '+')

                                {

                                        $word = substr($word, 1);

                                }

                                // a group of words of which at least one word should be in every resulting post

                                if ($word[0] == '(')

                                {

                                        $word = array_unique(explode('|', substr($word, 1, -1)));

                                }

                                $ignore_no_id = false;

                                $mode = 'must_contain';

                        }

                        if (empty($word))

                        {

                                continue;

                        }

                        // if this is an array of words then retrieve an id for each

                        if (is_array($word))

                        {

                                $non_common_words = array();

                                $id_words = array();

                                foreach ($word as $i => $word_part)

                                {

                                        if (strpos($word_part, '*') !== false)

                                        {

                                                $id_words[] = '\'' . $db->sql_escape(str_replace('*', '%', $word_part)) . '\'';

                                                $non_common_words[] = $word_part;

                                        }

                                        else if (isset($words[$word_part]))

                                        {

                                                $id_words[] = $words[$word_part];

                                                $non_common_words[] = $word_part;

                                        }

                                        else

                                        {

                                                $len = utf8_strlen($word_part);

                                                if ($len < $this->word_length['min'] || $len > $this->word_length['max'])

                                                {

                                                        $this->common_words[] = $word_part;

                                                }

                                        }

                                }

                                if (sizeof($id_words))

                                {

                                        sort($id_words);

                                        if (sizeof($id_words) > 1)

                                        {

                                                $this->{$mode . '_ids'}[] = $id_words;

                                        }

                                        else

                                        {

                                                $mode = ($mode == 'must_exclude_one') ? 'must_not_contain' : $mode;

                                                $this->{$mode . '_ids'}[] = $id_words[0];

                                        }

                                }

                                // throw an error if we shall not ignore unexistant words

                                else if (!$ignore_no_id && sizeof($non_common_words))

                                {

                                        trigger_error(sprintf($user->lang['WORDS_IN_NO_POST'], implode(', ', $non_common_words)));

                                }

                                unset($non_common_words);

                        }

                        // else we only need one id

                        else if (($wildcard = strpos($word, '*') !== false) || isset($words[$word]))

                        {

                                if ($wildcard)

                                {

                                        $len = utf8_strlen(str_replace('*', '', $word));

                                        if ($len >= $this->word_length['min'] && $len <= $this->word_length['max'])

                                        {

                                                $this->{$mode . '_ids'}[] = '\'' . $db->sql_escape(str_replace('*', '%', $word)) . '\'';

                                        }

                                        else

                                        {

                                                $this->common_words[] = $word;

                                        }

                                }

                                else

                                {

                                        $this->{$mode . '_ids'}[] = $words[$word];

                                }

                        }

                        // throw an error if we shall not ignore unexistant words

                        else if (!$ignore_no_id)

                        {

                                if (!isset($common_ids[$word]))

                                {

                                        $len = utf8_strlen($word);

                                        if ($len >= $this->word_length['min'] && $len <= $this->word_length['max'])

                                        {

                                                trigger_error(sprintf($user->lang['WORD_IN_NO_POST'], $word));

                                        }

                                        else

                                        {

                                                $this->common_words[] = $word;

                                        }

                                }

                        }

                        else

                        {

                                $len = utf8_strlen($word);

                                if ($len < $this->word_length['min'] || $len > $this->word_length['max'])

                                {

                                        $this->common_words[] = $word;

                                }

                        }

                }

                // we can't search for negatives only

                if (!sizeof($this->must_contain_ids))

                {

                        return false;

                }

                if (!empty($this->search_query))

                {

                        return true;

                }

                return false;

        }

        /**

        * Performs a search on keywords depending on display specific params. You have to run split_keywords() first.

        *

        * @param        string                $type                                contains either posts or topics depending on what should be searched for

        * @param        string                $fields                                contains either titleonly (topic titles should be searched), msgonly (only message bodies should be searched), firstpost (only subject and body of the first post should be searched) or all (all post bodies and subjects should be searched)

        * @param        string                $terms                                is either 'all' (use query as entered, words without prefix should default to "have to be in field") or 'any' (ignore search query parts and just return all posts that contain any of the specified words)

        * @param        array                $sort_by_sql                contains SQL code for the ORDER BY part of a query

        * @param        string                $sort_key                        is the key of $sort_by_sql for the selected sorting

        * @param        string                $sort_dir                        is either a or d representing ASC and DESC

        * @param        string                $sort_days                        specifies the maximum amount of days a post may be old

        * @param        array                $ex_fid_ary                        specifies an array of forum ids which should not be searched

        * @param        array                $m_approve_fid_ary        specifies an array of forum ids in which the searcher is allowed to view unapproved posts

        * @param        int                        $topic_id                        is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched

        * @param        array                $author_ary                        an array of author ids if the author should be ignored during the search the array is empty

        * @param        string                $author_name                specifies the author match, when ANONYMOUS is also a search-match

        * @param        array                &$id_ary                        passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered

        * @param        int                        $start                                indicates the first index of the page

        * @param        int                        $per_page                        number of ids each page is supposed to contain

        * @return        boolean|int                                                total number of results

        *

        * @access        public

        */

        function keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $m_approve_fid_ary, $topic_id, $author_ary, $author_name, &$id_ary, $start, $per_page)

        {

                global $config, $db;

                // No keywords? No posts.

                if (empty($this->search_query))

                {

                        return false;

                }

                $must_contain_ids = $this->must_contain_ids;

                $must_not_contain_ids = $this->must_not_contain_ids;

                $must_exclude_one_ids = $this->must_exclude_one_ids;

                sort($must_contain_ids);

                sort($must_not_contain_ids);

                sort($must_exclude_one_ids);

                // generate a search_key from all the options to identify the results

                $search_key = md5(implode('#', array(

                        serialize($must_contain_ids),

                        serialize($must_not_contain_ids),

                        serialize($must_exclude_one_ids),

                        $type,

                        $fields,

                        $terms,

                        $sort_days,

                        $sort_key,

                        $topic_id,

                        implode(',', $ex_fid_ary),

                        implode(',', $m_approve_fid_ary),

                        implode(',', $author_ary),

                        $author_name,

                )));

                // try reading the results from cache

                $total_results = 0;

                if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == SEARCH_RESULT_IN_CACHE)

                {

                        return $total_results;

                }

                $id_ary = array();

                $sql_where = array();

                $group_by = false;

                $m_num = 0;

                $w_num = 0;

                $sql_array = array(

                        'SELECT'        => ($type == 'posts') ? 'p.post_id' : 'p.topic_id',

                        'FROM'                => array(

                                SEARCH_WORDMATCH_TABLE        => array(),

                                SEARCH_WORDLIST_TABLE        => array(),

                        ),

                        'LEFT_JOIN' => array(array(

                                'FROM'        => array(POSTS_TABLE => 'p'),

                                'ON'        => 'm0.post_id = p.post_id',

                        )),

                );

                $title_match = '';

                $left_join_topics = false;

                $group_by = true;

                // Build some display specific sql strings

                switch ($fields)

                {

                        case 'titleonly':

                                $title_match = 'title_match = 1';

                                $group_by = false;

                        // no break

                        case 'firstpost':

                                $left_join_topics = true;

                                $sql_where[] = 'p.post_id = t.topic_first_post_id';

                        break;

                        case 'msgonly':

                                $title_match = 'title_match = 0';

                                $group_by = false;

                        break;

                }

                if ($type == 'topics')

                {

                        $left_join_topics = true;

                        $group_by = true;

                }

                /**

                * @todo Add a query optimizer (handle stuff like "+(4|3) +4")

                */

                foreach ($this->must_contain_ids as $subquery)

                {

                        if (is_array($subquery))

                        {

                                $group_by = true;

                                $word_id_sql = array();

                                $word_ids = array();

                                foreach ($subquery as $id)

                                {

                                        if (is_string($id))

                                        {

                                                $sql_array['LEFT_JOIN'][] = array(

                                                        'FROM'        => array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),

                                                        'ON'        => "w$w_num.word_text LIKE $id"

                                                );

                                                $word_ids[] = "w$w_num.word_id";

                                                $w_num++;

                                        }

                                        else

                                        {

                                                $word_ids[] = $id;

                                        }

                                }

                                $sql_where[] = $db->sql_in_set("m$m_num.word_id", $word_ids);

                                unset($word_id_sql);

                                unset($word_ids);

                        }

                        else if (is_string($subquery))

                        {

                                $sql_array['FROM'][SEARCH_WORDLIST_TABLE][] = 'w' . $w_num;

                                $sql_where[] = "w$w_num.word_text LIKE $subquery";

                                $sql_where[] = "m$m_num.word_id = w$w_num.word_id";

                                $group_by = true;

                                $w_num++;

                        }

                        else

                        {

                                $sql_where[] = "m$m_num.word_id = $subquery";

                        }

                        $sql_array['FROM'][SEARCH_WORDMATCH_TABLE][] = 'm' . $m_num;

                        if ($title_match)

                        {

                                $sql_where[] = "m$m_num.$title_match";

                        }

                        if ($m_num != 0)

                        {

                                $sql_where[] = "m$m_num.post_id = m0.post_id";

                        }

                        $m_num++;

                }

                foreach ($this->must_not_contain_ids as $key => $subquery)

                {

                        if (is_string($subquery))

                        {

                                $sql_array['LEFT_JOIN'][] = array(

                                        'FROM'        => array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),

                                        'ON'        => "w$w_num.word_text LIKE $subquery"

                                );

                                $this->must_not_contain_ids[$key] = "w$w_num.word_id";

                                $group_by = true;

                                $w_num++;

                        }

                }

                if (sizeof($this->must_not_contain_ids))

                {

                        $sql_array['LEFT_JOIN'][] = array(

                                'FROM'        => array(SEARCH_WORDMATCH_TABLE => 'm' . $m_num),

                                'ON'        => $db->sql_in_set("m$m_num.word_id", $this->must_not_contain_ids) . (($title_match) ? " AND m$m_num.$title_match" : '') . " AND m$m_num.post_id = m0.post_id"

                        );

                        $sql_where[] = "m$m_num.word_id IS NULL";

                        $m_num++;

                }

                foreach ($this->must_exclude_one_ids as $ids)

                {

                        $is_null_joins = array();

                        foreach ($ids as $id)

                        {

                                if (is_string($id))

                                {

                                        $sql_array['LEFT_JOIN'][] = array(

                                                'FROM'        => array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),

                                                'ON'        => "w$w_num.word_text LIKE $id"

                                        );

                                        $id = "w$w_num.word_id";

                                        $group_by = true;

                                        $w_num++;

                                }

                                $sql_array['LEFT_JOIN'][] = array(

                                        'FROM'        => array(SEARCH_WORDMATCH_TABLE => 'm' . $m_num),

                                        'ON'        => "m$m_num.word_id = $id AND m$m_num.post_id = m0.post_id" . (($title_match) ? " AND m$m_num.$title_match" : '')

                                );

                                $is_null_joins[] = "m$m_num.word_id IS NULL";

                                $m_num++;

                        }

                        $sql_where[] = '(' . implode(' OR ', $is_null_joins) . ')';

                }

                if (!sizeof($m_approve_fid_ary))

                {

                        $sql_where[] = 'p.post_approved = 1';

                }

                else if ($m_approve_fid_ary !== array(-1))

                {

                        $sql_where[] = '(p.post_approved = 1 OR ' . $db->sql_in_set('p.forum_id', $m_approve_fid_ary, true) . ')';

                }

                if ($topic_id)

                {

                        $sql_where[] = 'p.topic_id = ' . $topic_id;

                }

                if (sizeof($author_ary))

                {

                        if ($author_name)

                        {

                                // first one matches post of registered users, second one guests and deleted users

                                $sql_author = '(' . $db->sql_in_set('p.poster_id', array_diff($author_ary, array(ANONYMOUS)), false, true) . ' OR p.post_username ' . $author_name . ')';

                        }

                        else

                        {

                                $sql_author = $db->sql_in_set('p.poster_id', $author_ary);

                        }

                        $sql_where[] = $sql_author;

                }

                if (sizeof($ex_fid_ary))

                {

                        $sql_where[] = $db->sql_in_set('p.forum_id', $ex_fid_ary, true);

                }

                if ($sort_days)

                {

                        $sql_where[] = 'p.post_time >= ' . (time() - ($sort_days * 86400));

                }

                $sql_array['WHERE'] = implode(' AND ', $sql_where);

                $is_mysql = false;

                // if the total result count is not cached yet, retrieve it from the db

                if (!$total_results)

                {

                        $sql = '';

                        $sql_array_count = $sql_array;

                        if ($left_join_topics)

                        {

                                $sql_array_count['LEFT_JOIN'][] = array(

                                        'FROM'        => array(TOPICS_TABLE => 't'),

                                        'ON'        => 'p.topic_id = t.topic_id'

                                );

                        }

                        switch ($db->sql_layer)

                        {

                                case 'mysql4':

                                case 'mysqli':

                                        // 3.x does not support SQL_CALC_FOUND_ROWS

                                        // $sql_array['SELECT'] = 'SQL_CALC_FOUND_ROWS ' . $sql_array['SELECT'];

                                        $is_mysql = true;

                                break;

                                case 'sqlite':

                                        $sql_array_count['SELECT'] = ($type == 'posts') ? 'DISTINCT p.post_id' : 'DISTINCT p.topic_id';

                                        $sql = 'SELECT COUNT(' . (($type == 'posts') ? 'post_id' : 'topic_id') . ') as total_results

                                                        FROM (' . $db->sql_build_query('SELECT', $sql_array_count) . ')';

                                // no break

                                default:

                                        $sql_array_count['SELECT'] = ($type == 'posts') ? 'COUNT(DISTINCT p.post_id) AS total_results' : 'COUNT(DISTINCT p.topic_id) AS total_results';

                                        $sql = (!$sql) ? $db->sql_build_query('SELECT', $sql_array_count) : $sql;

                                        $result = $db->sql_query($sql);

                                        $total_results = (int) $db->sql_fetchfield('total_results');

                                        $db->sql_freeresult($result);

                                        if (!$total_results)

                                        {

                                                return false;

                                        }

                                break;

                        }

                        unset($sql_array_count, $sql);

                }

                // Build sql strings for sorting

                $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC');

                switch ($sql_sort[0])

                {

                        case 'u':

                                $sql_array['FROM'][USERS_TABLE] = 'u';

                                $sql_where[] = 'u.user_id = p.poster_id ';

                        break;

                        case 't':

                                $left_join_topics = true;

                        break;

                        case 'f':

                                $sql_array['FROM'][FORUMS_TABLE] = 'f';

                                $sql_where[] = 'f.forum_id = p.forum_id';

                        break;

                }

                if ($left_join_topics)

                {

                        $sql_array['LEFT_JOIN'][] = array(

                                'FROM'        => array(TOPICS_TABLE => 't'),

                                'ON'        => 'p.topic_id = t.topic_id'

                        );

                }

                $sql_array['WHERE'] = implode(' AND ', $sql_where);

                $sql_array['GROUP_BY'] = ($group_by) ? (($type == 'posts') ? 'p.post_id' : 'p.topic_id') . ', ' . $sort_by_sql[$sort_key] : '';