<?php

/*

 * Enano - an open-source CMS capable of wiki functions, Drupal-like sidebar blocks, and everything in between

 * Version 1.1.1

 * Copyright (C) 2006-2007 Dan Fuhry

 * search.php - algorithm used to search pages

 *

 * This program is Free Software; you can redistribute and/or modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied

 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for details.

 */

/**

 * Implementation of array_merge() that preserves key names. $arr2 takes precedence over $arr1.

 * @param array $arr1

 * @param array $arr2

 * @return array

 */

function enano_safe_array_merge($arr1, $arr2)

{

  $arr3 = $arr1;

  foreach($arr2 as $k => $v)

  {

    $arr3[$k] = $v;

  }

  return $arr3;

}

/**

 * In Enano versions prior to 1.0.2, this class provided a search function that was keyword-based and allowed boolean searches. It was

 * cut from Coblynau and replaced with perform_search(), later in this file, because of speed issues. Now mostly deprecated. The only

 * thing remaining is the buildIndex function, which is still used by the path manager and the new search framework.

 * 

 * @package Enano

 * @subpackage Page management frontend

 * @license GNU General Public License <http://enanocms.org/Special:GNU_General_Public_License>

 */

class Searcher

{

  var $results;

  var $index;

  var $warnings;

  var $match_case = false;

  function buildIndex($texts)

  {

    $this->index = Array();

    $stopwords = get_stopwords();

    foreach($texts as $i => $l)

    {

      $seed = md5(microtime(true) . mt_rand());

      $texts[$i] = str_replace("'", 'xxxApoS'.$seed.'xxx', $texts[$i]);

      $texts[$i] = preg_replace('#([\W_]+)#i', ' ', $texts[$i]);

      $texts[$i] = preg_replace('#([ ]+?)#', ' ', $texts[$i]);

      $texts[$i] = preg_replace('#([\']*){2,}#s', '', $texts[$i]);

      $texts[$i] = str_replace('xxxApoS'.$seed.'xxx', "'", $texts[$i]);

      $l = $texts[$i];

      $words = Array();

      $good_chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\' ';

      $good_chars = enano_str_split($good_chars, 1);

      $letters = enano_str_split($l, 1);

      foreach($letters as $x => $t)

      {

        if(!in_array($t, $good_chars))

          unset($letters[$x]);

      }

      $letters = implode('', $letters);

      $words = explode(' ', $letters);

      foreach($words as $c => $w)

      {

        if(strlen($w) < 2 || in_array($w, $stopwords))

          unset($words[$c]);

        else

          $words[$c] = $w;

      }

      $words = array_values($words);

      foreach($words as $c => $w)

      {

        if(isset($this->index[$w]))

        {

          if(!in_array($i, $this->index[$w]))

            $this->index[$w][] = $i;

        }

        else

        {

          $this->index[$w] = Array();

          $this->index[$w][] = $i;

        }

      }

    }

    foreach($this->index as $k => $v)

    {

      $this->index[$k] = implode(',', $this->index[$k]);

    }

  }

}

/**

 * Searches the site for the specified string and returns an array with each value being an array filled with the following:

 *   page_id: string, self-explanatory

 *   namespace: string, self-explanatory

 *   page_length: integer, the length of the full page in bytes

 *   page_text: string, the contents of the page (trimmed to ~150 bytes if necessary)

 *   score: numerical relevance score, 1-100, rounded to 2 digits and calculated based on which terms were present and which were not

 * @param string Search query

 * @param string Will be filled with any warnings encountered whilst parsing the query

 * @param bool Case sensitivity - defaults to false

 * @return array

 */

function perform_search($query, &$warnings, $case_sensitive = false)

{

  global $db, $session, $paths, $template, $plugins; // Common objects

  $warnings = array();

  $query = parse_search_query($query, $warnings);

  // Segregate search terms containing spaces

  $query_phrase = array(

    'any' => array(),

    'req' => array()

    );

  foreach ( $query['any'] as $i => $_ )

  {

    $term =& $query['any'][$i];

    $term = trim($term);

    // the indexer only indexes words a-z with apostrophes

    if ( preg_match('/[^A-Za-z\']/', $term) )

    {

      $query_phrase['any'][] = $term;

      unset($term, $query['any'][$i]);

    }

  }

  unset($term);

  $query['any'] = array_values($query['any']);

  foreach ( $query['req'] as $i => $_ )

  {

    $term =& $query['req'][$i];

    $term = trim($term);

    if ( preg_match('/[^A-Za-z\']/', $term) )

    {

      $query_phrase['req'][] = $term;

      unset($term, $query['req'][$i]);

    }

  }

  unset($term);

  $query['req'] = array_values($query['req']);

  $results = array();

  $scores = array();

  // FIXME: Update to use FULLTEXT algo when available.

  // Build an SQL query to load from the index table

  if ( count($query['any']) < 1 && count($query['req']) < 1 && count($query_phrase['any']) < 1 && count($query_phrase['req']) < 1 )

  {

    // This is both because of technical restrictions and devastation that would occur on shared servers/large sites.

    $warnings[] = 'You need to have at least one keyword in your search query. Searching only for pages not containing a term is not allowed.';

    return array();

  }

  //

  // STAGE 1

  // Get all possible result pages from the search index. Tally which pages have the most words, and later sort them by boolean relevance

  //

  // Skip this if no indexable words are included

  if ( count($query['any']) > 0 || count($query['req']) > 0 )

  {

    $where_any = array();

    foreach ( $query['any'] as $term )

    {

      $term = escape_string_like($term);

      if ( !$case_sensitive )

        $term = strtolower($term);

      $where_any[] = $term;

    }

    foreach ( $query['req'] as $term )

    {

      $term = escape_string_like($term);

      if ( !$case_sensitive )

        $term = strtolower($term);

      $where_any[] = $term;

    }

    $col_word = ( $case_sensitive ) ? 'word' : 'lcase(word)';

    $where_any = ( count($where_any) > 0 ) ? '( ' . $col_word . ' = \'' . implode('\' OR ' . $col_word . ' = \'', $where_any) . '\' )' : '';

    // generate query

    // using a GROUP BY here ensures that the same word with a different case isn't counted as 2 words - it's all melted back

    // into one later in the processing stages

    $group_by = ( $case_sensitive ) ? '' : ' GROUP BY lcase(word);';

    $sql = "SELECT word, page_names FROM " . table_prefix . "search_index WHERE {$where_any}{$group_by}";

    if ( !($q = $db->sql_unbuffered_query($sql)) )

      $db->_die('Error is in perform_search(), includes/search.php, query 1');

    $word_tracking = array();

    if ( $row = $db->fetchrow() )

    {

      do

      {

        // get page list

        $pages =& $row['page_names'];

        $ns_list = '(' . implode('|', array_keys($paths->nslist)) . ')';

        if ( strpos($pages, ',') )

        {

          // the term occurs in more than one page

          // Find page IDs that contain commas

          // This should never happen because commas are escaped by sanitize_page_id(). Nevertheless for compatibility with older

          // databases, and to alleviate the concerns of hackers, we'll accommodate for page IDs with commas here by checking for

          // IDs that don't match the pattern for stringified page ID + namespace. If it doesn't match, that means it's a continuation

          // of the previous ID and should be concatenated to the previous entry.

          $matches = explode(',', $pages);

          $prev = false;

          foreach ( $matches as $i => $_ )

          {

            $match =& $matches[$i];

            if ( !preg_match("/^ns=$ns_list;pid=(.+)$/", $match) && $prev )

            {

              $matches[$prev] .= ',' . $match;

              unset($match, $matches[$i]);

              continue;

            }

            $prev = $i;

          }

          unset($match);

          // Iterate through each of the results, assigning scores based on how many times the page has shown up.

          // This works because this phase of the search is strongly word-based not page-based. If a page shows up

          // multiple times while fetching the result rows from the search_index table, it simply means that page

          // contains more than one of the terms the user searched for.

          foreach ( $matches as $match )

          {

            if ( isset($scores[$match]) )

            {

              $scores[$match]++;

            }

            else

            {

              $scores[$match] = 1;

            }

            if ( isset($word_tracking[$match]) )

            {

              $word_tracking[$match][] = $row['word'];

            }

            else

            {

              $word_tracking[$match] = array($row['word']);

            }

          }

        }

        else

        {

          // the term only occurs in one page

          if ( isset($scores[$pages]) )

          {

            $scores[$pages]++;

          }

          else

          {

            $scores[$pages] = 1;

          }

          if ( isset($word_tracking[$pages]) )

          {

            $word_tracking[$pages][] = $row['word'];

          }

          else

          {

            $word_tracking[$pages] = array($row['word']);

          }

        }

      }

      while ( $row = $db->fetchrow() );

    }

    $db->free_result();

    //

    // STAGE 2: FIRST ELIMINATION ROUND

    // Iterate through the list of required terms. If a given page is not found to have the required term, eliminate it

    //

    foreach ( $query['req'] as $term )

    {

      foreach ( $word_tracking as $i => $page )

      {

        if ( !in_array($term, $page) )

        {

          unset($word_tracking[$i], $scores[$i]);

        }

      }

    }

  }

  //

  // STAGE 3: PHRASE SEARCHING

  // Use LIKE to find pages with specified phrases. We can do a super-picky single query without another elimination round because

  // at this stage we can search the full page_text column instead of relying on a word list.

  //

  // We can skip this stage if none of these special terms apply

  $text_col = ( $case_sensitive ) ? 'page_text' : 'lcase(page_text)';

  if ( count($query_phrase['any']) > 0 || count($query_phrase['req']) > 0 )

  {

    $where_any = array();

    foreach ( $query_phrase['any'] as $term )

    {

      $term = escape_string_like($term);

      if ( !$case_sensitive )

        $term = strtolower($term);

      $where_any[] = $term;

    }

    $where_any = ( count($where_any) > 0 ) ? "( $text_col LIKE '%" . implode("%' OR $text_col LIKE '%", $where_any) . "%' )" : '';

    // Also do required columns, but use AND to ensure that all required terms are included

    $where_req = array();

    foreach ( $query_phrase['req'] as $term )

    {

      $term = escape_string_like($term);

      if ( !$case_sensitive )

        $term = strtolower($term);

      $where_req[] = $term;

    }

    $and_clause = ( $where_any != '' ) ? 'AND ' : '';

    $where_req = ( count($where_req) > 0 ) ? "{$and_clause}$text_col LIKE '%" . implode("%' AND $text_col LIKE '%", $where_req) . "%'" : '';

    $sql = 'SELECT CONCAT("ns=",namespace,";pid=",page_id) AS id FROM ' . table_prefix . "page_text WHERE $where_any $where_req;";

    if ( !($q = $db->sql_unbuffered_query($sql)) )

      $db->_die('Error is in perform_search(), includes/search.php, query 2. Parsed query dump follows:<pre>(indexable) ' . htmlspecialchars(print_r($query, true)) . '(non-indexable) ' . htmlspecialchars(print_r($query_phrase, true)) . '</pre>');

    if ( $row = $db->fetchrow() )

    {

      do

      {

        $id =& $row['id'];

        if ( isset($scores[$id]) )

        {

          $scores[$id]++;

        }

        else

        {

          $scores[$id] = 1;

        }

      }

      while ( $row = $db->fetchrow() );

    }

    $db->free_result();

  }

  //

  // STAGE 4 - SELECT PAGE TEXT AND ELIMINATE NOTS

  // At this point, we have a complete list of all the possible pages. Now we want to obtain the page text, and within the same query

  // eliminate any terms that shouldn't be in there.

  //

  // Generate master word list for the highlighter

  $word_list = array_values(array_merge($query['any'], $query['req'], $query_phrase['any'], $query_phrase['req']));

  $text_where = array();

  foreach ( $scores as $page_id => $_ )

  {

    $text_where[] = $db->escape($page_id);

  }

  $text_where = '( CONCAT("ns=",t.namespace,";pid=",t.page_id) = \'' . implode('\' OR CONCAT("ns=",t.namespace,";pid=",t.page_id) = \'', $text_where) . '\' )';

  if ( count($query['not']) > 0 )

    $text_where .= ' AND';

  $where_not = array();

  foreach ( $query['not'] as $term )

  {

    $term = escape_string_like($term);

    if ( !$case_sensitive )

      $term = strtolower($term);

    $where_not[] = $term;

  }

  $where_not = ( count($where_not) > 0 ) ? "$text_col NOT LIKE '%" . implode("%' AND $text_col NOT LIKE '%", $where_not) . "%'" : '';

  $sql = 'SELECT CONCAT("ns=",t.namespace,";pid=",t.page_id) AS id, t.page_id, t.namespace, CHAR_LENGTH(t.page_text) AS page_length, t.page_text, p.name AS page_name FROM ' . table_prefix . "page_text AS t

            LEFT JOIN " . table_prefix . "pages AS p

              ON ( p.urlname = t.page_id AND p.namespace = t.namespace )

            WHERE $text_where $where_not;";

  if ( !($q = $db->sql_unbuffered_query($sql)) )

    $db->_die('Error is in perform_search(), includes/search.php, query 3');

  $page_data = array();

  if ( $row = $db->fetchrow() )

  {

    do

    {

      $row['page_text'] = htmlspecialchars($row['page_text']);

      $row['page_name'] = htmlspecialchars($row['page_name']);

      // Highlight results (this is wonderfully automated)

      $row['page_text'] = highlight_and_clip_search_result($row['page_text'], $word_list, $case_sensitive);

      if ( strlen($row['page_text']) > 250 && !preg_match('/^\.\.\.(.+)\.\.\.$/', $row['page_text']) )

      {

        $row['page_text'] = substr($row['page_text'], 0, 150) . '...';

      }

      $row['page_name'] = highlight_search_result($row['page_name'], $word_list, $case_sensitive);

      $page_data[$row['id']] = $row;

    }

    while ( $row = $db->fetchrow() );

  }

  $db->free_result();

  //

  // STAGE 5 - SPECIAL PAGE TITLE SEARCH

  // Iterate through $paths->pages and check the titles for search terms. Score accordingly.

  //

  foreach ( $paths->pages as $page )

  {

    if ( $page['namespace'] != 'Special' )

      continue;

    $idstring = 'ns=' . $page['namespace'] . ';pid=' . $page['urlname_nons'];

    $any = array_merge($query['any'], $query_phrase['any']);

    foreach ( $any as $term )

    {

      if ( $case_sensitive )

      {

        if ( strstr($page['name'], $term) || strstr($page['urlname_nons'], $term) )

        {

          ( isset($scores[$idstring]) ) ? $scores[$idstring]++ : $scores[$idstring] = 1;

        }

      }

      else

      {

        if ( strstr(strtolower($page['name']), strtolower($term)) || strstr(strtolower($page['urlname_nons']), strtolower($term)) )

        {

          ( isset($scores[$idstring]) ) ? $scores[$idstring]++ : $scores[$idstring] = 1;

        }

      }

    }

    if ( isset($scores[$idstring]) )

    {

      $page_data[$idstring] = array(

          'page_name' => $page['name'],

          'page_text' => '',

          'page_id' => $page['urlname_nons'],

          'namespace' => $page['namespace'],

          'score' => $scores[$idstring],

          'page_length' => 1,

          'page_note' => '[Special page]'

        );

    }

  }

  //

  // STAGE 6 - SECOND ELIMINATION ROUND

  // Iterate through the list of required terms. If a given page is not found to have the required term, eliminate it

  //

  $required = array_merge($query['req'], $query_phrase['req']);

  foreach ( $required as $term )

  {

    foreach ( $page_data as $id => $page )

    {

      if ( ( $page['namespace'] == 'Special' || ( $page['namespace'] != 'Special' && !strstr($page['page_text'], $term) ) ) && !strstr($page['page_id'], $term) && !strstr($page['page_name'], $term) )

      {

        unset($page_data[$id]);

      }

    }

  }

  // At this point, all of our normal results are in. However, we can also allow plugins to hook into the system and score their own

  // pages and add text, etc. as necessary.

  // Plugins are COMPLETELY responsible for using the search terms and handling Boolean logic properly

  $code = $plugins->setHook('search_global_inner');

  foreach ( $code as $cmd )

  {

    eval($cmd);

  }

  // a marvelous debugging aid :-)

  // die('<pre>' . htmlspecialchars(print_r($page_data, true)) . '</pre>');

  //

  // STAGE 7 - HIGHLIGHT, TRIM, AND SCORE RESULTS

  // We now have the complete results of the search. We need to trim text down to show only portions of the page containing search

  // terms, highlight any search terms within the page, and sort the final results array in descending order of score.

  //