From: Pierre Habouzit (MadCoder Date: Tue, 12 Apr 2005 14:50:38 +0000 (+0000) Subject: brand new search X-Git-Tag: xorg/old~230 X-Git-Url: http://git.polytechnique.org/?a=commitdiff_plain;h=5e6e9a067ddd82428a6931e808fff9a8c705a13b;p=platal.git brand new search with full support for admin changes of names/second names/... now has to cope with usage name and soundex things. git-archimport-id: opensource@polytechnique.org--2005/platal--mainline--0.9--patch-531 --- diff --git a/ChangeLog b/ChangeLog index e515c29..5a59d9b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,10 +1,23 @@ ================================================================================ +VERSION 0.9.6 -- May 2004 + +New : + + * Search : + - Last improvements (clean code for search now !). -MC + +Bug/Wish : + + +Fixes (from 0.9.5 branch) : + +================================================================================ VERSION 0.9.5 07 Apr 2004 New : * Core : - - Improve login/exit wrt cookie. -MC + - Improve login/exit wrt cookie. -MC * Docs : - use of secure SMTP on a pocket PC. -Car diff --git a/bin/cron/recherche.pl b/bin/cron/recherche.pl deleted file mode 100755 index 7ceb410..0000000 --- a/bin/cron/recherche.pl +++ /dev/null @@ -1,45 +0,0 @@ -#! /usr/bin/perl -w -#*************************************************************************** -#* Copyright (C) 2003-2004 Polytechnique.org * -#* http://opensource.polytechnique.org/ * -#* * -#* This program is free software; you can redistribute it and/or modify * -#* it under the terms of the GNU General Public License as published by * -#* the Free Software Foundation; either version 2 of the License, or * -#* (at your option) any later version. * -#* * -#* This program is distributed in the hope that it will be useful, * -#* but WITHOUT ANY WARRANTY; without even the implied warranty of * -#* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * -#* GNU General Public License for more details. * -#* * -#* You should have received a copy of the GNU General Public License * -#* along with this program; if not, write to the Free Software * -#* Foundation, Inc., * -#* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * -#*************************************************************************** - - -my $mode = shift; -#mode 1 = mise à jour obligatoire -#mode autre = mise à jour si flag - -my @args = ("mysql x4dat ; - if (/1/) { - system(@args); - system(@args2); - } - close INFILE; - open(OUTFILE,'>/tmp/flag_recherche'); - print OUTFILE "0"; - close OUTFILE; -} diff --git a/bin/cron/recherche.sql b/bin/cron/recherche.sql deleted file mode 100644 index f4007e4..0000000 --- a/bin/cron/recherche.sql +++ /dev/null @@ -1,10 +0,0 @@ -USE x4dat; -DROP TABLE IF EXISTS recherche; -CREATE TABLE recherche SELECT matricule AS matricule,LOWER(REPLACE(nom_ini,'-',' ')) AS nom1, -LOWER(REPLACE(nom,'-',' ')) AS nom2, LOWER(REPLACE(nom_usage,'-',' ')) AS nom3, -LOWER(REPLACE(prenom_ini,'-',' ')) AS prenom1, LOWER(REPLACE(prenom,'-',' ')) AS prenom2, promo AS -promo FROM auth_user_md5; -DROP TABLE IF EXISTS recherche_soundex; -CREATE TABLE recherche_soundex (matricule int(8) unsigned PRIMARY KEY,nom1_soundex -char(4),nom2_soundex char(4),nom3_soundex char(4),prenom1_soundex char(4),prenom2_soundex char(4), -promo smallint(4) unsigned); diff --git a/bin/search.rebuild_db.php b/bin/search.rebuild_db.php new file mode 100755 index 0000000..9812153 --- /dev/null +++ b/bin/search.rebuild_db.php @@ -0,0 +1,37 @@ +#!/usr/bin/php4 -q +xdb->execute('DELETE FROM search_name'); + +$res = $globals->xdb->iterRow('SELECT user_id, nom, prenom, nom_usage FROM auth_user_md5'); +$i = 0; +while ($tmp = $res->next()) { + $uid = array_shift($tmp); + _user_reindex($uid, $tmp); + printf ("%02.2f %%\n", ++$i*100/$res->total()); +} + +?> diff --git a/configs/platal.cron b/configs/platal.cron index 4c602d0..c00a822 100644 --- a/configs/platal.cron +++ b/configs/platal.cron @@ -20,9 +20,6 @@ WD=/home/web/prod/platal/bin/cron # validations 0 */3 * * * web cd $WD; ./cron_validations.php -# search -0 */3 * * * web cd $WD; ./recherche.pl 0 - # homonymes 0 0 4 * * * web cd $WD; ./homonymes.php # vim:set noet syntax=crontab ts=8 sw=8 sts=8: diff --git a/htdocs/admin/utilisateurs.php b/htdocs/admin/utilisateurs.php index ea98cf3..ce3117f 100644 --- a/htdocs/admin/utilisateurs.php +++ b/htdocs/admin/utilisateurs.php @@ -126,8 +126,7 @@ if ($login) { comment = '".addslashes($comm)."' WHERE user_id = '{$mr['user_id']}'"; if ($globals->xdb->execute($query)) { - // FIXME: recherche - system('echo 1 > /tmp/flag_recherche'); + user_reindex($mr['user_id']); require_once("diogenes/diogenes.hermes.inc.php"); $mailer = new HermesMailer(); diff --git a/htdocs/advanced_search.php b/htdocs/advanced_search.php index 8d6f438..3be1313 100644 --- a/htdocs/advanced_search.php +++ b/htdocs/advanced_search.php @@ -118,6 +118,9 @@ if (!Env::has('rechercher')) { global $globals; $where = $fields->get_where_statement(); + if ($where) { + $where = "WHERE $where"; + } $sql = 'SELECT SQL_CALC_FOUND_ROWS DISTINCT u.nom, u.prenom, '.$globals->search->result_fields.' @@ -126,15 +129,13 @@ if (!Env::has('rechercher')) { FROM auth_user_md5 AS u LEFT JOIN auth_user_quick AS q USING(user_id) '.$fields->get_select_statement().' - '.(Env::has('only_referent') ? - ' INNER JOIN mentor AS m ON (m.uid = u.user_id)' : - '').' + '.(Env::has('only_referent') ? ' INNER JOIN mentor AS m ON (m.uid = u.user_id)' : '').' LEFT JOIN aliases AS a ON (u.user_id = a.id AND a.type="a_vie") LEFT JOIN contacts AS c ON (c.uid='.Session::getInt('uid').' AND c.contact=u.user_id) LEFT JOIN watch_nonins AS w ON (w.ni_id=u.user_id AND w.uid='.Session::getInt('uid').') - '.$globals->search->result_where_statement.' - '.(empty($where) ? '' : "WHERE $where").' - ORDER BY '.($order?($order.', '):'') + '.$globals->search->result_where_statement." + $where + ORDER BY ".($order?($order.', '):'') .implode(',',array_filter(array($fields->get_order_statement(), 'promo DESC, NomSortKey, prenom'))).' LIMIT '.($offset * $limit).','.$limit; $liste = $globals->xdb->iterator($sql); diff --git a/htdocs/search.php b/htdocs/search.php index b11b267..266fce5 100644 --- a/htdocs/search.php +++ b/htdocs/search.php @@ -43,22 +43,22 @@ if (Env::has('quick')) { new ThrowError('Recherche trop générale.'); } - $sql = 'SELECT SQL_CALC_FOUND_ROWS DISTINCT + $sql = 'SELECT SQL_CALC_FOUND_ROWS UPPER(IF(u.nom!="",u.nom,u.nom_ini)) AS nom, IF(u.prenom!="",u.prenom,u.prenom_ini) AS prenom, '.$globals->search->result_fields.' c.uid AS contact, w.ni_id AS watch, - '.$qSearch->get_mark_statement().' - + '.$qSearch->get_score_statement().' FROM auth_user_md5 AS u - LEFT JOIN auth_user_quick AS q USING(user_id) + '.$fields->get_select_statement().' + LEFT JOIN auth_user_quick AS q ON (u.user_id = q.user_id) LEFT JOIN aliases AS a ON (u.user_id = a.id AND a.type="a_vie") LEFT JOIN contacts AS c ON (c.uid='.Session::getInt('uid').' AND c.contact=u.user_id) LEFT JOIN watch_nonins AS w ON (w.ni_id=u.user_id AND w.uid='.Session::getInt('uid').') '.$globals->search->result_where_statement.' WHERE '.$fields->get_where_statement().(logged() && Env::has('nonins') ? ' AND u.perms="pending" AND u.deces=0' : '').' - HAVING mark>0 + GROUP BY u.user_id ORDER BY '.($order?($order.', '):'') .implode(',',array_filter(array($fields->get_order_statement(), 'u.promo DESC, NomSortKey, prenom'))).' LIMIT '.$offset * $globals->search->per_page.','.$globals->search->per_page; @@ -72,7 +72,7 @@ if (Env::has('quick')) { $search = new XOrgSearch(get_list); $search->setNbLines($globals->search->per_page); - $search->addOrder('mark', 'mark', false, 'pertinence', AUTH_PUBLIC, true); + $search->addOrder('score', 'score', false, 'pertinence', AUTH_PUBLIC, true); $nb_tot = $search->show(); diff --git a/include/search/classes.inc.php b/include/search/classes.inc.php index dffb6c2..75d61ed 100644 --- a/include/search/classes.inc.php +++ b/include/search/classes.inc.php @@ -239,9 +239,9 @@ class QuickSearch extends SField parent::get_request(); $s = replace_accent(trim($this->value)); $s = preg_replace('!\d+!', ' ', $s); - $s = preg_replace('! - !', '', $s); + $s = preg_replace('![\'\-]!', '', $s); $s = str_replace('*','%',$s); - $this->strings = preg_split("![^a-zA-Z\-%]+!",$s, -1, PREG_SPLIT_NO_EMPTY); + $this->strings = preg_split("![^a-zA-Z%]+!",$s, -1, PREG_SPLIT_NO_EMPTY); $s = trim($this->value); $s = preg_replace('! *- *!', '-', $s); @@ -261,10 +261,10 @@ class QuickSearch extends SField function get_where_statement() { $where = Array(); - foreach ($this->strings as $s) { - $t = '%'.str_replace('*', '%', $s).'%'; + foreach ($this->strings as $i => $s) { + $t = str_replace('*', '%', $s).'%'; $t = str_replace('%%', '%', $t); - $where[] = "(u.nom LIKE '$t' OR u.nom_usage LIKE '$t' OR u.prenom LIKE '$t')"; + $where[] = "sn$i.token LIKE '$t'"; } $wherep = Array(); @@ -288,30 +288,33 @@ class QuickSearch extends SField } // }}} - // {{{ function get_mark_statement() - - function get_mark_statement() + // {{{ get_select_statement + function get_select_statement() { - if (empty($this->strings)) { - return "10 AS mark"; + $join = ""; + foreach ($this->strings as $i => $s) { + $join .= "INNER JOIN search_name AS sn$i ON (u.user_id = sn$i.uid)\n"; } - $order = "0"; - $sep = "[ \\'\\-]"; - foreach ($this->strings as $s) { - $order .= " + ( (u.nom='$s' OR u.nom_usage='$s') + (CONCAT(' ',u.nom,' ',u.nom_usage,' ') RLIKE '$sep{$s}$sep') )*1000 - + ( CONCAT(' ',u.nom,' ',u.nom_usage,' ') RLIKE '$sep{$s}' )*100 - + ( (u.prenom = '$s') + (CONCAT(' ',u.prenom,' ') RLIKE '$sep{$s}$sep') )*10 - + ( u.prenom RLIKE '(^|$sep){$s}' )"; - } - return $order.' AS mark'; + return $join; } - // }}} // {{{ function get_order_statement() function get_order_statement() { - return 'mark DESC'; + return false; + } + + // }}} + // {{{ function get_score_statement + + function get_score_statement() + { + $sum = array('0'); + foreach ($this->strings as $i => $s) { + $sum[] .= "SUM(sn$i.score)"; + } + return join('+', $sum).' AS score'; } // }}} diff --git a/include/user.func.inc.php b/include/user.func.inc.php index b9350fa..86cf616 100644 --- a/include/user.func.inc.php +++ b/include/user.func.inc.php @@ -253,6 +253,36 @@ function &get_user_details($login, $from_uid = '') } // }}} +// {{{ function _user_reindex +function _user_reindex($uid, $keys) { + global $globals; + foreach ($keys as $key) { + if ($key == '') { + continue; + } + $toks = preg_split('/[ \'\-]+/', $key); + $token = ""; + $first = 5; + while ($toks) { + $token = strtolower(replace_accent(array_pop($toks) . $token)); + $score = ($toks ? 0 : 10 + $first); + mysql_query("REPLACE INTO search_name (token, uid, score) VALUES('$token',$uid,$score)"); + $first = 0; + } + } +} + +// }}} +// {{{ function user_reindex + +function user_reindex($uid) { + global $globals; + $globals->xdb->execute("DELETE FROM search_name WHERE uid={?}", $uid); + $res = $globals->xdb->query("SELECT prenom, nom, nom_usage FROM auth_user_md5 WHERE user_id = {?}", $uid); + _user_reindex($uid, $res->fetchOneRow()); +} + +// }}} // vim:set et sw=4 sts=4 sws=4 foldmethod=marker: ?> diff --git a/include/xorg/database.inc.php b/include/xorg/database.inc.php index 52e7ee4..d223662 100644 --- a/include/xorg/database.inc.php +++ b/include/xorg/database.inc.php @@ -34,8 +34,7 @@ class XOrgDB function _prepare($args) { $query = array_map(Array($this, '_db_escape'), $args); - $query[0] = str_replace('%', '%%', $args[0]); - $query[0] = str_replace('{?}', '%s', $query[0]); + $query[0] = str_replace('{?}', '%s', str_replace('%', '%%', $args[0])); return call_user_func_array('sprintf', $query); } @@ -44,8 +43,7 @@ class XOrgDB function &query() { - $query = $this->_prepare(func_get_args()); - return new XOrgDBResult($query); + return new XOrgDBResult($this->_prepare(func_get_args())); } // }}} @@ -53,8 +51,7 @@ class XOrgDB function execute() { global $globals; - $query = $this->_prepare(func_get_args()); - return $globals->db->query($query); + return $globals->db->query($this->_prepare(func_get_args())); } // }}} @@ -62,8 +59,7 @@ class XOrgDB function &iterator() { - $query = $this->_prepare(func_get_args()); - return new XOrgDBIterator($query); + return new XOrgDBIterator($this->_prepare(func_get_args())); } // }}} @@ -71,8 +67,7 @@ class XOrgDB function &iterRow() { - $query = $this->_prepare(func_get_args()); - return new XOrgDBIterator($query, MYSQL_NUM); + return new XOrgDBIterator($this->_prepare(func_get_args()), MYSQL_NUM); } // }}} diff --git a/upgrade/0.9.6/10_search_name.sql b/upgrade/0.9.6/10_search_name.sql new file mode 100644 index 0000000..0c49a14 --- /dev/null +++ b/upgrade/0.9.6/10_search_name.sql @@ -0,0 +1,12 @@ + +CREATE TABLE `search_name` ( + `uid` int(11) NOT NULL default '0', + `token` char(255) NOT NULL default '', + `score` int(11) NOT NULL default '0', + PRIMARY KEY (uid, token), + INDEX (token) +); + + +drop table recherche; + diff --git a/upgrade/0.9.6/update.sh b/upgrade/0.9.6/update.sh new file mode 100755 index 0000000..de82fc0 --- /dev/null +++ b/upgrade/0.9.6/update.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +. ../inc/pervasive.sh + +mailman_stop +mailman_templates +mailman_start + + +########################################################### +for sql in *.sql +do + echo -n $sql + $MYSQL x4dat < $sql &>/dev/null || echo -n " ERROR" + echo . +done + +########################################################### + +echo "we will now upgrade the search table (this may be a long operation) + +please hit ^D to continue +" + +cat + +pushd ../../bin +./search.rebuild_db.php +popd + +########################################################### +