From 834fd0f608437a2621e437225a0f779b0cad25c8 Mon Sep 17 00:00:00 2001 From: x2003bruneau Date: Sun, 19 Aug 2007 12:57:42 +0000 Subject: [PATCH] Asynchronous mailing list mail moderation... This means: -> when the user moderate a mail, it just add an entry in a table of the database -> every minute, a cron lists the moderation requests and send as much mails as possible with two limits: * the cron should not run more than 1 minute * the cron should not send more than XXX mails (XXX is defined in the configuration file, and is set to 400 by default) ChangeLog | 3 + classes/xdb.php | 2 - configs/platal.cron.in | 4 ++ configs/platal.ini | 2 + modules/lists.php | 77 +++++++++++++++++++------------------------------ 5 files changed, 40 insertions(+), 48 deletions(-) git-svn-id: svn+ssh://murphy/home/svn/platal/trunk@1922 839d8a87-29fc-0310-9880-83ba4fa771e5 --- ChangeLog | 3 ++ bin/cron/cron_ml_moderate.php | 98 +++++++++++++++++++++++++++++++++++++++++++ classes/xdb.php | 2 +- configs/platal.cron.in | 4 ++ configs/platal.ini | 2 + modules/lists.php | 79 ++++++++++++++-------------------- upgrade/0.9.15/01_mailman.sql | 13 ++++++ upgrade/0.9.15/update.sh | 32 ++++++++++++++ 8 files changed, 184 insertions(+), 49 deletions(-) create mode 100755 bin/cron/cron_ml_moderate.php create mode 100644 upgrade/0.9.15/01_mailman.sql create mode 100755 upgrade/0.9.15/update.sh diff --git a/ChangeLog b/ChangeLog index 7da9302..e37c52a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,9 @@ New: * Core: - Auto-redirect HTML pages to HTTPS -FRU + * Lists: + - Asynchronous mail moderation (should avoid server overload) -FRU + * Search: - Shortcuts to open profiles or search in documentation -FRU diff --git a/bin/cron/cron_ml_moderate.php b/bin/cron/cron_ml_moderate.php new file mode 100755 index 0000000..febf47c --- /dev/null +++ b/bin/cron/cron_ml_moderate.php @@ -0,0 +1,98 @@ +#!/usr/bin/php5 -q +lists->max_mail_per_min + && time() - $handler < 60) { + // take a lock on a mail + XDB::execute("UPDATE ml_moderate + SET handler = {?} + WHERE handler IS NULL + ORDER BY ts + LIMIT 1", $handler); + if (XDB::affectedRows() == 0) { + break; + } + $query = XDB::query("SELECT nom, prenom, user_id, password, + ml, domain, mid, action, message + FROM auth_user_md5 AS u + INNER JOIN ml_moderate AS ml ON (u.user_id = ml.uid) + WHERE ml.handler = {?}", $handler); + list($nom, $prenom, $uid, $password, $list, $domain, $mid, $action, $reason) = $query->fetchOneRow(); + + // build the client + $client = new MMList($uid, $password, $domain); + + // send the mail + $mail = $client->get_pending_mail($list, $mid); + list($det,$mem,$own) = $client->get_members($list); + $count = 0; + switch ($action) { + case 'accept': + $action = 1; /** 1 = ACCEPT **/ + $subject = "Message accepté"; + $append = "a été accepté par $prenom $nom.\n"; + $count += count($mem) + count($own); + break; + case 'refuse': + $action = 2; /** 2 = REJECT **/ + $subject = "Message refusé"; + $append = "a été refusé par $prenom $nom avec la raison :\n\n" . $reason; + $count += count($own) + 1; + break; + case 'delete': + $action = 3; /** 3 = DISCARD **/ + $subject = "Message supprimé"; + $append = "a été supprimé par $prenom $nom.\n\n" + . "Rappel: il ne faut utiliser cette opération " + . "que dans le cas de spams ou de virus !\n"; + $count += $count($own); + break; + } + + if ($client->handle_request($list, $mid, $action, $reason)) { + $sent_mails += $count; + $texte = "le message suivant :\n\n" + . " Auteur: {$mail['sender']}\n" + . " Sujet : « {$mail['subj']} »\n" + . " Date : ".strftime("le %d %b %Y à %H:%M:%S", (int)$mail['stamp'])."\n\n" + . $append; + $mailer = new PlMailer(); + $mailer->addTo("$list-owner@{$domain}"); + $mailer->setFrom("$list-bounces@{$domain}"); + $mailer->addHeader('Reply-To', "$list-owner@{$domain}"); + $mailer->setSubject($subject); + $mailer->setTxtBody($texte); + $mailer->send(); + } + + // release the lock + XDB::execute("DELETE FROM ml_moderate WHERE handler = {?}", + $handler); + sleep(60 * $count / $globals->lists->max_mail_per_min); +} + +// vim:set et sw=4 sts=4 sws=4 foldmethod=marker enc=utf-8: +?> diff --git a/classes/xdb.php b/classes/xdb.php index 502af95..a431882 100644 --- a/classes/xdb.php +++ b/classes/xdb.php @@ -134,7 +134,7 @@ class XDB } public static function error() - { + { return XDB::$mysqli->error; } diff --git a/configs/platal.cron.in b/configs/platal.cron.in index ba914ed..c8c6196 100644 --- a/configs/platal.cron.in +++ b/configs/platal.cron.in @@ -26,4 +26,8 @@ WD=/home/web/prod/platal/bin/cron # homonymes 0 0 4 * * web cd $WD; ./homonymes.php + +# ml moderation +* * * * * web cd $WD; ./cron_ml_moderate.php > /dev/null + # vim:set noet syntax=crontab ts=8 sw=8 sts=8 enc=utf-8: diff --git a/configs/platal.ini b/configs/platal.ini index d8237f0..545ac84 100644 --- a/configs/platal.ini +++ b/configs/platal.ini @@ -39,6 +39,8 @@ rpcport = 4949 spool = "/var/lib/mailman/archives/private" vhost_sep = "_" +max_mail_per_min = 400 + [Manageurs] authorized_ips = "129.104.30.32 129.104.30.33 213.251.145.200" diff --git a/modules/lists.php b/modules/lists.php index 9c584dd..b2bacc0 100644 --- a/modules/lists.php +++ b/modules/lists.php @@ -62,6 +62,22 @@ class ListsModule extends PLModule return $globals->mail->domain; } + function get_pending_ops($domain, $list) + { + list($subs,$mails) = $this->client->get_pending_ops($list); + $res = XDB::query("SELECT mid + FROM ml_moderate + WHERE ml = {?} AND domain = {?}", + $list, $domain); + $mids = $res->fetchColumn(); + foreach ($mails as $key=>$mail) { + if (in_array($mail['id'], $mids)) { + unset($mails[$key]); + } + } + return array($subs, $mails); + } + function handler_lists(&$page) { function filter_owner($list) @@ -74,7 +90,7 @@ class ListsModule extends PLModule return $list['sub']; } - $this->prepare_client($page); + $domain = $this->prepare_client($page); $page->changeTpl('lists/index.tpl'); $page->addJsLink('ajax.js'); @@ -103,7 +119,7 @@ class ListsModule extends PLModule $member = array_filter($listes, 'filter_member'); $listes = array_diff_key($listes, $member); foreach ($owner as $key=>$liste) { - list($subs,$mails) = $this->client->get_pending_ops($liste['list']); + list($subs,$mails) = $this->get_pending_ops($domain, $liste['list']); $owner[$key]['subscriptions'] = $subs; $owner[$key]['mails'] = $mails; } @@ -133,7 +149,7 @@ class ListsModule extends PLModule list($liste, $members, $owners) = $this->client->get_members($list); if ($liste['own']) { - list($subs,$mails) = $this->client->get_pending_ops($list); + list($subs,$mails) = $this->get_pending_ops($domain, $list); $liste['subscriptions'] = $subs; $liste['mails'] = $mails; } @@ -367,47 +383,19 @@ class ListsModule extends PLModule function moderate_mail($domain, $liste, $mid) { - $mail = $this->client->get_pending_mail($liste, $mid); - $reason = ''; - - $prenom = S::v('prenom'); - $nom = S::v('nom'); - if (Env::has('mok')) { - $action = 1; /** 2 = ACCEPT **/ - $subject = "Message accepté"; - $append .= "a été accepté par $prenom $nom.\n"; + $action = 'accept'; } elseif (Env::has('mno')) { - $action = 2; /** 2 = REJECT **/ - $subject = "Message refusé"; - $reason = Post::v('reason'); - $append = "a été refusé par $prenom $nom avec la raison :\n\n" - . $reason; + $action = 'refuse'; } elseif (Env::has('mdel')) { - $action = 3; /** 3 = DISCARD **/ - $subject = "Message supprimé"; - $append = "a été supprimé par $prenom $nom.\n\n" - . "Rappel: il ne faut utiliser cette opération " - . "que dans le cas de spams ou de virus !\n"; - } - - if (isset($action) && $this->client->handle_request($liste, $mid, $action, $reason)) { - $texte = "le message suivant :\n\n" - ." Auteur: {$mail['sender']}\n" - ." Sujet : « {$mail['subj']} »\n" - ." Date : ".strftime("le %d %b %Y à %H:%M:%S", (int)$mail['stamp'])."\n\n" - .$append; - $mailer = new PlMailer(); - $mailer->addTo("$liste-owner@{$domain}"); - $mailer->setFrom("$liste-bounces@{$domain}"); - $mailer->addHeader('Reply-To', "$liste-owner@{$domain}"); - $mailer->setSubject($subject); - $mailer->setTxtBody(wordwrap($texte,72)); - $mailer->send(); - Get::kill('mid'); - } - - return $mail; + $action = 'delete'; + } else { + return false; + } + Get::kill('mid'); + return XDB::execute("INSERT IGNORE INTO ml_moderate + VALUES ({?}, {?}, {?}, {?}, {?}, NOW(), {?}, NULL)", + $liste, $domain, $mid, S::i('uid'), $action, Post::v('reason')); } function handler_moderate(&$page, $liste = null) @@ -454,13 +442,8 @@ class ListsModule extends PLModule if (Post::has('moderate_mails') && Post::has('select_mails')) { $mails = array_keys(Post::v('select_mails')); - if (count($mails) > 10) { - $page->trig("Le nombre d'actions qui peuvent être effectuées en un seul appel de cette page est limité à 10, car le temps de chargement de celle-ci est autrement trop long. Seules les dix premières actions demandées ont été effectuées."); - $mails = array_slice($mails, 0, 10); - } foreach($mails as $mail) { $this->moderate_mail($domain, $liste, $mail); - usleep(200000); } } elseif (Env::has('mid')) { if (Get::has('mid') && !Env::has('mok') && !Env::has('mdel')) { @@ -482,7 +465,7 @@ class ListsModule extends PLModule $mail = $this->moderate_mail($domain, $liste, Env::i('mid')); } elseif (Env::has('sid')) { - if (list($subs,$mails) = $this->client->get_pending_ops($liste)) { + if (list($subs,$mails) = $this->get_pending_ops($domain, $liste)) { foreach($subs as $user) { if ($user['id'] == Env::v('sid')) { $page->changeTpl('lists/moderate_sub.tpl'); @@ -494,7 +477,7 @@ class ListsModule extends PLModule } - if (list($subs,$mails) = $this->client->get_pending_ops($liste)) { + if (list($subs,$mails) = $this->get_pending_ops($domain, $liste)) { foreach ($mails as $key=>$mail) { $mails[$key]['stamp'] = strftime("%Y%m%d%H%M%S", $mail['stamp']); } diff --git a/upgrade/0.9.15/01_mailman.sql b/upgrade/0.9.15/01_mailman.sql new file mode 100644 index 0000000..1bb1c64 --- /dev/null +++ b/upgrade/0.9.15/01_mailman.sql @@ -0,0 +1,13 @@ +CREATE TABLE ml_moderate ( + ml VARCHAR(64) NOT NULL, + domain VARCHAR(64) NOT NULL, + mid SMALLINT(5) UNSIGNED NOT NULL, + uid SMALLINT(5) UNSIGNED NOT NULL, + action ENUM('accept', 'refuse', 'delete') NOT NULL, + ts TIMESTAMP NOT NULL, + message TEXT, + handler INT(8) UNSIGNED DEFAULT NULL, + PRIMARY KEY(ml, domain, mid) +) CHARSET=utf8; + +# vim:set syntax=mysql: diff --git a/upgrade/0.9.15/update.sh b/upgrade/0.9.15/update.sh new file mode 100755 index 0000000..de82fc0 --- /dev/null +++ b/upgrade/0.9.15/update.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +. ../inc/pervasive.sh + +mailman_stop +mailman_templates +mailman_start + + +########################################################### +for sql in *.sql +do + echo -n $sql + $MYSQL x4dat < $sql &>/dev/null || echo -n " ERROR" + echo . +done + +########################################################### + +echo "we will now upgrade the search table (this may be a long operation) + +please hit ^D to continue +" + +cat + +pushd ../../bin +./search.rebuild_db.php +popd + +########################################################### + -- 2.1.4