From: Stéphane Jacob Date: Tue, 2 Nov 2010 18:18:58 +0000 (+0100) Subject: Properly reorganises merge issues related scripts. X-Git-Tag: xorg/1.0.2~161 X-Git-Url: http://git.polytechnique.org/?a=commitdiff_plain;h=cee996432b4742bdbb3c3f2378f00dbbd8a61b58;p=platal.git Properly reorganises merge issues related scripts. Signed-off-by: Stéphane Jacob --- diff --git a/upgrade/1.0.1/README b/upgrade/1.0.1/README index 4b789df..955a527 100644 --- a/upgrade/1.0.1/README +++ b/upgrade/1.0.1/README @@ -1,3 +1,3 @@ 99_jobs.sql must be run after php scripts. Both merge.php and merge_issues_*.php must not be run during the release. They are only for merge purpose. -On top of that, merge_issues_*.php require to be run in a screen in parallel as they take a couple of days/weeks to complete. +On top of that, merge_issues_geocoding.php requires to be run in a screen *after* merge_issues.php as it take a couple of weeks to complete. diff --git a/upgrade/1.0.1/merge_issues_phones.php b/upgrade/1.0.1/merge_issues.php similarity index 56% rename from upgrade/1.0.1/merge_issues_phones.php rename to upgrade/1.0.1/merge_issues.php index af8b01b..ac0c495 100755 --- a/upgrade/1.0.1/merge_issues_phones.php +++ b/upgrade/1.0.1/merge_issues.php @@ -4,10 +4,70 @@ require_once 'connect.db.inc.php'; require_once '../../classes/phone.php'; +require_once '../../classes/address.php'; $globals->debug = 0; // Do not store backtraces. -print "Formats non formated phones. (1/2)\n"; +print "Deletes duplicated addresses. (1/3)\n"; +$pids = XDB::rawFetchColumn("SELECT DISTINCT(pid) + FROM profile_addresses AS a1 + WHERE type = 'home' AND EXISTS (SELECT * + FROM profile_addresses AS a2 + WHERE a2.type = 'home' AND a2.pid = a1.pid AND a2.id != a1.id) + ORDER BY pid"); +$total = count($pids); +$done = 0; +$aux = 0; +$deleted = 0; +$addresses = array(); +$rawAddresses = array(); +$duplicates = array(); +foreach ($pids as $pid) { + $count = 0; + $it = Address::iterate(array($pid), array(Address::LINK_PROFILE), array(0)); + while ($item = $it->next()) { + $addresses[] = $item; + $rawAddresses[] = preg_replace('/[^a-zA-Z0-9]/', '', replace_accent($item->text)); + ++$count; + } + for ($i = 0; $i < $count; ++$i) { + for ($j = $i + 1; $j < $count; ++$j) { + if ($rawAddresses[$i] == $rawAddresses[$j]) { + $duplicates[$i] = true; + } + } + } + foreach ($duplicates as $key => $bool) { + unset($addresses[$key]); + } + if (count($addresses) != $count) { + $deleted += ($count - count($addresses)); + Address::deleteAddresses($pid, 'home'); + $id = 0; + foreach ($addresses as $address) { + $address->setId($id); + $address->save(); + ++$id; + } + XDB::execute('UPDATE IGNORE profile_merge_issues + SET issues = REPLACE(issues, \'address\', \'\') + WHERE pid = {?}', $pid); + } + unset($rawAddresses); + unset($addresses); + unset($duplicates); + + ++$done; + ++$aux; + if ($aux == 100) { + $aux = 0; + printf("\r%u / %u", $done, $total); + } +} +printf("\r%u / %u", $done, $total); +print "\n$deleted addresses deleted.\n\n"; + +print "Formats non formated phones. (2/3)\n"; $it = XDB::rawIterator("SELECT search_tel AS search, display_tel AS display, comment, link_id, tel_type AS type, link_type, tel_id AS id, pid, pub FROM profile_phones @@ -31,7 +91,7 @@ while ($item = $it->next()) { printf("\r%u / %u", $i, $total); print "\nFormating done.\n\n"; -print "Deletes duplicated phones. (2/2)\n"; +print "Deletes duplicated phones. (3/3)\n"; $pids = XDB::rawFetchColumn("SELECT DISTINCT(pid) FROM profile_phones AS p1 WHERE link_type = 'user' AND EXISTS (SELECT * diff --git a/upgrade/1.0.1/merge_issues_addresses.php b/upgrade/1.0.1/merge_issues_addresses.php deleted file mode 100755 index 4c876f7..0000000 --- a/upgrade/1.0.1/merge_issues_addresses.php +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/php5 -debug = 0; // Do not store backtraces. - -print "Tries to geocode addresses (due a bug in the previous release, all addresses must run once again). (1/2)\n"; -$time = XDB::fetchOneCell('SELECT COUNT(distinct(pid), jobid) - FROM profile_addresses'); -$time = ceil($time / 60 / 24); -print "It will approximately take $time days.\n"; - -$it = XDB::rawIterator('SELECT * - FROM profile_addresses - ORDER BY pid, jobid, type, id'); -$total = $it->total(); -$i = 0; -$j = 0; -printf("\r%u / %u", $i, $total); -$pid = 0; -$jobid = 0; -while ($item = $it->next()) { - $address = new Address($item); - $address->format(array(true, true)); - $address->delete(); - $address->save(); - if (!($pid == $address->pid && $jobid == $address->jobid)) { - $pid = $address->pid; - $jobid = $address->jobid; - sleep(60); - } - - ++$i; - ++$j; - if ($j == 10) { - $j = 0; - printf("\r%u / %u", $i, $total); - } -} -printf("\r%u / %u", $i, $total); -print "\nGeocoding done.\n\n"; - -print "Deletes duplicated addresses. (2/2)\n"; -$pids = XDB::rawFetchColumn("SELECT DISTINCT(pid) - FROM profile_addresses AS a1 - WHERE type = 'home' AND EXISTS (SELECT * - FROM profile_addresses AS a2 - WHERE a2.type = 'home' AND a2.pid = a1.pid AND a2.id != a1.id) - ORDER BY pid"); -$total = count($pids); -$done = 0; -$aux = 0; -$deleted = 0; -$addresses = array(); -$rawAddresses = array(); -$duplicates = array(); -foreach ($pids as $pid) { - $count = 0; - $it = Address::iterate(array($pid), array(Address::LINK_PROFILE), array(0)); - while ($item = $it->next()) { - $addresses[] = $item; - $rawAddresses[] = preg_replace('/[^a-zA-Z0-9]/', '', replace_accent($item->text)); - ++$count; - } - for ($i = 0; $i < $count; ++$i) { - for ($j = $i + 1; $j < $count; ++$j) { - if ($rawAddresses[$i] == $rawAddresses[$j]) { - $duplicates[$i] = true; - } - } - } - foreach ($duplicates as $key => $bool) { - unset($addresses[$key]); - } - if (count($addresses) != $count) { - $deleted += ($count - count($addresses)); - Address::deleteAddresses($pid, 'home'); - $id = 0; - foreach ($addresses as $address) { - $address->setId($id); - $address->save(); - ++$id; - } - XDB::execute('UPDATE IGNORE profile_merge_issues - SET issues = REPLACE(issues, \'address\', \'\') - WHERE pid = {?}', $pid); - } - unset($rawAddresses); - unset($addresses); - unset($duplicates); - - ++$done; - ++$aux; - if ($aux == 100) { - $aux = 0; - printf("\r%u / %u", $done, $total); - } -} -printf("\r%u / %u", $done, $total); -print "\n$deleted addresses deleted.\n\n"; - -print "That's all folks!\n"; - -/* vim:set et sw=4 sts=4 ts=4: */ -?> diff --git a/upgrade/1.0.1/merge_issues_geocoding.php b/upgrade/1.0.1/merge_issues_geocoding.php new file mode 100755 index 0000000..8ec2ba3 --- /dev/null +++ b/upgrade/1.0.1/merge_issues_geocoding.php @@ -0,0 +1,48 @@ +#!/usr/bin/php5 +debug = 0; // Do not store backtraces. + +print "Tries to geocode addresses (due a bug in the previous release, all addresses must run once again).\n"; +$time = XDB::fetchOneCell('SELECT COUNT(distinct(pid), jobid) + FROM profile_addresses'); +$time = ceil($time / 60 / 24); +print "It will approximately take $time days.\n"; + +$it = XDB::rawIterator('SELECT * + FROM profile_addresses + ORDER BY pid, jobid, type, id'); +$total = $it->total(); +$i = 0; +$j = 0; +printf("\r%u / %u", $i, $total); +$pid = 0; +$jobid = 0; +while ($item = $it->next()) { + $address = new Address($item); + $address->format(array(true, true)); + $address->delete(); + $address->save(); + if (!($pid == $address->pid && $jobid == $address->jobid)) { + $pid = $address->pid; + $jobid = $address->jobid; + sleep(60); + } + + ++$i; + ++$j; + if ($j == 10) { + $j = 0; + printf("\r%u / %u", $i, $total); + } +} +printf("\r%u / %u", $i, $total); +print "\nGeocoding done.\n\n"; +print "That's all folks!\n"; + +/* vim:set et sw=4 sts=4 ts=4: */ +?>