But cruel reality differs from theory.
Problem : I 've tried to add 35K users to cohort using "bulk user actions" page and
get memory exhausted php exception:
Fatal error: Allowed memory size of 134217728 bytes exhausted (tried to allocate 123 bytes)
in /www/moodle/lib/dml/mysqli_native_moodle_database.php on line 832
Environment:
1. Win 7
2. wamp
3. moodle 2.3.9+ installed
Solving:
Lets check original php script - user_bulk_cohortadd.php to allocate memory eater code.
It located
/moodle/admin/user/user_bulk_cohortadd.php
Note:marker - my comments
//ORIGINAL SCRIPT START
require('../../config.php');
require_once($CFG->libdir.'/adminlib.php');
require_once('user_bulk_cohortadd_form.php');
require_once("$CFG->dirroot/cohort/lib.php");
$sort = optional_param('sort', 'fullname', PARAM_ALPHA);
$dir = optional_param('dir', 'asc', PARAM_ALPHA);
admin_externalpage_setup('userbulk');
require_capability('moodle/cohort:assign', get_context_instance(CONTEXT_SYSTEM));
$users = $SESSION->bulk_users; //This array has 35K elements - input data
$strnever = get_string('never');
$cohorts = array(''=>get_string('choosedots'));
$allcohorts = $DB->get_records('cohort');
foreach ($allcohorts as $c) {
if (!empty($c->component)) {
// external cohorts can not be modified
continue;
}
$context = get_context_instance_by_id($c->contextid);
if (!has_capability('moodle/cohort:assign', $context)) {
continue;
}
if (empty($c->idnumber)) {
$cohorts[$c->id] = format_string($c->name);
} else {
$cohorts[$c->id] = format_string($c->name) . ' [' . $c->idnumber . ']';
}
}
unset($allcohorts);
if (count($cohorts) < 2) {
echo $OUTPUT->header();
echo $OUTPUT->heading(get_string('bulkadd', 'core_cohort'));
echo $OUTPUT->notification(get_string('bulknocohort', 'core_cohort'));
echo $OUTPUT->continue_button(new moodle_url('/admin/user/user_bulk.php'));
echo $OUTPUT->footer();
die;
}
$countries = get_string_manager()->get_list_of_countries(true);
foreach ($users as $key => $id) {
$user = $DB->get_record('user', array('id'=>$id, 'deleted'=>0), 'id, firstname, lastname, username, email, country, lastaccess, city'); // This is memory eater No 1
$user->fullname = fullname($user, true);
$user->country = @$countries[$user->country];
unset($user->firstname);
unset($user->lastname);
$users[$key] = $user;
}
unset($countries);
$mform = new user_bulk_cohortadd_form(null, $cohorts);
if (empty($users) or $mform->is_cancelled()) {
redirect(new moodle_url('/admin/user/user_bulk.php'));
} else if ($data = $mform->get_data()) {
// process request
foreach ($users as $user) {
if (!$DB->record_exists('cohort_members', array('cohortid'=>$data->cohort, 'userid'=>$user->id))) {
cohort_add_member($data->cohort, $user->id);
}
}
redirect(new moodle_url('/admin/user/user_bulk.php'));
}
// Need to sort by date
function sort_compare($a, $b) {
global $sort, $dir;
if ($sort == 'lastaccess') {
$rez = $b->lastaccess - $a->lastaccess;
} else {
$rez = strcasecmp(@$a->$sort, @$b->$sort);
}
return $dir == 'desc' ? -$rez : $rez;
}
usort($users, 'sort_compare');
$table = new html_table();
$table->width = "95%";
$columns = array('fullname', 'email', 'city', 'country', 'lastaccess');
foreach ($columns as $column) {
$strtitle = get_string($column);
if ($sort != $column) {
$columnicon = '';
$columndir = 'asc';
} else {
$columndir = ($dir == 'asc') ? 'desc' : 'asc';
$columnicon = ' <img src="'.$OUTPUT->pix_url('t/'.($dir == 'asc' ? 'down' : 'up' )).'" alt="" />';
}
$table->head[] = '<a href="user_bulk_cohortadd.php?sort='.$column.'&dir='.$columndir.'">'.$strtitle.'</a>'.$columnicon;
$table->align[] = 'left';
}
foreach ($users as $user) {
$table->data[] = array (
'<a href="'.$CFG->wwwroot.'/user/view.php?id='.$user->id.'&course='.SITEID.'">'.$user->fullname.'</a>',
$user->email,
$user->city,
$user->country,
$user->lastaccess ? format_time(time() - $user->lastaccess) : $strnever
);
} //This is memory eater No 2
echo $OUTPUT->header();
echo $OUTPUT->heading(get_string('bulkadd', 'core_cohort'));
echo html_writer::table($table);
echo $OUTPUT->box_start();
$mform->display();
echo $OUTPUT->box_end();
echo $OUTPUT->footer();
//ORIGINAL SCRIPT END
//FIXED SCRIPT START
require('../../config.php');
require_once($CFG->libdir.'/adminlib.php');
require_once('user_bulk_cohortadd_form.php');
require_once("$CFG->dirroot/cohort/lib.php");
$sort = optional_param('sort', 'fullname', PARAM_ALPHA);
$dir = optional_param('dir', 'asc', PARAM_ALPHA);
admin_externalpage_setup('userbulk');
require_capability('moodle/cohort:assign', get_context_instance(CONTEXT_SYSTEM));
$users = $SESSION->bulk_users;
$strnever = get_string('never');
$cohorts = array(''=>get_string('choosedots'));
$allcohorts = $DB->get_records('cohort');
foreach ($allcohorts as $c) {
if (!empty($c->component)) {
// external cohorts can not be modified
continue;
}
$context = get_context_instance_by_id($c->contextid);
if (!has_capability('moodle/cohort:assign', $context)) {
continue;
}
if (empty($c->idnumber)) {
$cohorts[$c->id] = format_string($c->name);
} else {
$cohorts[$c->id] = format_string($c->name) . ' [' . $c->idnumber . ']';
}
}
unset($allcohorts);
if (count($cohorts) < 2) {
echo $OUTPUT->header();
echo $OUTPUT->heading(get_string('bulkadd', 'core_cohort'));
echo $OUTPUT->notification(get_string('bulknocohort', 'core_cohort'));
echo $OUTPUT->continue_button(new moodle_url('/admin/user/user_bulk.php'));
echo $OUTPUT->footer();
die;
}
$mform = new user_bulk_cohortadd_form(null, $cohorts);
$countries = get_string_manager()->get_list_of_countries(true);
if (empty($users) or $mform->is_cancelled()) {
redirect(new moodle_url('/admin/user/user_bulk.php'));
} else if ($data = $mform->get_data()) {
// process request
//Fixing memory eater No 1
try {
//Huge amount of inserts optimizations
$transaction = $DB->start_delegated_transaction();
foreach ($users as $key => $id) {
$user = $DB->get_record('user', array('id'=>$id, 'deleted'=>0), 'id');
if($user){
if (!$DB->record_exists('cohort_members', array('cohortid'=>$data->cohort, 'userid'=>$user->id))) {
cohort_add_member($data->cohort, $user->id);
}
}
}
$transaction->allow_commit();
} catch(Exception $e) {
$transaction->rollback($e);
}
redirect(new moodle_url('/admin/user/user_bulk.php'));
}
unset($countries);
function sort_compare($a, $b) {
global $sort, $dir;
if ($sort == 'lastaccess') {
$rez = $b->lastaccess - $a->lastaccess;
} else {
$rez = strcasecmp(@$a->$sort, @$b->$sort);
}
return $dir == 'desc' ? -$rez : $rez;
}
usort($users, 'sort_compare');
echo $OUTPUT->header();
echo $OUTPUT->heading(get_string('bulkadd', 'core_cohort'));
echo '<table width="95%" >';
$columns = array(
'fullname' => get_string('fullname')
, 'email' => get_string('email')
, 'city' => get_string('city')
, 'country' => get_string('country')
, 'lastaccess' => get_string('lastaccess')
);
echo '<tr>';
echo '<th>';
echo $columns['fullname'];
echo '</th>';
echo '<th>';
echo $columns['email'];
echo '</th>';
echo '<th>';
echo $columns['city'];
echo '</th>';
echo '<th>';
echo $columns['country'];
echo '</th>';
echo '<th>';
echo $columns['lastaccess'];
echo '</th>';
echo '</tr>';
//Fixing memory eater No 2
foreach ($users as $key => $id) {
$user = $DB->get_record('user', array('id'=>$id, 'deleted'=>0), 'id, firstname, lastname, username, email, country, lastaccess, city');
$user->fullname = fullname($user, true);
$user->country = @$countries[$user->country];
unset($user->firstname);
unset($user->lastname);
echo '<tr>';
echo '<td>';
echo '<a href="'.$CFG->wwwroot.'/user/view.php?id='.$user->id.'&course='.SITEID.'">'.$user->fullname.'</a>';
echo '</td>';
echo '<td>';
echo $user->email;
echo '</td>';
echo '<td>';
echo $user->city;
echo '</td>';
echo '<td>';
echo $user->country;
echo '</td>';
echo '<td>';
echo $user->lastaccess ? format_time(time() - $user->lastaccess) : $strnever;
echo '</td>';
echo '</tr>';
}
echo '</table>';
echo $OUTPUT->box_start();
$mform->display();
echo $OUTPUT->box_end();
echo $OUTPUT->footer();
//FIXED SCRIPT END
Tested with 35K. It works, yeah!!!
No comments:
Post a Comment