From 4ee009425dcf0715afa9dfc9136a31e020abb039 Mon Sep 17 00:00:00 2001 From: Glenn Rice Date: Tue, 12 May 2026 19:52:09 -0500 Subject: [PATCH 1/2] Improve the speed of the database user retrieval for the student nav. Now that the student nav is on every set and problem page there is a need to make the retrieval of all users assigned to the set faster. I noticed in a large class with more than 25,000 users this is quite slow now. It takes about 8 seconds for each page to load. The problem is the query is passing all of the user ids of those assigned to the set to the `getUsersWhere` method. That means that `SQL::Abstract` has to process all of those and compile a rather long sql statement. That is slow. So this breaks the list of user ids into chunks of 500. For the class with more than 25,000 users this drops the page load time to just under 2 seconds. Stil not great, but considerably better. On my production placement exam server that has just over 32,000 users it currently takes more than 35 seconds for each page to load. With this pull request it drops the time to about 8 seconds. Again, not great but considerably better. Clearly the placement server is not as fast as my local computer also. Note that for classes with less than 500 users this won't change anything. Also note that this is only an issue for those that have the student nav shown. --- lib/WeBWorK/ContentGenerator/GatewayQuiz.pm | 8 ++++- lib/WeBWorK/HTML/StudentNav.pm | 35 ++++++++++++--------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/lib/WeBWorK/ContentGenerator/GatewayQuiz.pm b/lib/WeBWorK/ContentGenerator/GatewayQuiz.pm index 428f0071ca..af068a65b6 100644 --- a/lib/WeBWorK/ContentGenerator/GatewayQuiz.pm +++ b/lib/WeBWorK/ContentGenerator/GatewayQuiz.pm @@ -1378,7 +1378,13 @@ sub nav ($c, $args) { # Find all versions of this set that have been taken (excluding those taken by the current user). my @users = $db->listSetVersionsWhere({ user_id => { '!=' => $userID }, set_id => { like => "$setID,v\%" } }); - my @allUserRecords = $db->getUsers(map { $_->[0] } @users); + my @allUserRecords; + my $i = 0; + while ($i < @users) { + push(@allUserRecords, + $db->getUsers(map { $_->[0] } @users[ $i .. ($i + 499 < $#users ? $i + 499 : $#users) ])); + $i += 500; + } if (@allUserRecords) { my $filter = $c->param('studentNavFilter'); diff --git a/lib/WeBWorK/HTML/StudentNav.pm b/lib/WeBWorK/HTML/StudentNav.pm index 7591712a8d..1f8c2f2464 100644 --- a/lib/WeBWorK/HTML/StudentNav.pm +++ b/lib/WeBWorK/HTML/StudentNav.pm @@ -15,13 +15,24 @@ sub studentNav ($c, $setID) { return '' unless $c->authz->hasPermissions($userID, 'become_student'); # Find all users for the given set (except the current user) sorted by last_name, then first_name, then user_id. - my @allUserRecords = $c->db->getUsersWhere( - { - user_id => - [ map { $_->[0] } $c->db->listUserSetsWhere({ set_id => $setID, user_id => { '!=' => $userID } }) ] - }, - [qw/last_name first_name user_id/] - ); + my @users = map { $_->[0] } $c->db->listUserSetsWhere({ set_id => $setID, user_id => { '!=' => $userID } }); + my @allUserRecords; + my $i = 0; + while ($i < @users) { + push( + @allUserRecords, + $c->db->getUsersWhere( + { user_id => [ @users[ $i .. ($i + 499 < $#users ? $i + 499 : $#users) ] ] }, + [qw/last_name first_name user_id/] + ) + ); + $i += 500; + } + if (@allUserRecords > 500) { + @allUserRecords = + sort { $a->last_name cmp $b->last_name || $a->first_name cmp $b->first_name || $a->user_id cmp $b->user_id } + @allUserRecords; + } return '' unless @allUserRecords; @@ -50,15 +61,11 @@ sub studentNav ($c, $setID) { || ($filter =~ /^section:(.*)$/ && $_->section eq $1) || ($filter =~ /^recitation:(.*)$/ && $_->recitation eq $1); - my $addRecord = $_; - $currentUserIndex = @userRecords if $addRecord->user_id eq $eUserID; - push @userRecords, $addRecord; + $currentUserIndex = @userRecords if $_->user_id eq $eUserID; + push @userRecords, $_; # Construct a display name. - $addRecord->{displayName} = - ($addRecord->last_name || $addRecord->first_name - ? $addRecord->last_name . ', ' . $addRecord->first_name - : $addRecord->user_id); + $_->{displayName} = ($_->last_name || $_->first_name ? $_->last_name . ', ' . $_->first_name : $_->user_id); } my $prevUser = $currentUserIndex > 0 ? $userRecords[ $currentUserIndex - 1 ] : 0; my $nextUser = $currentUserIndex < $#userRecords ? $userRecords[ $currentUserIndex + 1 ] : 0; From a3d5379c7e6d35100e1bd52563360c9eebd72724 Mon Sep 17 00:00:00 2001 From: Glenn Rice Date: Mon, 18 May 2026 15:14:01 -0500 Subject: [PATCH 2/2] Switch to getting all users from the database and filtering via Perl. This was suggested by @Alex-Jordan, and does seem to be much better. --- lib/WeBWorK/ContentGenerator/GatewayQuiz.pm | 24 ++++++++++----------- lib/WeBWorK/HTML/StudentNav.pm | 23 +++++--------------- 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/lib/WeBWorK/ContentGenerator/GatewayQuiz.pm b/lib/WeBWorK/ContentGenerator/GatewayQuiz.pm index af068a65b6..f23b9656d5 100644 --- a/lib/WeBWorK/ContentGenerator/GatewayQuiz.pm +++ b/lib/WeBWorK/ContentGenerator/GatewayQuiz.pm @@ -1376,15 +1376,13 @@ sub nav ($c, $args) { my $setVersion = $c->{set}->version_id; # Find all versions of this set that have been taken (excluding those taken by the current user). - my @users = + my @userVersions = $db->listSetVersionsWhere({ user_id => { '!=' => $userID }, set_id => { like => "$setID,v\%" } }); - my @allUserRecords; - my $i = 0; - while ($i < @users) { - push(@allUserRecords, - $db->getUsers(map { $_->[0] } @users[ $i .. ($i + 499 < $#users ? $i + 499 : $#users) ])); - $i += 500; - } + my %users = map { $_->user_id => 1 } @userVersions; + my @allUserRecords = + grep { $users{ $_->{user_id} } } + $c->db->getUsersWhere({ -and => { user_id => { not_like => 'set_id:%' } }, user_id => { '!=' => $userID } }, + [qw/last_name first_name user_id/]); if (@allUserRecords) { my $filter = $c->param('studentNavFilter'); @@ -1396,13 +1394,15 @@ sub nav ($c, $args) { # Add to the sections and recitations if defined. Also store the first user found in that section or # recitation. This user will be switched to when the filter is selected. my $section = $allUserRecords[$_]->section; - $filters{"section:$section"} = - [ $c->maketext('Filter by section [_1]', $section), $allUserRecords[$_]->user_id, $users[$_][2] ] + $filters{"section:$section"} = [ + $c->maketext('Filter by section [_1]', $section), $allUserRecords[$_]->user_id, + $userVersions[$_][2] + ] if $section && !$filters{"section:$section"}; my $recitation = $allUserRecords[$_]->recitation; $filters{"recitation:$recitation"} = [ $c->maketext('Filter by recitation [_1]', $recitation), $allUserRecords[$_]->user_id, - $users[$_][2] + $userVersions[$_][2] ] if $recitation && !$filters{"recitation:$recitation"}; @@ -1419,7 +1419,7 @@ sub nav ($c, $args) { ($addRecord->last_name || $addRecord->first_name ? $addRecord->last_name . ', ' . $addRecord->first_name : $addRecord->user_id); - $addRecord->{setVersion} = $users[$_][2]; + $addRecord->{setVersion} = $userVersions[$_][2]; } # Sort by last name, then first name, then user_id, then set version. diff --git a/lib/WeBWorK/HTML/StudentNav.pm b/lib/WeBWorK/HTML/StudentNav.pm index 1f8c2f2464..afa7f3c058 100644 --- a/lib/WeBWorK/HTML/StudentNav.pm +++ b/lib/WeBWorK/HTML/StudentNav.pm @@ -15,24 +15,11 @@ sub studentNav ($c, $setID) { return '' unless $c->authz->hasPermissions($userID, 'become_student'); # Find all users for the given set (except the current user) sorted by last_name, then first_name, then user_id. - my @users = map { $_->[0] } $c->db->listUserSetsWhere({ set_id => $setID, user_id => { '!=' => $userID } }); - my @allUserRecords; - my $i = 0; - while ($i < @users) { - push( - @allUserRecords, - $c->db->getUsersWhere( - { user_id => [ @users[ $i .. ($i + 499 < $#users ? $i + 499 : $#users) ] ] }, - [qw/last_name first_name user_id/] - ) - ); - $i += 500; - } - if (@allUserRecords > 500) { - @allUserRecords = - sort { $a->last_name cmp $b->last_name || $a->first_name cmp $b->first_name || $a->user_id cmp $b->user_id } - @allUserRecords; - } + my %users = map { $_->[0] => 1 } $c->db->listUserSetsWhere({ set_id => $setID, user_id => { '!=' => $userID } }); + my @allUserRecords = + grep { $users{ $_->{user_id} } } + $c->db->getUsersWhere({ -and => { user_id => { not_like => 'set_id:%' } }, user_id => { '!=' => $userID } }, + [qw/last_name first_name user_id/]); return '' unless @allUserRecords;