From c8946ab379745c2061acb27cf5beaa9ec79df3cd Mon Sep 17 00:00:00 2001 From: Marko Glaubitz Date: Fri, 24 Jul 2026 13:34:06 +0200 Subject: [PATCH] Performance(TestQuestionPool): two-phase query for question browser + index The 'Add from pool' question browser (ilObjTestGUI -> ilTestQuestionBrowserTableGUI -> ilAssQuestionList::load) built one large query with 7 correlated EXISTS subqueries (feedback x4, hints, taxonomies) as SELECT fields. These were evaluated for every candidate row BEFORE ORDER BY/LIMIT, so on large instances (bug report: ~199k rows) the query ran 30-40 min and blocked other queries. Fix: when a Range is set and neither a HAVING filter nor an ORDER BY on a computed column (feedback/hints/taxonomies) is active, load in two phases: Phase A: SELECT question_id + required JOINs/filters + GROUP BY + ORDER BY + LIMIT (no EXISTS subqueries) -> small paginated id set Phase B: full SELECT incl. feedback/hints/taxonomies EXISTS, restricted to the paginated ids via IN (...) -> flags computed only for the ~800 visible rows instead of the full candidate set. Fallback to the original single-phase query for: no range, HAVING filter (feedback/hints = true|false), ORDER BY feedback/hints/taxonomies. buildOrderQueryExpression now qualifies columns for phase A (qpl_questions.* not selected -> ambiguous title) and backticks qualified names per segment (`qpl_questions`.`title`). Adds composite index i6 (obj_fi, original_id, title) on qpl_questions via ilTestQuestionPool10DBUpdateSteps::step_3() to support the phase A filter (obj_fi IN ... AND original_id IS NULL) + default title ordering. Adds ilAssQuestionListTwoPhaseTest (15 tests) covering the two-phase path, all fallback conditions, column qualification and SQL shape. Verified with EXPLAIN against the local docker DB: phase A has no DEPENDENT SUBQUERY (3 simple JOINs only), phase B's subqueries are evaluated over rows=2 (paginated set) instead of the full candidate set. --- ...lass.ilTestQuestionPool10DBUpdateSteps.php | 19 ++ .../classes/class.ilAssQuestionList.php | 211 +++++++++++++++++- .../tests/ilAssQuestionListTwoPhaseTest.php | 200 +++++++++++++++++ 3 files changed, 428 insertions(+), 2 deletions(-) create mode 100644 components/ILIAS/TestQuestionPool/tests/ilAssQuestionListTwoPhaseTest.php diff --git a/components/ILIAS/TestQuestionPool/classes/Setup/class.ilTestQuestionPool10DBUpdateSteps.php b/components/ILIAS/TestQuestionPool/classes/Setup/class.ilTestQuestionPool10DBUpdateSteps.php index 7b9b56281927..3d3827969f7e 100755 --- a/components/ILIAS/TestQuestionPool/classes/Setup/class.ilTestQuestionPool10DBUpdateSteps.php +++ b/components/ILIAS/TestQuestionPool/classes/Setup/class.ilTestQuestionPool10DBUpdateSteps.php @@ -47,4 +47,23 @@ public function step_2(): void $this->db->dropTableColumn('qpl_questionpool', 'show_taxonomies'); } } + + /** + * Composite index supporting the paginated question browser query + * (ilAssQuestionList two-phase loading, phase A) which filters by + * obj_fi (eq / IN) and original_id IS NULL and commonly orders by + * title. Covers both the pool-internal view (obj_fi = X) and the + * "add from pool" browser (obj_fi IN (...)) of ilObjTestGUI. + * + * Note: ILIAS' ilDBPdoFieldDefinition::checkIndexName limits index + * names to 3 characters, hence the short name "i6" (the existing + * i1_idx..i5_idx on this table predate that constraint). + */ + public function step_3(): void + { + $fields = ['obj_fi', 'original_id', 'title']; + if (!$this->db->indexExistsByFields('qpl_questions', $fields)) { + $this->db->addIndex('qpl_questions', $fields, 'i6'); + } + } } diff --git a/components/ILIAS/TestQuestionPool/classes/class.ilAssQuestionList.php b/components/ILIAS/TestQuestionPool/classes/class.ilAssQuestionList.php index d7046c9e6df0..fe295adf1fa2 100755 --- a/components/ILIAS/TestQuestionPool/classes/class.ilAssQuestionList.php +++ b/components/ILIAS/TestQuestionPool/classes/class.ilAssQuestionList.php @@ -545,7 +545,13 @@ private function getHavingFilterExpression(): string return $having !== '' ? "HAVING $having" : ''; } - private function buildOrderQueryExpression(): string + /** + * @param bool $qualify In the two-phase ID query (phase A) qpl_questions.* + * is not selected, so ambiguous order columns like + * `title` (exists on qpl_questions and object_data) + * must be qualified to their table. + */ + private function buildOrderQueryExpression(bool $qualify = false): string { $order = $this->order; if ($order === null) { @@ -562,7 +568,37 @@ private function buildOrderQueryExpression(): string $order_direction = Order::ASC; } - return " ORDER BY `$order_field` $order_direction"; + if ($qualify) { + $order_field = $this->qualifyOrderField($order_field); + } + + // Backtick each segment separately so qualified names like + // qpl_questions.title become `qpl_questions`.`title` (a single + // pair of backticks around the dotted name would be parsed as one + // oddly-named column). + $order_field = implode('.', array_map( + static fn(string $seg): string => '`' . $seg . '`', + explode('.', $order_field) + )); + + return " ORDER BY $order_field $order_direction"; + } + + /** + * Map an order field to its fully qualified column. Needed in phase A + * where qpl_questions.* is not selected and columns like `title` would + * be ambiguous between qpl_questions and object_data. + */ + private function qualifyOrderField(string $field): string + { + return match ($field) { + 'title', 'description', 'author', 'lifecycle', 'points', + 'created', 'tstamp', 'complete', 'question_id', 'original_id' => "qpl_questions.$field", + 'max_points' => 'qpl_questions.points', + 'type_tag' => 'qpl_qst_type.type_tag', + 'parent_title' => 'object_data.title', + default => $field, + }; } private function buildLimitQueryExpression(): string @@ -589,10 +625,181 @@ private function buildQuery(): string ])); } + /** + * Two-phase query loading can be used when the result is paginated + * (range set) and neither a HAVING filter nor an ORDER BY on a computed + * column (feedback/hints/taxonomies) is active. In that case the + * expensive correlated EXISTS subqueries are evaluated only for the + * small paginated set of question ids instead of the full candidate + * set before LIMIT/OFFSET is applied. + */ + private function canUseTwoPhaseQuery(): bool + { + if ($this->range === null) { + return false; + } + if ($this->getHavingFilterExpression() !== '') { + return false; + } + if ($this->isOrderByComputedField()) { + return false; + } + return true; + } + + /** + * ORDER BY targets one of the computed columns (feedback/hints/taxonomies) + * that only exist in the single-phase query. Two-phase loading cannot + * sort by these and must fall back to the single-phase query. + */ + private function isOrderByComputedField(): bool + { + if ($this->order === null) { + return false; + } + [$order_field] = $this->order->join( + '', + static fn(string $index, string $key, string $value): array => [$key, $value] + ); + return in_array($order_field, ['feedback', 'hints', 'taxonomies'], true); + } + + /** + * Phase A: determine the paginated, ordered set of question ids using + * only the required JOINs/filters — without the expensive correlated + * EXISTS subqueries for feedback/hints/taxonomies. + * + * GROUP BY question_id (instead of DISTINCT) is used so that ORDER BY + * may reference any column functionally dependent on question_id (e.g. + * qpl_questions.title) without having to add it to the SELECT list — + * MySQL rejects `SELECT DISTINCT question_id ... ORDER BY title`. + * GROUP BY also deduplicates rows that the tst_test_question / + * feedback / hints filter JOINs may produce. + */ + private function buildPaginatedIdsQuery(): string + { + return implode(PHP_EOL, array_filter([ + "SELECT qpl_questions.question_id FROM qpl_questions {$this->getTableJoinExpression()}", + "WHERE qpl_questions.tstamp > 0", + $this->getConditionalFilterExpression(), + "GROUP BY qpl_questions.question_id", + $this->buildOrderQueryExpression(true), + $this->buildLimitQueryExpression(), + ])); + } + + /** + * Phase B: enrich the small paginated set of question ids with the full + * select fields including the computed feedback/hints/taxonomies flags. + * Filter JOINs (feedback/hints) are NOT applied here — they were already + * honoured in phase A. + */ + private function buildEnrichmentQuery(array $question_ids): string + { + $in = $this->db->in('qpl_questions.question_id', $question_ids, false, ilDBConstants::T_INTEGER); + return implode(PHP_EOL, array_filter([ + "{$this->getSelectFieldsExpression()} FROM qpl_questions {$this->getBaseTableJoinExpression()}", + "WHERE qpl_questions.tstamp > 0 AND $in", + ])); + } + + /** + * Base table joins without the feedback/hints filter JOINs added by + * handleFeedbackJoin()/handleHintJoin(). Used in phase B where those + * filter JOINs are not needed (filtering happened in phase A). + */ + private function getBaseTableJoinExpression(): string + { + $table_join = ' + INNER JOIN qpl_qst_type + ON qpl_qst_type.question_type_id = qpl_questions.question_type_fi + '; + + if ($this->join_obj_data) { + $table_join .= ' + INNER JOIN object_data + ON object_data.obj_id = qpl_questions.obj_fi + '; + } + + if ( + $this->parentObjType === 'tst' + && $this->questionInstanceTypeFilter === self::QUESTION_INSTANCE_TYPE_ALL + ) { + $table_join .= 'INNER JOIN tst_test_question tstquest ON tstquest.question_fi = qpl_questions.question_id'; + } + + if ($this->answerStatusActiveId) { + $table_join .= " + LEFT JOIN tst_test_result + ON tst_test_result.question_fi = qpl_questions.question_id + AND tst_test_result.active_fi = {$this->db->quote($this->answerStatusActiveId, ilDBConstants::T_INTEGER)} + "; + } + + return $table_join; + } + + private function loadTwoPhase(): void + { + $tags_trafo = $this->refinery->encode()->htmlSpecialCharsAsEntities(); + + $ids_res = $this->db->query($this->buildPaginatedIdsQuery()); + $ordered_ids = []; + while ($row = $this->db->fetchAssoc($ids_res)) { + $ordered_ids[] = (int) $row['question_id']; + } + + if ($ordered_ids === []) { + return; + } + + $rows_by_id = []; + $res = $this->db->query($this->buildEnrichmentQuery($ordered_ids)); + while ($row = $this->db->fetchAssoc($res)) { + $rows_by_id[(int) $row['question_id']] = $row; + } + + foreach ($ordered_ids as $question_id) { + if (!isset($rows_by_id[$question_id])) { + continue; + } + $row = $rows_by_id[$question_id]; + $row = ilAssQuestionType::completeMissingPluginName($row); + + if (!$this->isActiveQuestionType($row)) { + continue; + } + + $row['title'] = $tags_trafo->transform($row['title'] ?? ' '); + $row['description'] = $tags_trafo->transform($row['description'] ?? ''); + $row['author'] = $tags_trafo->transform($row['author']); + $row['taxonomies'] = $this->loadTaxonomyAssignmentData($row['obj_fi'], $row['question_id']); + $row['ttype'] = $this->lng->txt($row['type_tag']); + $row['feedback'] = $row['feedback'] === 1; + $row['hints'] = $row['hints'] === 1; + $row['comments'] = $this->getNumberOfCommentsForQuestion($row['question_id']); + + if ( + $this->filter_comments === self::QUESTION_COMMENTED_ONLY && $row['comments'] === 0 + || $this->filter_comments === self::QUESTION_COMMENTED_EXCLUDED && $row['comments'] > 0 + ) { + continue; + } + + $this->questions[$row['question_id']] = $row; + } + } + public function load(): void { $this->checkFilters(); + if ($this->canUseTwoPhaseQuery()) { + $this->loadTwoPhase(); + return; + } + $tags_trafo = $this->refinery->encode()->htmlSpecialCharsAsEntities(); $res = $this->db->query($this->buildQuery()); diff --git a/components/ILIAS/TestQuestionPool/tests/ilAssQuestionListTwoPhaseTest.php b/components/ILIAS/TestQuestionPool/tests/ilAssQuestionListTwoPhaseTest.php new file mode 100644 index 000000000000..96c8750b5f0e --- /dev/null +++ b/components/ILIAS/TestQuestionPool/tests/ilAssQuestionListTwoPhaseTest.php @@ -0,0 +1,200 @@ +createMock(ilDBInterface::class); + $db->method('like')->willReturn('1'); + $db->method('in')->willReturnCallback(function (string $field, array $values, bool $negate = false): string { + if ($values === []) { + return $negate ? ' 1=1 ' : ' 1=2 '; + } + $list = implode(',', array_map('intval', $values)); + return ($negate ? 'NOT ' : '') . "$field IN ($list)"; + }); + $db->method('quote')->willReturnCallback(fn($v) => is_numeric($v) ? (string) $v : "'$v'"); + + $lng = $this->createMock(ilLanguage::class); + $refinery = $this->createMock(ILIAS\Refinery\Factory::class); + $component_repository = $this->createMock(ilComponentRepository::class); + $notes_service = $this->createMock(ILIAS\Notes\Service::class); + + $this->object = new ilAssQuestionList($db, $lng, $refinery, $component_repository, $notes_service); + } + + private function setPrivateProperty(string $name, mixed $value): void + { + $prop = new ReflectionProperty(ilAssQuestionList::class, $name); + $prop->setAccessible(true); + $prop->setValue($this->object, $value); + } + + private function invokePrivate(string $name, array $args = []): mixed + { + $method = new ReflectionMethod(ilAssQuestionList::class, $name); + $method->setAccessible(true); + return $method->invokeArgs($this->object, $args); + } + + public function testCanUseTwoPhaseQueryReturnsFalseWithoutRange(): void + { + $this->setPrivateProperty('range', null); + $this->assertFalse($this->invokePrivate('canUseTwoPhaseQuery')); + } + + public function testCanUseTwoPhaseQueryReturnsTrueWithRangeAndSimpleOrder(): void + { + $this->setPrivateProperty('range', new Range(0, 800)); + $this->setPrivateProperty('order', new Order('title', Order::ASC)); + $this->assertTrue($this->invokePrivate('canUseTwoPhaseQuery')); + } + + public function testCanUseTwoPhaseQueryReturnsFalseForOrderByFeedback(): void + { + $this->setPrivateProperty('range', new Range(0, 800)); + $this->setPrivateProperty('order', new Order('feedback', Order::ASC)); + $this->assertFalse($this->invokePrivate('canUseTwoPhaseQuery')); + } + + public function testCanUseTwoPhaseQueryReturnsFalseForOrderByHints(): void + { + $this->setPrivateProperty('range', new Range(0, 800)); + $this->setPrivateProperty('order', new Order('hints', Order::ASC)); + $this->assertFalse($this->invokePrivate('canUseTwoPhaseQuery')); + } + + public function testCanUseTwoPhaseQueryReturnsFalseForOrderByTaxonomies(): void + { + $this->setPrivateProperty('range', new Range(0, 800)); + $this->setPrivateProperty('order', new Order('taxonomies', Order::ASC)); + $this->assertFalse($this->invokePrivate('canUseTwoPhaseQuery')); + } + + public function testCanUseTwoPhaseQueryReturnsFalseWithFeedbackFalseFilter(): void + { + $this->setPrivateProperty('range', new Range(0, 800)); + $this->setPrivateProperty('order', new Order('title', Order::ASC)); + $this->setPrivateProperty('fieldFilters', ['feedback' => 'false']); + $this->assertFalse($this->invokePrivate('canUseTwoPhaseQuery')); + } + + public function testCanUseTwoPhaseQueryReturnsFalseWithFeedbackTrueFilter(): void + { + // feedback=true uses an INNER JOIN but ALSO a HAVING clause -> fallback + $this->setPrivateProperty('range', new Range(0, 800)); + $this->setPrivateProperty('order', new Order('title', Order::ASC)); + $this->setPrivateProperty('fieldFilters', ['feedback' => 'true']); + $this->assertFalse($this->invokePrivate('canUseTwoPhaseQuery')); + } + + public function testQualifyOrderFieldMapsKnownFields(): void + { + $this->assertSame('qpl_questions.title', $this->invokePrivate('qualifyOrderField', ['title'])); + $this->assertSame('qpl_questions.description', $this->invokePrivate('qualifyOrderField', ['description'])); + $this->assertSame('qpl_questions.author', $this->invokePrivate('qualifyOrderField', ['author'])); + $this->assertSame('qpl_questions.points', $this->invokePrivate('qualifyOrderField', ['points'])); + $this->assertSame('qpl_questions.points', $this->invokePrivate('qualifyOrderField', ['max_points'])); + $this->assertSame('qpl_questions.created', $this->invokePrivate('qualifyOrderField', ['created'])); + $this->assertSame('qpl_questions.tstamp', $this->invokePrivate('qualifyOrderField', ['tstamp'])); + $this->assertSame('qpl_qst_type.type_tag', $this->invokePrivate('qualifyOrderField', ['type_tag'])); + $this->assertSame('object_data.title', $this->invokePrivate('qualifyOrderField', ['parent_title'])); + } + + public function testQualifyOrderFieldLeavesUnknownFieldsUnchanged(): void + { + $this->assertSame('feedback', $this->invokePrivate('qualifyOrderField', ['feedback'])); + $this->assertSame('hints', $this->invokePrivate('qualifyOrderField', ['hints'])); + $this->assertSame('taxonomies', $this->invokePrivate('qualifyOrderField', ['taxonomies'])); + } + + public function testBuildPaginatedIdsQueryDoesNotContainExistsSubqueries(): void + { + $this->setPrivateProperty('range', new Range(0, 800)); + $this->setPrivateProperty('order', new Order('title', Order::ASC)); + $this->setPrivateProperty('join_obj_data', true); + + $sql = $this->invokePrivate('buildPaginatedIdsQuery'); + + $this->assertStringContainsString('SELECT qpl_questions.question_id', $sql); + $this->assertStringNotContainsString('EXISTS', $sql); + $this->assertStringNotContainsString('qpl_fb_generic', $sql); + $this->assertStringNotContainsString('qpl_hints', $sql); + $this->assertStringNotContainsString('tax_node_assignment', $sql); + $this->assertStringContainsString('`qpl_questions`.`title`', $sql); + $this->assertStringContainsString('GROUP BY qpl_questions.question_id', $sql); + $this->assertStringContainsString('LIMIT 800 OFFSET 0', $sql); + } + + public function testBuildEnrichmentQueryContainsExistsSubqueriesAndInClause(): void + { + $this->setPrivateProperty('join_obj_data', true); + + $sql = $this->invokePrivate('buildEnrichmentQuery', [[10, 20, 30]]); + + $this->assertStringContainsString('EXISTS', $sql); + $this->assertStringContainsString('qpl_fb_generic', $sql); + $this->assertStringContainsString('qpl_hints', $sql); + $this->assertStringContainsString('tax_node_assignment', $sql); + $this->assertStringContainsString('qpl_questions.question_id IN (10,20,30)', $sql); + $this->assertStringNotContainsString('LIMIT', $sql); + } + + public function testBuildEnrichmentQueryHandlesEmptyIdList(): void + { + $this->setPrivateProperty('join_obj_data', true); + $sql = $this->invokePrivate('buildEnrichmentQuery', [[]]); + $this->assertStringContainsString('1=2', $sql); + } + + public function testBuildOrderQueryExpressionQualifiesWhenRequested(): void + { + $this->setPrivateProperty('order', new Order('title', Order::ASC)); + $sql = $this->invokePrivate('buildOrderQueryExpression', [true]); + $this->assertStringContainsString('`qpl_questions`.`title`', $sql); + } + + public function testBuildOrderQueryExpressionDoesNotQualifyByDefault(): void + { + $this->setPrivateProperty('order', new Order('title', Order::ASC)); + $sql = $this->invokePrivate('buildOrderQueryExpression'); + $this->assertStringNotContainsString('qpl_questions', $sql); + $this->assertStringContainsString('`title`', $sql); + } + + public function testIsOrderByComputedFieldReturnsFalseWithoutOrder(): void + { + $this->setPrivateProperty('order', null); + $this->assertFalse($this->invokePrivate('isOrderByComputedField')); + } +}