Skip to content

Improve rollup ranking queries #23868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mneudert merged 4 commits into from
Dec 8, 2025
Merged

Improve rollup ranking queries #23868

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4a21c37
Improve "Others" aggregation for rollup ranking queries
mneudert Dec 3, 2025
ad4b172
Always sort rollup results to ensure ordering
mneudert Dec 3, 2025
93170f8
Remove no longer needed double rollup result iteration
mneudert Dec 3, 2025
1376a0c
Add test to ensure rollup ranking query result structure
mneudert Dec 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 51 additions & 32 deletions core/RankingQuery.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,33 +293,49 @@ public function generateRankingQuery($innerQuery, bool $withRollup = false)

// generate select clauses for label columns
$labelColumnsString = '`' . implode('`, `', array_keys($this->labelColumns)) . '`';
$labelColumnsOthersSwitch = array();
$withRollupColumns = array();

$labelColumnsOthersSwitch = [];
$withRollupColumns = [];
$withRollupOthersGroupBy = [];

foreach (array_keys($this->labelColumns) as $column) {
$rollupWhen = '';
if ($withRollup) {
$rollupLimitValue = empty($withRollupColumns) ?
"'" . $this->othersLabelValue . "'"
:
'NULL';

$rollupWhen = "
WHEN counterRollup = $limit THEN $rollupLimitValue
WHEN counterRollup > 0 THEN `$column`
if ([] === $withRollupColumns) {
// support "Others" row for first label column
$rollupWhen = "
WHEN counterRollup = $limit THEN '" . $this->othersLabelValue . "'
WHEN counterRollup > 0 THEN `$column`
WHEN counter = $limit AND counterRollup = 0 THEN `$column`
WHEN counter = $limit THEN '" . $this->othersLabelValue . "'
";
} else {
// support "Others" row for secondary label columns
$rollupWhen = "
WHEN `$column` IS NULL THEN NULL
WHEN counter = $limit AND counterRollup = 0 THEN '" . $this->othersLabelValue . "'
";
}

$switch = "
CASE
$rollupWhen
ELSE `$column`
END
";

$withRollupColumns[] = $column;
$labelColumnsOthersSwitch[] = "$switch AS `$column`";
$withRollupColumns[] = $column;
$withRollupOthersGroupBy[] = $switch;
} else {
$labelColumnsOthersSwitch[] = "
CASE
WHEN counter = $limit THEN '" . $this->othersLabelValue . "'
ELSE `$column`
END AS `$column`
";
}

$labelColumnsOthersSwitch[] = "
CASE
$rollupWhen
WHEN counter = $limit THEN '" . $this->othersLabelValue . "'
ELSE `$column`
END AS `$column`
";
}

$labelColumnsOthersSwitch = implode(', ', $labelColumnsOthersSwitch);

// generate select clauses for additional columns
Expand All @@ -346,29 +362,29 @@ public function generateRankingQuery($innerQuery, bool $withRollup = false)

$counterRollupExpression = '';

if ($withRollup && !empty($withRollupColumns)) {
if ($withRollup) {
$initCounter .= ' ( SELECT @counterRollup:=0 ) initCounterRollup,';
$counterRollupWhen = '';

if (count($withRollupColumns) >= 2) {
$counterRollupWhen = "
WHEN `" . implode('` IS NULL AND `', $withRollupColumns) . "` IS NULL THEN -1
";
";
}

foreach ($withRollupColumns as $withRollupColumn) {
$counterRollupWhen .= "
WHEN `$withRollupColumn` IS NULL AND @counterRollup = $limit THEN $limit
WHEN `$withRollupColumn` IS NULL THEN @counterRollup := @counterRollup + 1
";
";
}

$counterRollupExpression = "
, CASE
$counterRollupWhen
ELSE 0
END AS counterRollup
";
";
}

if (false === strpos($innerQuery, ' LIMIT ') && !Schema::getInstance()->supportsSortingInSubquery()) {
Expand All @@ -389,7 +405,7 @@ public function generateRankingQuery($innerQuery, bool $withRollup = false)
( $innerQuery ) actualQuery
";

if ($withRollup && !empty($withRollupColumns) && !Schema::getInstance()->supportsRankingRollupWithoutExtraSorting()) {
if ($withRollup && !Schema::getInstance()->supportsRankingRollupWithoutExtraSorting()) {
// MariaDB requires an additional sorting layer to return
// the counter/counterRollup values we expect
$rollupColumnSorts = [];
Expand All @@ -410,13 +426,16 @@ public function generateRankingQuery($innerQuery, bool $withRollup = false)
// group by the counter - this groups "Others" because the counter stops at $limit
$groupBy = 'counter';

if ($withRollup && !empty($counterRollupExpression)) {
$groupBy .= ', counterRollup';
if ($withRollup) {
// group rollups additionally by the rollup counter and the
// full "Others" switch to ensure correct secondary level "Others" calculation
$groupBy .= ', counterRollup, ' . implode(', ', $withRollupOthersGroupBy);
}

if ($this->partitionColumn !== false) {
$groupBy .= ', `' . $this->partitionColumn . '`';
}

$groupOthers = "
SELECT
$labelColumnsOthersSwitch
Expand All @@ -425,13 +444,13 @@ public function generateRankingQuery($innerQuery, bool $withRollup = false)
GROUP BY $groupBy
";

if (!Schema::getInstance()->supportsSortingInSubquery()) {
if ($withRollup) {
// Sort the final result if a rollup was used
// to ensure rollup values are returned first, and "Others" last
$groupOthers .= " ORDER BY counter, counterRollup";
} elseif (!Schema::getInstance()->supportsSortingInSubquery()) {
// When subqueries aren't sorted, we need to sort the result manually again
$groupOthers .= " ORDER BY counter";

if (!empty($counterRollupExpression)) {
$groupOthers .= ', counterRollup';
}
}

return $groupOthers;
Expand Down
88 changes: 32 additions & 56 deletions plugins/CustomDimensions/RecordBuilders/CustomDimension.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,89 +186,65 @@ protected function aggregateFromActions(DataTable $report, LogAggregator $logAgg
$metricIds[] = Metrics::INDEX_BOUNCE_COUNT;
$metricIds[] = Metrics::INDEX_PAGE_EXIT_NB_VISITS;

$actionRows = [];

while ($row = $resultSet->fetch()) {
if (!isset($row[Metrics::INDEX_NB_VISITS])) {
return;
}

$label = $row[$valueField];

if ($withRollup) {
$url = $row['url'];

if (is_null($label)) {
continue;
}

if (!is_null($url)) {
$actionRows[] = $row;
continue;
}
if ($withRollup && $label === null) {
// top-level rollup result
continue;
}

$columns = [];

foreach ($metricIds as $id) {
$columns[$id] = (float) ($row[$id] ?? 0);
}

$label = $this->cleanCustomDimensionValue($label);
$tableRow = $report->sumRowWithLabel($label, $columns);
$url = $row['url'];

if (!$withRollup) {
$url = $row['url'];
if (empty($url)) {
continue;
}

// make sure we always work with normalized URL no matter how the individual action stores it
$normalized = Tracker\PageUrl::normalizeUrl($url);
$url = $normalized['url'];

if (empty($url)) {
continue;
}

$tableRow->sumRowWithLabelToSubtable($url, $columns);
}
}

if ($withRollup) {
foreach ($actionRows as $row) {
if (!isset($row[Metrics::INDEX_NB_VISITS])) {
return;
}

$label = $row[$valueField];
$url = $row['url'];

if (is_null($label) || is_null($url)) {
$tableRow = $report->sumRowWithLabel($label, $columns);
} else {
if ($url === null) {
// second-level rollup result
$report->sumRowWithLabel($label, $columns);
continue;
}

$label = $this->cleanCustomDimensionValue($label);
$tableRow = $report->getRowFromLabel($label);

if (empty($tableRow)) {
continue;
}

// make sure we always work with normalized URL no matter how the individual action stores it
$normalized = Tracker\PageUrl::normalizeUrl($url);
$url = $normalized['url'];

if (empty($url)) {
if (false === $tableRow) {
// non-rollup row but rollup row is missing
// should not happen, but don't break
continue;
}
}

$columns = [];
// make sure we always work with normalized URL no matter how the individual action stores it
$normalized = Tracker\PageUrl::normalizeUrl($url);
$url = $normalized['url'];

foreach ($metricIds as $id) {
$columns[$id] = (float) ($row[$id] ?? 0);
}
if (empty($url)) {
continue;
}

$tableRow->sumRowWithLabelToSubtable($url, $columns);
if (
$withRollup
&& $url === RankingQuery::LABEL_SUMMARY_ROW
&& !$tableRow->isSubtableLoaded()
) {
// skip creating the subtable if:
// - we are using rollups
// - the only row would be "Others"
continue;
}

$tableRow->sumRowWithLabelToSubtable($url, $columns);
}
}

Expand Down
42 changes: 42 additions & 0 deletions ...ranking_limit_with_rollup_by_archiving_query__CustomDimensions.getCustomDimension_day.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,48 @@
<segment>dimension1==site3+group5;actionUrl=$example.com%2Fpage%2F5%2F5</segment>
<url>example.com%2Fpage%2F5%2F5</url>
</row>
<row>
<label>Others</label>
<nb_visits>2</nb_visits>
<nb_uniq_visitors>2</nb_uniq_visitors>
<nb_hits>53</nb_hits>
<sum_time_network>0</sum_time_network>
<nb_hits_with_time_network>0</nb_hits_with_time_network>
<min_time_network>0</min_time_network>
<max_time_network>0</max_time_network>
<sum_time_server>0</sum_time_server>
<nb_hits_with_time_server>0</nb_hits_with_time_server>
<min_time_server>0</min_time_server>
<max_time_server>0</max_time_server>
<sum_time_transfer>0</sum_time_transfer>
<nb_hits_with_time_transfer>0</nb_hits_with_time_transfer>
<min_time_transfer>0</min_time_transfer>
<max_time_transfer>0</max_time_transfer>
<sum_time_dom_processing>0</sum_time_dom_processing>
<nb_hits_with_time_dom_processing>0</nb_hits_with_time_dom_processing>
<min_time_dom_processing>0</min_time_dom_processing>
<max_time_dom_processing>0</max_time_dom_processing>
<sum_time_dom_completion>0</sum_time_dom_completion>
<nb_hits_with_time_dom_completion>0</nb_hits_with_time_dom_completion>
<min_time_dom_completion>0</min_time_dom_completion>
<max_time_dom_completion>0</max_time_dom_completion>
<sum_time_on_load>0</sum_time_on_load>
<nb_hits_with_time_on_load>0</nb_hits_with_time_on_load>
<min_time_on_load>0</min_time_on_load>
<max_time_on_load>0</max_time_on_load>
<sum_bandwidth>0</sum_bandwidth>
<nb_hits_with_bandwidth>0</nb_hits_with_bandwidth>
<min_bandwidth>0</min_bandwidth>
<max_bandwidth>0</max_bandwidth>
<sum_time_spent>0</sum_time_spent>
<bounce_count>0</bounce_count>
<exit_nb_visits>2</exit_nb_visits>
<avg_time_on_dimension>0</avg_time_on_dimension>
<bounce_rate>0%</bounce_rate>
<exit_rate>100%</exit_rate>
<segment>dimension1==site3+group5;actionUrl=$__mtm_ranking_query_others__</segment>
<url>__mtm_ranking_query_others__</url>
Comment thread
mneudert marked this conversation as resolved.
</row>
</subtable>
</row>
<row>
Expand Down
42 changes: 42 additions & 0 deletions ...imit_with_rollup_by_archiving_query_expanded__CustomDimensions.getCustomDimension_day.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,48 @@
<segment>dimension1==site3+group5;actionUrl=$example.com%2Fpage%2F5%2F5</segment>
<url>example.com%2Fpage%2F5%2F5</url>
</row>
<row>
<label>Others</label>
<nb_visits>2</nb_visits>
<nb_uniq_visitors>2</nb_uniq_visitors>
<nb_hits>53</nb_hits>
<sum_time_network>0</sum_time_network>
<nb_hits_with_time_network>0</nb_hits_with_time_network>
<min_time_network>0</min_time_network>
<max_time_network>0</max_time_network>
<sum_time_server>0</sum_time_server>
<nb_hits_with_time_server>0</nb_hits_with_time_server>
<min_time_server>0</min_time_server>
<max_time_server>0</max_time_server>
<sum_time_transfer>0</sum_time_transfer>
<nb_hits_with_time_transfer>0</nb_hits_with_time_transfer>
<min_time_transfer>0</min_time_transfer>
<max_time_transfer>0</max_time_transfer>
<sum_time_dom_processing>0</sum_time_dom_processing>
<nb_hits_with_time_dom_processing>0</nb_hits_with_time_dom_processing>
<min_time_dom_processing>0</min_time_dom_processing>
<max_time_dom_processing>0</max_time_dom_processing>
<sum_time_dom_completion>0</sum_time_dom_completion>
<nb_hits_with_time_dom_completion>0</nb_hits_with_time_dom_completion>
<min_time_dom_completion>0</min_time_dom_completion>
<max_time_dom_completion>0</max_time_dom_completion>
<sum_time_on_load>0</sum_time_on_load>
<nb_hits_with_time_on_load>0</nb_hits_with_time_on_load>
<min_time_on_load>0</min_time_on_load>
<max_time_on_load>0</max_time_on_load>
<sum_bandwidth>0</sum_bandwidth>
<nb_hits_with_bandwidth>0</nb_hits_with_bandwidth>
<min_bandwidth>0</min_bandwidth>
<max_bandwidth>0</max_bandwidth>
<sum_time_spent>0</sum_time_spent>
<bounce_count>0</bounce_count>
<exit_nb_visits>2</exit_nb_visits>
<avg_time_on_dimension>0</avg_time_on_dimension>
<bounce_rate>0%</bounce_rate>
<exit_rate>100%</exit_rate>
<segment>dimension1==site3+group5;actionUrl=$__mtm_ranking_query_others__</segment>
<url>__mtm_ranking_query_others__</url>
</row>
</subtable>
</row>
<row>
Expand Down
Loading
Loading
⚡