Skip to content

Commit a5ef0f1

Browse files
sgiehlsgiehl
committed
more cleanup
1 parent 399c87e commit a5ef0f1

3 files changed

Lines changed: 158 additions & 75 deletions

File tree

‎plugins/Actions/ArchivingHelper.php‎

Lines changed: 43 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -704,7 +704,28 @@ public static function getActionRow(
704704

705705
public static function mergeHierarchicalActionsTableIntoFlatTable(DataTable $hierarchicalTable, DataTable $flatTable): void
706706
{
707-
self::appendHierarchicalRowsToFlatTable($hierarchicalTable, [], $flatTable);
707+
self::appendHierarchicalRowsToFlatTable(
708+
$hierarchicalTable,
709+
[],
710+
$flatTable,
711+
[self::class, 'buildFlatRowLabel']
712+
);
713+
}
714+
715+
public static function mergeHierarchicalActionsTableIntoBestEffortFlatTable(
716+
DataTable $hierarchicalTable,
717+
DataTable $flatTable,
718+
int $actionType
719+
): void {
720+
self::appendHierarchicalRowsToFlatTable(
721+
$hierarchicalTable,
722+
[],
723+
$flatTable,
724+
function (array $path) use ($actionType): string {
725+
return self::buildBestEffortActionLabelFromPath($path, $actionType);
726+
},
727+
false
728+
);
708729
}
709730

710731
private static function getFlatActionRow($actionName, $actionType, $urlPrefix, DataTable $table): Row
@@ -757,8 +778,13 @@ public static function buildBestEffortActionLabelFromPath(array $actionPath, int
757778
return implode($delimiter, $segments);
758779
}
759780

760-
private static function appendHierarchicalRowsToFlatTable(DataTable $sourceTable, array $path, DataTable $flatTable): void
761-
{
781+
private static function appendHierarchicalRowsToFlatTable(
782+
DataTable $sourceTable,
783+
array $path,
784+
DataTable $flatTable,
785+
callable $flatLabelBuilder,
786+
bool $useDefaultFlatRowColumns = true
787+
): void {
762788
foreach ($sourceTable->getRowsWithoutSummaryRow() as $row) {
763789
$label = $row->getColumn('label');
764790
if (!is_string($label) || $label === '') {
@@ -770,15 +796,26 @@ private static function appendHierarchicalRowsToFlatTable(DataTable $sourceTable
770796

771797
$subtable = $row->getSubtable();
772798
if ($subtable) {
773-
self::appendHierarchicalRowsToFlatTable($subtable, $currentPath, $flatTable);
799+
self::appendHierarchicalRowsToFlatTable(
800+
$subtable,
801+
$currentPath,
802+
$flatTable,
803+
$flatLabelBuilder,
804+
$useDefaultFlatRowColumns
805+
);
774806
continue;
775807
}
776808

777-
$flatLabel = self::buildFlatRowLabel($currentPath);
809+
$flatLabel = call_user_func($flatLabelBuilder, $currentPath);
778810
$flatRow = $flatTable->getRowFromLabel($flatLabel);
779811
if ($flatRow === false) {
812+
$columns = array('label' => $flatLabel);
813+
if ($useDefaultFlatRowColumns) {
814+
$columns += self::getDefaultFlatRowColumns();
815+
}
816+
780817
$flatRow = new Row(array(
781-
Row::COLUMNS => array('label' => $flatLabel) + self::getDefaultFlatRowColumns(),
818+
Row::COLUMNS => $columns,
782819
));
783820
$flatRow->setMetadata(self::ACTION_FLAT_PATH_METADATA_NAME, $currentPath);
784821
$flatTable->addRow($flatRow);

‎plugins/Actions/RecordBuilders/ActionReports.php‎

Lines changed: 16 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -100,24 +100,32 @@ public function getRecordMetadata(ArchiveProcessor $archiveProcessor): array
100100
return $records;
101101
}
102102

103-
public function flatRowToUrlHierarchyPath(Row $flatRow, ArchiveProcessor $archiveProcessor, Record $record): ?array
103+
public function flatRowToUrlHierarchyPath(Row $flatRow): ?array
104104
{
105105
return $this->flatRowToHierarchyPath($flatRow, Action::TYPE_PAGE_URL);
106106
}
107107

108-
public function flatRowToTitleHierarchyPath(Row $flatRow, ArchiveProcessor $archiveProcessor, Record $record): ?array
108+
public function flatRowToTitleHierarchyPath(Row $flatRow): ?array
109109
{
110110
return $this->flatRowToHierarchyPath($flatRow, Action::TYPE_PAGE_TITLE);
111111
}
112112

113113
public function reduceLegacyUrlHierarchyIntoFlatTable(DataTable $legacyHierarchy, DataTable $flatTable, ArchiveProcessor $archiveProcessor, Record $record): void
114114
{
115-
$this->reduceLegacyHierarchyIntoFlatTable($legacyHierarchy, $flatTable, Action::TYPE_PAGE_URL);
115+
ArchivingHelper::mergeHierarchicalActionsTableIntoBestEffortFlatTable(
116+
$legacyHierarchy,
117+
$flatTable,
118+
Action::TYPE_PAGE_URL
119+
);
116120
}
117121

118122
public function reduceLegacyTitleHierarchyIntoFlatTable(DataTable $legacyHierarchy, DataTable $flatTable, ArchiveProcessor $archiveProcessor, Record $record): void
119123
{
120-
$this->reduceLegacyHierarchyIntoFlatTable($legacyHierarchy, $flatTable, Action::TYPE_PAGE_TITLE);
124+
ArchivingHelper::mergeHierarchicalActionsTableIntoBestEffortFlatTable(
125+
$legacyHierarchy,
126+
$flatTable,
127+
Action::TYPE_PAGE_TITLE
128+
);
121129
}
122130

123131
protected function aggregate(ArchiveProcessor $archiveProcessor): array
@@ -170,15 +178,11 @@ protected function aggregate(ArchiveProcessor $archiveProcessor): array
170178

171179
$tablesByType[Action::TYPE_PAGE_URL] = $this->buildDayHierarchicalTableFromFlatTable(
172180
$flatPageTablesByType[Action::TYPE_PAGE_URL],
173-
Archiver::PAGE_URLS_RECORD_NAME,
174-
[$this, 'flatRowToUrlHierarchyPath'],
175-
$archiveProcessor
181+
[$this, 'flatRowToUrlHierarchyPath']
176182
);
177183
$tablesByType[Action::TYPE_PAGE_TITLE] = $this->buildDayHierarchicalTableFromFlatTable(
178184
$flatPageTablesByType[Action::TYPE_PAGE_TITLE],
179-
Archiver::PAGE_TITLES_RECORD_NAME,
180-
[$this, 'flatRowToTitleHierarchyPath'],
181-
$archiveProcessor
185+
[$this, 'flatRowToTitleHierarchyPath']
182186
);
183187
$this->finalizeBuiltFromFlatHierarchyTable(
184188
$archiveProcessor,
@@ -284,11 +288,6 @@ private function flatRowToHierarchyPath(Row $flatRow, int $actionType): ?array
284288
return ArchivingHelper::getActionExplodedNames($label, $actionType);
285289
}
286290

287-
private function reduceLegacyHierarchyIntoFlatTable(DataTable $legacyHierarchy, DataTable $flatTable, int $actionType): void
288-
{
289-
$this->appendLegacyHierarchyRowsToFlatTable($legacyHierarchy, [], $flatTable, $actionType);
290-
}
291-
292291
private function removeFlatPathMetadataFromDataTable(DataTable $dataTable): void
293292
{
294293
$summaryRow = $dataTable->getRowFromId(DataTable::ID_SUMMARY_ROW);
@@ -325,52 +324,6 @@ private function finalizeBuiltFromFlatHierarchyTable(
325324
ArchivingHelper::deleteInvalidSummedColumnsFromDataTable($hierarchicalTable);
326325
}
327326

328-
private function appendLegacyHierarchyRowsToFlatTable(DataTable $sourceTable, array $path, DataTable $flatTable, int $actionType): void
329-
{
330-
foreach ($sourceTable->getRowsWithoutSummaryRow() as $row) {
331-
$label = $row->getColumn('label');
332-
if (!is_string($label) || $label === '') {
333-
continue;
334-
}
335-
336-
$currentPath = $path;
337-
$currentPath[] = $label;
338-
339-
$subtable = $row->getSubtable();
340-
if ($subtable) {
341-
$this->appendLegacyHierarchyRowsToFlatTable($subtable, $currentPath, $flatTable, $actionType);
342-
continue;
343-
}
344-
345-
$flatLabel = ArchivingHelper::buildBestEffortActionLabelFromPath($currentPath, $actionType);
346-
$flatRow = $flatTable->getRowFromLabel($flatLabel);
347-
if ($flatRow === false) {
348-
$flatRow = new Row([
349-
Row::COLUMNS => ['label' => $flatLabel],
350-
]);
351-
$flatRow->setMetadata(ArchivingHelper::ACTION_FLAT_PATH_METADATA_NAME, $currentPath);
352-
$flatTable->addRow($flatRow);
353-
}
354-
355-
$flatRow->sumRow(clone $row, true, Metrics::getColumnsAggregationOperation());
356-
}
357-
358-
$summaryRow = $sourceTable->getRowFromId(DataTable::ID_SUMMARY_ROW);
359-
if ($summaryRow === false) {
360-
return;
361-
}
362-
363-
$globalSummary = $flatTable->getRowFromId(DataTable::ID_SUMMARY_ROW);
364-
if ($globalSummary === false) {
365-
$globalSummary = clone $summaryRow;
366-
$globalSummary->setIsSummaryRow();
367-
$flatTable->addSummaryRow($globalSummary);
368-
return;
369-
}
370-
371-
$globalSummary->sumRow(clone $summaryRow, true, Metrics::getColumnsAggregationOperation());
372-
}
373-
374327
private function isFlatArchivingEnabled(): bool
375328
{
376329
ArchivingHelper::reloadConfig();
@@ -392,18 +345,12 @@ private function setHierarchyBuiltFromFlatRecord(
392345

393346
private function buildDayHierarchicalTableFromFlatTable(
394347
DataTable $flatTable,
395-
string $hierarchicalRecordName,
396-
callable $flatToHierarchyPathCallback,
397-
ArchiveProcessor $archiveProcessor
348+
callable $flatToHierarchyPathCallback
398349
): DataTable {
399-
$hierarchicalRecord = Record::make(Record::TYPE_BLOB, $hierarchicalRecordName);
400-
401350
return $this->buildHierarchicalTableFromFlatTable(
402351
$flatTable,
403352
Metrics::getColumnsAggregationOperation(),
404-
function (Row $flatRow) use ($flatToHierarchyPathCallback, $archiveProcessor, $hierarchicalRecord) {
405-
return call_user_func($flatToHierarchyPathCallback, $flatRow, $archiveProcessor, $hierarchicalRecord);
406-
},
353+
$flatToHierarchyPathCallback,
407354
$this->getDefaultHierarchyRowColumns()
408355
);
409356
}

‎plugins/Actions/tests/System/MixedArchivingAggregationTest.php‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
use Piwik\Db;
1717
use Piwik\Metrics;
1818
use Piwik\Plugins\Actions\Archiver;
19+
use Piwik\Plugins\Actions\ArchivingHelper;
1920
use Piwik\Tests\Framework\Fixture;
2021
use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
2122

@@ -106,6 +107,66 @@ public function testWeekAggregationCombinesLegacyAndFlatDaysAndMergesOthers()
106107
$this->assertSame($weekSummaryHits, (int) $weekHierarchicalSummary->getColumn(Metrics::INDEX_PAGE_NB_HITS));
107108
}
108109

110+
public function testWeekAggregationParityBetweenLegacyAndFlatFirst()
111+
{
112+
$legacySiteId = Fixture::createWebsite('2026-02-01 00:00:00');
113+
$flatSiteId = Fixture::createWebsite('2026-02-01 00:00:00');
114+
115+
$hitsByDay = [
116+
'2026-02-02' => [
117+
'/shop/shoes/nike' => 5,
118+
'/shop/shoes/adidas' => 3,
119+
'/about' => 2,
120+
],
121+
'2026-02-03' => [
122+
'/shop/shoes/nike' => 4,
123+
'/blog/article-1' => 3,
124+
'/contact' => 1,
125+
],
126+
];
127+
128+
foreach ($hitsByDay as $day => $urlHits) {
129+
$this->trackUrlHits($legacySiteId, $day, $urlHits);
130+
$this->trackUrlHits($flatSiteId, $day, $urlHits);
131+
}
132+
133+
$config = Config::getInstance();
134+
$configKeys = [
135+
'datatable_archiving_maximum_rows_actions_flat',
136+
'datatable_archiving_maximum_rows_actions',
137+
'datatable_archiving_maximum_rows_subtable_actions',
138+
'archiving_ranking_query_row_limit',
139+
];
140+
$configBackup = $this->backupGeneralConfig($configKeys);
141+
142+
try {
143+
$config->General['datatable_archiving_maximum_rows_actions'] = 50000;
144+
$config->General['datatable_archiving_maximum_rows_subtable_actions'] = 50000;
145+
$config->General['archiving_ranking_query_row_limit'] = 100000;
146+
147+
$config->General['datatable_archiving_maximum_rows_actions_flat'] = 0;
148+
$legacyWeekArchive = Archive::build($legacySiteId, 'week', '2026-02-03');
149+
$legacyWeekUrls = $legacyWeekArchive->getDataTable(Archiver::PAGE_URLS_RECORD_NAME);
150+
$legacyWeekTitles = $legacyWeekArchive->getDataTable(Archiver::PAGE_TITLES_RECORD_NAME);
151+
152+
$config->General['datatable_archiving_maximum_rows_actions_flat'] = 50000;
153+
$flatWeekArchive = Archive::build($flatSiteId, 'week', '2026-02-03');
154+
$flatWeekUrls = $flatWeekArchive->getDataTable(Archiver::PAGE_URLS_RECORD_NAME);
155+
$flatWeekTitles = $flatWeekArchive->getDataTable(Archiver::PAGE_TITLES_RECORD_NAME);
156+
} finally {
157+
$this->restoreGeneralConfig($configBackup);
158+
}
159+
160+
$this->assertSame(
161+
$this->exportHierarchyTableValues($legacyWeekUrls),
162+
$this->exportHierarchyTableValues($flatWeekUrls)
163+
);
164+
$this->assertSame(
165+
$this->exportHierarchyTableValues($legacyWeekTitles),
166+
$this->exportHierarchyTableValues($flatWeekTitles)
167+
);
168+
}
169+
109170
protected static function configureFixture($fixture)
110171
{
111172
parent::configureFixture($fixture);
@@ -211,4 +272,42 @@ private function sumHitsWithoutSummary(DataTable $table): int
211272

212273
return $sum;
213274
}
275+
276+
private function exportHierarchyTableValues(DataTable $hierarchicalTable): array
277+
{
278+
$flattened = new DataTable();
279+
ArchivingHelper::mergeHierarchicalActionsTableIntoFlatTable($hierarchicalTable, $flattened);
280+
281+
return $this->exportFlatTableValues($flattened);
282+
}
283+
284+
private function exportFlatTableValues(DataTable $flatTable): array
285+
{
286+
$rows = [];
287+
foreach ($flatTable->getRowsWithoutSummaryRow() as $row) {
288+
$label = $row->getColumn('label');
289+
if (!is_string($label)) {
290+
continue;
291+
}
292+
293+
$columns = $row->getColumns();
294+
unset($columns['label']);
295+
ksort($columns);
296+
$rows[$label] = $columns;
297+
}
298+
ksort($rows);
299+
300+
$summaryColumns = [];
301+
$summaryRow = $flatTable->getRowFromId(DataTable::ID_SUMMARY_ROW);
302+
if ($summaryRow !== false) {
303+
$summaryColumns = $summaryRow->getColumns();
304+
unset($summaryColumns['label']);
305+
ksort($summaryColumns);
306+
}
307+
308+
return [
309+
'rows' => $rows,
310+
'summary' => $summaryColumns,
311+
];
312+
}
214313
}

0 commit comments

Comments
 (0)
⚡