Merge pull request #22269 from owncloud/issue-22243-avoid-deadlock-with-lots-of-entries-to-cleanup

Chunk the cleanup queries to make sure they don't time out

Author: DeepDiver1975

apps/files/command/deleteorphanedfiles.php

@@ -33,6 +33,8 @@
  */
 class DeleteOrphanedFiles extends Command {
 
+	const CHUNK_SIZE = 200;
+
 	/**
 	 * @var IDBConnection
 	 */
@@ -50,13 +52,31 @@ protected function configure() {
 	}
 
 	public function execute(InputInterface $input, OutputInterface $output) {
+		$deletedEntries = 0;
+
+		$query = $this->connection->getQueryBuilder();
+		$query->select('fc.fileid')
+			->from('filecache', 'fc')
+			->where($query->expr()->isNull('s.numeric_id'))
+			->leftJoin('fc', 'storages', 's', $query->expr()->eq('fc.storage', 's.numeric_id'))
+			->setMaxResults(self::CHUNK_SIZE);
+
+		$deleteQuery = $this->connection->getQueryBuilder();
+		$deleteQuery->delete('filecache')
+			->where($deleteQuery->expr()->eq('fileid', $deleteQuery->createParameter('objectid')));
 
-		$sql =
-			'DELETE FROM `*PREFIX*filecache` ' .
-			'WHERE NOT EXISTS ' .
-			'(SELECT 1 FROM `*PREFIX*storages` WHERE `storage` = `numeric_id`)';
+		$deletedInLastChunk = self::CHUNK_SIZE;
+		while ($deletedInLastChunk === self::CHUNK_SIZE) {
+			$deletedInLastChunk = 0;
+			$result = $query->execute();
+			while ($row = $result->fetch()) {
+				$deletedInLastChunk++;
+				$deletedEntries += $deleteQuery->setParameter('objectid', (int) $row['fileid'])
+					->execute();
+			}
+			$result->closeCursor();
+		}
 
-		$deletedEntries = $this->connection->executeUpdate($sql);
 		$output->writeln("$deletedEntries orphaned file cache entries deleted");
 	}
 

apps/files/lib/backgroundjob/deleteorphaneditems.php

@@ -29,6 +29,8 @@
  */
 class DeleteOrphanedItems extends TimedJob {
 
+	const CHUNK_SIZE = 200;
+
 	/** @var \OCP\IDBConnection */
 	protected $connection;
 
@@ -66,19 +68,38 @@ public function run($argument) {
 	/**
 	 * Deleting orphaned system tag mappings
 	 *
+	 * @param string $table
+	 * @param string $idCol
+	 * @param string $typeCol
 	 * @return int Number of deleted entries
 	 */
 	protected function cleanUp($table, $idCol, $typeCol) {
-		$subQuery = $this->connection->getQueryBuilder();
-		$subQuery->select($subQuery->expr()->literal('1'))
-			->from('filecache', 'f')
-			->where($subQuery->expr()->eq($idCol, 'f.fileid'));
+		$deletedEntries = 0;
 
 		$query = $this->connection->getQueryBuilder();
-		$deletedEntries = $query->delete($table)
+		$query->select('t1.' . $idCol)
+			->from($table, 't1')
 			->where($query->expr()->eq($typeCol, $query->expr()->literal('files')))
-			->andWhere($query->expr()->isNull($query->createFunction('(' . $subQuery->getSql() . ')')))
-			->execute();
+			->andWhere($query->expr()->isNull('t2.fileid'))
+			->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid'))
+			->groupBy('t1.' . $idCol)
+			->setMaxResults(self::CHUNK_SIZE);
+
+		$deleteQuery = $this->connection->getQueryBuilder();
+		$deleteQuery->delete($table)
+			->where($deleteQuery->expr()->eq($idCol, $deleteQuery->createParameter('objectid')));
+
+		$deletedInLastChunk = self::CHUNK_SIZE;
+		while ($deletedInLastChunk === self::CHUNK_SIZE) {
+			$result = $query->execute();
+			$deletedInLastChunk = 0;
+			while ($row = $result->fetch()) {
+				$deletedInLastChunk++;
+				$deletedEntries += $deleteQuery->setParameter('objectid', (int) $row[$idCol])
+					->execute();
+			}
+			$result->closeCursor();
+		}
 
 		return $deletedEntries;
 	}
@@ -111,8 +132,7 @@ protected function cleanUserTags() {
 	 * @return int Number of deleted entries
 	 */
 	protected function cleanComments() {
-		$qb = $this->connection->getQueryBuilder();
-		$deletedEntries = $this->cleanUp('comments', $qb->expr()->castColumn('object_id', IQueryBuilder::PARAM_INT), 'object_type');
+		$deletedEntries = $this->cleanUp('comments', 'object_id', 'object_type');
 		$this->logger->debug("$deletedEntries orphaned comments deleted", ['app' => 'DeleteOrphanedItems']);
 		return $deletedEntries;
 	}
@@ -123,8 +143,7 @@ protected function cleanComments() {
 	 * @return int Number of deleted entries
 	 */
 	protected function cleanCommentMarkers() {
-		$qb = $this->connection->getQueryBuilder();
-		$deletedEntries = $this->cleanUp('comments_read_markers', $qb->expr()->castColumn('object_id', IQueryBuilder::PARAM_INT), 'object_type');
+		$deletedEntries = $this->cleanUp('comments_read_markers', 'object_id', 'object_type');
 		$this->logger->debug("$deletedEntries orphaned comment read marks deleted", ['app' => 'DeleteOrphanedItems']);
 		return $deletedEntries;
 	}

lib/private/repair/repairinvalidshares.php

@@ -30,6 +30,8 @@
  */
 class RepairInvalidShares extends BasicEmitter implements \OC\RepairStep {
 
+	const CHUNK_SIZE = 200;
+
 	/**
 	 * @var \OCP\IConfig
 	 */
@@ -83,18 +85,24 @@ private function removeSharesNonExistingParent() {
 			->where($query->expr()->isNotNull('s1.parent'))
 				->andWhere($query->expr()->isNull('s2.id'))
 			->leftJoin('s1', 'share', 's2', $query->expr()->eq('s1.parent', 's2.id'))
-			->groupBy('s1.parent');
+			->groupBy('s1.parent')
+			->setMaxResults(self::CHUNK_SIZE);
 
 		$deleteQuery = $this->connection->getQueryBuilder();
 		$deleteQuery->delete('share')
-			->where($query->expr()->eq('parent', $deleteQuery->createParameter('parent')));
-
-		$result = $query->execute();
-		while ($row = $result->fetch()) {
-			$deletedEntries += $deleteQuery->setParameter('parent', (int) $row['parent'])
-				->execute();
+			->where($deleteQuery->expr()->eq('parent', $deleteQuery->createParameter('parent')));
+
+		$deletedInLastChunk = self::CHUNK_SIZE;
+		while ($deletedInLastChunk === self::CHUNK_SIZE) {
+			$deletedInLastChunk = 0;
+			$result = $query->execute();
+			while ($row = $result->fetch()) {
+				$deletedInLastChunk++;
+				$deletedEntries += $deleteQuery->setParameter('parent', (int) $row['parent'])
+					->execute();
+			}
+			$result->closeCursor();
 		}
-		$result->closeCursor();
 
 		if ($deletedEntries) {
 			$this->emit('\OC\Repair', 'info', array('Removed ' . $deletedEntries . ' shares where the parent did not exist'));