/*
 * Syncany, www.syncany.org
 * Copyright (C) 2011-2016 Philipp C. Heckel <philipp.heckel@gmail.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.syncany.operations.cleanup;

import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.syncany.chunk.Chunk;
import org.syncany.chunk.MultiChunk;
import org.syncany.config.Config;
import org.syncany.database.DatabaseVersion;
import org.syncany.database.FileContent;
import org.syncany.database.FileVersion;
import org.syncany.database.MultiChunkEntry;
import org.syncany.database.MultiChunkEntry.MultiChunkId;
import org.syncany.database.PartialFileHistory;
import org.syncany.database.PartialFileHistory.FileHistoryId;
import org.syncany.database.SqlDatabase;
import org.syncany.database.dao.DatabaseXmlSerializer;
import org.syncany.database.dao.FileVersionSqlDao;
import org.syncany.operations.AbstractTransferOperation;
import org.syncany.operations.cleanup.CleanupOperationOptions.TimeUnit;
import org.syncany.operations.cleanup.CleanupOperationResult.CleanupResultCode;
import org.syncany.operations.daemon.messages.CleanupEndSyncExternalEvent;
import org.syncany.operations.daemon.messages.CleanupStartCleaningSyncExternalEvent;
import org.syncany.operations.daemon.messages.CleanupStartSyncExternalEvent;
import org.syncany.operations.down.DownOperation;
import org.syncany.operations.ls_remote.LsRemoteOperation;
import org.syncany.operations.ls_remote.LsRemoteOperationResult;
import org.syncany.operations.status.StatusOperation;
import org.syncany.operations.status.StatusOperationResult;
import org.syncany.operations.up.BlockingTransfersException;
import org.syncany.operations.up.UpOperation;
import org.syncany.plugins.transfer.RemoteTransaction;
import org.syncany.plugins.transfer.StorageException;
import org.syncany.plugins.transfer.files.CleanupRemoteFile;
import org.syncany.plugins.transfer.files.DatabaseRemoteFile;
import org.syncany.plugins.transfer.files.MultichunkRemoteFile;
import org.syncany.plugins.transfer.files.RemoteFile;

/**
 * The purpose of the cleanup operation is to keep the local database and the
 * remote repository clean -- thereby allowing it to be used indefinitely without
 * any performance issues or storage shortage.
 *
 * <p>The responsibilities of the cleanup operations include:
 * <ul>
 *   <li>Remove old {@link FileVersion} and their corresponding database entities.
 *       In particular, it also removes {@link PartialFileHistory}s, {@link FileContent}s,
 *       {@link Chunk}s and {@link MultiChunk}s.</li>
 *   <li>Merge metadata of a single client and remove old database version files
 *       from the remote storage.</li>
 * </ul>
 *
 * <p>High level strategy:
 * <ul>
 *    <li>Lock repo and start thread that renews the lock every X seconds</li>
 *    <li>Find old versions / contents / ... from database</li>
 *    <li>Delete these versions and contents locally</li>
 *    <li>Delete all remote metadata</li>
 *    <li>Obtain consistent database files from local database</li>
 *    <li>Upload new database files to repo</li>
 *    <li>Remotely delete unused multichunks</li>
 *    <li>Stop lock renewal thread and unlock repo</li>
 * </ul>
 *
 * <p><b>Important issues:</b>
 * All remote operations MUST check if the lock has been recently renewed. If it hasn't, the connection has been lost.
 *
 * @author Philipp C. Heckel (philipp.heckel@gmail.com)
 */
public class CleanupOperation extends AbstractTransferOperation {
        private static final Logger logger = Logger.getLogger(CleanupOperation.class.getSimpleName());

        public static final String ACTION_ID = "cleanup";
        private static final int BEFORE_DOUBLE_CHECK_TIME = 1200;

        private CleanupOperationOptions options;
        private CleanupOperationResult result;

        private SqlDatabase localDatabase;
        private RemoteTransaction remoteTransaction;

        public CleanupOperation(Config config) {
                this(config, new CleanupOperationOptions());
        }

        public CleanupOperation(Config config, CleanupOperationOptions options) {
                super(config, ACTION_ID);

                this.options = options;
                this.result = new CleanupOperationResult();
                this.localDatabase = new SqlDatabase(config);
        }

        @Override
        public CleanupOperationResult execute() throws Exception {
                logger.log(Level.INFO, "");
                logger.log(Level.INFO, "Running 'Cleanup' at client " + config.getMachineName() + " ...");
                logger.log(Level.INFO, "--------------------------------------------");

                // Do initial check out remote repository preconditions
                CleanupResultCode preconditionResult = checkPreconditions();

                fireStartEvent();
                if (preconditionResult != CleanupResultCode.OK) {
                        fireEndEvent();
                        return new CleanupOperationResult(preconditionResult);
                }

                fireCleanupNeededEvent();

                // At this point, the operation will lock the repository
                startOperation();

                // If there are any, rollback any existing/old transactions.
                // If other clients have unfinished transactions with deletions, do not proceed.
                try {
                        transferManager.cleanTransactions();
                }
                catch (BlockingTransfersException ignored) {
                        finishOperation();
                        fireEndEvent();
                        return new CleanupOperationResult(CleanupResultCode.NOK_REPO_BLOCKED);
                }

                // Wait two seconds (conservative cleanup, see #104)
                logger.log(Level.INFO, "Cleanup: Waiting a while to be sure that no other actions are running ...");
                Thread.sleep(BEFORE_DOUBLE_CHECK_TIME);

                // Check again. No other clients should be busy, because we waited BEFORE_DOUBLE_CHECK_TIME
                preconditionResult = checkPreconditions();

                if (preconditionResult != CleanupResultCode.OK) {
                        finishOperation();
                        fireEndEvent();
                        return new CleanupOperationResult(preconditionResult);
                }

                // If we do cleanup, we are no longer allowed to resume a transaction
                transferManager.clearResumableTransactions();
                transferManager.clearPendingTransactions();

                // Now do the actual work!
                logger.log(Level.INFO, "Cleanup: Starting transaction.");
                remoteTransaction = new RemoteTransaction(config, transferManager);

                removeOldVersions();

                if (options.isRemoveUnreferencedTemporaryFiles()) {
                        transferManager.removeUnreferencedTemporaryFiles();
                }

                mergeRemoteFiles();

                // We went succesfully through the entire operation and checked everything. Hence we update the last cleanup time.
                updateLastCleanupTime();

                finishOperation();
                fireEndEvent();

                return updateResultCode(result);
        }

        /**
         * This method checks if we have changed anything and sets the
         * {@link CleanupResultCode} of the given result accordingly.
         *
         * @param result The result so far in this operation.
         * @return result The original result, with the relevant {@link CleanupResultCode}
         */
        private CleanupOperationResult updateResultCode(CleanupOperationResult result) {
                if (result.getMergedDatabaseFilesCount() > 0 || result.getRemovedMultiChunksCount() > 0 || result.getRemovedOldVersionsCount() > 0) {
                        result.setResultCode(CleanupResultCode.OK);
                }
                else {
                        result.setResultCode(CleanupResultCode.OK_NOTHING_DONE);
                }

                return result;
        }

        private void fireStartEvent() {
                eventBus.post(new CleanupStartSyncExternalEvent(config.getLocalDir().getAbsolutePath()));
        }

        private void fireCleanupNeededEvent() {
                eventBus.post(new CleanupStartCleaningSyncExternalEvent(config.getLocalDir().getAbsolutePath()));
        }

        private void fireEndEvent() {
                eventBus.post(new CleanupEndSyncExternalEvent(config.getLocalDir().getAbsolutePath(), result));
        }

        /**
         * This method inspects the local database and remote repository to
         * see if cleanup should be performed.
         *
         * @return {@link CleanupResultCode.OK} if nothing prevents continuing, another relevant code otherwise.
         */
        private CleanupResultCode checkPreconditions() throws Exception {
                if (hasDirtyDatabaseVersions()) {
                        return CleanupResultCode.NOK_DIRTY_LOCAL;
                }

                if (!options.isForce() && wasCleanedRecently()) {
                        return CleanupResultCode.NOK_RECENTLY_CLEANED;
                }

                if (hasLocalChanges()) {
                        return CleanupResultCode.NOK_LOCAL_CHANGES;
                }

                if (hasRemoteChanges()) {
                        return CleanupResultCode.NOK_REMOTE_CHANGES;
                }

                if (otherRemoteOperationsRunning(CleanupOperation.ACTION_ID, UpOperation.ACTION_ID, DownOperation.ACTION_ID)) {
                        return CleanupResultCode.NOK_OTHER_OPERATIONS_RUNNING;
                }

                return CleanupResultCode.OK;
        }

        private boolean hasLocalChanges() throws Exception {
                StatusOperationResult statusOperationResult = new StatusOperation(config, options.getStatusOptions()).execute();
                return statusOperationResult.getChangeSet().hasChanges();
        }

        /**
         * This method checks if there exist {@link FileVersion}s which are to be deleted because the history they are a part
         * of is too long. It will collect these, remove them locally and add them to the {@link RemoteTransaction} for deletion.
         */
        private void removeOldVersions() throws Exception {
                Map<FileHistoryId, List<FileVersion>> purgeFileVersions = new TreeMap<FileHistoryId, List<FileVersion>>();
                Map<FileHistoryId, FileVersion> purgeBeforeFileVersions = new TreeMap<FileHistoryId, FileVersion>();

                if (options.isRemoveVersionsByInterval()) {
                        // Get file versions that should be purged according to the settings that are given. Time-based.
                        purgeFileVersions = collectPurgableFileVersions();
                }

                if (options.isRemoveOldVersions()) {
                        // Get all non-final fileversions and deleted (final) fileversions that we want to fully delete.
                        // purgeFileVersions is modified here!
                        purgeBeforeFileVersions = collectPurgeBeforeFileVersions(purgeFileVersions);
                }
                if (purgeFileVersions.isEmpty() && purgeBeforeFileVersions.isEmpty()) {
                        logger.log(Level.INFO, "- Old version removal: Not necessary.");
                        return;
                }

                logger.log(Level.INFO, "- Old version removal: Found {0} file histories and {1} file versions that need cleaning.", new Object[] {
                                purgeFileVersions.size(),
                                purgeBeforeFileVersions.size() });

                // Local: First, remove file versions that are not longer needed
                localDatabase.removeSmallerOrEqualFileVersions(purgeBeforeFileVersions);
                localDatabase.removeFileVersions(purgeFileVersions);

                // Local: Then, determine what must be changed remotely and remove it locally
                Map<MultiChunkId, MultiChunkEntry> unusedMultiChunks = localDatabase.getUnusedMultiChunks();

                localDatabase.removeUnreferencedDatabaseEntities();
                deleteUnusedRemoteMultiChunks(unusedMultiChunks);

                // Update stats
                long unusedMultiChunkSize = 0;

                for (MultiChunkEntry removedMultiChunk : unusedMultiChunks.values()) {
                        unusedMultiChunkSize += removedMultiChunk.getSize();
                }

                result.setRemovedOldVersionsCount(purgeBeforeFileVersions.size() + purgeFileVersions.size());
                result.setRemovedMultiChunksCount(unusedMultiChunks.size());
                result.setRemovedMultiChunksSize(unusedMultiChunkSize);
        }

        private Map<FileHistoryId, FileVersion> collectPurgeBeforeFileVersions(Map<FileHistoryId, List<FileVersion>> purgeFileVersions) {
                long deleteBeforeTimestamp = System.currentTimeMillis() - options.getMinKeepDeletedSeconds() * 1000;
                
                Map<FileHistoryId, FileVersion> deletedFileVersionsBeforeTimestamp = localDatabase.getDeletedFileVersionsBefore(deleteBeforeTimestamp);
                Map<FileHistoryId, List<FileVersion>> selectedPurgeFileVersions = localDatabase.getFileHistoriesToPurgeBefore(deleteBeforeTimestamp);
                
                Map<FileHistoryId, FileVersion> purgeBeforeFileVersions = new HashMap<FileHistoryId, FileVersion>();
                purgeBeforeFileVersions.putAll(deletedFileVersionsBeforeTimestamp);
                putAllFileVersionsInMap(selectedPurgeFileVersions, purgeFileVersions);
                
                return purgeBeforeFileVersions;
        }

        /**
         * For all time intervals defined in the purge file settings, determine the eligible file
         * versions to be purged -- namely all but the newest one.
         * 
         * @see CleanupOperation 
         * @see CleanupOperationOptions#getPurgeFileVersionSettings()
         * @see FileVersionSqlDao#getFileHistoriesToPurgeInInterval(long, long, TimeUnit)
         */
        private Map<FileHistoryId, List<FileVersion>> collectPurgableFileVersions() {
                Map<FileHistoryId, List<FileVersion>> purgeFileVersions = new HashMap<FileHistoryId, List<FileVersion>>();

                long currentTime = System.currentTimeMillis();
                long previousTruncateIntervalTimeMultiplier = 0;                
                
                for (Map.Entry<Long, TimeUnit> purgeFileVersionSetting : options.getPurgeFileVersionSettings().entrySet()) {
                        Long truncateIntervalMultiplier = purgeFileVersionSetting.getKey();
                        TimeUnit truncateIntervalTimeUnit = purgeFileVersionSetting.getValue();                 
                        
                        long beginIntervalTimestamp = currentTime - truncateIntervalMultiplier * 1000;
                        long endIntervalTimestamp = currentTime - previousTruncateIntervalTimeMultiplier * 1000;
                        
                        Map<FileHistoryId, List<FileVersion>> newPurgeFileVersions = localDatabase.getFileHistoriesToPurgeInInterval(
                                        beginIntervalTimestamp, endIntervalTimestamp, truncateIntervalTimeUnit);

                        putAllFileVersionsInMap(newPurgeFileVersions, purgeFileVersions);
                        previousTruncateIntervalTimeMultiplier = truncateIntervalMultiplier;
                }

                return purgeFileVersions;
        }

        private void putAllFileVersionsInMap(Map<FileHistoryId, List<FileVersion>> newFileVersions,
                        Map<FileHistoryId, List<FileVersion>> fileHistoryPurgeFileVersions) {
                
                for (FileHistoryId fileHistoryId : newFileVersions.keySet()) {
                        List<FileVersion> purgeFileVersions = fileHistoryPurgeFileVersions.get(fileHistoryId);
                        List<FileVersion> newPurgeFileVersions = newFileVersions.get(fileHistoryId);
                        
                        if (purgeFileVersions != null) {
                                purgeFileVersions.addAll(newPurgeFileVersions);
                        }
                        else {
                                fileHistoryPurgeFileVersions.put(fileHistoryId, newPurgeFileVersions);
                        }
                }
        }

        /**
         * This method adds unusedMultiChunks to the @{link RemoteTransaction} for deletion.
         *
         * @param unusedMultiChunks which are to be deleted because all references to them are gone.
         */
        private void deleteUnusedRemoteMultiChunks(Map<MultiChunkId, MultiChunkEntry> unusedMultiChunks) throws StorageException {
                logger.log(Level.INFO, "- Deleting remote multichunks ...");

                for (MultiChunkEntry multiChunkEntry : unusedMultiChunks.values()) {
                        logger.log(Level.FINE, "  + Deleting remote multichunk " + multiChunkEntry + " ...");
                        remoteTransaction.delete(new MultichunkRemoteFile(multiChunkEntry.getId()));
                }
        }

        private boolean hasDirtyDatabaseVersions() {
                Iterator<DatabaseVersion> dirtyDatabaseVersions = localDatabase.getDirtyDatabaseVersions();
                return dirtyDatabaseVersions.hasNext(); // TODO [low] Is this a resource creeper?
        }

        private boolean hasRemoteChanges() throws Exception {
                LsRemoteOperationResult lsRemoteOperationResult = new LsRemoteOperation(config).execute();
                return lsRemoteOperationResult.getUnknownRemoteDatabases().size() > 0;
        }

        /**
         * Checks if Cleanup has been performed less then a configurable time ago.
         */
        private boolean wasCleanedRecently() throws Exception {
                Long lastCleanupTime = localDatabase.getCleanupTime();

                if (lastCleanupTime == null) {
                        return false;
                }
                else {
                        return lastCleanupTime + options.getMinSecondsBetweenCleanups() > System.currentTimeMillis() / 1000;
                }
        }

        /**
         * This method deletes all remote database files and writes new ones for each client using the local database.
         * To make the state clear and prevent issues with replacing files, new database files are given a higher number
         * than all existing database files.
         * Both the deletions and the new files added to the current @{link RemoteTransaction}.
         */
        private void mergeRemoteFiles() throws Exception {
                // Retrieve all database versions
                Map<String, List<DatabaseRemoteFile>> allDatabaseFilesMap = retrieveAllRemoteDatabaseFiles();

                boolean needMerge = needMerge(allDatabaseFilesMap);

                if (!needMerge) {
                        logger.log(Level.INFO, "- No purging happened. Number of database files does not exceed threshold. Not merging remote files.");
                        return;
                }

                // Now do the merge!
                logger.log(Level.INFO, "- Merge remote files ...");

                List<DatabaseRemoteFile> allToDeleteDatabaseFiles = new ArrayList<DatabaseRemoteFile>();
                Map<File, DatabaseRemoteFile> allMergedDatabaseFiles = new TreeMap<File, DatabaseRemoteFile>();

                for (String client : allDatabaseFilesMap.keySet()) {
                        List<DatabaseRemoteFile> clientDatabaseFiles = allDatabaseFilesMap.get(client);
                        Collections.sort(clientDatabaseFiles);
                        logger.log(Level.INFO, "Databases: " + clientDatabaseFiles);

                        // 1. Determine files to delete remotely
                        List<DatabaseRemoteFile> toDeleteDatabaseFiles = new ArrayList<DatabaseRemoteFile>(clientDatabaseFiles);
                        allToDeleteDatabaseFiles.addAll(toDeleteDatabaseFiles);

                        // 2. Write new database file and save it in allMergedDatabaseFiles
                        writeMergeFile(client, allMergedDatabaseFiles);

                }

                rememberDatabases(allMergedDatabaseFiles);

                // 3. Prepare transaction

                // Queue old databases for deletion
                for (RemoteFile toDeleteRemoteFile : allToDeleteDatabaseFiles) {
                        logger.log(Level.INFO, "   + Deleting remote file " + toDeleteRemoteFile + " ...");
                        remoteTransaction.delete(toDeleteRemoteFile);
                }

                // Queue new databases for uploading
                for (File lastLocalMergeDatabaseFile : allMergedDatabaseFiles.keySet()) {
                        RemoteFile lastRemoteMergeDatabaseFile = allMergedDatabaseFiles.get(lastLocalMergeDatabaseFile);

                        logger.log(Level.INFO, "   + Uploading new file {0} from local file {1} ...", new Object[] { lastRemoteMergeDatabaseFile,
                                        lastLocalMergeDatabaseFile });

                        remoteTransaction.upload(lastLocalMergeDatabaseFile, lastRemoteMergeDatabaseFile);
                }

                finishMerging();

                // Update stats
                result.setMergedDatabaseFilesCount(allToDeleteDatabaseFiles.size());
        }

        /**
         * This method decides if a merge is needed. Most of the time it will be, since we need to merge every time we remove
         * any FileVersions to delete them remotely. Another reason for merging is if the number of files exceeds a certain threshold.
         * This threshold scales linearly with the number of clients that have database files.
         *
         * @param allDatabaseFilesMap used to determine if there are too many database files.
         *
         * @return true if there are too many database files or we have removed FileVersions, false otherwise.
         */
        private boolean needMerge(Map<String, List<DatabaseRemoteFile>> allDatabaseFilesMap) {
                int numberOfDatabaseFiles = 0;

                for (String client : allDatabaseFilesMap.keySet()) {
                        numberOfDatabaseFiles += allDatabaseFilesMap.get(client).size();
                }

                // A client will merge databases if the number of databases exceeds the maximum number per client times the amount of clients
                int maxDatabaseFiles = options.getMaxDatabaseFiles() * allDatabaseFilesMap.keySet().size();
                boolean tooManyDatabaseFiles = numberOfDatabaseFiles > maxDatabaseFiles;
                boolean removedOldVersions = result.getRemovedOldVersionsCount() > 0;

                return removedOldVersions || tooManyDatabaseFiles || options.isForce();
        }

        /**
         * This method writes the file with merged databases for a single client and adds it to a Map containing all merged
         * database files. This is done by querying the local database for all {@link DatabaseVersion}s by this client and
         * serializing them.
         *
         * @param clientName for which we want to write the merged dataabse file.
         * @param allMergedDatabaseFiles Map where we add the merged file once it is written.
         */
        private void writeMergeFile(String clientName, Map<File, DatabaseRemoteFile> allMergedDatabaseFiles)
                        throws StorageException, IOException {

                // Increment the version by 1, to signal cleanup has occurred

                long lastClientVersion = getNewestDatabaseFileVersion(clientName, localDatabase.getKnownDatabases());
                DatabaseRemoteFile newRemoteMergeDatabaseFile = new DatabaseRemoteFile(clientName, lastClientVersion + 1);

                File newLocalMergeDatabaseFile = config.getCache().getDatabaseFile(newRemoteMergeDatabaseFile.getName());

                logger.log(Level.INFO, "   + Writing new merge file (all files up to {0}) to {1} ...", new Object[] { lastClientVersion,
                                newLocalMergeDatabaseFile });

                Iterator<DatabaseVersion> lastNDatabaseVersions = localDatabase.getDatabaseVersionsTo(clientName, lastClientVersion);

                DatabaseXmlSerializer databaseDAO = new DatabaseXmlSerializer(config.getTransformer());
                databaseDAO.save(lastNDatabaseVersions, newLocalMergeDatabaseFile);
                allMergedDatabaseFiles.put(newLocalMergeDatabaseFile, newRemoteMergeDatabaseFile);
        }

        /**
         * This method locally remembers which databases were newly uploaded, such that they will not be downloaded in
         * future Downs.
         */
        private void rememberDatabases(Map<File, DatabaseRemoteFile> allMergedDatabaseFiles) throws SQLException {
                // Remember newly written files as so not to redownload them later.
                List<DatabaseRemoteFile> newRemoteMergeDatabaseFiles = new ArrayList<DatabaseRemoteFile>();
                newRemoteMergeDatabaseFiles.addAll(allMergedDatabaseFiles.values());

                logger.log(Level.INFO, "Writing new known databases table: " + newRemoteMergeDatabaseFiles);

                localDatabase.removeKnownDatabases();
                localDatabase.writeKnownRemoteDatabases(newRemoteMergeDatabaseFiles);
        }

        /**
         * This method finishes the merging of remote files, by attempting to commit the {@link RemoteTransaction}.
         * If this fails, it will roll back the local database.
         */
        private void finishMerging() throws Exception {
                updateCleanupFileInTransaction();

                try {
                        logger.log(Level.INFO, "Cleanup: COMMITTING TX ...");

                        remoteTransaction.commit();
                        localDatabase.commit();
                }
                catch (StorageException e) {
                        logger.log(Level.INFO, "Cleanup: FAILED TO COMMIT TX. Rolling back ...");

                        localDatabase.rollback();
                        throw e;
                }

                logger.log(Level.INFO, "Cleanup: SUCCESS COMMITTING TX.");
        }

        /**
         * This method obtains a Map with Lists of {@link DatabaseRemoteFile}s as values, by listing them in the remote repo and
         * collecting the files per client.
         *
         * @return a Map with clientNames as keys and lists of corresponding DatabaseRemoteFiles as values.
         */
        private Map<String, List<DatabaseRemoteFile>> retrieveAllRemoteDatabaseFiles() throws StorageException {
                SortedMap<String, List<DatabaseRemoteFile>> allDatabaseRemoteFilesMap = new TreeMap<String, List<DatabaseRemoteFile>>();
                Map<String, DatabaseRemoteFile> allDatabaseRemoteFiles = transferManager.list(DatabaseRemoteFile.class);

                for (Map.Entry<String, DatabaseRemoteFile> entry : allDatabaseRemoteFiles.entrySet()) {
                        String clientName = entry.getValue().getClientName();

                        if (allDatabaseRemoteFilesMap.get(clientName) == null) {
                                allDatabaseRemoteFilesMap.put(clientName, new ArrayList<DatabaseRemoteFile>());
                        }

                        allDatabaseRemoteFilesMap.get(clientName).add(entry.getValue());
                }

                return allDatabaseRemoteFilesMap;
        }

        /**
         * This method checks what the current cleanup number is, increments it by one and adds
         * a new cleanup file to the transaction, to signify to other clients that Cleanup has occurred.
         */
        private void updateCleanupFileInTransaction() throws StorageException, IOException {
                if (remoteTransaction.isEmpty()) {
                        // No need to bump numbers
                        return;
                }
                // Find all existing cleanup files
                Map<String, CleanupRemoteFile> cleanupFiles = transferManager.list(CleanupRemoteFile.class);

                long lastRemoteCleanupNumber = getLastRemoteCleanupNumber(cleanupFiles);

                // Schedule any existing cleanup files for deletion
                for (CleanupRemoteFile cleanupRemoteFile : cleanupFiles.values()) {
                        remoteTransaction.delete(cleanupRemoteFile);
                }

                // Upload a new cleanup file that indicates changes
                File newCleanupFile = config.getCache().createTempFile("cleanup");
                long newCleanupNumber = lastRemoteCleanupNumber + 1;

                remoteTransaction.upload(newCleanupFile, new CleanupRemoteFile(newCleanupNumber));
                localDatabase.writeCleanupNumber(newCleanupNumber);
        }

        /**
         * The cleanup time is used to check if cleanup has been done recently. If it has, we do not need
         * to clean again.
         */
        private void updateLastCleanupTime() throws SQLException {
                // Set cleanup number locally
                localDatabase.writeCleanupTime(System.currentTimeMillis() / 1000);
                localDatabase.commit();
        }
}