001/*
002 * Syncany, www.syncany.org
003 * Copyright (C) 2011-2016 Philipp C. Heckel <philipp.heckel@gmail.com> 
004 *
005 * This program is free software: you can redistribute it and/or modify
006 * it under the terms of the GNU General Public License as published by
007 * the Free Software Foundation, either version 3 of the License, or
008 * (at your option) any later version.
009 *
010 * This program is distributed in the hope that it will be useful,
011 * but WITHOUT ANY WARRANTY; without even the implied warranty of
012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
013 * GNU General Public License for more details.
014 *
015 * You should have received a copy of the GNU General Public License
016 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
017 */
018package org.syncany.database.dao;
019
020import java.sql.Connection;
021import java.sql.PreparedStatement;
022import java.sql.ResultSet;
023import java.sql.SQLException;
024import java.sql.Statement;
025import java.sql.Timestamp;
026import java.util.ArrayList;
027import java.util.Date;
028import java.util.Iterator;
029import java.util.List;
030import java.util.Map;
031import java.util.logging.Level;
032import java.util.logging.Logger;
033
034import org.syncany.database.ChunkEntry;
035import org.syncany.database.ChunkEntry.ChunkChecksum;
036import org.syncany.database.DatabaseConnectionFactory;
037import org.syncany.database.DatabaseVersion;
038import org.syncany.database.DatabaseVersion.DatabaseVersionStatus;
039import org.syncany.database.DatabaseVersionHeader;
040import org.syncany.database.FileContent;
041import org.syncany.database.FileContent.FileChecksum;
042import org.syncany.database.FileVersion;
043import org.syncany.database.MultiChunkEntry;
044import org.syncany.database.MultiChunkEntry.MultiChunkId;
045import org.syncany.database.PartialFileHistory;
046import org.syncany.database.PartialFileHistory.FileHistoryId;
047import org.syncany.database.VectorClock;
048import org.syncany.operations.down.DatabaseBranch;
049
050/**
051 * The database version data access object (DAO) writes and queries the SQL database for information
052 * on {@link DatabaseVersion}s. It translates the relational data in the "databaseversion" table to
053 * Java objects; but also uses the other DAOs to persist entire {@link DatabaseVersion} objects. 
054 * 
055 * @see ChunkSqlDao
056 * @see FileContentSqlDao
057 * @see FileVersionSqlDao
058 * @see FileHistorySqlDao
059 * @see MultiChunkSqlDao
060 * @author Philipp C. Heckel (philipp.heckel@gmail.com)
061 */
062public class DatabaseVersionSqlDao extends AbstractSqlDao {
063        protected static final Logger logger = Logger.getLogger(DatabaseVersionSqlDao.class.getSimpleName());
064
065        private ChunkSqlDao chunkDao;
066        private FileContentSqlDao fileContentDao;
067        private FileVersionSqlDao fileVersionDao;
068        private FileHistorySqlDao fileHistoryDao;
069        private MultiChunkSqlDao multiChunkDao;
070
071        public DatabaseVersionSqlDao(Connection connection, ChunkSqlDao chunkDao, FileContentSqlDao fileContentDao, FileVersionSqlDao fileVersionDao,
072                        FileHistorySqlDao fileHistoryDao,
073                        MultiChunkSqlDao multiChunkDao) {
074
075                super(connection);
076
077                this.chunkDao = chunkDao;
078                this.fileContentDao = fileContentDao;
079                this.fileVersionDao = fileVersionDao;
080                this.fileHistoryDao = fileHistoryDao;
081                this.multiChunkDao = multiChunkDao;
082        }
083
084        /**
085         * Marks the database version with the given vector clock as DIRTY, i.e.
086         * sets the {@link DatabaseVersionStatus} to {@link DatabaseVersionStatus#DIRTY DIRTY}.
087         * Marking a database version dirty will lead to a deletion in the next sync up
088         * cycle.
089         * 
090         * @param vectorClock Identifies the database version to mark dirty
091         */
092        public void markDatabaseVersionDirty(VectorClock vectorClock) {
093                try (PreparedStatement preparedStatement = getStatement("databaseversion.update.master.markDatabaseVersionDirty.sql")) {
094                        preparedStatement.setString(1, DatabaseVersionStatus.DIRTY.toString());
095                        preparedStatement.setString(2, vectorClock.toString());
096
097                        preparedStatement.executeUpdate();
098                        connection.commit();
099                }
100                catch (SQLException e) {
101                        throw new RuntimeException(e);
102                }
103        }
104
105        public long writeDatabaseVersion(DatabaseVersion databaseVersion) {
106                try {
107                        // Insert & commit database version
108                        long databaseVersionId = writeDatabaseVersion(connection, databaseVersion);
109                        
110                        // Commit & clear local caches
111                        clearCaches();
112
113                        return databaseVersionId;
114                }
115                catch (Exception e) {
116                        logger.log(Level.SEVERE, "SQL Error: ", e);
117
118                        throw new RuntimeException("Cannot persist database.", e);
119                }
120        }
121
122        private long writeDatabaseVersion(Connection connection, DatabaseVersion databaseVersion) throws SQLException {
123                long databaseVersionId = writeDatabaseVersionHeaderInternal(connection, databaseVersion.getHeader()); // TODO [low] Use writeDatabaseVersion()?
124                writeVectorClock(connection, databaseVersionId, databaseVersion.getHeader().getVectorClock());
125
126                chunkDao.writeChunks(connection, databaseVersionId, databaseVersion.getChunks());
127                multiChunkDao.writeMultiChunks(connection, databaseVersionId, databaseVersion.getMultiChunks());
128                fileContentDao.writeFileContents(connection, databaseVersionId, databaseVersion.getFileContents());
129                fileHistoryDao.writeFileHistories(connection, databaseVersionId, databaseVersion.getFileHistories());
130
131                return databaseVersionId;
132        }
133
134        private long writeDatabaseVersionHeaderInternal(Connection connection, DatabaseVersionHeader databaseVersionHeader) throws SQLException {
135                try (PreparedStatement preparedStatement = connection.prepareStatement(
136                                DatabaseConnectionFactory.getStatement("databaseversion.insert.all.writeDatabaseVersion.sql"), Statement.RETURN_GENERATED_KEYS)) {
137
138                        preparedStatement.setString(1, DatabaseVersionStatus.MASTER.toString());
139                        preparedStatement.setTimestamp(2, new Timestamp(databaseVersionHeader.getDate().getTime()));
140                        preparedStatement.setString(3, databaseVersionHeader.getClient());
141                        preparedStatement.setString(4, databaseVersionHeader.getVectorClock().toString());
142
143                        int affectedRows = preparedStatement.executeUpdate();
144
145                        if (affectedRows == 0) {
146                                throw new SQLException("Cannot add database version header. Affected rows is zero.");
147                        }
148
149                        try (ResultSet resultSet = preparedStatement.getGeneratedKeys()) {
150                                if (resultSet.next()) {
151                                        return resultSet.getLong(1);
152                                }
153                                else {
154                                        throw new SQLException("Cannot get new database version ID");
155                                }
156                        }
157                }
158        }
159
160        private void writeVectorClock(Connection connection, long databaseVersionId, VectorClock vectorClock) throws SQLException {
161                try (PreparedStatement preparedStatement = getStatement(connection, "databaseversion.insert.all.writeVectorClock.sql")) {
162                        for (Map.Entry<String, Long> vectorClockEntry : vectorClock.entrySet()) {
163                                preparedStatement.setLong(1, databaseVersionId);
164                                preparedStatement.setString(2, vectorClockEntry.getKey());
165                                preparedStatement.setLong(3, vectorClockEntry.getValue());
166
167                                preparedStatement.addBatch();
168                        }
169
170                        preparedStatement.executeBatch();
171                }
172        }
173
174        /**
175         * Removes dirty {@link DatabaseVersion}s, {@link FileVersion}s, {@link PartialFileHistory}s and {@link FileContent}s
176         * from the database, but leaves stale/unreferenced chunks/multichunks untouched (must be cleaned up at a later stage).
177         * @param newDatabaseVersionId 
178         */
179        public void removeDirtyDatabaseVersions(long newDatabaseVersionId) {
180                try {
181                        // IMPORTANT: The order is important, because of 
182                        //            the database foreign key consistencies!
183
184                        // First, remove dirty file histories, then file versions
185                        fileVersionDao.removeDirtyFileVersions();
186                        fileHistoryDao.removeDirtyFileHistories();
187
188                        // Now, remove all unreferenced file contents
189                        fileContentDao.removeUnreferencedFileContents();
190
191                        // Change foreign key of multichunks
192                        multiChunkDao.updateDirtyMultiChunksNewDatabaseId(newDatabaseVersionId);
193                        fileContentDao.updateDirtyFileContentsNewDatabaseId(newDatabaseVersionId);
194                        chunkDao.updateDirtyChunksNewDatabaseId(newDatabaseVersionId);
195
196                        // And the database versions
197                        removeDirtyVectorClocks();
198                        removeDirtyDatabaseVersionsInt();
199
200                        // Commit & clear local caches
201                        connection.commit();
202                        clearCaches();
203                }
204                catch (SQLException e) {
205                        throw new RuntimeException("Unable to remove dirty database versions.", e);
206                }
207        }
208
209        public void removeEmptyDatabaseVersionHeaders() {               
210                // Delete vector clocks
211                try (PreparedStatement preparedStatement = getStatement("databaseversion.delete.all.removeEmptyDatabaseVersionHeadersVectorClocks.sql")) {
212                        preparedStatement.executeUpdate();              
213                        preparedStatement.close();
214                }
215                catch (SQLException e) {
216                        throw new RuntimeException(e);
217                }
218                
219                // Delete database version headers
220                try (PreparedStatement preparedStatement = getStatement("databaseversion.delete.all.removeEmptyDatabaseVersionHeaders.sql")) {
221                        preparedStatement.executeUpdate();              
222                        preparedStatement.close();
223                }
224                catch (SQLException e) {
225                        throw new RuntimeException(e);
226                }
227        }
228
229        public void clearCaches() {
230                chunkDao.clearCache();
231        }
232
233        public Long getMaxDirtyVectorClock(String machineName) {
234                try (PreparedStatement preparedStatement = getStatement("databaseversion.select.dirty.getMaxDirtyVectorClock.sql")) {
235                        preparedStatement.setMaxRows(1);
236                        preparedStatement.setString(1, machineName);
237
238                        try (ResultSet resultSet = preparedStatement.executeQuery()) {
239                                if (resultSet.next()) {
240                                        return resultSet.getLong("logicaltime");
241                                }
242                        }
243
244                        return null;
245                }
246                catch (SQLException e) {
247                        throw new RuntimeException(e);
248                }
249        }
250
251        public List<DatabaseVersionHeader> getNonEmptyDatabaseVersionHeaders() {
252                List<DatabaseVersionHeader> databaseVersionHeaders = new ArrayList<>();
253
254                try (PreparedStatement preparedStatement = getStatement("databaseversion.select.all.getNonEmptyDatabaseVersionHeaders.sql")) {
255                        try (ResultSet resultSet = preparedStatement.executeQuery()) {
256                                while (resultSet.next()) {
257                                        DatabaseVersionHeader databaseVersionHeader = createDatabaseVersionHeaderFromRow(resultSet);
258                                        databaseVersionHeaders.add(databaseVersionHeader);
259                                }
260                        }
261
262                        return databaseVersionHeaders;
263                }
264                catch (SQLException e) {
265                        throw new RuntimeException(e);
266                }
267        }
268
269        public Iterator<DatabaseVersion> getDirtyDatabaseVersions() {
270                try (PreparedStatement preparedStatement = getStatement("databaseversion.select.dirty.getDirtyDatabaseVersions.sql")) {
271                        preparedStatement.setString(1, DatabaseVersionStatus.DIRTY.toString());
272
273                        return new DatabaseVersionIterator(preparedStatement.executeQuery());
274                }
275                catch (SQLException e) {
276                        throw new RuntimeException(e);
277                }
278        }
279
280        public Iterator<DatabaseVersion> getDatabaseVersionsTo(String machineName, long maxLocalClientVersion) {
281                try (PreparedStatement preparedStatement = getStatement("databaseversion.select.master.getDatabaseVersionsTo.sql")) {
282                        preparedStatement.setString(1, machineName);
283                        preparedStatement.setLong(2, maxLocalClientVersion);
284
285                        return new DatabaseVersionIterator(preparedStatement.executeQuery());
286                }
287                catch (SQLException e) {
288                        throw new RuntimeException(e);
289                }
290        }
291        
292        public Iterator<DatabaseVersion> getLastDatabaseVersions(int maxDatabaseVersionCount, int startDatabaseVersionIndex, int maxFileHistoryCount) {
293                try (PreparedStatement preparedStatement = getStatement("databaseversion.select.master.getLastDatabaseVersions.sql")) {
294                        maxDatabaseVersionCount = (maxDatabaseVersionCount > 0) ? maxDatabaseVersionCount : Integer.MAX_VALUE;
295                        startDatabaseVersionIndex = (startDatabaseVersionIndex > 0) ? startDatabaseVersionIndex : 0;
296                        
297                        preparedStatement.setInt(1, maxDatabaseVersionCount);
298                        preparedStatement.setInt(2, startDatabaseVersionIndex);
299                        
300                        try (ResultSet resultSet = preparedStatement.executeQuery()) {
301                                return new DatabaseVersionIterator(preparedStatement.executeQuery(), true, maxFileHistoryCount);
302                        }
303                }
304                catch (SQLException e) {
305                        throw new RuntimeException(e);
306                }
307        }
308
309
310        private class DatabaseVersionIterator implements Iterator<DatabaseVersion> {
311                private ResultSet resultSet;
312                private boolean excludeChunkData;
313                private int fileHistoryMaxCount;
314
315                private boolean hasNext;
316
317                public DatabaseVersionIterator(ResultSet resultSet) throws SQLException {
318                        this(resultSet, false, -1);
319                }
320
321                public DatabaseVersionIterator(ResultSet resultSet, boolean excludeChunkData, int fileHistoryMaxCount) throws SQLException {
322                        this.resultSet = resultSet;
323                        this.excludeChunkData = excludeChunkData;
324                        this.fileHistoryMaxCount = fileHistoryMaxCount;
325                                        
326                        this.hasNext = resultSet.next();                                        
327                }
328
329                @Override
330                public boolean hasNext() {
331                        return hasNext;
332                }
333
334                @Override
335                public DatabaseVersion next() {
336                        if (hasNext) {
337                                try {
338                                        DatabaseVersion databaseVersion = createDatabaseVersionFromRow(resultSet, excludeChunkData, fileHistoryMaxCount);
339                                        hasNext = resultSet.next();
340
341                                        return databaseVersion;
342                                }
343                                catch (Exception e) {
344                                        throw new RuntimeException("Cannot load next SQL row.", e);
345                                }
346                        }
347                        else {
348                                return null;
349                        }
350                }
351
352                @Override
353                public void remove() {
354                        throw new RuntimeException("Not implemented.");
355                }
356
357        }
358
359        protected DatabaseVersion createDatabaseVersionFromRow(ResultSet resultSet, boolean excludeChunkData, int fileHistoryMaxCount) throws SQLException {
360                DatabaseVersionHeader databaseVersionHeader = createDatabaseVersionHeaderFromRow(resultSet);
361
362                DatabaseVersion databaseVersion = new DatabaseVersion();
363                databaseVersion.setHeader(databaseVersionHeader);
364
365                // Add chunk/multichunk/filecontent data
366                if (!excludeChunkData) {
367                        Map<ChunkChecksum, ChunkEntry> chunks = chunkDao.getChunks(databaseVersionHeader.getVectorClock());
368                        Map<MultiChunkId, MultiChunkEntry> multiChunks = multiChunkDao.getMultiChunks(databaseVersionHeader.getVectorClock());
369                        Map<FileChecksum, FileContent> fileContents = fileContentDao.getFileContents(databaseVersionHeader.getVectorClock());
370                        
371                        for (ChunkEntry chunk : chunks.values()) {
372                                databaseVersion.addChunk(chunk);
373                        }
374
375                        for (MultiChunkEntry multiChunk : multiChunks.values()) {
376                                databaseVersion.addMultiChunk(multiChunk);
377                        }
378
379                        for (FileContent fileContent : fileContents.values()) {
380                                databaseVersion.addFileContent(fileContent);
381                        }
382                }
383                
384                // Add file histories
385                Map<FileHistoryId, PartialFileHistory> fileHistories = fileHistoryDao
386                                .getFileHistoriesWithFileVersions(databaseVersionHeader.getVectorClock(), fileHistoryMaxCount);
387
388                for (PartialFileHistory fileHistory : fileHistories.values()) {
389                        databaseVersion.addFileHistory(fileHistory);
390                }
391
392                return databaseVersion;
393        }
394
395        public DatabaseVersionHeader getLastDatabaseVersionHeader() {
396                try (PreparedStatement preparedStatement = getStatement("databaseversion.select.master.getLastDatabaseVersionHeader.sql")) {
397                        preparedStatement.setMaxRows(1);
398
399                        try (ResultSet resultSet = preparedStatement.executeQuery()) {
400                                if (resultSet.next()) {
401                                        DatabaseVersionHeader databaseVersionHeader = createDatabaseVersionHeaderFromRow(resultSet);
402                                        return databaseVersionHeader;
403                                }
404                        }
405
406                        return null;
407                }
408                catch (SQLException e) {
409                        throw new RuntimeException(e);
410                }
411        }
412        
413        private DatabaseVersionHeader createDatabaseVersionHeaderFromRow(ResultSet resultSet) throws SQLException {
414                DatabaseVersionHeader databaseVersionHeader = new DatabaseVersionHeader();
415
416                databaseVersionHeader.setClient(resultSet.getString("client"));
417                databaseVersionHeader.setDate(new Date(resultSet.getTimestamp("localtime").getTime()));
418                databaseVersionHeader.setVectorClock(getVectorClockByDatabaseVersionId(resultSet.getInt("id")));
419
420                return databaseVersionHeader;
421        }
422
423        public DatabaseBranch getLocalDatabaseBranch() {
424                DatabaseBranch databaseBranch = new DatabaseBranch();
425
426                try (PreparedStatement preparedStatement = getStatement("databaseversion.select.master.getLocalDatabaseBranch.sql")) {
427                        try (ResultSet resultSet = preparedStatement.executeQuery()) {
428                                DatabaseVersionHeader currentDatabaseVersionHeader = null;
429                                int currentDatabaseVersionHeaderId = -1;
430
431                                while (resultSet.next()) {
432                                        int databaseVersionHeaderId = resultSet.getInt("id");
433
434                                        // Row does NOT belong to the current database version
435                                        if (currentDatabaseVersionHeader == null || currentDatabaseVersionHeaderId != databaseVersionHeaderId) {
436                                                // Add to database branch
437                                                if (currentDatabaseVersionHeader != null) {
438                                                        databaseBranch.add(currentDatabaseVersionHeader);
439                                                }
440
441                                                // Make a new database version header
442                                                currentDatabaseVersionHeader = new DatabaseVersionHeader();
443                                                currentDatabaseVersionHeader.setClient(resultSet.getString("client"));
444                                                currentDatabaseVersionHeader.setDate(new Date(resultSet.getTimestamp("localtime").getTime()));
445
446                                                currentDatabaseVersionHeaderId = databaseVersionHeaderId;
447                                        }
448
449                                        currentDatabaseVersionHeader.getVectorClock().setClock(resultSet.getString("vc_client"), resultSet.getLong("vc_logicaltime"));
450                                }
451
452                                // Add to database branch
453                                if (currentDatabaseVersionHeader != null) {
454                                        databaseBranch.add(currentDatabaseVersionHeader);
455                                }
456
457                                return databaseBranch;
458                        }
459                }
460                catch (SQLException e) {
461                        throw new RuntimeException(e);
462                }
463        }
464
465        protected VectorClock getVectorClockByDatabaseVersionId(int databaseVersionId) throws SQLException {
466                PreparedStatement preparedStatement = getStatement("databaseversion.select.all.getVectorClockByDatabaseVersionId.sql");
467                preparedStatement.setInt(1, databaseVersionId);
468
469                ResultSet resultSet = preparedStatement.executeQuery();
470
471                VectorClock vectorClock = new VectorClock();
472
473                while (resultSet.next()) {
474                        vectorClock.setClock(resultSet.getString("client"), resultSet.getLong("logicaltime"));
475                }
476
477                resultSet.close();
478                preparedStatement.close();
479
480                return vectorClock;
481        }
482
483        private void removeDirtyVectorClocks() throws SQLException {
484                PreparedStatement preparedStatement = getStatement("databaseversion.delete.dirty.removeDirtyVectorClocks.sql");
485                preparedStatement.executeUpdate();
486                preparedStatement.close();
487        }
488
489        private void removeDirtyDatabaseVersionsInt() throws SQLException {
490                PreparedStatement preparedStatement = getStatement("databaseversion.delete.dirty.removeDirtyDatabaseVersionsInt.sql");
491                preparedStatement.executeUpdate();
492                preparedStatement.close();
493        }
494}