001/* 002 * Syncany, www.syncany.org 003 * Copyright (C) 2011-2016 Philipp C. Heckel <philipp.heckel@gmail.com> 004 * 005 * This program is free software: you can redistribute it and/or modify 006 * it under the terms of the GNU General Public License as published by 007 * the Free Software Foundation, either version 3 of the License, or 008 * (at your option) any later version. 009 * 010 * This program is distributed in the hope that it will be useful, 011 * but WITHOUT ANY WARRANTY; without even the implied warranty of 012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 013 * GNU General Public License for more details. 014 * 015 * You should have received a copy of the GNU General Public License 016 * along with this program. If not, see <http://www.gnu.org/licenses/>. 017 */ 018package org.syncany.database.dao; 019 020import java.sql.Connection; 021import java.sql.PreparedStatement; 022import java.sql.ResultSet; 023import java.sql.SQLException; 024import java.sql.Statement; 025import java.sql.Timestamp; 026import java.util.ArrayList; 027import java.util.Date; 028import java.util.Iterator; 029import java.util.List; 030import java.util.Map; 031import java.util.logging.Level; 032import java.util.logging.Logger; 033 034import org.syncany.database.ChunkEntry; 035import org.syncany.database.ChunkEntry.ChunkChecksum; 036import org.syncany.database.DatabaseConnectionFactory; 037import org.syncany.database.DatabaseVersion; 038import org.syncany.database.DatabaseVersion.DatabaseVersionStatus; 039import org.syncany.database.DatabaseVersionHeader; 040import org.syncany.database.FileContent; 041import org.syncany.database.FileContent.FileChecksum; 042import org.syncany.database.FileVersion; 043import org.syncany.database.MultiChunkEntry; 044import org.syncany.database.MultiChunkEntry.MultiChunkId; 045import org.syncany.database.PartialFileHistory; 046import org.syncany.database.PartialFileHistory.FileHistoryId; 047import org.syncany.database.VectorClock; 048import org.syncany.operations.down.DatabaseBranch; 049 050/** 051 * The database version data access object (DAO) writes and queries the SQL database for information 052 * on {@link DatabaseVersion}s. It translates the relational data in the "databaseversion" table to 053 * Java objects; but also uses the other DAOs to persist entire {@link DatabaseVersion} objects. 054 * 055 * @see ChunkSqlDao 056 * @see FileContentSqlDao 057 * @see FileVersionSqlDao 058 * @see FileHistorySqlDao 059 * @see MultiChunkSqlDao 060 * @author Philipp C. Heckel (philipp.heckel@gmail.com) 061 */ 062public class DatabaseVersionSqlDao extends AbstractSqlDao { 063 protected static final Logger logger = Logger.getLogger(DatabaseVersionSqlDao.class.getSimpleName()); 064 065 private ChunkSqlDao chunkDao; 066 private FileContentSqlDao fileContentDao; 067 private FileVersionSqlDao fileVersionDao; 068 private FileHistorySqlDao fileHistoryDao; 069 private MultiChunkSqlDao multiChunkDao; 070 071 public DatabaseVersionSqlDao(Connection connection, ChunkSqlDao chunkDao, FileContentSqlDao fileContentDao, FileVersionSqlDao fileVersionDao, 072 FileHistorySqlDao fileHistoryDao, 073 MultiChunkSqlDao multiChunkDao) { 074 075 super(connection); 076 077 this.chunkDao = chunkDao; 078 this.fileContentDao = fileContentDao; 079 this.fileVersionDao = fileVersionDao; 080 this.fileHistoryDao = fileHistoryDao; 081 this.multiChunkDao = multiChunkDao; 082 } 083 084 /** 085 * Marks the database version with the given vector clock as DIRTY, i.e. 086 * sets the {@link DatabaseVersionStatus} to {@link DatabaseVersionStatus#DIRTY DIRTY}. 087 * Marking a database version dirty will lead to a deletion in the next sync up 088 * cycle. 089 * 090 * @param vectorClock Identifies the database version to mark dirty 091 */ 092 public void markDatabaseVersionDirty(VectorClock vectorClock) { 093 try (PreparedStatement preparedStatement = getStatement("databaseversion.update.master.markDatabaseVersionDirty.sql")) { 094 preparedStatement.setString(1, DatabaseVersionStatus.DIRTY.toString()); 095 preparedStatement.setString(2, vectorClock.toString()); 096 097 preparedStatement.executeUpdate(); 098 connection.commit(); 099 } 100 catch (SQLException e) { 101 throw new RuntimeException(e); 102 } 103 } 104 105 public long writeDatabaseVersion(DatabaseVersion databaseVersion) { 106 try { 107 // Insert & commit database version 108 long databaseVersionId = writeDatabaseVersion(connection, databaseVersion); 109 110 // Commit & clear local caches 111 clearCaches(); 112 113 return databaseVersionId; 114 } 115 catch (Exception e) { 116 logger.log(Level.SEVERE, "SQL Error: ", e); 117 118 throw new RuntimeException("Cannot persist database.", e); 119 } 120 } 121 122 private long writeDatabaseVersion(Connection connection, DatabaseVersion databaseVersion) throws SQLException { 123 long databaseVersionId = writeDatabaseVersionHeaderInternal(connection, databaseVersion.getHeader()); // TODO [low] Use writeDatabaseVersion()? 124 writeVectorClock(connection, databaseVersionId, databaseVersion.getHeader().getVectorClock()); 125 126 chunkDao.writeChunks(connection, databaseVersionId, databaseVersion.getChunks()); 127 multiChunkDao.writeMultiChunks(connection, databaseVersionId, databaseVersion.getMultiChunks()); 128 fileContentDao.writeFileContents(connection, databaseVersionId, databaseVersion.getFileContents()); 129 fileHistoryDao.writeFileHistories(connection, databaseVersionId, databaseVersion.getFileHistories()); 130 131 return databaseVersionId; 132 } 133 134 private long writeDatabaseVersionHeaderInternal(Connection connection, DatabaseVersionHeader databaseVersionHeader) throws SQLException { 135 try (PreparedStatement preparedStatement = connection.prepareStatement( 136 DatabaseConnectionFactory.getStatement("databaseversion.insert.all.writeDatabaseVersion.sql"), Statement.RETURN_GENERATED_KEYS)) { 137 138 preparedStatement.setString(1, DatabaseVersionStatus.MASTER.toString()); 139 preparedStatement.setTimestamp(2, new Timestamp(databaseVersionHeader.getDate().getTime())); 140 preparedStatement.setString(3, databaseVersionHeader.getClient()); 141 preparedStatement.setString(4, databaseVersionHeader.getVectorClock().toString()); 142 143 int affectedRows = preparedStatement.executeUpdate(); 144 145 if (affectedRows == 0) { 146 throw new SQLException("Cannot add database version header. Affected rows is zero."); 147 } 148 149 try (ResultSet resultSet = preparedStatement.getGeneratedKeys()) { 150 if (resultSet.next()) { 151 return resultSet.getLong(1); 152 } 153 else { 154 throw new SQLException("Cannot get new database version ID"); 155 } 156 } 157 } 158 } 159 160 private void writeVectorClock(Connection connection, long databaseVersionId, VectorClock vectorClock) throws SQLException { 161 try (PreparedStatement preparedStatement = getStatement(connection, "databaseversion.insert.all.writeVectorClock.sql")) { 162 for (Map.Entry<String, Long> vectorClockEntry : vectorClock.entrySet()) { 163 preparedStatement.setLong(1, databaseVersionId); 164 preparedStatement.setString(2, vectorClockEntry.getKey()); 165 preparedStatement.setLong(3, vectorClockEntry.getValue()); 166 167 preparedStatement.addBatch(); 168 } 169 170 preparedStatement.executeBatch(); 171 } 172 } 173 174 /** 175 * Removes dirty {@link DatabaseVersion}s, {@link FileVersion}s, {@link PartialFileHistory}s and {@link FileContent}s 176 * from the database, but leaves stale/unreferenced chunks/multichunks untouched (must be cleaned up at a later stage). 177 * @param newDatabaseVersionId 178 */ 179 public void removeDirtyDatabaseVersions(long newDatabaseVersionId) { 180 try { 181 // IMPORTANT: The order is important, because of 182 // the database foreign key consistencies! 183 184 // First, remove dirty file histories, then file versions 185 fileVersionDao.removeDirtyFileVersions(); 186 fileHistoryDao.removeDirtyFileHistories(); 187 188 // Now, remove all unreferenced file contents 189 fileContentDao.removeUnreferencedFileContents(); 190 191 // Change foreign key of multichunks 192 multiChunkDao.updateDirtyMultiChunksNewDatabaseId(newDatabaseVersionId); 193 fileContentDao.updateDirtyFileContentsNewDatabaseId(newDatabaseVersionId); 194 chunkDao.updateDirtyChunksNewDatabaseId(newDatabaseVersionId); 195 196 // And the database versions 197 removeDirtyVectorClocks(); 198 removeDirtyDatabaseVersionsInt(); 199 200 // Commit & clear local caches 201 connection.commit(); 202 clearCaches(); 203 } 204 catch (SQLException e) { 205 throw new RuntimeException("Unable to remove dirty database versions.", e); 206 } 207 } 208 209 public void removeEmptyDatabaseVersionHeaders() { 210 // Delete vector clocks 211 try (PreparedStatement preparedStatement = getStatement("databaseversion.delete.all.removeEmptyDatabaseVersionHeadersVectorClocks.sql")) { 212 preparedStatement.executeUpdate(); 213 preparedStatement.close(); 214 } 215 catch (SQLException e) { 216 throw new RuntimeException(e); 217 } 218 219 // Delete database version headers 220 try (PreparedStatement preparedStatement = getStatement("databaseversion.delete.all.removeEmptyDatabaseVersionHeaders.sql")) { 221 preparedStatement.executeUpdate(); 222 preparedStatement.close(); 223 } 224 catch (SQLException e) { 225 throw new RuntimeException(e); 226 } 227 } 228 229 public void clearCaches() { 230 chunkDao.clearCache(); 231 } 232 233 public Long getMaxDirtyVectorClock(String machineName) { 234 try (PreparedStatement preparedStatement = getStatement("databaseversion.select.dirty.getMaxDirtyVectorClock.sql")) { 235 preparedStatement.setMaxRows(1); 236 preparedStatement.setString(1, machineName); 237 238 try (ResultSet resultSet = preparedStatement.executeQuery()) { 239 if (resultSet.next()) { 240 return resultSet.getLong("logicaltime"); 241 } 242 } 243 244 return null; 245 } 246 catch (SQLException e) { 247 throw new RuntimeException(e); 248 } 249 } 250 251 public List<DatabaseVersionHeader> getNonEmptyDatabaseVersionHeaders() { 252 List<DatabaseVersionHeader> databaseVersionHeaders = new ArrayList<>(); 253 254 try (PreparedStatement preparedStatement = getStatement("databaseversion.select.all.getNonEmptyDatabaseVersionHeaders.sql")) { 255 try (ResultSet resultSet = preparedStatement.executeQuery()) { 256 while (resultSet.next()) { 257 DatabaseVersionHeader databaseVersionHeader = createDatabaseVersionHeaderFromRow(resultSet); 258 databaseVersionHeaders.add(databaseVersionHeader); 259 } 260 } 261 262 return databaseVersionHeaders; 263 } 264 catch (SQLException e) { 265 throw new RuntimeException(e); 266 } 267 } 268 269 public Iterator<DatabaseVersion> getDirtyDatabaseVersions() { 270 try (PreparedStatement preparedStatement = getStatement("databaseversion.select.dirty.getDirtyDatabaseVersions.sql")) { 271 preparedStatement.setString(1, DatabaseVersionStatus.DIRTY.toString()); 272 273 return new DatabaseVersionIterator(preparedStatement.executeQuery()); 274 } 275 catch (SQLException e) { 276 throw new RuntimeException(e); 277 } 278 } 279 280 public Iterator<DatabaseVersion> getDatabaseVersionsTo(String machineName, long maxLocalClientVersion) { 281 try (PreparedStatement preparedStatement = getStatement("databaseversion.select.master.getDatabaseVersionsTo.sql")) { 282 preparedStatement.setString(1, machineName); 283 preparedStatement.setLong(2, maxLocalClientVersion); 284 285 return new DatabaseVersionIterator(preparedStatement.executeQuery()); 286 } 287 catch (SQLException e) { 288 throw new RuntimeException(e); 289 } 290 } 291 292 public Iterator<DatabaseVersion> getLastDatabaseVersions(int maxDatabaseVersionCount, int startDatabaseVersionIndex, int maxFileHistoryCount) { 293 try (PreparedStatement preparedStatement = getStatement("databaseversion.select.master.getLastDatabaseVersions.sql")) { 294 maxDatabaseVersionCount = (maxDatabaseVersionCount > 0) ? maxDatabaseVersionCount : Integer.MAX_VALUE; 295 startDatabaseVersionIndex = (startDatabaseVersionIndex > 0) ? startDatabaseVersionIndex : 0; 296 297 preparedStatement.setInt(1, maxDatabaseVersionCount); 298 preparedStatement.setInt(2, startDatabaseVersionIndex); 299 300 try (ResultSet resultSet = preparedStatement.executeQuery()) { 301 return new DatabaseVersionIterator(preparedStatement.executeQuery(), true, maxFileHistoryCount); 302 } 303 } 304 catch (SQLException e) { 305 throw new RuntimeException(e); 306 } 307 } 308 309 310 private class DatabaseVersionIterator implements Iterator<DatabaseVersion> { 311 private ResultSet resultSet; 312 private boolean excludeChunkData; 313 private int fileHistoryMaxCount; 314 315 private boolean hasNext; 316 317 public DatabaseVersionIterator(ResultSet resultSet) throws SQLException { 318 this(resultSet, false, -1); 319 } 320 321 public DatabaseVersionIterator(ResultSet resultSet, boolean excludeChunkData, int fileHistoryMaxCount) throws SQLException { 322 this.resultSet = resultSet; 323 this.excludeChunkData = excludeChunkData; 324 this.fileHistoryMaxCount = fileHistoryMaxCount; 325 326 this.hasNext = resultSet.next(); 327 } 328 329 @Override 330 public boolean hasNext() { 331 return hasNext; 332 } 333 334 @Override 335 public DatabaseVersion next() { 336 if (hasNext) { 337 try { 338 DatabaseVersion databaseVersion = createDatabaseVersionFromRow(resultSet, excludeChunkData, fileHistoryMaxCount); 339 hasNext = resultSet.next(); 340 341 return databaseVersion; 342 } 343 catch (Exception e) { 344 throw new RuntimeException("Cannot load next SQL row.", e); 345 } 346 } 347 else { 348 return null; 349 } 350 } 351 352 @Override 353 public void remove() { 354 throw new RuntimeException("Not implemented."); 355 } 356 357 } 358 359 protected DatabaseVersion createDatabaseVersionFromRow(ResultSet resultSet, boolean excludeChunkData, int fileHistoryMaxCount) throws SQLException { 360 DatabaseVersionHeader databaseVersionHeader = createDatabaseVersionHeaderFromRow(resultSet); 361 362 DatabaseVersion databaseVersion = new DatabaseVersion(); 363 databaseVersion.setHeader(databaseVersionHeader); 364 365 // Add chunk/multichunk/filecontent data 366 if (!excludeChunkData) { 367 Map<ChunkChecksum, ChunkEntry> chunks = chunkDao.getChunks(databaseVersionHeader.getVectorClock()); 368 Map<MultiChunkId, MultiChunkEntry> multiChunks = multiChunkDao.getMultiChunks(databaseVersionHeader.getVectorClock()); 369 Map<FileChecksum, FileContent> fileContents = fileContentDao.getFileContents(databaseVersionHeader.getVectorClock()); 370 371 for (ChunkEntry chunk : chunks.values()) { 372 databaseVersion.addChunk(chunk); 373 } 374 375 for (MultiChunkEntry multiChunk : multiChunks.values()) { 376 databaseVersion.addMultiChunk(multiChunk); 377 } 378 379 for (FileContent fileContent : fileContents.values()) { 380 databaseVersion.addFileContent(fileContent); 381 } 382 } 383 384 // Add file histories 385 Map<FileHistoryId, PartialFileHistory> fileHistories = fileHistoryDao 386 .getFileHistoriesWithFileVersions(databaseVersionHeader.getVectorClock(), fileHistoryMaxCount); 387 388 for (PartialFileHistory fileHistory : fileHistories.values()) { 389 databaseVersion.addFileHistory(fileHistory); 390 } 391 392 return databaseVersion; 393 } 394 395 public DatabaseVersionHeader getLastDatabaseVersionHeader() { 396 try (PreparedStatement preparedStatement = getStatement("databaseversion.select.master.getLastDatabaseVersionHeader.sql")) { 397 preparedStatement.setMaxRows(1); 398 399 try (ResultSet resultSet = preparedStatement.executeQuery()) { 400 if (resultSet.next()) { 401 DatabaseVersionHeader databaseVersionHeader = createDatabaseVersionHeaderFromRow(resultSet); 402 return databaseVersionHeader; 403 } 404 } 405 406 return null; 407 } 408 catch (SQLException e) { 409 throw new RuntimeException(e); 410 } 411 } 412 413 private DatabaseVersionHeader createDatabaseVersionHeaderFromRow(ResultSet resultSet) throws SQLException { 414 DatabaseVersionHeader databaseVersionHeader = new DatabaseVersionHeader(); 415 416 databaseVersionHeader.setClient(resultSet.getString("client")); 417 databaseVersionHeader.setDate(new Date(resultSet.getTimestamp("localtime").getTime())); 418 databaseVersionHeader.setVectorClock(getVectorClockByDatabaseVersionId(resultSet.getInt("id"))); 419 420 return databaseVersionHeader; 421 } 422 423 public DatabaseBranch getLocalDatabaseBranch() { 424 DatabaseBranch databaseBranch = new DatabaseBranch(); 425 426 try (PreparedStatement preparedStatement = getStatement("databaseversion.select.master.getLocalDatabaseBranch.sql")) { 427 try (ResultSet resultSet = preparedStatement.executeQuery()) { 428 DatabaseVersionHeader currentDatabaseVersionHeader = null; 429 int currentDatabaseVersionHeaderId = -1; 430 431 while (resultSet.next()) { 432 int databaseVersionHeaderId = resultSet.getInt("id"); 433 434 // Row does NOT belong to the current database version 435 if (currentDatabaseVersionHeader == null || currentDatabaseVersionHeaderId != databaseVersionHeaderId) { 436 // Add to database branch 437 if (currentDatabaseVersionHeader != null) { 438 databaseBranch.add(currentDatabaseVersionHeader); 439 } 440 441 // Make a new database version header 442 currentDatabaseVersionHeader = new DatabaseVersionHeader(); 443 currentDatabaseVersionHeader.setClient(resultSet.getString("client")); 444 currentDatabaseVersionHeader.setDate(new Date(resultSet.getTimestamp("localtime").getTime())); 445 446 currentDatabaseVersionHeaderId = databaseVersionHeaderId; 447 } 448 449 currentDatabaseVersionHeader.getVectorClock().setClock(resultSet.getString("vc_client"), resultSet.getLong("vc_logicaltime")); 450 } 451 452 // Add to database branch 453 if (currentDatabaseVersionHeader != null) { 454 databaseBranch.add(currentDatabaseVersionHeader); 455 } 456 457 return databaseBranch; 458 } 459 } 460 catch (SQLException e) { 461 throw new RuntimeException(e); 462 } 463 } 464 465 protected VectorClock getVectorClockByDatabaseVersionId(int databaseVersionId) throws SQLException { 466 PreparedStatement preparedStatement = getStatement("databaseversion.select.all.getVectorClockByDatabaseVersionId.sql"); 467 preparedStatement.setInt(1, databaseVersionId); 468 469 ResultSet resultSet = preparedStatement.executeQuery(); 470 471 VectorClock vectorClock = new VectorClock(); 472 473 while (resultSet.next()) { 474 vectorClock.setClock(resultSet.getString("client"), resultSet.getLong("logicaltime")); 475 } 476 477 resultSet.close(); 478 preparedStatement.close(); 479 480 return vectorClock; 481 } 482 483 private void removeDirtyVectorClocks() throws SQLException { 484 PreparedStatement preparedStatement = getStatement("databaseversion.delete.dirty.removeDirtyVectorClocks.sql"); 485 preparedStatement.executeUpdate(); 486 preparedStatement.close(); 487 } 488 489 private void removeDirtyDatabaseVersionsInt() throws SQLException { 490 PreparedStatement preparedStatement = getStatement("databaseversion.delete.dirty.removeDirtyDatabaseVersionsInt.sql"); 491 preparedStatement.executeUpdate(); 492 preparedStatement.close(); 493 } 494}