001/* 002 * Syncany, www.syncany.org 003 * Copyright (C) 2011-2016 Philipp C. Heckel <philipp.heckel@gmail.com> 004 * 005 * This program is free software: you can redistribute it and/or modify 006 * it under the terms of the GNU General Public License as published by 007 * the Free Software Foundation, either version 3 of the License, or 008 * (at your option) any later version. 009 * 010 * This program is distributed in the hope that it will be useful, 011 * but WITHOUT ANY WARRANTY; without even the implied warranty of 012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 013 * GNU General Public License for more details. 014 * 015 * You should have received a copy of the GNU General Public License 016 * along with this program. If not, see <http://www.gnu.org/licenses/>. 017 */ 018package org.syncany.plugins.transfer.features; 019 020import java.io.File; 021import java.lang.reflect.Constructor; 022import java.lang.reflect.InvocationTargetException; 023import java.nio.file.Path; 024import java.nio.file.Paths; 025import java.util.List; 026import java.util.Map; 027import java.util.logging.Level; 028import java.util.logging.Logger; 029 030import org.syncany.config.Config; 031import org.syncany.plugins.transfer.FileType; 032import org.syncany.plugins.transfer.StorageException; 033import org.syncany.plugins.transfer.StorageTestResult; 034import org.syncany.plugins.transfer.TransferManager; 035import org.syncany.plugins.transfer.TransferPlugin; 036import org.syncany.plugins.transfer.files.MultichunkRemoteFile; 037import org.syncany.plugins.transfer.files.RemoteFile; 038import org.syncany.plugins.transfer.files.RemoteFileAttributes; 039import org.syncany.util.ReflectionUtil; 040import org.syncany.util.StringUtil; 041 042import com.google.common.base.Charsets; 043import com.google.common.collect.ImmutableList; 044import com.google.common.collect.Maps; 045import com.google.common.hash.Hashing; 046 047/** 048 * The path aware transfer manager can be used to extend a backend storage 049 * with the ability to add subfolders to the folders with many files (e.g. multichunks 050 * or temporary files). This is especially critical if the backend storage has a limit 051 * on how many files can be stored in a single folder (e.g. the Dropbox plugin). 052 * 053 * <p>To enable subfoldering in {@link TransferPlugin}s, the plugin's {@link TransferManager} 054 * has to be annotated with the {@link PathAware} annotation, and a {@link PathAwareFeatureExtension} 055 * has to be provided. 056 * 057 * <p>The sub-path for a {@link RemoteFile} can then be accessed via the 058 * {@link PathAwareRemoteFileAttributes} using the {@link RemoteFile#getAttributes(Class)} method. 059 * 060 * @see PathAware 061 * @see PathAwareFeatureExtension 062 * @see PathAwareRemoteFileAttributes 063 * @author Christian Roth (christian.roth@port17.de) 064 */ 065public class PathAwareFeatureTransferManager implements FeatureTransferManager { 066 private static final Logger logger = Logger.getLogger(PathAwareFeatureTransferManager.class.getSimpleName()); 067 068 private final TransferManager underlyingTransferManager; 069 070 private final int subfolderDepth; 071 private final int bytesPerFolder; 072 private final char folderSeparator; 073 private final List<Class<? extends RemoteFile>> affectedFiles; 074 private final PathAwareFeatureExtension pathAwareFeatureExtension; 075 076 public PathAwareFeatureTransferManager(TransferManager originalTransferManager, TransferManager underlyingTransferManager, Config config, PathAware pathAwareAnnotation) { 077 this.underlyingTransferManager = underlyingTransferManager; 078 079 this.subfolderDepth = pathAwareAnnotation.subfolderDepth(); 080 this.bytesPerFolder = pathAwareAnnotation.bytesPerFolder(); 081 this.folderSeparator = pathAwareAnnotation.folderSeparator(); 082 this.affectedFiles = ImmutableList.copyOf(pathAwareAnnotation.affected()); 083 084 this.pathAwareFeatureExtension = getPathAwareFeatureExtension(originalTransferManager, pathAwareAnnotation); 085 } 086 087 @SuppressWarnings("unchecked") 088 private PathAwareFeatureExtension getPathAwareFeatureExtension(TransferManager originalTransferManager, PathAware pathAwareAnnotation) { 089 Class<? extends TransferManager> originalTransferManagerClass = originalTransferManager.getClass(); 090 Class<PathAwareFeatureExtension> pathAwareFeatureExtensionClass = (Class<PathAwareFeatureExtension>) pathAwareAnnotation.extension(); 091 092 try { 093 Constructor<?> constructor = ReflectionUtil.getMatchingConstructorForClass(pathAwareFeatureExtensionClass, originalTransferManagerClass); 094 095 if (constructor != null) { 096 return (PathAwareFeatureExtension) constructor.newInstance(originalTransferManager); 097 } 098 099 return pathAwareFeatureExtensionClass.newInstance(); 100 } 101 catch (InvocationTargetException | InstantiationException | IllegalAccessException | NullPointerException e) { 102 throw new RuntimeException("Cannot instantiate PathAwareFeatureExtension (perhaps " + pathAwareFeatureExtensionClass + " does not exist?)", e); 103 } 104 } 105 106 @Override 107 public void connect() throws StorageException { 108 underlyingTransferManager.connect(); 109 } 110 111 @Override 112 public void disconnect() throws StorageException { 113 underlyingTransferManager.disconnect(); 114 } 115 116 @Override 117 public void init(final boolean createIfRequired) throws StorageException { 118 underlyingTransferManager.init(createIfRequired); 119 } 120 121 @Override 122 public void download(final RemoteFile remoteFile, final File localFile) throws StorageException { 123 underlyingTransferManager.download(createPathAwareRemoteFile(remoteFile), localFile); 124 } 125 126 @Override 127 public void move(final RemoteFile sourceFile, final RemoteFile targetFile) throws StorageException { 128 final RemoteFile pathAwareSourceFile = createPathAwareRemoteFile(sourceFile); 129 final RemoteFile pathAwareTargetFile = createPathAwareRemoteFile(targetFile); 130 131 if (!createFolder(pathAwareTargetFile)) { 132 throw new StorageException("Unable to create path for " + pathAwareTargetFile); 133 } 134 135 underlyingTransferManager.move(pathAwareSourceFile, pathAwareTargetFile); 136 removeFolder(pathAwareSourceFile); 137 } 138 139 @Override 140 public void upload(final File localFile, final RemoteFile remoteFile) throws StorageException { 141 final RemoteFile pathAwareRemoteFile = createPathAwareRemoteFile(remoteFile); 142 143 if (!createFolder(pathAwareRemoteFile)) { 144 throw new StorageException("Unable to create path for " + pathAwareRemoteFile); 145 } 146 147 underlyingTransferManager.upload(localFile, pathAwareRemoteFile); 148 } 149 150 @Override 151 public boolean delete(final RemoteFile remoteFile) throws StorageException { 152 RemoteFile pathAwareRemoteFile = createPathAwareRemoteFile(remoteFile); 153 154 boolean fileDeleted = underlyingTransferManager.delete(pathAwareRemoteFile); 155 boolean folderDeleted = removeFolder(pathAwareRemoteFile); 156 157 return fileDeleted && folderDeleted; 158 } 159 160 @Override 161 public <T extends RemoteFile> Map<String, T> list(final Class<T> remoteFileClass) throws StorageException { 162 Map<String, T> filesInFolder = Maps.newHashMap(); 163 String remoteFilePath = getRemoteFilePath(remoteFileClass); 164 165 list(remoteFilePath, filesInFolder, remoteFileClass); 166 167 return filesInFolder; 168 } 169 170 private <T extends RemoteFile> void list(String remoteFilePath, Map<String, T> remoteFiles, Class<T> remoteFileClass) throws StorageException { 171 logger.log(Level.INFO, "Listing folder for files matching " + remoteFileClass.getSimpleName() + ": " + remoteFilePath); 172 Map<String, FileType> folderList = pathAwareFeatureExtension.listFolder(remoteFilePath); 173 174 for (Map.Entry<String, FileType> folderListEntry : folderList.entrySet()) { 175 String fileName = folderListEntry.getKey(); 176 FileType fileType = folderListEntry.getValue(); 177 178 if (fileType == FileType.FILE) { 179 try { 180 remoteFiles.put(fileName, RemoteFile.createRemoteFile(fileName, remoteFileClass)); 181 logger.log(Level.INFO, "- File: " + fileName); 182 } 183 catch (StorageException e) { 184 // We don't care and ignore non-matching files! 185 } 186 } 187 else if (fileType == FileType.FOLDER) { 188 logger.log(Level.INFO, "- Folder: " + fileName); 189 190 String newRemoteFilePath = remoteFilePath + folderSeparator + fileName; 191 list(newRemoteFilePath, remoteFiles, remoteFileClass); 192 } 193 } 194 } 195 196 @Override 197 public String getRemoteFilePath(Class<? extends RemoteFile> remoteFileClass) { 198 return underlyingTransferManager.getRemoteFilePath(remoteFileClass); 199 } 200 201 @Override 202 public StorageTestResult test(boolean testCreateTarget) { 203 return underlyingTransferManager.test(testCreateTarget); 204 } 205 206 @Override 207 public boolean testTargetExists() throws StorageException { 208 return underlyingTransferManager.testTargetExists(); 209 } 210 211 @Override 212 public boolean testTargetCanWrite() throws StorageException { 213 return underlyingTransferManager.testTargetCanWrite(); 214 } 215 216 @Override 217 public boolean testTargetCanCreate() throws StorageException { 218 return underlyingTransferManager.testTargetCanCreate(); 219 } 220 221 @Override 222 public boolean testRepoFileExists() throws StorageException { 223 return underlyingTransferManager.testRepoFileExists(); 224 } 225 226 private boolean isFolderizable(Class<? extends RemoteFile> remoteFileClass) { 227 return affectedFiles.contains(remoteFileClass); 228 } 229 230 private RemoteFile createPathAwareRemoteFile(RemoteFile remoteFile) throws StorageException { 231 PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = new PathAwareRemoteFileAttributes(); 232 remoteFile.setAttributes(pathAwareRemoteFileAttributes); 233 234 if (isFolderizable(remoteFile.getClass())) { 235 // If remote file is folderizable, i.e. an 'affected file', 236 // get the sub-path for it 237 238 String subPathId = getSubPathId(remoteFile); 239 String subPath = getSubPath(subPathId); 240 241 pathAwareRemoteFileAttributes.setPath(subPath); 242 } 243 244 return remoteFile; 245 } 246 247 /** 248 * Returns the subpath identifier for this file. For {@link MultichunkRemoteFile}s, this is the 249 * hex string of the multichunk identifier. For all other files, this is the 128-bit murmur3 hash 250 * of the full filename (fast algorithm!). 251 */ 252 private String getSubPathId(RemoteFile remoteFile) { 253 if (remoteFile.getClass() == MultichunkRemoteFile.class) { 254 return StringUtil.toHex(((MultichunkRemoteFile) remoteFile).getMultiChunkId()); 255 } 256 else { 257 return StringUtil.toHex(Hashing.murmur3_128().hashString(remoteFile.getName(), Charsets.UTF_8).asBytes()); 258 } 259 } 260 261 private String getSubPath(String fileId) { 262 StringBuilder path = new StringBuilder(); 263 264 for (int i = 0; i < subfolderDepth; i++) { 265 String subPathPart = fileId.substring(i * bytesPerFolder * 2, (i + 1) * bytesPerFolder * 2); 266 267 path.append(subPathPart); 268 path.append(folderSeparator); 269 } 270 271 return path.toString(); 272 } 273 274 private String pathToString(Path path) { 275 return path.toString().replace(File.separator, String.valueOf(folderSeparator)); 276 } 277 278 private boolean createFolder(RemoteFile remoteFile) throws StorageException { 279 PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = remoteFile.getAttributes(PathAwareRemoteFileAttributes.class); 280 boolean notAPathAwareRemoteFile = pathAwareRemoteFileAttributes == null || !pathAwareRemoteFileAttributes.hasPath(); 281 282 if (notAPathAwareRemoteFile) { 283 return true; 284 } 285 else { 286 String remoteFilePath = pathToString(Paths.get(underlyingTransferManager.getRemoteFilePath(remoteFile.getClass()), pathAwareRemoteFileAttributes.getPath())); 287 288 logger.log(Level.INFO, "Remote file is path aware, creating folder " + remoteFilePath); 289 boolean success = pathAwareFeatureExtension.createPath(remoteFilePath); 290 291 return success; 292 } 293 } 294 295 private boolean removeFolder(RemoteFile remoteFile) throws StorageException { 296 PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = remoteFile.getAttributes(PathAwareRemoteFileAttributes.class); 297 boolean notAPathAwareRemoteFile = pathAwareRemoteFileAttributes == null || !pathAwareRemoteFileAttributes.hasPath(); 298 299 if (notAPathAwareRemoteFile) { 300 return true; 301 } 302 else { 303 String remoteFilePath = pathToString(Paths.get(underlyingTransferManager.getRemoteFilePath(remoteFile.getClass()), pathAwareRemoteFileAttributes.getPath())); 304 305 logger.log(Level.INFO, "Remote file is path aware, cleaning empty folders at " + remoteFilePath); 306 boolean success = removeFolder(remoteFilePath); 307 308 return success; 309 } 310 } 311 312 private boolean removeFolder(String folder) throws StorageException { 313 for(int i = 0; i < subfolderDepth; i++) { 314 logger.log(Level.FINE, "Removing folder " + folder); 315 316 if (pathAwareFeatureExtension.listFolder(folder).size() != 0) { 317 return true; 318 } 319 320 if (!pathAwareFeatureExtension.removeFolder(folder)) { 321 return false; 322 } 323 324 folder = folder.substring(0, folder.lastIndexOf(folderSeparator)); 325 } 326 327 return true; 328 } 329 330 public static class PathAwareRemoteFileAttributes extends RemoteFileAttributes { 331 private String path; 332 333 public boolean hasPath() { 334 return path != null; 335 } 336 337 public String getPath() { 338 return path; 339 } 340 341 public void setPath(String path) { 342 this.path = path; 343 } 344 } 345 346}