001/*
002 * Syncany, www.syncany.org
003 * Copyright (C) 2011-2016 Philipp C. Heckel <philipp.heckel@gmail.com>
004 *
005 * This program is free software: you can redistribute it and/or modify
006 * it under the terms of the GNU General Public License as published by
007 * the Free Software Foundation, either version 3 of the License, or
008 * (at your option) any later version.
009 *
010 * This program is distributed in the hope that it will be useful,
011 * but WITHOUT ANY WARRANTY; without even the implied warranty of
012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
013 * GNU General Public License for more details.
014 *
015 * You should have received a copy of the GNU General Public License
016 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
017 */
018package org.syncany.plugins.transfer.features;
019
020import java.io.File;
021import java.lang.reflect.Constructor;
022import java.lang.reflect.InvocationTargetException;
023import java.nio.file.Path;
024import java.nio.file.Paths;
025import java.util.List;
026import java.util.Map;
027import java.util.logging.Level;
028import java.util.logging.Logger;
029
030import org.syncany.config.Config;
031import org.syncany.plugins.transfer.FileType;
032import org.syncany.plugins.transfer.StorageException;
033import org.syncany.plugins.transfer.StorageTestResult;
034import org.syncany.plugins.transfer.TransferManager;
035import org.syncany.plugins.transfer.TransferPlugin;
036import org.syncany.plugins.transfer.files.MultichunkRemoteFile;
037import org.syncany.plugins.transfer.files.RemoteFile;
038import org.syncany.plugins.transfer.files.RemoteFileAttributes;
039import org.syncany.util.ReflectionUtil;
040import org.syncany.util.StringUtil;
041
042import com.google.common.base.Charsets;
043import com.google.common.collect.ImmutableList;
044import com.google.common.collect.Maps;
045import com.google.common.hash.Hashing;
046
047/**
048 * The path aware transfer manager can be used to extend a backend storage
049 * with the ability to add subfolders to the folders with many files (e.g. multichunks
050 * or temporary files). This is especially critical if the backend storage has a limit
051 * on how many files can be stored in a single folder (e.g. the Dropbox plugin). 
052 * 
053 * <p>To enable subfoldering in {@link TransferPlugin}s, the plugin's {@link TransferManager}
054 * has to be annotated with the {@link PathAware} annotation, and a {@link PathAwareFeatureExtension}
055 * has to be provided.
056 * 
057 * <p>The sub-path for a {@link RemoteFile} can then be accessed via the
058 * {@link PathAwareRemoteFileAttributes} using the {@link RemoteFile#getAttributes(Class)} method.
059 * 
060 * @see PathAware
061 * @see PathAwareFeatureExtension
062 * @see PathAwareRemoteFileAttributes
063 * @author Christian Roth (christian.roth@port17.de)
064 */
065public class PathAwareFeatureTransferManager implements FeatureTransferManager {
066        private static final Logger logger = Logger.getLogger(PathAwareFeatureTransferManager.class.getSimpleName());
067
068        private final TransferManager underlyingTransferManager;
069        
070        private final int subfolderDepth;
071        private final int bytesPerFolder;
072        private final char folderSeparator;
073        private final List<Class<? extends RemoteFile>> affectedFiles;
074        private final PathAwareFeatureExtension pathAwareFeatureExtension;
075
076        public PathAwareFeatureTransferManager(TransferManager originalTransferManager, TransferManager underlyingTransferManager, Config config, PathAware pathAwareAnnotation) {
077                this.underlyingTransferManager = underlyingTransferManager;
078
079                this.subfolderDepth = pathAwareAnnotation.subfolderDepth();
080                this.bytesPerFolder = pathAwareAnnotation.bytesPerFolder();
081                this.folderSeparator = pathAwareAnnotation.folderSeparator();
082                this.affectedFiles = ImmutableList.copyOf(pathAwareAnnotation.affected());
083
084                this.pathAwareFeatureExtension = getPathAwareFeatureExtension(originalTransferManager, pathAwareAnnotation);
085        }
086
087        @SuppressWarnings("unchecked")
088        private PathAwareFeatureExtension getPathAwareFeatureExtension(TransferManager originalTransferManager, PathAware pathAwareAnnotation) {
089                Class<? extends TransferManager> originalTransferManagerClass = originalTransferManager.getClass();
090                Class<PathAwareFeatureExtension> pathAwareFeatureExtensionClass = (Class<PathAwareFeatureExtension>) pathAwareAnnotation.extension();
091
092                try {
093                        Constructor<?> constructor = ReflectionUtil.getMatchingConstructorForClass(pathAwareFeatureExtensionClass, originalTransferManagerClass);
094
095                        if (constructor != null) {
096                                return (PathAwareFeatureExtension) constructor.newInstance(originalTransferManager);
097                        }
098
099                        return pathAwareFeatureExtensionClass.newInstance();
100                }
101                catch (InvocationTargetException | InstantiationException | IllegalAccessException | NullPointerException e) {
102                        throw new RuntimeException("Cannot instantiate PathAwareFeatureExtension (perhaps " + pathAwareFeatureExtensionClass + " does not exist?)", e);
103                }
104        }
105
106        @Override
107        public void connect() throws StorageException {
108                underlyingTransferManager.connect();
109        }
110
111        @Override
112        public void disconnect() throws StorageException {
113                underlyingTransferManager.disconnect();
114        }
115
116        @Override
117        public void init(final boolean createIfRequired) throws StorageException {
118                underlyingTransferManager.init(createIfRequired);
119        }
120
121        @Override
122        public void download(final RemoteFile remoteFile, final File localFile) throws StorageException {
123                underlyingTransferManager.download(createPathAwareRemoteFile(remoteFile), localFile);
124        }
125
126        @Override
127        public void move(final RemoteFile sourceFile, final RemoteFile targetFile) throws StorageException {
128                final RemoteFile pathAwareSourceFile = createPathAwareRemoteFile(sourceFile);
129                final RemoteFile pathAwareTargetFile = createPathAwareRemoteFile(targetFile);
130
131                if (!createFolder(pathAwareTargetFile)) {
132                        throw new StorageException("Unable to create path for " + pathAwareTargetFile);
133                }
134
135                underlyingTransferManager.move(pathAwareSourceFile, pathAwareTargetFile);
136                removeFolder(pathAwareSourceFile);
137        }
138
139        @Override
140        public void upload(final File localFile, final RemoteFile remoteFile) throws StorageException {
141                final RemoteFile pathAwareRemoteFile = createPathAwareRemoteFile(remoteFile);
142
143                if (!createFolder(pathAwareRemoteFile)) {
144                        throw new StorageException("Unable to create path for " + pathAwareRemoteFile);
145                }
146
147                underlyingTransferManager.upload(localFile, pathAwareRemoteFile);
148        }
149
150        @Override
151        public boolean delete(final RemoteFile remoteFile) throws StorageException {
152                RemoteFile pathAwareRemoteFile = createPathAwareRemoteFile(remoteFile);
153                
154                boolean fileDeleted = underlyingTransferManager.delete(pathAwareRemoteFile);
155                boolean folderDeleted = removeFolder(pathAwareRemoteFile);
156
157                return fileDeleted && folderDeleted;
158        }
159
160        @Override
161        public <T extends RemoteFile> Map<String, T> list(final Class<T> remoteFileClass) throws StorageException {
162                Map<String, T> filesInFolder = Maps.newHashMap();
163                String remoteFilePath = getRemoteFilePath(remoteFileClass);
164
165                list(remoteFilePath, filesInFolder, remoteFileClass);
166
167                return filesInFolder;
168        }
169
170        private <T extends RemoteFile> void list(String remoteFilePath, Map<String, T> remoteFiles, Class<T> remoteFileClass) throws StorageException {
171                logger.log(Level.INFO, "Listing folder for files matching " + remoteFileClass.getSimpleName() + ": " + remoteFilePath);
172                Map<String, FileType> folderList = pathAwareFeatureExtension.listFolder(remoteFilePath);
173                
174                for (Map.Entry<String, FileType> folderListEntry : folderList.entrySet()) {
175                        String fileName = folderListEntry.getKey();
176                        FileType fileType = folderListEntry.getValue();
177                        
178                        if (fileType == FileType.FILE) {
179                                try {
180                                        remoteFiles.put(fileName, RemoteFile.createRemoteFile(fileName, remoteFileClass));
181                                        logger.log(Level.INFO, "- File: " + fileName);                                  
182                                }
183                                catch (StorageException e) {
184                                        // We don't care and ignore non-matching files!
185                                }
186                        }
187                        else if (fileType == FileType.FOLDER) {
188                                logger.log(Level.INFO, "- Folder: " + fileName);
189
190                                String newRemoteFilePath = remoteFilePath + folderSeparator + fileName;
191                                list(newRemoteFilePath, remoteFiles, remoteFileClass);
192                        }
193                }
194        }
195
196        @Override
197        public String getRemoteFilePath(Class<? extends RemoteFile> remoteFileClass) {
198                return underlyingTransferManager.getRemoteFilePath(remoteFileClass);
199        }
200
201        @Override
202        public StorageTestResult test(boolean testCreateTarget) {
203                return underlyingTransferManager.test(testCreateTarget);
204        }
205
206        @Override
207        public boolean testTargetExists() throws StorageException {
208                return underlyingTransferManager.testTargetExists();
209        }
210
211        @Override
212        public boolean testTargetCanWrite() throws StorageException {
213                return underlyingTransferManager.testTargetCanWrite();
214        }
215
216        @Override
217        public boolean testTargetCanCreate() throws StorageException {
218                return underlyingTransferManager.testTargetCanCreate();
219        }
220
221        @Override
222        public boolean testRepoFileExists() throws StorageException {
223                return underlyingTransferManager.testRepoFileExists();
224        }
225
226        private boolean isFolderizable(Class<? extends RemoteFile> remoteFileClass) {
227                return affectedFiles.contains(remoteFileClass);
228        }
229
230        private RemoteFile createPathAwareRemoteFile(RemoteFile remoteFile) throws StorageException {           
231                PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = new PathAwareRemoteFileAttributes();
232                remoteFile.setAttributes(pathAwareRemoteFileAttributes);
233
234                if (isFolderizable(remoteFile.getClass())) {
235                        // If remote file is folderizable, i.e. an 'affected file', 
236                        // get the sub-path for it
237                        
238                        String subPathId = getSubPathId(remoteFile);    
239                        String subPath = getSubPath(subPathId);
240                        
241                        pathAwareRemoteFileAttributes.setPath(subPath);
242                }
243                
244                return remoteFile;
245        }
246
247        /**
248         * Returns the subpath identifier for this file. For {@link MultichunkRemoteFile}s, this is the
249         * hex string of the multichunk identifier. For all other files, this is the 128-bit murmur3 hash
250         * of the full filename (fast algorithm!).
251         */
252        private String getSubPathId(RemoteFile remoteFile) {
253                if (remoteFile.getClass() == MultichunkRemoteFile.class) {
254                        return StringUtil.toHex(((MultichunkRemoteFile) remoteFile).getMultiChunkId());
255                }
256                else {
257                        return StringUtil.toHex(Hashing.murmur3_128().hashString(remoteFile.getName(), Charsets.UTF_8).asBytes());
258                }
259        }
260
261        private String getSubPath(String fileId) {
262                StringBuilder path = new StringBuilder();
263
264                for (int i = 0; i < subfolderDepth; i++) {
265                        String subPathPart = fileId.substring(i * bytesPerFolder * 2, (i + 1) * bytesPerFolder * 2); 
266                        
267                        path.append(subPathPart);
268                        path.append(folderSeparator);
269                }
270                
271                return path.toString();
272        }
273
274        private String pathToString(Path path) {
275                return path.toString().replace(File.separator, String.valueOf(folderSeparator));
276        }
277
278        private boolean createFolder(RemoteFile remoteFile) throws StorageException {
279                PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = remoteFile.getAttributes(PathAwareRemoteFileAttributes.class);            
280                boolean notAPathAwareRemoteFile = pathAwareRemoteFileAttributes == null || !pathAwareRemoteFileAttributes.hasPath();
281                
282                if (notAPathAwareRemoteFile) {
283                        return true;
284                } 
285                else {
286                        String remoteFilePath = pathToString(Paths.get(underlyingTransferManager.getRemoteFilePath(remoteFile.getClass()), pathAwareRemoteFileAttributes.getPath()));
287        
288                        logger.log(Level.INFO, "Remote file is path aware, creating folder " + remoteFilePath);
289                        boolean success = pathAwareFeatureExtension.createPath(remoteFilePath);
290                        
291                        return success;
292                }
293        }
294
295        private boolean removeFolder(RemoteFile remoteFile) throws StorageException {
296                PathAwareRemoteFileAttributes pathAwareRemoteFileAttributes = remoteFile.getAttributes(PathAwareRemoteFileAttributes.class);
297                boolean notAPathAwareRemoteFile = pathAwareRemoteFileAttributes == null || !pathAwareRemoteFileAttributes.hasPath();
298                
299                if (notAPathAwareRemoteFile) {
300                        return true;
301                }
302                else {
303                        String remoteFilePath = pathToString(Paths.get(underlyingTransferManager.getRemoteFilePath(remoteFile.getClass()), pathAwareRemoteFileAttributes.getPath()));
304
305                        logger.log(Level.INFO, "Remote file is path aware, cleaning empty folders at " + remoteFilePath);
306                        boolean success = removeFolder(remoteFilePath);
307                
308                        return success;
309                }
310        }
311
312        private boolean removeFolder(String folder) throws StorageException {
313                for(int i = 0; i < subfolderDepth; i++) {
314                        logger.log(Level.FINE, "Removing folder " + folder);
315
316                        if (pathAwareFeatureExtension.listFolder(folder).size() != 0) {
317                                return true;
318                        }
319
320                        if (!pathAwareFeatureExtension.removeFolder(folder)) {
321                                return false;
322                        }
323
324                        folder = folder.substring(0, folder.lastIndexOf(folderSeparator));
325                }
326
327                return true;
328        }
329
330        public static class PathAwareRemoteFileAttributes extends RemoteFileAttributes {
331                private String path;
332
333                public boolean hasPath() {
334                        return path != null;
335                }
336
337                public String getPath() {
338                        return path;
339                }
340
341                public void setPath(String path) {
342                        this.path = path;
343                }
344        }
345
346}