Source code

001package co.codewizards.cloudstore.local;
002
003import static co.codewizards.cloudstore.core.io.StreamUtil.*;
004import static co.codewizards.cloudstore.core.objectfactory.ObjectFactoryUtil.*;
005import static java.util.Objects.*;
006
007import java.io.IOException;
008import java.io.InputStream;
009import java.security.MessageDigest;
010import java.security.NoSuchAlgorithmException;
011import java.util.ArrayList;
012import java.util.Collection;
013import java.util.Collections;
014import java.util.Date;
015import java.util.HashMap;
016import java.util.HashSet;
017import java.util.List;
018import java.util.Map;
019import java.util.Set;
020
021import javax.jdo.PersistenceManager;
022
023import org.slf4j.Logger;
024import org.slf4j.LoggerFactory;
025
026import co.codewizards.cloudstore.core.dto.FileChunkDto;
027import co.codewizards.cloudstore.core.ignore.IgnoreRuleManagerImpl;
028import co.codewizards.cloudstore.core.oio.File;
029import co.codewizards.cloudstore.core.progress.ProgressMonitor;
030import co.codewizards.cloudstore.core.progress.SubProgressMonitor;
031import co.codewizards.cloudstore.core.repo.local.LocalRepoTransaction;
032import co.codewizards.cloudstore.core.util.HashUtil;
033import co.codewizards.cloudstore.local.persistence.CopyModification;
034import co.codewizards.cloudstore.local.persistence.DeleteModification;
035import co.codewizards.cloudstore.local.persistence.DeleteModificationDao;
036import co.codewizards.cloudstore.local.persistence.Directory;
037import co.codewizards.cloudstore.local.persistence.FileChunk;
038import co.codewizards.cloudstore.local.persistence.ModificationDao;
039import co.codewizards.cloudstore.local.persistence.NormalFile;
040import co.codewizards.cloudstore.local.persistence.NormalFileDao;
041import co.codewizards.cloudstore.local.persistence.RemoteRepository;
042import co.codewizards.cloudstore.local.persistence.RemoteRepositoryDao;
043import co.codewizards.cloudstore.local.persistence.RepoFile;
044import co.codewizards.cloudstore.local.persistence.RepoFileDao;
045import co.codewizards.cloudstore.local.persistence.Symlink;
046
047public class LocalRepoSync {
048
049        private static final Logger logger = LoggerFactory.getLogger(LocalRepoSync.class);
050
051        protected final LocalRepoTransaction transaction;
052        protected final File localRoot;
053        protected final RepoFileDao repoFileDao;
054        protected final NormalFileDao normalFileDao;
055        protected final RemoteRepositoryDao remoteRepositoryDao;
056        protected final ModificationDao modificationDao;
057        protected final DeleteModificationDao deleteModificationDao;
058        private Collection<RemoteRepository> remoteRepositories;
059        private boolean ignoreRulesEnabled;
060
061        private final Map<String, Set<String>> sha1AndLength2Paths = new HashMap<String, Set<String>>();
062
063        protected LocalRepoSync(final LocalRepoTransaction transaction) {
064                this.transaction = requireNonNull(transaction, "transaction");
065                localRoot = this.transaction.getLocalRepoManager().getLocalRoot();
066                repoFileDao = this.transaction.getDao(RepoFileDao.class);
067                normalFileDao = this.transaction.getDao(NormalFileDao.class);
068                remoteRepositoryDao = this.transaction.getDao(RemoteRepositoryDao.class);
069                modificationDao = this.transaction.getDao(ModificationDao.class);
070                deleteModificationDao = this.transaction.getDao(DeleteModificationDao.class);
071        }
072
073        public static LocalRepoSync create(final LocalRepoTransaction transaction) {
074                return createObject(LocalRepoSync.class, transaction);
075        }
076
077        public void sync(final ProgressMonitor monitor) {
078                sync(null, localRoot, monitor, true);
079        }
080
081        public RepoFile sync(final File file, final ProgressMonitor monitor, final boolean recursiveChildren) {
082                if (!(requireNonNull(file, "file").isAbsolute()))
083                        throw new IllegalArgumentException("file is not absolute: " + file);
084
085                if (localRoot.equals(file)) {
086                        return sync(null, file, monitor, recursiveChildren);
087                }
088
089                monitor.beginTask("Local sync...", 100);
090                try {
091                        final File parentFile = file.getParentFile();
092                        RepoFile parentRepoFile = repoFileDao.getRepoFile(localRoot, parentFile);
093                        if (parentRepoFile == null) {
094                                // If the file does not exist and its RepoFile neither exists, then
095                                // this is in sync already and we can simply leave. This regularly
096                                // happens during the deletion of a directory which is the connection-point
097                                // of a remote-repository. The following re-up-sync then leads us here.
098                                // To speed up things, we simply quit as this is a valid state.
099                                if (!file.isSymbolicLink() && !file.exists() && repoFileDao.getRepoFile(localRoot, file) == null)
100                                        return null;
101
102                                // In the unlikely event, that this is not a valid state, we simply sync all
103                                // and return.
104                                sync(null, localRoot, new SubProgressMonitor(monitor, 99), true);
105                                final RepoFile repoFile = repoFileDao.getRepoFile(localRoot, file);
106                                if (repoFile != null) // if it still does not exist, we run into the re-sync below and this might quickly return null, if that is correct or otherwise sync what's needed.
107                                        return repoFile;
108
109                                parentRepoFile = repoFileDao.getRepoFile(localRoot, parentFile);
110                                if (parentRepoFile == null && parentFile.exists())
111                                        throw new IllegalStateException("RepoFile not found for existing file/dir: " + parentFile.getAbsolutePath());
112                        }
113
114                        monitor.worked(1);
115
116                        return sync(parentRepoFile, file, new SubProgressMonitor(monitor, 99), recursiveChildren);
117                } finally {
118                        monitor.done();
119                }
120        }
121
122        /**
123         * Sync the single given {@code file}.
124         * <p>
125         * If {@code file} is a directory, it recursively syncs all its children.
126         * @param parentRepoFile the parent. May be <code>null</code>, if the file is the repository's root-directory.
127         * For non-root files, this must not be <code>null</code>!
128         * @param file the file to be synced. Must not be <code>null</code>.
129         * @param monitor the progress-monitor. Must not be <code>null</code>.
130         * @param recursiveChildren TODO
131         * @return the {@link RepoFile} corresponding to the given {@code file}. Is <code>null</code>, if the given
132         * {@code file} does not exist; otherwise it is never <code>null</code>.
133         */
134        protected RepoFile sync(final RepoFile parentRepoFile, final File file, final ProgressMonitor monitor, final boolean recursiveChildren) {
135                requireNonNull(file, "file");
136                requireNonNull(monitor, "monitor");
137                monitor.beginTask("Local sync...", 100);
138                try {
139                        RepoFile repoFile = repoFileDao.getRepoFile(localRoot, file);
140
141                        if (parentRepoFile != null) {
142                                boolean ignored = isIgnoreRulesEnabled()
143                                                ? IgnoreRuleManagerImpl.getInstanceForDirectory(file.getParentFile()).isIgnored(file)
144                                                                : false;
145                                if (ignored) {
146                                        if (repoFile != null) {
147                                                deleteRepoFile(repoFile, false);
148                                                repoFile = null;
149                                        }
150                                        return null;
151                                }
152                        }
153
154                        // If the type changed - e.g. from normal file to directory - or if the file was deleted
155                        // we must delete the old instance.
156                        if (repoFile != null && !isRepoFileTypeCorrect(repoFile, file)) {
157                                deleteRepoFile(repoFile, false);
158                                repoFile = null;
159                        }
160
161                        final boolean fileIsSymlink = file.isSymbolicLink();
162                        if (repoFile == null) {
163                                if (!fileIsSymlink && !file.exists())
164                                        return null;
165
166                                repoFile = createRepoFile(parentRepoFile, file, new SubProgressMonitor(monitor, 50));
167                                if (repoFile == null) { // ignoring non-normal files.
168                                        return null;
169                                }
170                        } else if (isModified(repoFile, file))
171                                updateRepoFile(repoFile, file, new SubProgressMonitor(monitor, 50));
172                        else
173                                monitor.worked(50);
174
175                        final Set<String> childNames = new HashSet<String>();
176                        if (!fileIsSymlink) {
177                                final SubProgressMonitor childSubProgressMonitor = new SubProgressMonitor(monitor, 50);
178                                final File[] children = file.listFiles(new FilenameFilterSkipMetaDir());
179                                if (children != null && children.length > 0) {
180                                        childSubProgressMonitor.beginTask("Local sync...", children.length);
181                                        for (final File child : children) {
182                                                childNames.add(child.getName());
183
184                                                if (recursiveChildren)
185                                                        sync(repoFile, child, new SubProgressMonitor(childSubProgressMonitor, 1), recursiveChildren);
186                                        }
187                                }
188                                childSubProgressMonitor.done();
189                        }
190
191                        final Collection<RepoFile> childRepoFiles = repoFileDao.getChildRepoFiles(repoFile);
192                        for (final RepoFile childRepoFile : childRepoFiles) {
193                                if (!childNames.contains(childRepoFile.getName())) {
194                                        deleteRepoFile(childRepoFile);
195                                }
196                        }
197
198                        transaction.flush();
199                        return repoFile;
200                } finally {
201                        monitor.done();
202                }
203        }
204
205        /**
206         * Determines, if the type of the given {@code repoFile} matches the type
207         * of the file in the file system referenced by the given {@code file}.
208         * @param repoFile the {@link RepoFile} currently representing the given {@code file} in the database.
209         * Must not be <code>null</code>.
210         * @param file the file in the file system. Must not be <code>null</code>.
211         * @return <code>true</code>, if both types correspond to each other; <code>false</code> otherwise. If
212         * the file does not exist (anymore) in the file system, <code>false</code> is returned, too.
213         */
214        public boolean isRepoFileTypeCorrect(final RepoFile repoFile, final File file) {
215                requireNonNull(repoFile, "repoFile");
216                requireNonNull(file, "file");
217
218                if (file.isSymbolicLink())
219                        return repoFile instanceof Symlink;
220
221                if (file.isFile())
222                        return repoFile instanceof NormalFile;
223
224                if (file.isDirectory())
225                        return repoFile instanceof Directory;
226
227                return false;
228        }
229
230        public boolean isModified(final RepoFile repoFile, final File file) {
231                final long fileLastModified = file.getLastModifiedNoFollow();
232                if (repoFile.getLastModified().getTime() != fileLastModified) {
233                        if (logger.isDebugEnabled()) {
234                                logger.debug("isModified: repoFile.lastModified != file.lastModified: repoFile.lastModified={} file.lastModified={} file={}",
235                                                repoFile.getLastModified(), new Date(fileLastModified), file);
236                        }
237                        return true;
238                }
239
240                if (file.isSymbolicLink()) {
241                        if (!(repoFile instanceof Symlink))
242                                throw new IllegalArgumentException("repoFile is not an instance of Symlink! file=" + file);
243
244                        final Symlink symlink = (Symlink) repoFile;
245                        String fileSymlinkTarget;
246                        try {
247                                fileSymlinkTarget = file.readSymbolicLinkToPathString();
248                        } catch (final IOException e) {
249                                //as this is already checked as symbolicLink, this should never happen!
250                                throw new IllegalArgumentException(e);
251                        }
252                        return !fileSymlinkTarget.equals(symlink.getTarget());
253                }
254                else if (file.isFile()) {
255                        if (!(repoFile instanceof NormalFile))
256                                throw new IllegalArgumentException("repoFile is not an instance of NormalFile! file=" + file);
257
258                        final NormalFile normalFile = (NormalFile) repoFile;
259                        if (normalFile.getLength() != file.length()) {
260                                if (logger.isDebugEnabled()) {
261                                        logger.debug("isModified: normalFile.length != file.length: repoFile.length={} file.length={} file={}",
262                                                        normalFile.getLength(), file.length(), file);
263                                }
264                                return true;
265                        }
266
267                        if (normalFile.getFileChunks().isEmpty()) // TODO remove this - only needed for downward compatibility!
268                                return true;
269                }
270
271                return false;
272        }
273
274        protected RepoFile createRepoFile(final RepoFile parentRepoFile, final File file, final ProgressMonitor monitor) {
275                if (parentRepoFile == null)
276                        throw new IllegalStateException("Creating the root this way is not possible! Why is it not existing, yet?!???");
277
278                monitor.beginTask("Local sync...", 100);
279                try {
280                        final RepoFile repoFile = _createRepoFile(parentRepoFile, file, new SubProgressMonitor(monitor, 98));
281
282                        if (repoFile instanceof NormalFile)
283                                createCopyModificationsIfPossible((NormalFile)repoFile);
284
285                        monitor.worked(1);
286
287                        return repoFileDao.makePersistent(repoFile);
288                } finally {
289                        monitor.done();
290                }
291        }
292
293        protected RepoFile _createRepoFile(final RepoFile parentRepoFile, final File file, final ProgressMonitor monitor) {
294                monitor.beginTask("Local sync...", 100);
295                try {
296                        RepoFile repoFile;
297                        if (file.isSymbolicLink()) {
298                                final Symlink symlink = (Symlink) (repoFile = createObject(Symlink.class));
299                                try {
300                                        symlink.setTarget(file.readSymbolicLinkToPathString());
301                                } catch (final IOException e) {
302                                        throw new RuntimeException(e);
303                                }
304                        } else if (file.isDirectory()) {
305                                repoFile = createObject(Directory.class);
306                        } else if (file.isFile()) {
307                                final NormalFile normalFile = (NormalFile) (repoFile = createObject(NormalFile.class));
308                                sha(normalFile, file, new SubProgressMonitor(monitor, 99));
309                        } else {
310                                if (file.exists())
311                                        logger.warn("createRepoFile: File exists, but is neither a directory nor a normal file! Skipping: {}", file);
312                                else
313                                        logger.warn("createRepoFile: File does not exist! Skipping: {}", file);
314
315                                return null;
316                        }
317
318                        repoFile.setParent(parentRepoFile);
319                        repoFile.setName(file.getName());
320                        repoFile.setLastModified(new Date(file.getLastModifiedNoFollow()));
321
322                        return repoFile;
323                } finally {
324                        monitor.done();
325                }
326        }
327
328        public void updateRepoFile(final RepoFile repoFile, final File file, final ProgressMonitor monitor) {
329                logger.debug("updateRepoFile: id={} file={}", repoFile.getId(), file);
330                monitor.beginTask("Local sync...", 100);
331                try {
332                        if (file.isSymbolicLink()) {
333                                if (!(repoFile instanceof Symlink))
334                                        throw new IllegalArgumentException("repoFile is not an instance of Symlink! file=" + file);
335
336                                final Symlink symlink = (Symlink) repoFile;
337                                try {
338                                        symlink.setTarget(file.readSymbolicLinkToPathString());
339                                } catch (final IOException e) {
340                                        throw new RuntimeException(e);
341                                }
342                        }
343                        else if (file.isFile()) {
344                                if (!(repoFile instanceof NormalFile))
345                                        throw new IllegalArgumentException("repoFile is not an instance of NormalFile!");
346
347                                final NormalFile normalFile = (NormalFile) repoFile;
348                                sha(normalFile, file, new SubProgressMonitor(monitor, 100));
349                        }
350                        repoFile.setLastSyncFromRepositoryId(null);
351                        repoFile.setLastModified(new Date(file.getLastModifiedNoFollow()));
352                } finally {
353                        monitor.done();
354                }
355        }
356
357        public void deleteRepoFile(final RepoFile repoFile) {
358                deleteRepoFile(repoFile, true);
359        }
360
361        public void deleteRepoFile(final RepoFile repoFile, final boolean createDeleteModifications) {
362                final RepoFile parentRepoFile = requireNonNull(repoFile, "repoFile").getParent();
363                if (parentRepoFile == null)
364                        throw new IllegalStateException("Deleting the root is not possible!");
365
366                final PersistenceManager pm = ((co.codewizards.cloudstore.local.LocalRepoTransactionImpl)transaction).getPersistenceManager();
367
368                // We make sure, nothing interferes with our deletions (see comment below).
369                pm.flush();
370
371                if (createDeleteModifications)
372                        createDeleteModifications(repoFile);
373
374                deleteRepoFileWithAllChildrenRecursively(repoFile);
375
376                // DN batches UPDATE and DELETE statements. This sometimes causes foreign key violations and other errors in
377                // certain situations. Additionally, the deleted objects still linger in the 1st-level-cache and re-using them
378                // causes "javax.jdo.JDOUserException: Cannot read fields from a deleted object". This happens when switching
379                // from a directory to a file (or vice versa).
380                // We therefore must flush to be on the safe side. And to be extra-safe, we flush before and after deletion.
381                pm.flush();
382        }
383
384        private int getMaxCopyModificationCount(final NormalFile newNormalFile) {
385                final long fileLength = newNormalFile.getLength();
386                if (fileLength < 10 * 1024) // 10 KiB
387                        return 0;
388
389                if (fileLength < 100 * 1024) // 100 KiB
390                        return 1;
391
392                if (fileLength < 1024 * 1024) // 1 MiB
393                        return 2;
394
395                if (fileLength < 10 * 1024 * 1024) // 10 MiB
396                        return 3;
397
398                if (fileLength < 100 * 1024 * 1024) // 100 MiB
399                        return 5;
400
401                if (fileLength < 1024 * 1024 * 1024) // 1 GiB
402                        return 7;
403
404                if (fileLength < 10 * 1024 * 1024 * 1024) // 10 GiB
405                        return 9;
406
407                return 11;
408        }
409
410        protected void createCopyModificationsIfPossible(final NormalFile newNormalFile) {
411                // A CopyModification is not necessary for an empty file. And since this method is called
412                // during RepoTransport.beginPutFile(...), we easily filter out this unwanted case already.
413                // Changed to dynamic limit of CopyModifications depending on file size.
414                // The bigger the file, the more it's worth the overhead.
415                final int maxCopyModificationCount = getMaxCopyModificationCount(newNormalFile);
416                if (maxCopyModificationCount < 1)
417                        return;
418
419                final Set<String> fromPaths = new HashSet<String>();
420
421                final Set<String> paths = sha1AndLength2Paths.get(getSha1AndLength(newNormalFile.getSha1(), newNormalFile.getLength()));
422                if (paths != null) {
423                        final List<String> pathList = new ArrayList<>(paths);
424                        Collections.shuffle(pathList);
425
426                        for (final String path : pathList) {
427                                createCopyModifications(path, newNormalFile, fromPaths);
428                                if (fromPaths.size() >= maxCopyModificationCount)
429                                        return;
430                        }
431                }
432
433                final List<NormalFile> normalFiles = new ArrayList<>(normalFileDao.getNormalFilesForSha1(newNormalFile.getSha1(), newNormalFile.getLength()));
434                Collections.shuffle(normalFiles);
435                for (final NormalFile normalFile : normalFiles) {
436//                      if (normalFile.isInProgress()) // Additional check. Do we really want this? I don't think so!
437//                              continue;
438
439                        if (newNormalFile.equals(normalFile)) // should never happen, because newNormalFile is not yet persisted, but we write robust code that doesn't break easily after refactoring.
440                                continue;
441
442                        createCopyModifications(normalFile, newNormalFile, fromPaths);
443                        if (fromPaths.size() >= maxCopyModificationCount)
444                                return;
445                }
446
447                final List<DeleteModification> deleteModifications = new ArrayList<>(deleteModificationDao.getDeleteModificationsForSha1(newNormalFile.getSha1(), newNormalFile.getLength()));
448                Collections.shuffle(deleteModifications);
449                for (final DeleteModification deleteModification : deleteModifications) {
450                        createCopyModifications(deleteModification, newNormalFile, fromPaths);
451                        if (fromPaths.size() >= maxCopyModificationCount)
452                                return;
453                }
454        }
455
456        private void createCopyModifications(final DeleteModification deleteModification, final NormalFile toNormalFile, final Set<String> fromPaths) {
457                requireNonNull(deleteModification, "deleteModification");
458                requireNonNull(toNormalFile, "toNormalFile");
459                requireNonNull(fromPaths, "fromPaths");
460
461                if (deleteModification.getLength() != toNormalFile.getLength())
462                        throw new IllegalArgumentException("fromNormalFile.length != toNormalFile.length");
463
464                if (!deleteModification.getSha1().equals(toNormalFile.getSha1()))
465                        throw new IllegalArgumentException("fromNormalFile.sha1 != toNormalFile.sha1");
466
467                createCopyModifications(deleteModification.getPath(), toNormalFile, fromPaths);
468        }
469
470        private void createCopyModifications(final String fromPath, final NormalFile toNormalFile, final Set<String> fromPaths) {
471                requireNonNull(fromPath, "fromPath");
472                requireNonNull(toNormalFile, "toNormalFile");
473                requireNonNull(fromPaths, "fromPaths");
474
475                if (!fromPaths.add(fromPath)) // already done before => prevent duplicates.
476                        return;
477
478                for (final RemoteRepository remoteRepository : getRemoteRepositories()) {
479                        final CopyModification modification = new CopyModification();
480                        modification.setRemoteRepository(remoteRepository);
481                        modification.setFromPath(fromPath);
482                        modification.setToPath(toNormalFile.getPath());
483                        modification.setLength(toNormalFile.getLength());
484                        modification.setSha1(toNormalFile.getSha1());
485                        modificationDao.makePersistent(modification);
486                }
487        }
488
489        private void createCopyModifications(final NormalFile fromNormalFile, final NormalFile toNormalFile, final Set<String> fromPaths) {
490                requireNonNull(fromNormalFile, "fromNormalFile");
491                requireNonNull(toNormalFile, "toNormalFile");
492                requireNonNull(fromPaths, "fromPaths");
493
494                if (fromNormalFile.getLength() != toNormalFile.getLength())
495                        throw new IllegalArgumentException("fromNormalFile.length != toNormalFile.length");
496
497                if (!fromNormalFile.getSha1().equals(toNormalFile.getSha1()))
498                        throw new IllegalArgumentException("fromNormalFile.sha1 != toNormalFile.sha1");
499
500                createCopyModifications(fromNormalFile.getPath(), toNormalFile, fromPaths);
501        }
502
503        protected void createDeleteModifications(final RepoFile repoFile) {
504                requireNonNull(repoFile, "repoFile");
505
506                for (final RemoteRepository remoteRepository : getRemoteRepositories())
507                        createDeleteModification(repoFile, remoteRepository);
508        }
509
510        protected DeleteModification createDeleteModification(final RepoFile repoFile, final RemoteRepository remoteRepository) {
511                requireNonNull(repoFile, "repoFile");
512                requireNonNull(remoteRepository, "remoteRepository");
513                final DeleteModification modification = createObject(DeleteModification.class);
514                populateDeleteModification(modification, repoFile, remoteRepository);
515                return modificationDao.makePersistent(modification);
516        }
517
518        protected void populateDeleteModification(final DeleteModification modification, final RepoFile repoFile, final RemoteRepository remoteRepository) {
519                requireNonNull(modification, "modification");
520                requireNonNull(repoFile, "repoFile");
521                requireNonNull(remoteRepository, "remoteRepository");
522
523                final NormalFile normalFile = (repoFile instanceof NormalFile) ? (NormalFile) repoFile : null;
524
525                modification.setRemoteRepository(remoteRepository);
526                modification.setPath(repoFile.getPath());
527                modification.setLength(normalFile == null ? -1 : normalFile.getLength());
528                modification.setSha1(normalFile == null ? null : normalFile.getSha1());
529        }
530
531        private Collection<RemoteRepository> getRemoteRepositories() {
532                if (remoteRepositories == null)
533                        remoteRepositories = Collections.unmodifiableCollection(remoteRepositoryDao.getObjects());
534
535                return remoteRepositories;
536        }
537
538        protected void deleteRepoFileWithAllChildrenRecursively(final RepoFile repoFile) {
539                requireNonNull(repoFile, "repoFile");
540                for (final RepoFile childRepoFile : repoFileDao.getChildRepoFiles(repoFile)) {
541                        deleteRepoFileWithAllChildrenRecursively(childRepoFile);
542                }
543                putIntoSha1AndLength2PathsIfNormalFile(repoFile);
544                repoFileDao.deletePersistent(repoFile);
545        }
546
547        protected void putIntoSha1AndLength2PathsIfNormalFile(final RepoFile repoFile) {
548                if (repoFile instanceof NormalFile) {
549                        final NormalFile normalFile = (NormalFile) repoFile;
550                        final String sha1AndLength = getSha1AndLength(normalFile.getSha1(), normalFile.getLength());
551                        Set<String> paths = sha1AndLength2Paths.get(sha1AndLength);
552                        if (paths == null) {
553                                paths = new HashSet<>(1);
554                                sha1AndLength2Paths.put(sha1AndLength, paths);
555                        }
556                        paths.add(normalFile.getPath());
557                }
558        }
559
560        private String getSha1AndLength(final String sha1, final long length) {
561                return sha1 + ':' + length;
562        }
563
564        protected void sha(final NormalFile normalFile, final File file, final ProgressMonitor monitor) {
565                monitor.beginTask("Local sync...", (int)Math.min(file.length(), Integer.MAX_VALUE));
566                try {
567                        normalFile.getFileChunks().clear();
568                        transaction.flush();
569
570                        final MessageDigest mdAll = MessageDigest.getInstance(HashUtil.HASH_ALGORITHM_SHA);
571                        final MessageDigest mdChunk = MessageDigest.getInstance(HashUtil.HASH_ALGORITHM_SHA);
572
573                        final int bufLength = 32 * 1024;
574
575                        long offset = 0;
576
577                        try (final InputStream in = castStream(file.createInputStream())) {
578                                FileChunk fileChunk = null;
579
580                                final byte[] buf = new byte[bufLength];
581                                while (true) {
582                                        if (fileChunk == null) {
583                                                fileChunk = createObject(FileChunk.class);
584                                                fileChunk.setNormalFile(normalFile);
585                                                fileChunk.setOffset(offset);
586                                                fileChunk.setLength(0);
587                                                mdChunk.reset();
588                                        }
589
590                                        final int bytesRead = in.read(buf, 0, buf.length);
591
592                                        if (bytesRead > 0) {
593                                                mdAll.update(buf, 0, bytesRead);
594                                                mdChunk.update(buf, 0, bytesRead);
595                                                offset += bytesRead;
596                                                fileChunk.setLength(fileChunk.getLength() + bytesRead);
597                                        }
598
599                                        if (bytesRead < 0 || fileChunk.getLength() >= FileChunkDto.MAX_LENGTH) {
600                                                fileChunk.setSha1(HashUtil.encodeHexStr(mdChunk.digest()));
601                                                onFinalizeFileChunk(fileChunk);
602                                                fileChunk.makeReadOnly();
603                                                normalFile.getFileChunks().add(fileChunk);
604                                                fileChunk = null;
605
606                                                if (bytesRead < 0) {
607                                                        break;
608                                                }
609                                        }
610
611                                        if (bytesRead > 0)
612                                                monitor.worked(bytesRead);
613                                }
614                        }
615                        normalFile.setSha1(HashUtil.encodeHexStr(mdAll.digest()));
616                        normalFile.setLength(offset);
617
618                        final long fileLength = file.length(); // Important to check it now at the end.
619                        if (fileLength != offset) {
620                                logger.warn("sha: file.length() != bytesReadTotal :: File seems to be written concurrently! file='{}' file.length={} bytesReadTotal={}",
621                                                file, fileLength, offset);
622                        }
623                } catch (final NoSuchAlgorithmException e) {
624                        throw new RuntimeException(e);
625                } catch (final IOException e) {
626                        throw new RuntimeException(e);
627                } finally {
628                        monitor.done();
629                }
630        }
631
632        protected void onFinalizeFileChunk(FileChunk fileChunk) {
633                // can be extended by sub-classes to handle FileChunk-subclasses specifically.
634        }
635
636        public boolean isIgnoreRulesEnabled() {
637                return ignoreRulesEnabled;
638        }
639        public void setIgnoreRulesEnabled(boolean ignoreRulesEnabled) {
640                this.ignoreRulesEnabled = ignoreRulesEnabled;
641        }
642        public LocalRepoSync ignoreRulesEnabled(boolean ignoreRulesEnabled) {
643                setIgnoreRulesEnabled(ignoreRulesEnabled);
644                return this;
645        }
646}