diff options
Diffstat (limited to 'libs/io/src/main/java/org')
3 files changed, 221 insertions, 7 deletions
diff --git a/libs/io/src/main/java/org/the_jk/cleversync/io/Hasher.kt b/libs/io/src/main/java/org/the_jk/cleversync/io/Hasher.kt new file mode 100644 index 0000000..5d48a98 --- /dev/null +++ b/libs/io/src/main/java/org/the_jk/cleversync/io/Hasher.kt @@ -0,0 +1,113 @@ +package org.the_jk.cleversync.io + +import java.io.FilterOutputStream +import java.io.IOException +import java.io.InputStream +import java.io.OutputStream +import java.lang.ref.WeakReference +import java.security.MessageDigest + +object Hasher { + val DEFAULT = Format.SHA256 + + fun hash(input: InputStream, format: Format = DEFAULT): String? { + val cache = digestCache.get()!! + val md = cache.acquire(format) + try { + val buffer = ByteArray(BUFFER_SIZE) + while (true) { + val got = input.read(buffer) + if (got == -1) break + md.update(buffer, 0, got) + } + return formatHash(md.digest()) + } catch (_: IOException) { + return null + } finally { + cache.release(format, md) + } + } + + fun wrap(outputStream: OutputStream, format: Format = DEFAULT): HashOutputStream { + return HashOutputStream(outputStream, format) + } + + @Suppress("MagicNumber") + fun getFormat(hash: String): Format { + return when (hash.length) { + 40 -> Format.SHA160 + 56 -> Format.SHA224 + 64 -> Format.SHA256 + 96 -> Format.SHA384 + 128 -> Format.SHA512 + else -> throw IllegalArgumentException("Unknown hash length: ${hash.length}") + } + } + + enum class Format(val algorithm: String) { + SHA160("SHA-1"), // Supported API 1+ + SHA224("SHA-224"), // Supported API 22+ + SHA256("SHA-256"), // Supported API 1+ + SHA384("SHA-384"), // Supported API 1+ + SHA512("SHA-512"), // Supported API 1+ + } + + class HashOutputStream(wrapped: OutputStream, private val format: Format) : FilterOutputStream(wrapped) { + private val cache = digestCache.get()!! + private var digest: MessageDigest? = cache.acquire(format) + private var hash: ByteArray? = null + + override fun write(b: Int) { + super.write(b) + + digest!!.update(b.toByte()) + } + + override fun write(b: ByteArray?, off: Int, len: Int) { + out.write(b, off, len) + if (len > 0) { + digest!!.update(b!!, off, len) + } + } + + override fun close() { + super.close() + + if (digest == null) return + hash = digest!!.digest() + cache.release(format, digest!!) + digest = null + } + + fun hash(): String { + return formatHash(hash ?: digest!!.digest()) + } + } + + private const val BUFFER_SIZE = 65536 + + private val digestCache = ThreadLocal.withInitial { DigestCache() } + + @OptIn(ExperimentalStdlibApi::class) + private fun formatHash(data: ByteArray): String { + return data.toHexString() + } + + private class DigestCache { + private val cache = mutableMapOf<Format, WeakReference<MessageDigest>>() + + fun acquire(format: Format): MessageDigest { + val ref = cache.remove(format) + if (ref != null) { + val md = ref.get() + if (md != null) return md + } + return MessageDigest.getInstance(format.algorithm) + } + + fun release(format: Format, digest: MessageDigest) { + digest.reset() + cache[format] = WeakReference(digest) + } + } +} diff --git a/libs/io/src/main/java/org/the_jk/cleversync/io/Modifier.kt b/libs/io/src/main/java/org/the_jk/cleversync/io/Modifier.kt index 84881cd..f9235e4 100644 --- a/libs/io/src/main/java/org/the_jk/cleversync/io/Modifier.kt +++ b/libs/io/src/main/java/org/the_jk/cleversync/io/Modifier.kt @@ -4,9 +4,14 @@ import org.the_jk.cleversync.PathUtils import java.io.IOException object Modifier { - fun apply(target: ModifiableDirectory, source: Directory, actions: List<Action>): List<String> { + fun apply( + target: ModifiableDirectory, + source: Directory, + actions: List<Action>, + memory: Verifier.Memory? = null, + ): List<String> { val errors = mutableListOf<String>() - apply(target, source, actions, "", errors) + apply(target, source, actions, memory, "", errors) return errors } @@ -14,6 +19,7 @@ object Modifier { target: ModifiableDirectory, source: Directory, actions: List<Action>, + memory: Verifier.Memory? = null, path: String, errors: MutableList<String>, ) { @@ -47,6 +53,7 @@ object Modifier { targetDir, sourceDir, action.actions, + memory, newPath, errors ) @@ -70,12 +77,24 @@ object Modifier { }?.let { targetFile -> try { val buffer = ByteArray(BUFFER_SIZE) - targetFile.write().use { output -> - sourceFile.read().use { input -> - val got = input.read(buffer) - output.write(buffer, 0, got) + if (memory != null) { + val stream = Hasher.wrap(targetFile.write()) + stream.use { output -> + sourceFile.read().use { input -> + val got = input.read(buffer) + output.write(buffer, 0, got) + } + output.flush() + } + memory.put("$path/${action.name}", stream.hash()) + } else { + targetFile.write().use { output -> + sourceFile.read().use { input -> + val got = input.read(buffer) + output.write(buffer, 0, got) + } + output.flush() } - output.flush() } } catch (e: IOException) { errors.add("$path/${action.name}: Error writing file: ${e.message}") diff --git a/libs/io/src/main/java/org/the_jk/cleversync/io/Verifier.kt b/libs/io/src/main/java/org/the_jk/cleversync/io/Verifier.kt new file mode 100644 index 0000000..d1d85b9 --- /dev/null +++ b/libs/io/src/main/java/org/the_jk/cleversync/io/Verifier.kt @@ -0,0 +1,82 @@ +package org.the_jk.cleversync.io + +object Verifier { + fun calculate(target: Directory, source: Directory, memory: Memory): List<Action> { + return visit(target, source, Context(memory), "") + } + + interface Memory { + fun get(path: String): String? + fun put(path: String, hash: String) + } + + private fun visit(target: Directory, source: Directory, context: Context, path: String): List<Action> { + val targetContent = target.list() + return buildList { + for (targetDir in targetContent.directories) { + val sourceDir = source.openDir(targetDir.name) + if (sourceDir != null) { + val actions = visit(targetDir, sourceDir, context, "$path/${targetDir.name}") + if (actions.isNotEmpty()) { + add( + Action.ChangeDir( + name = targetDir.name, + actions = actions, + create = false, + ), + ) + } + } + } + for (targetFile in targetContent.files) { + val sourceFile = source.openFile(targetFile.name) + if (sourceFile != null) { + val storedHash = context.memory.get("$path/${targetFile.name}") + if (storedHash == null) { + // Can happen if hash setting has changed since the + // merge that copied the file. + val targetHash = hash(targetFile, Hasher.DEFAULT) + val sourceHash = hash(sourceFile, Hasher.DEFAULT) + if (sourceHash != null) { + context.memory.put("$path/${targetFile.name}", sourceHash) + if (sourceHash != targetHash) { + add( + Action.Copy( + name = targetFile.name, + overwrite = true, + ) + ) + } + } else { + // No hash stored and unable to read source file, + // really nothing we can do. + } + } else { + val targetHash = hash(targetFile, Hasher.getFormat(storedHash)) + if (storedHash != targetHash) { + add( + Action.Copy( + name = targetFile.name, + overwrite = true, + ) + ) + } else { + // Hashes match, all is good. + } + } + } + } + // Ignoring links as they can only link to other files inside + // the same root or, if they point outside, we can't find a "source" + // so let them be. + } + } + + private fun hash(file: File, format: Hasher.Format): String? { + return file.read().use { Hasher.hash(it, format) } + } + + private data class Context( + val memory: Memory, + ) +} |